r10k 3.3.2 → 3.5.1

data/lib/r10k/action/deploy/environment.rb

@@ -140,14 +140,14 @@ module R10K
         end
 
         def visit_module(mod)
-          logger.info _("Deploying Puppetfile content %{mod_path}") % {mod_path: mod.path}
+          logger.info _("Deploying %{origin} content %{path}") % {origin: mod.origin, path: mod.path}
           mod.sync(force: @force)
         end
 
         def write_environment_info!(environment, started_at, success)
           module_deploys = []
           begin
-            environment.puppetfile.modules.each do |mod|
+            environment.modules.each do |mod|
               name = mod.name
               version = mod.version
               sha = mod.repo.head rescue nil
@@ -157,7 +157,10 @@ module R10K
             logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}")
           end
 
-          File.open("#{environment.path}/.r10k-deploy.json", 'w') do |f|
+          # make this file write as atomic as possible in pure ruby
+          final   = "#{environment.path}/.r10k-deploy.json"
+          staging = "#{environment.path}/.r10k-deploy.json~"
+          File.open(staging, 'w') do |f|
             deploy_info = environment.info.merge({
               :started_at => started_at,
               :finished_at => Time.new,
@@ -167,6 +170,7 @@ module R10K
 
             f.puts(JSON.pretty_generate(deploy_info))
           end
+          FileUtils.mv(staging, final)
         end
 
         def undeployable_environment_names(environments, expected_names)

data/lib/r10k/action/deploy/module.rb

@@ -72,7 +72,11 @@ module R10K
         end
 
         def allowed_initialize_opts
-          super.merge(environment: true, 'no-force': :self, 'generate-types': :self, 'puppet-path': :self)
+          super.merge(environment: true,
+                      cachedir: :self,
+                      'no-force': :self,
+                      'generate-types': :self,
+                      'puppet-path': :self)
         end
       end
     end

data/lib/r10k/action/runner.rb

@@ -43,10 +43,10 @@ module R10K
         config_settings = settings_from_config(@opts[:config])
 
         overrides = {}
-        overrides[:cachedir] = @opts[:cachedir] unless @opts[:cachedir].nil?
-        overrides[:deploy] = {} if @opts[:'puppet-path'] || @opts[:'generate-types']
-        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] unless @opts[:'puppet-path'].nil?
-        overrides[:deploy][:generate_types] = @opts[:'generate-types'] unless @opts[:'generate-types'].nil?
+        overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
+        overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
+        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+        overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash

data/lib/r10k/cli/deploy.rb

@@ -18,7 +18,7 @@ module R10K::CLI
 
         description <<-DESCRIPTION
 `r10k deploy` implements the Git branch to Puppet environment workflow
-(https://puppetlabs.com/blog/git-workflow-and-puppet-environments/).
+(https://puppet.com/docs/puppet/latest/environments_about.html).
         DESCRIPTION
 
         required nil, :cachedir, 'Specify a cachedir, overriding the value in config'

data/lib/r10k/environment.rb

@@ -1,6 +1,36 @@
 module R10K
   module Environment
+    def self.factory
+      @factory ||= R10K::KeyedFactory.new
+    end
+
+    def self.register(key, klass)
+      factory.register(key, klass)
+    end
+
+    def self.retrieve(key)
+      factory.retrieve(key)
+    end
+
+    def self.generate(type, name, basedir, dirname, options = {})
+      factory.generate(type, name, basedir, dirname, options)
+    end
+
+    def self.from_hash(name, hash)
+      R10K::Util::SymbolizeKeys.symbolize_keys!(hash)
+
+      basedir = hash.delete(:basedir)
+      dirname = hash.delete(:dirname) || name
+
+      type = hash.delete(:type)
+      type = type.is_a?(String) ? type.to_sym : type
+
+      generate(type, name, basedir, dirname, hash)
+    end
+
     require 'r10k/environment/base'
+    require 'r10k/environment/with_modules'
+    require 'r10k/environment/bare'
     require 'r10k/environment/git'
     require 'r10k/environment/svn'
   end

data/lib/r10k/environment/bare.rb

@@ -0,0 +1,16 @@
+class R10K::Environment::Bare < R10K::Environment::WithModules
+
+  R10K::Environment.register(:bare, self)
+
+  def sync
+    path.mkpath
+  end
+
+  def status
+    :not_applicable
+  end
+
+  def signature
+    'bare-default'
+  end
+end

data/lib/r10k/environment/git.rb

@@ -6,10 +6,14 @@ require 'forwardable'
 # This class implements an environment based on a Git branch.
 #
 # @since 1.3.0
-class R10K::Environment::Git < R10K::Environment::Base
+class R10K::Environment::Git < R10K::Environment::WithModules
 
   include R10K::Logging
 
+  R10K::Environment.register(:git, self)
+  # Register git as the default environment type
+  R10K::Environment.register(nil, self)
+
   # @!attribute [r] remote
   #   @return [String] The URL to the remote git repository
   attr_reader :remote
@@ -66,15 +70,12 @@ class R10K::Environment::Git < R10K::Environment::Base
 
   include R10K::Util::Purgeable
 
-  def managed_directories
-    [@full_path]
-  end
-
   # Returns an array of the full paths to all the content being managed.
   # @note This implements a required method for the Purgeable mixin
   # @return [Array<String>]
   def desired_contents
     desired = [File.join(@full_path, '.git')]
     desired += @repo.tracked_paths.map { |entry| File.join(@full_path, entry) }
+    desired += super
   end
 end

data/lib/r10k/environment/svn.rb

@@ -9,6 +9,8 @@ class R10K::Environment::SVN < R10K::Environment::Base
 
   include R10K::Logging
 
+  R10K::Environment.register(:svn, self)
+
   # @!attribute [r] remote
   #   @return [String] The URL to the remote SVN branch to check out
   attr_reader :remote

data/lib/r10k/environment/with_modules.rb

@@ -0,0 +1,139 @@
+require 'r10k/logging'
+require 'r10k/util/purgeable'
+
+# This abstract base class implements an environment that can include module
+# content
+#
+# @since 3.4.0
+class R10K::Environment::WithModules < R10K::Environment::Base
+
+  include R10K::Logging
+
+  # @!attribute [r] moduledir
+  #   @return [String] The directory to install environment-defined modules
+  #     into (default: #{basedir}/modules)
+  attr_reader :moduledir
+
+  # Initialize the given environment.
+  #
+  # @param name [String] The unique name describing this environment.
+  # @param basedir [String] The base directory where this environment will be created.
+  # @param dirname [String] The directory name for this environment.
+  # @param options [Hash] An additional set of options for this environment.
+  #
+  # @param options [String] :moduledir The path to install modules to
+  # @param options [Hash] :modules Modules to add to the environment
+  def initialize(name, basedir, dirname, options = {})
+    super(name, basedir, dirname, options)
+
+    @managed_content = {}
+    @modules = []
+    @moduledir = case options[:moduledir]
+                 when nil
+                   File.join(@basedir, @dirname, 'modules')
+                 when File.absolute_path(options[:moduledir])
+                   options.delete(:moduledir)
+                 else
+                   File.join(@basedir, @dirname, options.delete(:moduledir))
+                 end
+
+    modhash = options.delete(:modules)
+    load_modules(modhash) unless modhash.nil?
+  end
+
+  # @return [Array<R10K::Module::Base>] All modules associated with this environment.
+  #   Modules may originate from either:
+  #     - The r10k environment object
+  #     - A Puppetfile in the environment's content
+  def modules
+    return @modules if @puppetfile.nil?
+
+    @puppetfile.load unless @puppetfile.loaded?
+    @modules + @puppetfile.modules
+  end
+
+  def accept(visitor)
+    visitor.visit(:environment, self) do
+      @modules.each do |mod|
+        mod.accept(visitor)
+      end
+
+      puppetfile.accept(visitor)
+      validate_no_module_conflicts
+    end
+  end
+
+  def load_modules(module_hash)
+    module_hash.each do |name, args|
+      add_module(name, args)
+    end
+  end
+
+  # @param [String] name
+  # @param [*Object] args
+  def add_module(name, args)
+    if args.is_a?(Hash)
+      # symbolize keys in the args hash
+      args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo }
+    end
+
+    if args.is_a?(Hash) && install_path = args.delete(:install_path)
+      install_path = resolve_install_path(install_path)
+      validate_install_path(install_path, name)
+    else
+      install_path = @moduledir
+    end
+
+    # Keep track of all the content this environment is managing to enable purging.
+    @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
+
+    mod = R10K::Module.new(name, install_path, args, self.name)
+    mod.origin = 'Environment'
+
+    @managed_content[install_path] << mod.name
+    @modules << mod
+  end
+
+  def validate_no_module_conflicts
+    @puppetfile.load unless @puppetfile.loaded?
+    conflicts = (@modules + @puppetfile.modules)
+                .group_by { |mod| mod.name }
+                .select { |_, v| v.size > 1 }
+                .map(&:first)
+    unless conflicts.empty?
+      msg = _('Puppetfile cannot contain module names defined by environment %{name}') % {name: self.name}
+      msg += ' '
+      msg += _("Remove the conflicting definitions of the following modules: %{conflicts}" % { conflicts: conflicts.join(' ') })
+      raise R10K::Error.new(msg)
+    end
+  end
+
+  include R10K::Util::Purgeable
+
+  # Returns an array of the full paths that can be purged.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def managed_directories
+    [@full_path]
+  end
+
+  # Returns an array of the full paths of filenames that should exist. Files
+  # inside managed_directories that are not listed in desired_contents will
+  # be purged.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def desired_contents
+    list = @managed_content.keys
+    list += @managed_content.flat_map do |install_path, modnames|
+      modnames.collect { |name| File.join(install_path, name) }
+    end
+  end
+
+  def purge_exclusions
+    super + @managed_content.flat_map do |install_path, modnames|
+      modnames.map do |name|
+        File.join(install_path, name, '**', '*')
+      end
+    end
+  end
+end

data/lib/r10k/logging/terminaloutputter.rb

@@ -1,4 +1,4 @@
-require 'colored'
+require 'colored2'
 require 'r10k/logging'
 require 'log4r/outputter/iooutputter'
 

data/lib/r10k/module/base.rb

@@ -31,6 +31,10 @@ class R10K::Module::Base
   #   @return [R10K::Environment, nil] The parent environment of the module
   attr_reader :environment
 
+  # @!attribute [rw] origin
+  #   @return [String] Where the module was sourced from. E.g., "Puppetfile"
+  attr_accessor :origin
+
   # There's been some churn over `author` vs `owner` and `full_name` over
   # `title`, so in the short run it's easier to support both and deprecate one
   # later.
@@ -47,6 +51,7 @@ class R10K::Module::Base
     @owner, @name = parse_title(@title)
     @path = Pathname.new(File.join(@dirname, @name))
     @environment = environment
+    @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value
   end
 
   # @deprecated

data/lib/r10k/module/forge.rb

@@ -75,7 +75,11 @@ class R10K::Module::Forge < R10K::Module::Base
 
   # @return [String] The version of the currently installed module
   def current_version
-    @metadata ? @metadata.version : nil
+    if insync?
+      (@metadata ||= @metadata_file.read).nil? ? nil : @metadata.version
+    else
+      nil
+    end
   end
 
   alias version current_version

data/lib/r10k/puppetfile.rb

@@ -64,6 +64,7 @@ class Puppetfile
   end
 
   def load(default_branch_override = nil)
+    return true if self.loaded?
     if File.readable? @puppetfile_path
       self.load!(default_branch_override)
     else
@@ -83,6 +84,10 @@ class Puppetfile
     raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path})
   end
 
+  def loaded?
+    @loaded
+  end
+
   # @param [Array<String>] modules
   def validate_no_duplicate_names(modules)
     dupes = modules
@@ -129,6 +134,7 @@ class Puppetfile
     @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
 
     mod = R10K::Module.new(name, install_path, args, @environment)
+    mod.origin = 'Puppetfile'
 
     @managed_content[install_path] << mod.name
     @modules << mod

data/lib/r10k/source.rb

@@ -32,7 +32,11 @@ module R10K
     end
 
     require 'r10k/source/base'
+    require 'r10k/source/hash'
     require 'r10k/source/git'
     require 'r10k/source/svn'
+    require 'r10k/source/yaml'
+    require 'r10k/source/yamldir'
+    require 'r10k/source/exec'
   end
 end

data/lib/r10k/source/exec.rb

@@ -0,0 +1,51 @@
+require 'r10k/util/subprocess'
+require 'json'
+require 'yaml'
+
+class R10K::Source::Exec < R10K::Source::Hash
+  R10K::Source.register(:exec, self)
+
+  def initialize(name, basedir, options = {})
+    unless @command = options[:command]
+      raise ConfigError, _('Environment source %{name} missing required parameter: command') % {name: name}
+    end
+
+    # We haven't set the environments option yet. We will do that by
+    # overloading the #environments method.
+    super(name, basedir, options)
+  end
+
+  def environments_hash
+    @environments_hash ||= set_environments_hash(run_environments_command)
+  end
+
+  private
+
+  def run_environments_command
+    subproc = R10K::Util::Subprocess.new([@command])
+    subproc.raise_on_fail = true
+    subproc.logger = self.logger
+    procresult = subproc.execute
+
+    begin
+      environments = JSON.parse(procresult.stdout)
+    rescue JSON::ParserError => json_err
+      begin
+        environments = YAML.safe_load(procresult.stdout)
+      rescue Psych::SyntaxError => yaml_err
+        raise R10K::Error, _("Error parsing command output for exec source %{name}:\n" \
+                             "Not valid JSON: %{j_msg}\n" \
+                             "Not valid YAML: %{y_msg}\n" \
+                             "Stdout:\n%{out}") % {name: name, j_msg: json_err.message, y_msg: yaml_err.message, out: procresult.stdout}
+      end
+    end
+
+    unless R10K::Source::Hash.valid_environments_hash?(environments)
+      raise R10K::Error, _("Environment source %{name} command %{cmd} did not return valid environment data.\n" \
+                           'Returned: %{data}') % {name: name, cmd: @command, data: environments}
+    end
+
+    # Return the resulting environments hash
+    environments
+  end
+end

data/lib/r10k/source/hash.rb

@@ -0,0 +1,182 @@
+# This class implements an environment source based on recieving a hash of
+# environments
+#
+# @since 3.4.0
+#
+# DESCRIPTION
+#
+# This class implements environments defined by a hash having the following
+# schema:
+#
+#     ---
+#     type: object
+#     additionalProperties:
+#       type: object
+#       properties:
+#         type:
+#           type: string
+#         basedir:
+#           type: string
+#         modules:
+#           type: object
+#           additionalProperties:
+#             type: object
+#         moduledir:
+#           type: string
+#       additionalProperties:
+#         type: string
+#
+# The top-level keys in the hash are environment names. Keys in individual
+# environments should be the same as those which would be given to define a
+# single source in r10k.yaml. Additionally, the "modules" key (and moduledir)
+# can be used to designate module content for the environment, independent of
+# the base source parameters.
+#
+# Example:
+#
+#     ---
+#     production:
+#       type: git
+#       remote: 'https://github.com/reidmv/control-repo.git'
+#       ref: '1.0.0'
+#       modules:
+#         geoffwilliams-r_profile: '1.1.0'
+#         geoffwilliams-r_role: '2.0.0'
+#
+#     development:
+#       type: git
+#       remote: 'https://github.com/reidmv/control-repo.git'
+#       ref: 'master'
+#       modules:
+#         geoffwilliams-r_profile: '1.1.0'
+#         geoffwilliams-r_role: '2.0.0'
+#
+# USAGE
+#
+# The following is an example implementation class showing how to use the
+# R10K::Source::Hash abstract base class. Assume an r10k.yaml file such as:
+#
+#     ---
+#     sources:
+#       proof-of-concept:
+#         type: demo
+#         basedir: '/etc/puppetlabs/code/environments'
+#
+# Class implementation:
+#
+#     class R10K::Source::Demo < R10K::Source::Hash
+#       R10K::Source.register(:demo, self)
+#
+#       def initialize(name, basedir, options = {})
+#         # This is just a demo class, so we hard-code an example :environments
+#         # hash here. In a real class, we might do something here such as
+#         # perform an API call to retrieve an :environments hash.
+#         options[:environments] = {
+#           'production' => {
+#             'remote'  => 'https://git.example.com/puppet/control-repo.git',
+#             'ref'     => 'release-141',
+#             'modules' => {
+#               'puppetlabs-stdlib' => '6.1.0',
+#               'puppetlabs-ntp' => '8.1.0',
+#               'example-myapp1' => {
+#                 'git' => 'https://git.example.com/puppet/example-myapp1.git',
+#                 'ref' => 'v1.3.0',
+#               },
+#             },
+#           },
+#           'development' => {
+#             'remote'  => 'https://git.example.com/puppet/control-repo.git',
+#             'ref'     => 'master',
+#             'modules' => {
+#               'puppetlabs-stdlib' => '6.1.0',
+#               'puppetlabs-ntp' => '8.1.0',
+#               'example-myapp1' => {
+#                 'git' => 'https://git.example.com/puppet/example-myapp1.git',
+#                 'ref' => 'v1.3.1',
+#               },
+#             },
+#           },
+#         }
+#
+#         # All we need to do is supply options with the :environments hash.
+#         # The R10K::Source::Hash parent class takes care of the rest.
+#         super(name, basedir, options)
+#       end
+#     end
+#
+# Example output:
+#
+#     [root@master:~] % r10k deploy environment production -pv
+#     INFO     -> Using Puppetfile '/etc/puppetlabs/code/environments/production/Puppetfile'
+#     INFO     -> Using Puppetfile '/etc/puppetlabs/code/environments/development/Puppetfile'
+#     INFO     -> Deploying environment /etc/puppetlabs/code/environments/production
+#     INFO     -> Environment production is now at 74ea2e05bba796918e4ff1803018c526337ef5f3
+#     INFO     -> Deploying Environment content /etc/puppetlabs/code/environments/production/modules/stdlib
+#     INFO     -> Deploying Environment content /etc/puppetlabs/code/environments/production/modules/ntp
+#     INFO     -> Deploying Environment content /etc/puppetlabs/code/environments/production/modules/myapp1
+#     INFO     -> Deploying Puppetfile content /etc/puppetlabs/code/environments/production/modules/ruby_task_helper
+#     INFO     -> Deploying Puppetfile content /etc/puppetlabs/code/environments/production/modules/bolt_shim
+#     INFO     -> Deploying Puppetfile content /etc/puppetlabs/code/environments/production/modules/apply_helpers
+#
+class R10K::Source::Hash < R10K::Source::Base
+
+  include R10K::Logging
+
+  # @param hash [Hash] A hash to validate.
+  # @return [Boolean] False if the hash is obviously invalid. A true return
+  #   means _maybe_ it's valid.
+  def self.valid_environments_hash?(hash)
+    # TODO: more robust schema valiation
+    hash.is_a?(Hash)
+  end
+
+  # @param name [String] The identifier for this source.
+  # @param basedir [String] The base directory where the generated environments will be created.
+  # @param options [Hash] An additional set of options for this source. The
+  #   semantics of this hash may depend on the source implementation.
+  #
+  # @option options [Boolean, String] :prefix If a String this becomes the prefix.
+  #   If true, will use the source name as the prefix. All sources should respect this option.
+  #   Defaults to false for no environment prefix.
+  # @option options [Hash] :environments The hash definition of environments
+  def initialize(name, basedir, options = {})
+    super(name, basedir, options)
+  end
+
+  # Set the environment hash for the source. The environment hash is what the
+  # source uses to generate enviroments.
+  # @param hash [Hash] The hash to sanitize and use as the source's environments.
+  #   Should be formatted for use with R10K::Environment#from_hash.
+  def set_environments_hash(hash)
+    @environments_hash = hash.reduce({}) do |memo,(name,opts)|
+      R10K::Util::SymbolizeKeys.symbolize_keys!(opts)
+      memo.merge({ 
+        name => opts.merge({
+          :basedir => @basedir,
+          :dirname => R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname
+        })
+      })
+    end
+  end
+
+  # Return the sanitized environments hash for this source. The environments
+  # hash should contain objects formatted for use with R10K::Environment#from_hash.
+  # If the hash does not exist it will be built based on @options.
+  def environments_hash
+    @environments_hash ||= set_environments_hash(@options.fetch(:environments, {}))
+  end
+
+  def environments
+    @environments ||= environments_hash.map do |name, hash|
+      R10K::Environment.from_hash(name, hash)
+    end
+  end
+
+  # List all environments that should exist in the basedir for this source
+  # @note This is required by {R10K::Util::Basedir}
+  # @return [Array<String>]
+  def desired_contents
+    environments.map {|env| env.dirname }
+  end
+
+end