r10k 3.11.0 → 3.14.0

data/lib/r10k/module_loader/puppetfile.rb

@@ -1,4 +1,9 @@
+require 'r10k/errors'
 require 'r10k/logging'
+require 'r10k/module'
+require 'r10k/module_loader/puppetfile/dsl'
+
+require 'pathname'
 
 module R10K
   module ModuleLoader
@@ -8,11 +13,11 @@ module R10K
 
       DEFAULT_MODULEDIR = 'modules'
       DEFAULT_PUPPETFILE_NAME = 'Puppetfile'
-      DEFAULT_FORGE_API = 'forgeapi.puppetlabs.com'
 
       attr_accessor :default_branch_override, :environment
       attr_reader :modules, :moduledir, :puppetfile_path,
-        :managed_directories, :desired_contents, :purge_exclusions
+        :managed_directories, :desired_contents, :purge_exclusions,
+        :environment_name
 
       # @param basedir [String] The path that contains the moduledir &
       #     Puppetfile by default. May be an environment, project, or
@@ -28,34 +33,41 @@ module R10K
       def initialize(basedir:,
                      moduledir: DEFAULT_MODULEDIR,
                      puppetfile: DEFAULT_PUPPETFILE_NAME,
-                     forge: DEFAULT_FORGE_API,
                      overrides: {},
                      environment: nil)
 
         @basedir     = cleanpath(basedir)
         @moduledir   = resolve_path(@basedir, moduledir)
         @puppetfile_path  = resolve_path(@basedir, puppetfile)
-        @forge       = forge
         @overrides   = overrides
         @environment = environment
+        @environment_name = @environment&.name
         @default_branch_override = @overrides.dig(:environments, :default_branch_override)
+        @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override)
 
+        @existing_module_metadata = []
+        @existing_module_versions_by_name = {}
         @modules = []
 
         @managed_directories = []
         @desired_contents = []
         @purge_exclusions = []
+      end
 
+      def load
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load!
+        end
+      end
+
+      def load!
         logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
         logger.debug _("Using moduledir '%{moduledir}'") % {moduledir: @moduledir}
-      end
 
-      def load
         dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self)
         dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
 
         validate_no_duplicate_names(@modules)
-        @modules
 
         managed_content = @modules.group_by(&:dirname)
 
@@ -74,6 +86,30 @@ module R10K
         raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path})
       end
 
+      def load_metadata
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load_metadata!
+        end
+      end
+
+      def load_metadata!
+        dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self, metadata_only: true)
+        dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
+
+        @existing_module_versions_by_name = @existing_module_metadata.map {|mod| [ mod.name, mod.version ] }.to_h
+        empty_load_output.merge(modules: @existing_module_metadata)
+
+      rescue SyntaxError, LoadError, ArgumentError, NameError => e
+        logger.warn _("Unable to preload Puppetfile because of %{msg}" % { msg: e.message })
+      end
+
+      def add_module_metadata(name, info)
+        install_path, metadata_info, _ = parse_module_definition(name, info)
+
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
+
+        @existing_module_metadata << mod
+      end
 
       ##
       ## set_forge, set_moduledir, and add_module are used directly by the DSL class
@@ -81,7 +117,12 @@ module R10K
 
       # @param [String] forge
       def set_forge(forge)
-        @forge = forge
+        if @allow_puppetfile_forge
+          logger.debug _("Using Forge from Puppetfile: %{forge}") % { forge: forge }
+          PuppetForge.host = forge
+        else
+          logger.debug _("Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}.") % { forge: PuppetForge.host }
+        end
       end
 
       # @param [String] moduledir
@@ -90,7 +131,7 @@ module R10K
       end
 
       # @param [String] name
-      # @param [Hash, String, Symbol, nil] module_info Calling with
+      # @param [Hash, String, Symbol, nil] info Calling with
       #   anything but a Hash is deprecated. The DSL will now convert
       #   String and Symbol versions to Hashes of the shape
       #     { version: <String or Symbol> }
@@ -103,28 +144,10 @@ module R10K
       #   DSL class, not the Puppetfile author) to do this conversion
       #   itself.
       #
-      def add_module(name, module_info)
-        if !module_info.is_a?(Hash)
-          module_info = { version: module_info }
-        end
-
-        module_info[:overrides] = @overrides
-
-        spec_deletable = false
-
-        if install_path = module_info.delete(:install_path)
-          install_path = resolve_path(@basedir, install_path)
-          validate_install_path(install_path, name)
-        else
-          install_path = @moduledir
-          spec_deletable = true
-        end
+      def add_module(name, info)
+        install_path, metadata_info, spec_deletable = parse_module_definition(name, info)
 
-        if @default_branch_override
-          module_info[:default_branch_override] = @default_branch_override
-        end
-
-        mod = R10K::Module.new(name, install_path, module_info, @environment)
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
         mod.origin = :puppetfile
         mod.spec_deletable = spec_deletable
 
@@ -134,11 +157,67 @@ module R10K
           return @modules
         end
 
+        # If this module's metadata has a static version and that version
+        # matches the existing module declaration use it, otherwise create
+        # a regular module to sync.
+        unless mod.version && (mod.version == @existing_module_versions_by_name[mod.name])
+          mod = mod.to_implementation
+        end
+
         @modules << mod
       end
 
+      # @deprecated
+      # @return [String] The base directory that contains the Puppetfile
+      def basedir
+        logger.warn _('"basedir" is deprecated. Please use "environment_name" instead. "basedir" will be removed in a future version.')
+        @basedir
+      end
+
      private
 
+      def empty_load_output
+        {
+          modules: [],
+          managed_directories: [],
+          desired_contents: [],
+          purge_exclusions: []
+        }
+      end
+
+      def with_readable_puppetfile(puppetfile_path, &block)
+        if File.readable?(puppetfile_path)
+          block.call
+        else
+          logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect}
+
+          empty_load_output
+        end
+      end
+
+      def parse_module_definition(name, info)
+        if !info.is_a?(Hash)
+          info = { version: info }
+        end
+
+        info[:overrides] = @overrides
+
+        if @default_branch_override
+          info[:default_branch_override] = @default_branch_override
+        end
+
+        spec_deletable = false
+        if install_path = info.delete(:install_path)
+          install_path = resolve_path(@basedir, install_path)
+          validate_install_path(install_path, name)
+        else
+          install_path = @moduledir
+          spec_deletable = true
+        end
+
+        return [ install_path, info, spec_deletable ]
+      end
+
       # @param [Array<R10K::Module>] modules
       def validate_no_duplicate_names(modules)
         dupes = modules

data/lib/r10k/puppetfile.rb

@@ -78,7 +78,6 @@ class Puppetfile
       basedir: @basedir,
       moduledir: @moduledir,
       puppetfile: @puppetfile,
-      forge: @forge,
       overrides: @overrides,
       environment: @environment
     )
@@ -120,7 +119,7 @@ class Puppetfile
       @loader.default_branch_override = default_branch_override
     end
 
-    @loaded_content = @loader.load
+    @loaded_content = @loader.load!
     @loaded = true
 
     @loaded_content

data/lib/r10k/settings.rb

@@ -126,8 +126,19 @@ module R10K
         URIDefinition.new(:baseurl, {
           :desc => "The URL to the Puppet Forge to use for downloading modules."
         }),
+
         Definition.new(:authorization_token, {
           :desc => "The token for Puppet Forge authorization. Leave blank for unauthorized or license-based connections."
+        }),
+
+        Definition.new(:allow_puppetfile_override, {
+          :desc => "Whether to use `forge` declarations in the Puppetfile as an override of `baseurl`.",
+          :default => false,
+          :validate => lambda do |value|
+            unless !!value == value
+              raise ArgumentError, "`allow_puppetfile_override` can only be a boolean value, not '#{value}'"
+            end
+          end
         })
       ])
     end
@@ -206,6 +217,39 @@ module R10K
         })])
     end
 
+    def self.logging_settings
+      R10K::Settings::Collection.new(:logging, [
+        Definition.new(:level, {
+          desc: 'What logging level should R10k run on if not specified at runtime.',
+          validate: lambda do |value|
+            if R10K::Logging.parse_level(value).nil?
+              raise ArgumentError, "`level` must be a valid log level.
+                                    Valid levels are #{R10K::Logging::LOG_LEVELS.map(&:downcase).inspect}"
+            end
+          end
+        }),
+
+        Definition.new(:outputs, {
+          desc: 'Additional log outputs to use.',
+          validate: lambda do |value|
+            unless value.is_a?(Array)
+              raise ArgumentError, "The `outputs` setting should be an array of outputs, not a #{value.class}"
+            end
+          end
+        }),
+
+        Definition.new(:disable_default_stderr, {
+          desc: 'Disable the default stderr logging output',
+          default: false,
+          validate: lambda do |value|
+            unless !!value == value
+              raise ArgumentError, "`disable_default_stderr` can only be a boolean value, not '#{value}'"
+            end
+          end
+        })
+      ])
+    end
+
     def self.global_settings
       R10K::Settings::Collection.new(:global, [
         Definition.new(:sources, {
@@ -260,6 +304,8 @@ module R10K
         R10K::Settings.git_settings,
 
         R10K::Settings.deploy_settings,
+
+        R10K::Settings.logging_settings
       ])
     end
   end

data/lib/r10k/source/git.rb

@@ -100,26 +100,26 @@ class R10K::Source::Git < R10K::Source::Base
 
   def generate_environments
     envs = []
-    branch_names.each do |bn|
-      if bn.valid?
-        envs << R10K::Environment::Git.new(bn.name,
+    environment_names.each do |en|
+      if en.valid?
+        envs << R10K::Environment::Git.new(en.name,
                                            @basedir,
-                                           bn.dirname,
+                                           en.dirname,
                                            {remote: remote,
-                                            ref: bn.name,
+                                            ref: en.original_name,
                                             puppetfile_name: puppetfile_name,
                                             overrides: @options[:overrides]})
-      elsif bn.correct?
-       logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: bn.name.inspect, corrected_env_name: bn.dirname}
-        envs << R10K::Environment::Git.new(bn.name,
+      elsif en.correct?
+       logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: en.name.inspect, corrected_env_name: en.dirname}
+        envs << R10K::Environment::Git.new(en.name,
                                            @basedir,
-                                           bn.dirname,
+                                           en.dirname,
                                            {remote: remote,
-                                            ref: bn.name,
+                                            ref: en.original_name,
                                             puppetfile_name: puppetfile_name,
                                             overrides: @options[:overrides]})
-      elsif bn.validate?
-       logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: bn.name.inspect}
+      elsif en.validate?
+       logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: en.name.inspect}
       end
     end
 
@@ -157,22 +157,22 @@ class R10K::Source::Git < R10K::Source::Base
 
   private
 
-  def branch_names
+  def environment_names
     opts = {prefix: @prefix,
             invalid: @invalid_branches,
             source: @name,
             strip_component: @strip_component}
-    branches = @cache.branches
+    branch_names = @cache.branches
     if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
-      branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes)
+      branch_names = filter_branches_by_regexp(branch_names, @ignore_branch_prefixes)
     end
 
     if @filter_command && !@filter_command.empty?
-      branches = filter_branches_by_command(branches, @filter_command)
+      branch_names = filter_branches_by_command(branch_names, @filter_command)
     end
 
-    branches.map do |branch|
-      R10K::Environment::Name.new(branch, opts)
+    branch_names.map do |branch_name|
+      R10K::Environment::Name.new(branch_name, opts)
     end
   end
 end

data/lib/r10k/source/yaml.rb

@@ -7,7 +7,7 @@ class R10K::Source::Yaml < R10K::Source::Hash
     begin
       contents = ::YAML.load_file(config)
     rescue => e
-      raise ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message}
+      raise R10K::ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message}
     end
 
     # Set the environments key for the parent class to consume

data/lib/r10k/tarball.rb

@@ -0,0 +1,183 @@
+require 'fileutils'
+require 'find'
+require 'minitar'
+require 'tempfile'
+require 'uri'
+require 'zlib'
+require 'r10k/settings'
+require 'r10k/settings/mixin'
+require 'r10k/util/platform'
+require 'r10k/util/cacheable'
+require 'r10k/util/downloader'
+
+module R10K
+  class Tarball
+
+    include R10K::Settings::Mixin
+    include R10K::Util::Cacheable
+    include R10K::Util::Downloader
+
+    def_setting_attr :proxy      # Defaults to global proxy setting
+    def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir
+
+    # @!attribute [rw] name
+    #   @return [String] The tarball's name
+    attr_accessor :name
+
+    # @!attribute [rw] source
+    #   @return [String] The tarball's source
+    attr_accessor :source
+
+    # @!attribute [rw] checksum
+    #   @return [String] The tarball's expected sha256 digest
+    attr_accessor :checksum
+
+    # @param name [String] The name of the tarball content
+    # @param source [String] The source for the tarball content
+    # @param checksum [String] The sha256 digest of the tarball content
+    def initialize(name, source, checksum: nil)
+      @name = name
+      @source = source
+      @checksum = checksum
+
+      # At this time, the only checksum type supported is sha256. In the future,
+      # we may decide to support other algorithms if a use case arises. TBD.
+      checksum_algorithm = :SHA256
+    end
+
+    # @return [String] Directory. Where the cache_basename file will be created.
+    def cache_dirname
+      File.join(settings[:cache_root], 'tarball')
+    end
+
+    # The final cache_path should match one of the templates:
+    #
+    #   - {cachedir}/{checksum}.tar.gz
+    #   - {cachedir}/{source}.tar.gz
+    #
+    # @return [String] File. The full file path the tarball will be cached to.
+    def cache_path
+      File.join(cache_dirname, cache_basename)
+    end
+
+    # @return [String] The basename of the tarball cache file.
+    def cache_basename
+      if checksum.nil?
+        sanitized_dirname(source) + '.tar.gz'
+      else
+        checksum + '.tar.gz'
+      end
+    end
+
+    # Extract the cached tarball to the target directory.
+    #
+    # @param target_dir [String] Where to unpack the tarball
+    def unpack(target_dir)
+      file = File.open(cache_path, 'rb')
+      reader = Zlib::GzipReader.new(file)
+      begin
+        Minitar.unpack(reader, target_dir)
+      ensure
+        reader.close
+      end
+    end
+
+    # @param target_dir [String] The directory to check if is in sync with the
+    #        tarball content
+    # @param ignore_untracked_files [Boolean] If true, consider the target
+    #        dir to be in sync as long as all tracked content matches.
+    #
+    # @return [Boolean]
+    def insync?(target_dir, ignore_untracked_files: false)
+      target_tree_entries = Find.find(target_dir).map(&:to_s) - [target_dir]
+      each_tarball_entry do |entry|
+        found = target_tree_entries.delete(File.join(target_dir, entry.full_name.chomp('/')))
+        return false if found.nil?
+        next if entry.directory?
+        return false unless file_digest(found) == reader_digest(entry)
+      end
+
+      if ignore_untracked_files
+        # We wouldn't have gotten this far if there were discrepancies in
+        # tracked content
+        true
+      else
+        # If there are still files in target_tree_entries, then there is
+        # untracked content present in the target tree. If not, we're in sync.
+        target_tree_entries.empty?
+      end
+    end
+
+    # Download the tarball from @source to @cache_path
+    def get
+      Tempfile.open(cache_basename) do |tempfile|
+        tempfile.binmode
+        src_uri = URI.parse(source)
+
+        temp_digest = case src_uri.scheme
+                      when 'file', nil
+                        copy(src_uri.path, tempfile)
+                      when %r{^[a-z]$} # Windows drive letter
+                        copy(src_uri.to_s, tempfile)
+                      when %r{^https?$}
+                        download(src_uri, tempfile)
+                      else
+                        raise "Unexpected source scheme #{src_uri.scheme}"
+                      end
+
+        # Verify the download
+        unless (checksum == temp_digest) || checksum.nil?
+          raise 'Downloaded file does not match checksum'
+        end
+
+        # Move the download to cache_path
+        FileUtils::mkdir_p(cache_dirname)
+        begin
+          FileUtils.mv(tempfile.path, cache_path)
+        rescue Errno::EACCES
+          # It may be the case that permissions don't permit moving the file
+          # into place, but do permit overwriting an existing in-place file.
+          FileUtils.cp(tempfile.path, cache_path)
+        end
+      end
+    end
+
+    # Checks the cached tarball's digest against the expected checksum. Returns
+    # false if no cached file is present. If the tarball has no expected
+    # checksum, any cached file is assumed to be valid.
+    #
+    # @return [Boolean]
+    def cache_valid?
+      return false unless File.exist?(cache_path)
+      return true if checksum.nil?
+      checksum == file_digest(cache_path)
+    end
+
+    # List all of the files contained in the tarball and their paths. This is
+    # useful for implementing R10K::Purgable
+    #
+    # @return [Array] A normalized list of file paths contained in the archive
+    def paths
+      names = Array.new
+      each_tarball_entry { |entry| names << Pathname.new(entry).cleanpath.to_s }
+      names - ['.']
+    end
+
+    def cache_checksum
+      raise R10K::Error, _("Cache not present at %{path}") % {path: cache_path} unless File.exist?(cache_path)
+      file_digest(cache_path)
+    end
+
+    private
+
+    def each_tarball_entry(&block)
+      File.open(cache_path, 'rb') do |file|
+        Zlib::GzipReader.wrap(file) do |reader|
+          Archive::Tar::Minitar::Input.each_entry(reader) do |entry|
+            yield entry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/r10k/util/cacheable.rb

@@ -0,0 +1,31 @@
+module R10K
+  module Util
+
+    # Utility mixin for classes that need to implement caches
+    #
+    # @abstract Classes using this mixin need to implement {#managed_directory} and
+    #   {#desired_contents}
+    module Cacheable
+
+      # Provide a default cachedir location. This is consumed by R10K::Settings
+      # for appropriate global default values.
+      #
+      # @return [String] Path to the default cache directory
+      def self.default_cachedir(basename = 'cache')
+        if R10K::Util::Platform.windows?
+          File.join(ENV['LOCALAPPDATA'], 'r10k', basename)
+        else
+          File.join(ENV['HOME'] || '/root', '.r10k', basename)
+        end
+      end
+
+      # Reformat a string into something that can be used as a directory
+      #
+      # @param string [String] An identifier to create a sanitized dirname for
+      # @return [String] A sanitized dirname for the given string
+      def sanitized_dirname(string)
+        string.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
+      end
+    end
+  end
+end