r10k 3.11.0 → 3.14.0

data/lib/r10k/git/stateful_repository.rb

@@ -35,6 +35,7 @@ class R10K::Git::StatefulRepository
     @cache.resolve(ref)
   end
 
+  # Returns true if the sync actually updated the repo, false otherwise
   def sync(ref, force=true)
     @cache.sync if sync_cache?(ref)
 
@@ -46,6 +47,7 @@ class R10K::Git::StatefulRepository
 
     workdir_status = status(ref)
 
+    updated = true
     case workdir_status
     when :absent
       logger.debug(_("Cloning %{repo_path} and checking out %{ref}") % {repo_path: @repo.path, ref: ref })
@@ -64,21 +66,28 @@ class R10K::Git::StatefulRepository
         @repo.checkout(sha, {:force => force})
       else
         logger.warn(_("Skipping %{repo_path} due to local modifications") % {repo_path: @repo.path})
+        updated = false
       end
     else
       logger.debug(_("%{repo_path} is already at Git ref %{ref}") % {repo_path: @repo.path, ref: ref })
+      updated = false
     end
+    updated
   end
 
   def status(ref)
     if !@repo.exist?
       :absent
+    elsif !@cache.exist?
+      :mismatched
     elsif !@repo.git_dir.exist?
       :mismatched
     elsif !@repo.git_dir.directory?
       :mismatched
     elsif !(@repo.origin == @remote)
       :mismatched
+    elsif @repo.head.nil?
+      :mismatched
     elsif @repo.dirty?
       :dirty
     elsif !(@repo.head == @cache.resolve(ref))

data/lib/r10k/initializers.rb

@@ -4,6 +4,7 @@ require 'r10k/git'
 require 'r10k/git/cache'
 
 require 'r10k/forge/module_release'
+require 'r10k/tarball'
 
 module R10K
   module Initializers
@@ -30,14 +31,27 @@ module R10K
           logger.warn(_("the purgedirs key in r10k.yaml is deprecated. it is currently ignored."))
         end
 
+        with_setting(:logging) { |value| LoggingInitializer.new(value).call }
+
         with_setting(:deploy) { |value| DeployInitializer.new(value).call }
 
         with_setting(:cachedir) { |value| R10K::Git::Cache.settings[:cache_root] = value }
         with_setting(:cachedir) { |value| R10K::Forge::ModuleRelease.settings[:cache_root] = value }
+        with_setting(:cachedir) { |value| R10K::Tarball.settings[:cache_root] = value }
         with_setting(:pool_size) { |value| R10K::Puppetfile.settings[:pool_size] = value }
+        with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value }
 
         with_setting(:git) { |value| GitInitializer.new(value).call }
         with_setting(:forge) { |value| ForgeInitializer.new(value).call }
+        with_setting(:tarball) { |value| TarballInitializer.new(value).call }
+      end
+    end
+
+    class LoggingInitializer < BaseInitializer
+      def call
+        with_setting(:level) { |value| R10K::Logging.level = value }
+        with_setting(:disable_default_stderr) { |value| R10K::Logging.disable_default_stderr = value }
+        with_setting(:outputs) { |value| R10K::Logging.add_outputters(value) }
       end
     end
 
@@ -66,13 +80,13 @@ module R10K
       def call
         with_setting(:baseurl) { |value| PuppetForge.host = value }
         with_setting(:proxy) { |value| PuppetForge::Connection.proxy = value }
-        with_setting(:authorization_token) { |value|
-          if @settings[:baseurl]
-            PuppetForge::Connection.authorization = value
-          else
-            raise R10K::Error, "Cannot specify a Forge authorization token without configuring a custom baseurl."
-          end
-        }
+        with_setting(:authorization_token) { |value| PuppetForge::Connection.authorization = value }
+      end
+    end
+
+    class TarballInitializer < BaseInitializer
+      def call
+        with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value }
       end
     end
   end

data/lib/r10k/logging.rb

@@ -8,6 +8,16 @@ require 'r10k/logging/terminaloutputter'
 module R10K::Logging
 
   LOG_LEVELS = %w{DEBUG2 DEBUG1 DEBUG INFO NOTICE WARN ERROR FATAL}
+  SYSLOG_LEVELS_MAP = {
+    'DEBUG2' => 'DEBUG',
+    'DEBUG1' => 'DEBUG',
+    'DEBUG' => 'DEBUG',
+    'INFO' => 'INFO',
+    'NOTICE' => 'INFO',
+    'WARN' => 'WARN',
+    'ERROR' => 'ERROR',
+    'FATAL' => 'FATAL',
+  }.freeze
 
   def logger_name
     self.class.to_s
@@ -21,6 +31,9 @@ module R10K::Logging
       else
         @logger = Log4r::Logger.new(name)
         @logger.add(R10K::Logging.outputter)
+        R10K::Logging.outputters.each do |output|
+          @logger.add(output)
+        end
       end
     end
     @logger
@@ -59,7 +72,7 @@ module R10K::Logging
       if level.nil?
         raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % {val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect}
       end
-      outputter.level = level
+      outputter.level = level unless @disable_default_stderr
       @level = level
 
       if level < Log4r::INFO
@@ -69,6 +82,58 @@ module R10K::Logging
       end
     end
 
+    def disable_default_stderr=(val)
+      @disable_default_stderr = val
+      outputter.level = val ? Log4r::OFF : @level
+    end
+
+    def add_outputters(outputs)
+      outputs.each do |output|
+        type = output.fetch(:type)
+        # Support specifying both short as well as full names
+        type = type.to_s[0..-10] if type.to_s.downcase.end_with? 'outputter'
+
+        name = output.fetch(:name, 'r10k')
+        if output[:level]
+          level = parse_level(output[:level])
+          if level.nil?
+            raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: output[:level], log_levels: LOG_LEVELS.map(&:downcase).inspect }
+          end
+        else
+          level = self.level
+        end
+        only_at = output[:only_at]
+        only_at&.map! do |val|
+          lv = parse_level(val)
+          if lv.nil?
+            raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect }
+          end
+
+          lv
+        end
+        parameters = output.fetch(:parameters, {}).merge({ level: level })
+
+        begin
+          # Try to load the outputter file if possible
+          require "log4r/outputter/#{type.to_s.downcase}outputter"
+        rescue LoadError
+          false
+        end
+        outputtertype = Log4r.constants
+                             .select { |klass| klass.to_s.end_with? 'Outputter' }
+                             .find { |klass| klass.to_s.downcase == "#{type.to_s.downcase}outputter" }
+        raise ArgumentError, "Unable to find a #{output[:type]} outputter." unless outputtertype
+
+        outputter = Log4r.const_get(outputtertype).new(name, parameters)
+        outputter.only_at(*only_at) if only_at
+        # Handle log4r's syslog mapping correctly
+        outputter.map_levels_by_name_to_syslog(SYSLOG_LEVELS_MAP) if outputter.respond_to? :map_levels_by_name_to_syslog
+
+        @outputters << outputter
+        Log4r::Logger.global.add outputter
+      end
+    end
+
     extend Forwardable
     def_delegators :@outputter, :use_color, :use_color=
 
@@ -87,6 +152,16 @@ module R10K::Logging
     #   @return [Log4r::Outputter]
     attr_reader :outputter
 
+    # @!attribute [r] outputters
+    #   @api private
+    #   @return [Array[Log4r::Outputter]]
+    attr_reader :outputters
+
+    # @!attribute [r] disable_default_stderr
+    #   @api private
+    #   @return [Boolean]
+    attr_reader :disable_default_stderr
+
     def default_formatter
       Log4r::PatternFormatter.new(:pattern => '%l\t -> %m')
     end
@@ -106,4 +181,6 @@ module R10K::Logging
   @level     = Log4r::WARN
   @formatter = default_formatter
   @outputter = default_outputter
+  @outputters = []
+  @disable_default_stderr = false
 end

data/lib/r10k/module/base.rb

@@ -1,9 +1,12 @@
 require 'r10k/module'
+require 'r10k/logging'
 require 'puppet_forge'
 
 # This class defines a common interface for module implementations.
 class R10K::Module::Base
 
+  include R10K::Logging
+
   # @!attribute [r] title
   #   @return [String] The forward slash separated owner and name of the module
   attr_reader :title
@@ -47,7 +50,7 @@ class R10K::Module::Base
 
   # @param title [String]
   # @param dirname [String]
-  # @param args [Array]
+  # @param args [Hash]
   def initialize(title, dirname, args, environment=nil)
     @title   = PuppetForge::V3.normalize_name(title)
     @dirname = dirname
@@ -103,6 +106,7 @@ class R10K::Module::Base
 
   # Synchronize this module with the indicated state.
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
     raise NotImplementedError
   end

data/lib/r10k/module/definition.rb

@@ -0,0 +1,64 @@
+require 'r10k/module'
+
+class R10K::Module::Definition < R10K::Module::Base
+
+  attr_reader :version
+
+  def initialize(name, dirname:, args:, implementation:, environment: nil)
+    @original_name  = name
+    @original_args  = args.dup
+    @implementation = implementation
+    @version        = implementation.statically_defined_version(name, args)
+
+    super(name, dirname, args, environment)
+  end
+
+  def to_implementation
+    mod = @implementation.new(@title, @dirname, @original_args, @environment)
+
+    mod.origin = origin
+    mod.spec_deletable = spec_deletable
+
+    mod
+  end
+
+  # syncing is a noop for module definitions
+  # Returns false to inidicate the module was not updated
+  def sync(args = {})
+    logger.debug1(_("Not updating module %{name}, assuming content unchanged") % {name: name})
+    false
+  end
+
+  def status
+    :insync
+  end
+
+  def properties
+    type = nil
+
+    if @args[:type]
+      type = @args[:type]
+    elsif @args[:ref] || @args[:commit] || @args[:branch] || @args[:tag]
+      type = 'git'
+    elsif @args[:svn]
+      # This logic is clear and included for completeness sake, though at
+      # this time module definitions do not support SVN versions.
+      type = 'svn'
+    else
+      type = 'forge'
+    end
+
+    {
+      expected: version,
+      # We can't get the value for `actual` here because that requires the
+      # implementation (and potentially expensive operations by the
+      # implementation). Some consumers will check this value, if it exists
+      # and if not, fall back to the expected version. That is the correct
+      # behavior when assuming modules are unchanged, and why `actual` is set
+      # to `nil` here.
+      actual: nil,
+      type: type
+    }
+  end
+end
+

data/lib/r10k/module/forge.rb

@@ -25,6 +25,10 @@ class R10K::Module::Forge < R10K::Module::Base
     expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version)
   end
 
+  def self.statically_defined_version(name, args)
+    args[:version] if args[:version].is_a?(String)
+  end
+
   # @!attribute [r] metadata
   #   @api private
   #   @return [PuppetForge::Metadata]
@@ -35,8 +39,6 @@ class R10K::Module::Forge < R10K::Module::Base
   #   @return [PuppetForge::V3::Module] The Puppet Forge module metadata
   attr_reader :v3_module
 
-  include R10K::Logging
-
   include R10K::Util::Setopts
 
   def initialize(title, dirname, opts, environment=nil)
@@ -58,18 +60,24 @@ class R10K::Module::Forge < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
+    updated = false
     if should_sync?
       case status
       when :absent
         install
+        updated = true
       when :outdated
         upgrade
+        updated = true
       when :mismatched
         reinstall
+        updated = true
       end
       maybe_delete_spec_dir
     end
+    updated
   end
 
   def properties

data/lib/r10k/module/git.rb

@@ -13,6 +13,21 @@ class R10K::Module::Git < R10K::Module::Base
     false
   end
 
+  # Will be called if self.implement? above returns true. Will return
+  # the version info, if version is statically defined in the modules
+  # declaration.
+  def self.statically_defined_version(name, args)
+    if !args[:type] && (args[:ref] || args[:tag] || args[:commit])
+      if args[:ref] && args[:ref].to_s.match(/[0-9a-f]{40}/)
+        args[:ref]
+      else
+        args[:tag] || args[:commit]
+      end
+    elsif args[:type].to_s == 'git' && args[:version] && args[:version].to_s.match(/[0-9a-f]{40}/)
+      args[:version]
+    end
+  end
+
   # @!attribute [r] repo
   #   @api private
   #   @return [R10K::Git::StatefulRepository]
@@ -83,10 +98,16 @@ class R10K::Module::Git < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
     force = opts[:force] || @force
-    @repo.sync(version, force) if should_sync?
+    if should_sync?
+      updated = @repo.sync(version, force)
+    else
+      updated = false
+    end
     maybe_delete_spec_dir
+    updated
   end
 
   def status

data/lib/r10k/module/local.rb

@@ -1,5 +1,4 @@
 require 'r10k/module'
-require 'r10k/logging'
 
 # A dummy module type that can be used to "protect" Puppet modules that exist
 # inside of the Puppetfile "moduledir" location. Local modules will not be
@@ -12,10 +11,12 @@ class R10K::Module::Local < R10K::Module::Base
     args.is_a?(Hash) && (args[:local] || args[:type].to_s == 'local')
   end
 
-  include R10K::Logging
+  def self.statically_defined_version(*)
+    "0.0.0"
+  end
 
   def version
-    "0.0.0"
+    self.class.statically_defined_version
   end
 
   def properties
@@ -31,7 +32,9 @@ class R10K::Module::Local < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] false, because local modules are always considered in-sync
   def sync(opts={})
     logger.debug1 _("Module %{title} is a local module, always indicating synced.") % {title: title}
+    false
   end
 end

data/lib/r10k/module/svn.rb

@@ -10,6 +10,10 @@ class R10K::Module::SVN < R10K::Module::Base
     args.has_key?(:svn) || args[:type].to_s == 'svn'
   end
 
+  def self.statically_defined_version(name, args)
+    nil
+  end
+
   # @!attribute [r] expected_revision
   #   @return [String] The SVN revision that the repo should have checked out
   attr_reader :expected_revision
@@ -70,18 +74,24 @@ class R10K::Module::SVN < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
+    updated = false
     if should_sync?
       case status
       when :absent
         install
+        updated = true
       when :mismatched
         reinstall
+        updated = true
       when :outdated
         update
+        updated = true
       end
       maybe_delete_spec_dir
     end
+    updated
   end
 
   def exist?

data/lib/r10k/module/tarball.rb

@@ -0,0 +1,101 @@
+require 'r10k/module'
+require 'r10k/util/setopts'
+require 'r10k/tarball'
+
+# This class defines a tarball source module implementation
+class R10K::Module::Tarball < R10K::Module::Base
+
+  R10K::Module.register(self)
+
+  def self.implement?(name, args)
+    args.is_a?(Hash) && args[:type].to_s == 'tarball'
+  rescue
+    false
+  end
+
+  def self.statically_defined_version(name, args)
+    args[:version] || args[:checksum]
+  end
+
+  # @!attribute [r] tarball
+  #   @api private
+  #   @return [R10K::Tarball]
+  attr_reader :tarball
+
+  include R10K::Util::Setopts
+
+  def initialize(name, dirname, opts, environment=nil)
+    super
+    setopts(opts, {
+      # Standard option interface
+      :source    => :self,
+      :version   => :checksum,
+      :type      => ::R10K::Util::Setopts::Ignore,
+      :overrides => :self,
+
+      # Type-specific options
+      :checksum => :self,
+    })
+
+    @tarball = R10K::Tarball.new(name, @source, checksum: @checksum)
+  end
+
+  # Return the status of the currently installed module.
+  #
+  # @return [Symbol]
+  def status
+    if not path.exist?
+      :absent
+    elsif not (tarball.cache_valid? && tarball.insync?(path.to_s))
+      :mismatched
+    else
+      :insync
+    end
+  end
+
+  # Synchronize this module with the indicated state.
+  # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
+  def sync(opts={})
+    tarball.get unless tarball.cache_valid?
+    if should_sync?
+      case status
+      when :absent
+        tarball.unpack(path.to_s)
+      when :mismatched
+        path.rmtree
+        tarball.unpack(path.to_s)
+      end
+      maybe_delete_spec_dir
+      true
+    else
+      false
+    end
+  end
+
+  # Return the desired version of this module
+  def version
+    @checksum || '(present)'
+  end
+
+  # Return the properties of the module
+  #
+  # @return [Hash]
+  # @abstract
+  def properties
+    {
+      :expected => version,
+      :actual   => ((state = status) == :insync) ? version : state,
+      :type     => :tarball,
+    }
+  end
+
+  # Tarball caches are files, not directories. An important purpose of this
+  # method is to indicate where the cache "path" is, for locking/parallelism,
+  # so for the Tarball module type, the relevant path location is returned.
+  #
+  # @return [String] The path this module will cache its tarball source to
+  def cachedir
+    tarball.cache_path
+  end
+end

data/lib/r10k/module.rb

@@ -22,17 +22,36 @@ module R10K::Module
   #
   # @return [Object < R10K::Module] A member of the implementing subclass
   def self.new(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      implementation.new(name, basedir, args, environment)
+    end
+  end
+
+  # Takes the same signature as Module.new but returns an metadata module
+  def self.from_metadata(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      R10K::Module::Definition.new(name,
+                                   dirname: basedir,
+                                   args: args,
+                                   implementation: implementation,
+                                   environment: environment)
+    end
+  end
+
+  def self.with_implementation(name, args, &block)
     if implementation = @klasses.find { |klass| klass.implement?(name, args) }
-      obj = implementation.new(name, basedir, args, environment)
-      obj
+      block.call(implementation)
     else
       raise _("Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)") % {name: name, args: args.inspect}
     end
   end
 
+
   require 'r10k/module/base'
   require 'r10k/module/git'
   require 'r10k/module/svn'
   require 'r10k/module/local'
   require 'r10k/module/forge'
+  require 'r10k/module/definition'
+  require 'r10k/module/tarball'
 end

data/lib/r10k/module_loader/puppetfile/dsl.rb

@@ -6,8 +6,9 @@ module R10K
         #
         # @api private
 
-        def initialize(librarian)
-          @librarian = librarian
+        def initialize(librarian, metadata_only: false)
+          @librarian     = librarian
+          @metadata_only = metadata_only
         end
 
         def mod(name, args = nil)
@@ -17,7 +18,11 @@ module R10K
             opts = { version: args }
           end
 
-          @librarian.add_module(name, opts)
+          if @metadata_only
+            @librarian.add_module_metadata(name, opts)
+          else
+            @librarian.add_module(name, opts)
+          end
         end
 
         def forge(location)