r10k 3.9.2 → 3.12.0

data/lib/r10k/module/definition.rb

@@ -0,0 +1,64 @@
+require 'r10k/module'
+
+class R10K::Module::Definition < R10K::Module::Base
+
+  attr_reader :version
+
+  def initialize(name, dirname:, args:, implementation:, environment: nil)
+    @original_name  = name
+    @original_args  = args.dup
+    @implementation = implementation
+    @version        = implementation.statically_defined_version(name, args)
+
+    super(name, dirname, args, environment)
+  end
+
+  def to_implementation
+    mod = @implementation.new(@title, @dirname, @original_args, @environment)
+
+    mod.origin = origin
+    mod.spec_deletable = spec_deletable
+
+    mod
+  end
+
+  # syncing is a noop for module definitions
+  # Returns false to inidicate the module was not updated
+  def sync(args = {})
+    logger.debug1(_("Not updating module %{name}, assuming content unchanged") % {name: name})
+    false
+  end
+
+  def status
+    :insync
+  end
+
+  def properties
+    type = nil
+
+    if @args[:type]
+      type = @args[:type]
+    elsif @args[:ref] || @args[:commit] || @args[:branch] || @args[:tag]
+      type = 'git'
+    elsif @args[:svn]
+      # This logic is clear and included for completeness sake, though at
+      # this time module definitions do not support SVN versions.
+      type = 'svn'
+    else
+      type = 'forge'
+    end
+
+    {
+      expected: version,
+      # We can't get the value for `actual` here because that requires the
+      # implementation (and potentially expensive operations by the
+      # implementation). Some consumers will check this value, if it exists
+      # and if not, fall back to the expected version. That is the correct
+      # behavior when assuming modules are unchanged, and why `actual` is set
+      # to `nil` here.
+      actual: nil,
+      type: type
+    }
+  end
+end
+

data/lib/r10k/module/forge.rb

@@ -25,6 +25,10 @@ class R10K::Module::Forge < R10K::Module::Base
     expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version)
   end
 
+  def self.statically_defined_version(name, args)
+    args[:version] if args[:version].is_a?(String)
+  end
+
   # @!attribute [r] metadata
   #   @api private
   #   @return [PuppetForge::Metadata]
@@ -35,8 +39,6 @@ class R10K::Module::Forge < R10K::Module::Base
   #   @return [PuppetForge::V3::Module] The Puppet Forge module metadata
   attr_reader :v3_module
 
-  include R10K::Logging
-
   include R10K::Util::Setopts
 
   def initialize(title, dirname, opts, environment=nil)
@@ -50,7 +52,7 @@ class R10K::Module::Forge < R10K::Module::Base
       :version => :expected_version,
       :source  => ::R10K::Util::Setopts::Ignore,
       :type    => ::R10K::Util::Setopts::Ignore,
-    })
+    }, :raise_on_unhandled => false)
 
     @expected_version ||= current_version || :latest
 
@@ -58,17 +60,24 @@ class R10K::Module::Forge < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
+    updated = false
     if should_sync?
       case status
       when :absent
         install
+        updated = true
       when :outdated
         upgrade
+        updated = true
       when :mismatched
         reinstall
+        updated = true
       end
+      maybe_delete_spec_dir
     end
+    updated
   end
 
   def properties
@@ -83,7 +92,11 @@ class R10K::Module::Forge < R10K::Module::Base
   def expected_version
     if @expected_version == :latest
       begin
-        @expected_version = @v3_module.current_release.version
+        if @v3_module.current_release
+          @expected_version = @v3_module.current_release.version
+        else
+          raise PuppetForge::ReleaseNotFound, _("The module %{title} does not appear to have any published releases, cannot determine latest version.") % { title: @title }
+        end
       rescue Faraday::ResourceNotFound => e
         raise PuppetForge::ReleaseNotFound, _("The module %{title} does not exist on %{url}.") % {title: @title, url: PuppetForge::V3::Release.conn.url_prefix}, e.backtrace
       end

data/lib/r10k/module/git.rb

@@ -13,6 +13,21 @@ class R10K::Module::Git < R10K::Module::Base
     false
   end
 
+  # Will be called if self.implement? above returns true. Will return
+  # the version info, if version is statically defined in the modules
+  # declaration.
+  def self.statically_defined_version(name, args)
+    if !args[:type] && (args[:ref] || args[:tag] || args[:commit])
+      if args[:ref] && args[:ref].to_s.match(/[0-9a-f]{40}/)
+        args[:ref]
+      else
+        args[:tag] || args[:commit]
+      end
+    elsif args[:type].to_s == 'git' && args[:version] && args[:version].to_s.match(/[0-9a-f]{40}/)
+      args[:version]
+    end
+  end
+
   # @!attribute [r] repo
   #   @api private
   #   @return [R10K::Git::StatefulRepository]
@@ -52,7 +67,7 @@ class R10K::Module::Git < R10K::Module::Base
       :git                     => :remote,
       :default_branch          => :default_ref,
       :default_branch_override => :default_override_ref,
-    })
+    }, :raise_on_unhandled => false)
 
     force = @overrides.dig(:modules, :force)
     @force = force == false ? false : true
@@ -83,9 +98,16 @@ class R10K::Module::Git < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
     force = opts[:force] || @force
-    @repo.sync(version, force) if should_sync?
+    if should_sync?
+      updated = @repo.sync(version, force)
+    else
+      updated = false
+    end
+    maybe_delete_spec_dir
+    updated
   end
 
   def status

data/lib/r10k/module/local.rb

@@ -1,5 +1,4 @@
 require 'r10k/module'
-require 'r10k/logging'
 
 # A dummy module type that can be used to "protect" Puppet modules that exist
 # inside of the Puppetfile "moduledir" location. Local modules will not be
@@ -12,8 +11,6 @@ class R10K::Module::Local < R10K::Module::Base
     args.is_a?(Hash) && (args[:local] || args[:type].to_s == 'local')
   end
 
-  include R10K::Logging
-
   def version
     "0.0.0"
   end
@@ -31,7 +28,9 @@ class R10K::Module::Local < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] false, because local modules are always considered in-sync
   def sync(opts={})
     logger.debug1 _("Module %{title} is a local module, always indicating synced.") % {title: title}
+    false
   end
 end

data/lib/r10k/module/svn.rb

@@ -10,6 +10,10 @@ class R10K::Module::SVN < R10K::Module::Base
     args.has_key?(:svn) || args[:type].to_s == 'svn'
   end
 
+  def self.statically_defined_version(name, args)
+    nil
+  end
+
   # @!attribute [r] expected_revision
   #   @return [String] The SVN revision that the repo should have checked out
   attr_reader :expected_revision
@@ -50,7 +54,7 @@ class R10K::Module::SVN < R10K::Module::Base
       :revision => :expected_revision,
       :username => :self,
       :password => :self
-    })
+    }, :raise_on_unhandled => false)
 
     @working_dir = R10K::SVN::WorkingDir.new(@path, :username => @username, :password => @password)
   end
@@ -70,17 +74,24 @@ class R10K::Module::SVN < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
+    updated = false
     if should_sync?
       case status
       when :absent
         install
+        updated = true
       when :mismatched
         reinstall
+        updated = true
       when :outdated
         update
+        updated = true
       end
+      maybe_delete_spec_dir
     end
+    updated
   end
 
   def exist?

data/lib/r10k/module.rb

@@ -22,17 +22,35 @@ module R10K::Module
   #
   # @return [Object < R10K::Module] A member of the implementing subclass
   def self.new(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      implementation.new(name, basedir, args, environment)
+    end
+  end
+
+  # Takes the same signature as Module.new but returns an metadata module
+  def self.from_metadata(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      R10K::Module::Definition.new(name,
+                                   dirname: basedir,
+                                   args: args,
+                                   implementation: implementation,
+                                   environment: environment)
+    end
+  end
+
+  def self.with_implementation(name, args, &block)
     if implementation = @klasses.find { |klass| klass.implement?(name, args) }
-      obj = implementation.new(name, basedir, args, environment)
-      obj
+      block.call(implementation)
     else
       raise _("Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)") % {name: name, args: args.inspect}
     end
   end
 
+
   require 'r10k/module/base'
   require 'r10k/module/git'
   require 'r10k/module/svn'
   require 'r10k/module/local'
   require 'r10k/module/forge'
+  require 'r10k/module/definition'
 end

data/lib/r10k/module_loader/puppetfile/dsl.rb

@@ -0,0 +1,42 @@
+module R10K
+  module ModuleLoader
+    class Puppetfile
+      class DSL
+        # A barebones implementation of the Puppetfile DSL
+        #
+        # @api private
+
+        def initialize(librarian, metadata_only: false)
+          @librarian     = librarian
+          @metadata_only = metadata_only
+        end
+
+        def mod(name, args = nil)
+          if args.is_a?(Hash)
+            opts = args
+          else
+            opts = { version: args }
+          end
+
+          if @metadata_only
+            @librarian.add_module_metadata(name, opts)
+          else
+            @librarian.add_module(name, opts)
+          end
+        end
+
+        def forge(location)
+          @librarian.set_forge(location)
+        end
+
+        def moduledir(location)
+          @librarian.set_moduledir(location)
+        end
+
+        def method_missing(method, *args)
+          raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method}
+        end
+      end
+    end
+  end
+end

data/lib/r10k/module_loader/puppetfile.rb

@@ -0,0 +1,272 @@
+require 'r10k/logging'
+require 'r10k/module'
+
+module R10K
+  module ModuleLoader
+    class Puppetfile
+
+      include R10K::Logging
+
+      DEFAULT_MODULEDIR = 'modules'
+      DEFAULT_PUPPETFILE_NAME = 'Puppetfile'
+
+      attr_accessor :default_branch_override, :environment
+      attr_reader :modules, :moduledir, :puppetfile_path,
+        :managed_directories, :desired_contents, :purge_exclusions
+
+      # @param basedir [String] The path that contains the moduledir &
+      #     Puppetfile by default. May be an environment, project, or
+      #     simple directory.
+      # @param puppetfile [String] The path to the Puppetfile, either an
+      #     absolute full path or a relative path with regards to the basedir.
+      # @param moduledir [String] The path to the moduledir, either an
+      #     absolute full path or a relative path with regards to the basedir.
+      # @param forge [String] The url (without protocol) to the Forge
+      # @param overrides [Hash] Configuration for loaded modules' behavior
+      # @param environment [R10K::Environment] When provided, the environment
+      #     in which loading takes place
+      def initialize(basedir:,
+                     moduledir: DEFAULT_MODULEDIR,
+                     puppetfile: DEFAULT_PUPPETFILE_NAME,
+                     overrides: {},
+                     environment: nil)
+
+        @basedir     = cleanpath(basedir)
+        @moduledir   = resolve_path(@basedir, moduledir)
+        @puppetfile_path  = resolve_path(@basedir, puppetfile)
+        @overrides   = overrides
+        @environment = environment
+        @default_branch_override = @overrides.dig(:environments, :default_branch_override)
+        @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override)
+
+        @existing_module_metadata = []
+        @existing_module_versions_by_name = {}
+        @modules = []
+
+        @managed_directories = []
+        @desired_contents = []
+        @purge_exclusions = []
+      end
+
+      def load
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load!
+        end
+      end
+
+      def load!
+        logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
+        logger.debug _("Using moduledir '%{moduledir}'") % {moduledir: @moduledir}
+
+        dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self)
+        dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
+
+        validate_no_duplicate_names(@modules)
+
+        managed_content = @modules.group_by(&:dirname)
+
+        @managed_directories = determine_managed_directories(managed_content)
+        @desired_contents = determine_desired_contents(managed_content)
+        @purge_exclusions = determine_purge_exclusions(@managed_directories)
+
+        {
+          modules: @modules,
+          managed_directories: @managed_directories,
+          desired_contents: @desired_contents,
+          purge_exclusions: @purge_exclusions
+        }
+
+      rescue SyntaxError, LoadError, ArgumentError, NameError => e
+        raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path})
+      end
+
+      def load_metadata
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load_metadata!
+        end
+      end
+
+      def load_metadata!
+        dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self, metadata_only: true)
+        dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
+
+        @existing_module_versions_by_name = @existing_module_metadata.map {|mod| [ mod.name, mod.version ] }.to_h
+        empty_load_output.merge(modules: @existing_module_metadata)
+
+      rescue SyntaxError, LoadError, ArgumentError, NameError => e
+        logger.warn _("Unable to preload Puppetfile because of %{msg}" % { msg: e.message })
+      end
+
+      def add_module_metadata(name, info)
+        install_path, metadata_info, _ = parse_module_definition(name, info)
+
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
+
+        @existing_module_metadata << mod
+      end
+
+      ##
+      ## set_forge, set_moduledir, and add_module are used directly by the DSL class
+      ##
+
+      # @param [String] forge
+      def set_forge(forge)
+        if @allow_puppetfile_forge
+          logger.debug _("Using Forge from Puppetfile: %{forge}") % { forge: forge }
+          PuppetForge.host = forge
+        else
+          logger.debug _("Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}.") % { forge: PuppetForge.host }
+        end
+      end
+
+      # @param [String] moduledir
+      def set_moduledir(moduledir)
+        @moduledir = resolve_path(@basedir, moduledir)
+      end
+
+      # @param [String] name
+      # @param [Hash, String, Symbol, nil] info Calling with
+      #   anything but a Hash is deprecated. The DSL will now convert
+      #   String and Symbol versions to Hashes of the shape
+      #     { version: <String or Symbol> }
+      #
+      #   String inputs should be valid module versions, the Symbol
+      #   `:latest` is allowed, as well as `nil`.
+      #
+      #   Non-Hash inputs are only ever used by Forge modules. In
+      #   future versions this method will require the caller (the
+      #   DSL class, not the Puppetfile author) to do this conversion
+      #   itself.
+      #
+      def add_module(name, info)
+        install_path, metadata_info, spec_deletable = parse_module_definition(name, info)
+
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
+        mod.origin = :puppetfile
+        mod.spec_deletable = spec_deletable
+
+        # Do not save modules if they would conflict with the attached
+        # environment
+        if @environment && @environment.module_conflicts?(mod)
+          return @modules
+        end
+
+        # If this module's metadata has a static version and that version
+        # matches the existing module declaration use it, otherwise create
+        # a regular module to sync.
+        unless mod.version && (mod.version == @existing_module_versions_by_name[mod.name])
+          mod = mod.to_implementation
+        end
+
+        @modules << mod
+      end
+
+     private
+
+      def empty_load_output
+        {
+          modules: [],
+          managed_directories: [],
+          desired_contents: [],
+          purge_exclusions: []
+        }
+      end
+
+      def with_readable_puppetfile(puppetfile_path, &block)
+        if File.readable?(puppetfile_path)
+          block.call
+        else
+          logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect}
+
+          empty_load_output
+        end
+      end
+
+      def parse_module_definition(name, info)
+        if !info.is_a?(Hash)
+          info = { version: info }
+        end
+
+        info[:overrides] = @overrides
+
+        if @default_branch_override
+          info[:default_branch_override] = @default_branch_override
+        end
+
+        spec_deletable = false
+        if install_path = info.delete(:install_path)
+          install_path = resolve_path(@basedir, install_path)
+          validate_install_path(install_path, name)
+        else
+          install_path = @moduledir
+          spec_deletable = true
+        end
+
+        return [ install_path, info, spec_deletable ]
+      end
+
+      # @param [Array<R10K::Module>] modules
+      def validate_no_duplicate_names(modules)
+        dupes = modules
+                .group_by { |mod| mod.name }
+                .select { |_, mods| mods.size > 1 }
+                .map(&:first)
+        unless dupes.empty?
+          msg = _('Puppetfiles cannot contain duplicate module names.')
+          msg += ' '
+          msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') })
+          raise R10K::Error.new(msg)
+        end
+      end
+
+      def resolve_path(base, path)
+        if Pathname.new(path).absolute?
+          cleanpath(path)
+        else
+          cleanpath(File.join(base, path))
+        end
+      end
+
+      def validate_install_path(path, modname)
+        unless /^#{Regexp.escape(@basedir)}.*/ =~ path
+          raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{@basedir}")
+        end
+
+        true
+      end
+
+      def determine_managed_directories(managed_content)
+        managed_content.keys.reject { |dir| dir == @basedir }
+      end
+
+      # Returns an array of the full paths to all the content being managed.
+      # @return [Array<String>]
+      def determine_desired_contents(managed_content)
+        managed_content.flat_map do |install_path, mods|
+          mods.collect { |mod| File.join(install_path, mod.name) }
+        end
+      end
+
+      def determine_purge_exclusions(managed_dirs)
+        if environment && environment.respond_to?(:desired_contents)
+          managed_dirs + environment.desired_contents
+        else
+          managed_dirs
+        end
+      end
+
+      # .cleanpath is as close to a canonical path as we can do without touching
+      # the filesystem. The .realpath methods will choke if some of the
+      # intermediate paths are missing, even though in some cases we will create
+      # them later as needed.
+      def cleanpath(path)
+        Pathname.new(path).cleanpath.to_s
+      end
+
+      # For testing purposes only
+      def puppetfile_content(path)
+        File.read(path)
+      end
+    end
+  end
+end