pdk 1.16.0 → 1.17.0

data/lib/pdk/util.rb

@@ -146,6 +146,15 @@ module PDK
     end
     module_function :configdir
 
+    def system_configdir
+      return @system_configdir unless @system_configdir.nil?
+      return @system_configdir = File.join('opt', 'puppetlabs', 'pdk', 'config') unless Gem.win_platform?
+
+      return @system_configdir = File.join(PDK::Util::Env['ProgramData'], 'PuppetLabs', 'PDK') unless PDK::Util::Env['ProgramData'].nil?
+      @system_configdir = File.join(PDK::Util::Env['AllUsersProfile'], 'PuppetLabs', 'PDK')
+    end
+    module_function :system_configdir
+
     # Returns path to the root of the module being worked on.
     #
     # @return [String, nil] Fully qualified base path to module, or nil if
@@ -248,16 +257,18 @@ module PDK
     module_function :targets_relative_to_pwd
 
     # TO-DO: Refactor replacement of lib/pdk/module/build.rb:metadata to use this function instead
-    def module_metadata
+    # @param module_path [String] The path to the root of the module. Default is determine the module root automatically
+    def module_metadata(module_path = nil)
       require 'pdk/module/metadata'
-
-      PDK::Module::Metadata.from_file(File.join(module_root, 'metadata.json')).data
+      module_path ||= module_root
+      PDK::Module::Metadata.from_file(File.join(module_path, 'metadata.json')).data
     end
     module_function :module_metadata
 
     # TO-DO: Refactor replacement of lib/pdk/module/build.rb:module_pdk_compatible? to use this function instead
-    def module_pdk_compatible?
-      ['pdk-version', 'template-url'].any? { |key| module_metadata.key?(key) }
+    # @param module_path [String] The path to the root of the module. Default is determine the module root automatically
+    def module_pdk_compatible?(module_path = nil)
+      ['pdk-version', 'template-url'].any? { |key| module_metadata(module_path).key?(key) }
     end
     module_function :module_pdk_compatible?
 
@@ -274,5 +285,24 @@ module PDK
       nil
     end
     module_function :module_pdk_version
+
+    # Does a deep copy instead of a shallow copy of an object.
+    #
+    # @param object [Object] The object to duplicate
+    #
+    # @return [Object] duplicate of the original object
+    #   the current working dir does not appear to be within a module.
+    def deep_duplicate(object)
+      if object.is_a?(Array)
+        object.map { |item| deep_duplicate(item) }
+      elsif object.is_a?(Hash)
+        hash = object.dup
+        hash.each_pair { |key, value| hash[key] = deep_duplicate(value) }
+        hash
+      else
+        object
+      end
+    end
+    module_function :deep_duplicate
   end
 end

data/lib/pdk/util/bundler.rb

@@ -210,6 +210,7 @@ module PDK
         end
 
         def binstubs!(gems)
+          raise PDK::CLI::FatalError, _('Unable to install requested binstubs as the Gemfile is missing') if gemfile.nil?
           binstub_dir = File.join(File.dirname(gemfile), 'bin')
           return true if gems.all? { |gem| PDK::Util::Filesystem.file?(File.join(binstub_dir, gem)) }
 

data/lib/pdk/util/changelog_generator.rb

@@ -5,11 +5,16 @@ module PDK
     module ChangelogGenerator
       # Taken from the version regex in https://forgeapi.puppet.com/schemas/module.json
       VERSION_REGEX = %r{^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$}
+      GEM = 'github_changelog_generator'.freeze
 
       # Raises if the github_changelog_generator is not available
       def self.github_changelog_generator_available!
         require 'bundler'
-        raise PDK::CLI::ExitWithError, _('Unable to generate the changelog as the github_changelog_generator gem is not installed') unless Bundler.rubygems.find_name('github_changelog_generator').any?
+        return if ::Bundler.rubygems.find_name(GEM).any?
+
+        raise PDK::CLI::ExitWithError, _(
+          'Unable to generate the changelog as the %{gem} gem is not installed',
+        ) % { gem: GEM }
       end
 
       # Runs the Changelog Generator gem (in the module's context) to automatically create a CHANGLELOG.MD file

data/lib/pdk/util/template_uri.rb

@@ -218,10 +218,11 @@ module PDK
                        else
                          nil
                        end
-        answers_uri = if [PACKAGED_TEMPLATE_KEYWORD, DEPRECATED_TEMPLATE_URL].include?(PDK.answers['template-url'])
+        default_template_url = PDK.config.get_within_scopes('module_defaults.template-url')
+        answers_uri = if [PACKAGED_TEMPLATE_KEYWORD, DEPRECATED_TEMPLATE_URL].include?(default_template_url)
                         Addressable::URI.parse(default_template_uri)
-                      elsif PDK.answers['template-url']
-                        new(uri_safe(PDK.answers['template-url'])).uri
+                      elsif default_template_url
+                        new(uri_safe(default_template_url)).uri
                       else
                         nil
                       end

data/lib/pdk/validate.rb

@@ -2,37 +2,84 @@ require 'pdk'
 
 module PDK
   module Validate
-    # TODO: Fix validator namespacing
-    autoload :BaseValidator, 'pdk/validate/base_validator'
-    autoload :MetadataJSONLint, 'pdk/validate/metadata/metadata_json_lint'
-    autoload :MetadataSyntax, 'pdk/validate/metadata/metadata_syntax'
-    autoload :MetadataValidator, 'pdk/validate/metadata_validator'
-    autoload :PuppetEPP, 'pdk/validate/puppet/puppet_epp'
-    autoload :PuppetLint, 'pdk/validate/puppet/puppet_lint'
-    autoload :PuppetSyntax, 'pdk/validate/puppet/puppet_syntax'
-    autoload :PuppetValidator, 'pdk/validate/puppet_validator'
-    autoload :Rubocop, 'pdk/validate/ruby/rubocop'
-    autoload :RubyValidator, 'pdk/validate/ruby_validator'
-    autoload :TasksValidator, 'pdk/validate/tasks_validator'
-    autoload :YAMLValidator, 'pdk/validate/yaml_validator'
+    autoload :ExternalCommandValidator, 'pdk/validate/external_command_validator'
+    autoload :InternalRubyValidator, 'pdk/validate/internal_ruby_validator'
+    autoload :InvokableValidator, 'pdk/validate/invokable_validator'
+    autoload :Validator, 'pdk/validate/validator'
+    autoload :ValidatorGroup, 'pdk/validate/validator_group'
 
-    class Tasks
-      autoload :Name, 'pdk/validate/tasks/name'
-      autoload :MetadataLint, 'pdk/validate/tasks/metadata_lint'
+    module Metadata
+      autoload :MetadataJSONLintValidator, 'pdk/validate/metadata/metadata_json_lint_validator'
+      autoload :MetadataSyntaxValidator, 'pdk/validate/metadata/metadata_syntax_validator'
+      autoload :MetadataValidatorGroup, 'pdk/validate/metadata/metadata_validator_group'
     end
 
-    class YAML
-      autoload :Syntax, 'pdk/validate/yaml/syntax'
+    module YAML
+      autoload :YAMLSyntaxValidator, 'pdk/validate/yaml/yaml_syntax_validator'
+      autoload :YAMLValidatorGroup, 'pdk/validate/yaml/yaml_validator_group'
+    end
+
+    module Ruby
+      autoload :RubyRubocopValidator, 'pdk/validate/ruby/ruby_rubocop_validator'
+      autoload :RubyValidatorGroup, 'pdk/validate/ruby/ruby_validator_group'
+    end
+
+    module Tasks
+      autoload :TasksNameValidator, 'pdk/validate/tasks/tasks_name_validator'
+      autoload :TasksMetadataLintValidator, 'pdk/validate/tasks/tasks_metadata_lint_validator'
+      autoload :TasksValidatorGroup, 'pdk/validate/tasks/tasks_validator_group'
+    end
+
+    module Puppet
+      autoload :PuppetEPPValidator, 'pdk/validate/puppet/puppet_epp_validator'
+      autoload :PuppetLintValidator, 'pdk/validate/puppet/puppet_lint_validator'
+      autoload :PuppetSyntaxValidator, 'pdk/validate/puppet/puppet_syntax_validator'
+      autoload :PuppetValidatorGroup, 'pdk/validate/puppet/puppet_validator_group'
     end
 
     def self.validators
-      @validators ||= [
-        MetadataValidator,
-        YAMLValidator,
-        PuppetValidator,
-        RubyValidator,
-        TasksValidator,
-      ].freeze
+      validator_hash.values
+    end
+
+    def self.validator_names
+      validator_hash.keys
+    end
+
+    # @api private
+    def self.validator_hash
+      # TODO: This isn't the most performant... But with only 5 items, it's fine
+      @validator_hash ||= [
+        Metadata::MetadataValidatorGroup,
+        Puppet::PuppetValidatorGroup,
+        Ruby::RubyValidatorGroup,
+        Tasks::TasksValidatorGroup,
+        YAML::YAMLValidatorGroup,
+      ].map { |klass| [klass.new.name, klass] }.to_h.freeze
+    end
+
+    def self.invoke_validators_by_name(context, names, parallel = false, options = {})
+      instances = names.select { |name| validator_names.include?(name) }
+                       .map { |name| validator_hash[name].new(context, options) }
+                       .each { |instance| instance.prepare_invoke! }
+      report = PDK::Report.new
+
+      # Nothing to validate then nothing to do.
+      return [0, report] if instances.empty?
+
+      require 'pdk/cli/exec_group'
+      exec_group = PDK::CLI::ExecGroup.create(
+        _('Validating module using %{num_of_threads} threads' % { num_of_threads: instances.count }),
+        { parallel: parallel },
+        options,
+      )
+
+      instances.each do |validator|
+        exec_group.register do
+          validator.invoke(report)
+        end
+      end
+
+      [exec_group.exit_code, report]
     end
 
     class ParseOutputError < StandardError; end

data/lib/pdk/validate/external_command_validator.rb

@@ -0,0 +1,208 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    # An abstract validator that runs external commands within a Ruby Bundled environment
+    # e.g. `puppet-lint`, or `puppet validate`
+    #
+    # At a a minimum child classes should implment the `name`, `cmd`, `pattern` and `parse_output` methods
+    #
+    # An example concrete implementation could look like:
+    #
+    # module PDK
+    #   module Validate
+    #     module Ruby
+    #       class RubyRubocopValidator < ExternalCommandValidator
+    #         def name
+    #           'rubocop'
+    #         end
+    #
+    #         def cmd
+    #           'rubocop'
+    #         end
+    #
+    #         def pattern
+    #           '**/**.rb'
+    #         end
+    #
+    #         def parse_options(targets)
+    #           ['--format', 'json']
+    #         end
+    #
+    #         def parse_output(report, result, _targets)
+    #    ... ruby code ...
+    #           report.add_event(
+    #             line:     offense['location']['line'],
+    #             column:   offense['location']['column'],
+    #             message:  offense['message'],
+    #             severity: offense['corrected'] ? 'corrected' : offense['severity'],
+    #             test:     offense['cop_name'],
+    #             state:    :failure,
+    #           )
+    #         end
+    #       end
+    #     end
+    #   end
+    # end
+    #
+    # @see PDK::Validate::InvokableValidator
+    class ExternalCommandValidator < InvokableValidator
+      # @return Array[PDK::CLI::Exec::Command] This is a private implementation attribute used for unit testing
+      # @api private
+      attr_reader :commands
+
+      # @see PDK::Validate::Validator.spinner
+      def spinner
+        # The validator has sub-commands with their own spinners.
+        nil
+      end
+
+      # Calculates the text of the spinner based on the target list
+      # @return [String]
+      # @abstract
+      def spinner_text_for_targets(targets); end
+
+      # The name of the command to be run for validation
+      # @return [String]
+      # @abstract
+      def cmd; end
+
+      # Alternate paths which the command (cmd) may exist in. Typically other ruby gem caches,
+      # or packaged installation bin directories.
+      # @return [Array[String]]
+      # @api private
+      def alternate_bin_paths
+        [
+          PDK::Util::RubyVersion.bin_path,
+          File.join(PDK::Util::RubyVersion.gem_home, 'bin'),
+          PDK::Util::RubyVersion.gem_paths_raw.map { |gem_path_raw| File.join(gem_path_raw, 'bin') },
+          PDK::Util.package_install? ? File.join(PDK::Util.pdk_package_basedir, 'bin') : nil,
+        ].flatten.compact
+      end
+
+      # The full path to the command (cmd)
+      # Can be overridden in child classes to a non-default path
+      # @return [String]
+      # @api private
+      def cmd_path
+        return @cmd_path unless @cmd_path.nil?
+        @cmd_path = File.join(context.root_path, 'bin', cmd)
+        # Return the path to the command if it exists on disk, or we have a gemfile (i.e. Bundled install)
+        # The Bundle may be created after the prepare_invoke so if the file doesn't exist, it may not be an error
+        return @cmd_path if PDK::Util::Filesystem.exist?(@cmd_path) || !PDK::Util::Bundler::BundleHelper.new.gemfile.nil?
+        # But if there is no Gemfile AND cmd doesn't exist in the default path, we need to go searching...
+        @cmd_path = alternate_bin_paths.map { |alternate_path| File.join(alternate_path, cmd) }
+                                       .find { |path| PDK::Util::Filesystem.exist?(path) }
+        return @cmd_path unless @cmd_path.nil?
+        # If we can't find it anywhere, just let the OS find it
+        @cmd_path = cmd
+      end
+
+      # An array of command line arguments to pass to the command for validation
+      # @return Array[String]
+      # @abstract
+      def parse_options(_targets)
+        []
+      end
+
+      # Parses the output from the command and appends formatted events to the report.
+      # This is called for each command, which is a group of targets
+      #
+      # @param report [PDK::Report] The report to add events to
+      # @param result [Hash[Symbol => Object]] The result of validation command process
+      # @param targets [Array[String]] The targets for this command result
+      # @api private
+      # @see PDK::CLI::Exec::Command.execute!
+      # @abstract
+      def parse_output(_report, _result, _targets); end
+
+      # Prepares for invokation by parsing targets and creating the needed commands.
+      # @api private
+      # @see PDK::Validate::Validator.prepare_invoke!
+      def prepare_invoke!
+        return if @prepared
+        super
+
+        @targets, @skipped, @invalid = parse_targets
+        @targets = [] if @targets.nil?
+
+        target_groups = if @targets.empty? && allow_empty_targets?
+                          # If we have no targets and we allow empty targets, create an empty target group list
+                          [[]]
+                        elsif invoke_style == :per_target
+                          # If invoking :per_target, split the targets array into an array of
+                          # single element arrays (one per target).
+                          @targets.combination(1).to_a.compact
+                        else
+                          # Else we're invoking :once, wrap the targets array in another array. This is so we
+                          # can loop through the invokes with the same logic, regardless of which invoke style
+                          # is needed.
+                          @targets.each_slice(1000).to_a.compact
+                        end
+
+        # Register all of the commands for all of the targets
+        @commands = []
+        target_groups.each do |invokation_targets|
+          next if invokation_targets.empty? && !allow_empty_targets?
+          cmd_argv = parse_options(invokation_targets).unshift(cmd_path).compact
+          cmd_argv.unshift(File.join(PDK::Util::RubyVersion.bin_path, 'ruby.exe'), '-W0') if Gem.win_platform?
+
+          command = PDK::CLI::Exec::Command.new(*cmd_argv).tap do |c|
+            c.context = :module
+            c.environment = { 'PUPPET_GEM_VERSION' => options[:puppet] } if options[:puppet]
+
+            if spinners_enabled?
+              parent_validator = options[:parent_validator]
+              if parent_validator.nil? || parent_validator.spinner.nil? || !parent_validator.spinner.is_a?(TTY::Spinner::Multi)
+                c.add_spinner(spinner_text_for_targets(invokation_targets))
+              else
+                spinner = TTY::Spinner.new("[:spinner] #{spinner_text_for_targets(invokation_targets)}", PDK::CLI::Util.spinner_opts_for_platform)
+                parent_validator.spinner.register(spinner)
+                c.register_spinner(spinner, PDK::CLI::Util.spinner_opts_for_platform)
+              end
+            end
+          end
+
+          @commands << { command: command, invokation_targets: invokation_targets }
+        end
+        nil
+      end
+
+      # Invokes the prepared commands as an ExecGroup
+      # @see PDK::Validate::Validator.invoke
+      def invoke(report)
+        prepare_invoke!
+
+        process_skipped(report, @skipped)
+        process_invalid(report, @invalid)
+
+        # Nothing to execute so return success
+        return 0 if @commands.empty?
+
+        # If there's no Gemfile, then we can't ensure the binstubs are correct
+        PDK::Util::Bundler.ensure_binstubs!(cmd) unless PDK::Util::Bundler::BundleHelper.new.gemfile.nil?
+
+        exec_group = PDK::CLI::ExecGroup.create(name, { parallel: false }, options)
+
+        # Register all of the commands for all of the targets
+        @commands.each do |item|
+          command = item[:command]
+          invokation_targets = item[:invokation_targets]
+
+          exec_group.register do
+            result = command.execute!
+            begin
+              parse_output(report, result, invokation_targets.compact)
+            rescue PDK::Validate::ParseOutputError => e
+              $stderr.puts e.message
+            end
+            result[:exit_code]
+          end
+        end
+
+        # Now execute and get the return code
+        exec_group.exit_code
+      end
+    end
+  end
+end

data/lib/pdk/validate/internal_ruby_validator.rb

@@ -0,0 +1,100 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    # An abstract validator that runs ruby code internal to the PDK e.g. JSON and YAML validation, on a single file.
+    # The validator code must run within the PDK Ruby environment as opposed to the bundled Ruby environment for a module.
+    #
+    # At a a minimum child classes should implment the `name`, `pattern` and `validate_target` methods
+    #
+    # An example concrete implementation could look like:
+    #
+    # module PDK
+    #   module Validate
+    #     module Tasks
+    #       class TasksNameValidator < InternalRubyValidator
+    #         def name
+    #           'task-name'
+    #         end
+    #
+    #         def pattern
+    #           'tasks/**/*'
+    #         end
+    #
+    #         def validate_target(report, target)
+    #           task_name = File.basename(target, File.extname(target))
+    #     ... ruby code ...
+    #           success ? 0 : 1
+    #         end
+    #       end
+    #     end
+    #   end
+    # end
+    #
+    #
+    # @abstract
+    # @see PDK::Validate::InvokableValidator
+    class InternalRubyValidator < InvokableValidator
+      # @see PDK::Validate::Validator.prepare_invoke!
+      def prepare_invoke!
+        return if @prepared
+        super
+
+        # Parse the targets
+        @targets, @skipped, @invalid = parse_targets
+
+        nil
+      end
+
+      # Invokes the validator to call `validate_target` on each target
+      # @see PDK::Validate::Validator.invoke
+      def invoke(report)
+        prepare_invoke!
+
+        process_skipped(report, @skipped)
+        process_invalid(report, @invalid)
+
+        return 0 if @targets.empty?
+
+        return_val = 0
+
+        before_validation
+
+        start_spinner
+        @targets.each do |target|
+          validation_result = validate_target(report, target)
+          if validation_result.nil?
+            report.add_event(
+              file:     target,
+              source:   name,
+              state:    :failure,
+              severity: 'error',
+              message:  "Validation did not return an exit code for #{target}",
+            )
+            validation_result = 1
+          end
+          return_val = validation_result if validation_result > return_val
+        end
+
+        stop_spinner(return_val.zero?)
+        return_val
+      end
+
+      # Validates a single target
+      # It is the responsibility of this method to populate the report with validation messages
+      #
+      # @param report [PDK::Report] The report to add the events to
+      # @param target [String] The target to validate
+      #
+      # @return [Integer] The exitcode of the validation. Zero indicates success.  A non-zero code indicates failure
+      # @api private
+      # @abstract
+      def validate_target(report, target); end
+
+      # Tasks to run before validation occurs. This is run once every time `.invoke` is called
+      # @api private
+      # @abstract
+      def before_validation; end
+    end
+  end
+end