pdk 1.16.0 → 1.17.0

data/lib/pdk/validate/invokable_validator.rb

@@ -0,0 +1,216 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    # A base class for file based validators.
+    # This class provides base methods and helpers to help determine the file targets to validate against.
+    # Acutal validator implementation should inherit from other child abstract classes e.g. ExternalCommandValdiator
+    # @see PDK::Validate::Validator
+    # @abstract
+    class InvokableValidator < Validator
+      # Controls how many times the validator is invoked.
+      #
+      #   :once -       The validator will be invoked once and passed all the
+      #                 targets.
+      #   :per_target - The validator will be invoked for each target
+      #                 separately.
+      # @abstract
+      def invoke_style
+        :once
+      end
+
+      # Whether this Validator can be invoked in this context. By default any Validator can work in any Context, except ::None
+      # @return [Boolean]
+      # @abstract
+      def valid_in_context?
+        !context.is_a?(PDK::Context::None)
+      end
+
+      # An array, or a string, of glob patterns to use to find targets
+      # @return [Array[String], String]
+      # @abstract
+      def pattern; end
+
+      # An array, or a string, of glob patterns to use to ignore targets
+      # @return [Array[String], String, Nil]
+      # @abstract
+      def pattern_ignore; end
+
+      # @see PDK::Validate::Validator.prepare_invoke!
+      def prepare_invoke!
+        return if @prepared
+        super
+
+        # Register the spinner
+        spinner
+        nil
+      end
+
+      # Parses the target strings provided from the CLI
+      #
+      # @param options [Hash] A Hash containing the input options from the CLI.
+      #
+      # @return targets [Array] An Array of Strings containing target file paths
+      #                         for the validator to validate.
+      # @return skipped [Array] An Array of Strings containing targets
+      #                         that are skipped due to target not containing
+      #                         any files that can be validated by the validator.
+      # @return invalid [Array] An Array of Strings containing targets that do
+      #                         not exist, and will not be run by validator.
+      def parse_targets
+        requested_targets = options.fetch(:targets, [])
+        # If no targets are specified and empty targets are allowed return with an empty list.
+        # It will be up to the validator (and whatever validation tool it uses) to determine the
+        # targets. For example, using rubocop with no targets, will allow rubocop to determine the
+        # target list using it's .rubocop.yml file
+        return [[], [], []] if requested_targets.empty? && allow_empty_targets?
+        # If no targets are specified, then we will run validations from the base context directory.
+        targets = requested_targets.empty? ? [context.root_path] : requested_targets
+        targets.map! { |r| r.gsub(File::ALT_SEPARATOR, File::SEPARATOR) } if File::ALT_SEPARATOR
+
+        # If this validator is not valid in this context then skip all of the targets
+        return [[], targets, []] unless valid_in_context?
+
+        skipped = []
+        invalid = []
+        matched = targets.map { |target|
+          if pattern.nil?
+            target
+          else
+            if PDK::Util::Filesystem.directory?(target) # rubocop:disable Style/IfInsideElse
+              target_root = context.root_path
+              pattern_glob = Array(pattern).map { |p| PDK::Util::Filesystem.glob(File.join(target_root, p), File::FNM_DOTMATCH) }
+              target_list = pattern_glob.flatten
+                                        .select { |glob| PDK::Util::Filesystem.fnmatch(File.join(PDK::Util::Filesystem.expand_path(PDK::Util.canonical_path(target)), '*'), glob, File::FNM_DOTMATCH) }
+                                        .map { |glob| Pathname.new(glob).relative_path_from(Pathname.new(context.root_path)).to_s }
+
+              ignore_list = ignore_pathspec
+              target_list = target_list.reject { |file| ignore_list.match(file) }
+
+              skipped << target if target_list.flatten.empty?
+              target_list
+            elsif PDK::Util::Filesystem.file?(target)
+              if Array(pattern).include? target
+                target
+              elsif Array(pattern).any? { |p| PDK::Util::Filesystem.fnmatch(PDK::Util::Filesystem.expand_path(p), PDK::Util::Filesystem.expand_path(target), File::FNM_DOTMATCH) }
+                target
+              else
+                skipped << target
+                next
+              end
+            else
+              invalid << target
+              next
+            end
+          end
+        }.compact.flatten.uniq
+        [matched, skipped, invalid]
+      end
+
+      # Whether the target parsing ignores "dotfiles" (e.g. .gitignore or .pdkignore) which are considered hidden files in POSIX
+      # @return [Boolean]
+      # @abstract
+      def ignore_dotfiles?
+        true
+      end
+
+      # @see PDK::Validate::Validator.spinner_text
+      # @abstract
+      def spinner_text
+        _('Running %{name} validator ...') % { name: name }
+      end
+
+      # @see PDK::Validate::Validator.spinner
+      def spinner
+        return nil unless spinners_enabled?
+        return @spinner unless @spinner.nil?
+        require 'pdk/cli/util/spinner'
+
+        @spinner = TTY::Spinner.new("[:spinner] #{spinner_text}", PDK::CLI::Util.spinner_opts_for_platform)
+      end
+
+      # Process any targets that were skipped by the validator and add the events to the validation report
+      # @param report [PDK::Report] The report to add the events to
+      # @param skipped [Array[String]] The list of skipped targets
+      def process_skipped(report, skipped = [])
+        skipped.each do |skipped_target|
+          PDK.logger.debug(_('%{validator}: Skipped \'%{target}\'. Target does not contain any files to validate (%{pattern}).') % { validator: name, target: skipped_target, pattern: pattern })
+          report.add_event(
+            file:     skipped_target,
+            source:   name,
+            message:  _('Target does not contain any files to validate (%{pattern}).') % { pattern: pattern },
+            severity: :info,
+            state:    :skipped,
+          )
+        end
+      end
+
+      # Process any targets that were invalid by the validator and add the events to the validation report
+      # @param report [PDK::Report] The report to add the events to
+      # @param invalid [Array[String]] The list of invalid targets
+      def process_invalid(report, invalid = [])
+        invalid.each do |invalid_target|
+          PDK.logger.debug(_('%{validator}: Skipped \'%{target}\'. Target file not found.') % { validator: name, target: invalid_target })
+          report.add_event(
+            file:     invalid_target,
+            source:   name,
+            message:  _('File does not exist.'),
+            severity: :error,
+            state:    :error,
+          )
+        end
+      end
+
+      # Controls how the validator behaves if not passed any targets.
+      #
+      #   true  - PDK will not pass the globbed targets to the validator command
+      #           and it will instead rely on the underlying tool to find its
+      #           own default targets.
+      #   false - PDK will pass the globbed targets to the validator command.
+      # @abstract
+      def allow_empty_targets?
+        false
+      end
+
+      protected
+
+      # Takes the pattern used in a module context and transforms it depending on the
+      # context e.g. A Control Repo will use the module pattern in each module path
+      #
+      # @param [Array, String] The pattern when used in the module root. Does not start with '/'
+      #
+      # @return [Array[String]]
+      def contextual_pattern(module_pattern)
+        module_pattern = [module_pattern] unless module_pattern.is_a?(Array)
+        return module_pattern unless context.is_a?(PDK::Context::ControlRepo)
+        context.actualized_module_paths.map { |mod_path| module_pattern.map { |pat_path| mod_path + '/*/' + pat_path } }.flatten
+      end
+
+      private
+
+      # Helper method to collate the default ignored paths
+      # @return [PathSpec] Paths to ignore
+      def ignore_pathspec
+        ignore_pathspec = if context.is_a?(PDK::Context::Module)
+                            require 'pdk/module'
+                            PDK::Module.default_ignored_pathspec(ignore_dotfiles?)
+                          elsif context.is_a?(PDK::Context::ControlRepo)
+                            require 'pdk/control_repo'
+                            PDK::ControlRepo.default_ignored_pathspec(ignore_dotfiles?)
+                          else
+                            PathSpec.new.tap do |ps|
+                              ps.add('.*') if ignore_dotfiles?
+                            end
+                          end
+
+        unless pattern_ignore.nil?
+          Array(pattern_ignore).each do |pattern|
+            ignore_pathspec.add(pattern)
+          end
+        end
+
+        ignore_pathspec
+      end
+    end
+  end
+end

data/lib/pdk/validate/metadata/metadata_json_lint_validator.rb

@@ -0,0 +1,86 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Metadata
+      class MetadataJSONLintValidator < ExternalCommandValidator
+        # Validate each metadata file separately, as metadata-json-lint does not
+        # support multiple targets.
+        def invoke_style
+          :per_target
+        end
+
+        def name
+          'metadata-json-lint'
+        end
+
+        def cmd
+          'metadata-json-lint'
+        end
+
+        def spinner_text_for_targets(targets)
+          _('Checking module metadata style (%{targets}).') % {
+            targets: PDK::Util.targets_relative_to_pwd(targets.flatten).join(' '),
+          }
+        end
+
+        def pattern
+          contextual_pattern('metadata.json')
+        end
+
+        def parse_options(targets)
+          cmd_options = ['--format', 'json']
+          cmd_options << '--strict-dependencies'
+
+          cmd_options.concat(targets)
+        end
+
+        def parse_output(report, result, targets)
+          raise ArgumentError, _('More than 1 target provided to PDK::Validate::MetadataJSONLintValidator.') if targets.count > 1
+
+          if result[:stdout].strip.empty?
+            # metadata-json-lint will print nothing if there are no problems with
+            # the file being linted. This should be handled separately to
+            # metadata-json-lint generating output that can not be parsed as JSON
+            # (unhandled exception in metadata-json-lint).
+            json_data = {}
+          else
+            begin
+              json_data = JSON.parse(result[:stdout])
+            rescue JSON::ParserError
+              raise PDK::Validate::ParseOutputError, result[:stdout]
+            end
+          end
+
+          if json_data.empty?
+            report.add_event(
+              file:     targets.first,
+              source:   name,
+              state:    :passed,
+              severity: :ok,
+            )
+          else
+            json_data.delete('result')
+            json_data.keys.each do |type|
+              json_data[type].each do |offense|
+                # metadata-json-lint groups the offenses by type, so the type ends
+                # up being `warnings` or `errors`. We want to convert that to the
+                # singular noun for the event.
+                event_type = type[%r{\A(.+?)s?\Z}, 1]
+
+                report.add_event(
+                  file:     targets.first,
+                  source:   name,
+                  message:  offense['msg'],
+                  test:     offense['check'],
+                  severity: event_type,
+                  state:    :failure,
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/metadata/metadata_syntax_validator.rb

@@ -0,0 +1,78 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Metadata
+      class MetadataSyntaxValidator < InternalRubyValidator
+        def name
+          'metadata-syntax'
+        end
+
+        def pattern
+          contextual_pattern(['metadata.json', 'tasks/*.json'])
+        end
+
+        def spinner_text
+          _('Checking metadata syntax (%{patterns}).') % {
+            patterns: pattern.join(' '),
+          }
+        end
+
+        def invoke(report)
+          super
+        ensure
+          JSON.parser = JSON::Ext::Parser if defined?(JSON::Ext::Parser)
+        end
+
+        def before_validation
+          # The pure ruby JSON parser gives much nicer parse error messages than
+          # the C extension at the cost of slightly slower parsing. We require it
+          # here and restore the C extension at the end of the method (if it was
+          # being used).
+          require 'json/pure'
+          JSON.parser = JSON::Pure::Parser
+        end
+
+        def validate_target(report, target)
+          unless PDK::Util::Filesystem.readable?(target)
+            report.add_event(
+              file: target,
+              source: name,
+              state: :failure,
+              severity: 'error',
+              message: _('Could not be read.'),
+            )
+            return 1
+          end
+
+          begin
+            JSON.parse(PDK::Util::Filesystem.read_file(target))
+
+            report.add_event(
+              file:     target,
+              source:   name,
+              state:    :passed,
+              severity: 'ok',
+            )
+            return 0
+          rescue JSON::ParserError => e
+            # Because the message contains a raw segment of the file, we use
+            # String#dump here to unescape any escape characters like newlines.
+            # We then strip out the surrounding quotes and the exclaimation
+            # point that json_pure likes to put in exception messages.
+            sane_message = e.message.dump[%r{\A"(.+?)!?"\Z}, 1]
+
+            report.add_event(
+              file:     target,
+              source:   name,
+              state:    :failure,
+              severity: 'error',
+              message:  sane_message,
+            )
+            return 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/metadata/metadata_validator_group.rb

@@ -0,0 +1,20 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Metadata
+      class MetadataValidatorGroup < ValidatorGroup
+        def name
+          'metadata'
+        end
+
+        def validators
+          [
+            MetadataSyntaxValidator,
+            MetadataJSONLintValidator,
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/puppet/puppet_epp_validator.rb

@@ -0,0 +1,133 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Puppet
+      class PuppetEPPValidator < ExternalCommandValidator
+        # In Puppet >= 5.3.4, the error context formatting was changed to facilitate localization
+        ERROR_CONTEXT = %r{(?:file:\s(?<file>.+?)|line:\s(?<line>.+?)|column:\s(?<column>.+?))}
+        # In Puppet < 5.3.3, the error context was formatted in these variations:
+        #   - "at file_path:line_num:col_num"
+        #   - "at file_path:line_num"
+        #   - "at line line_num"
+        #   - "in file_path"
+        ERROR_CONTEXT_LEGACY = %r{(?:at\sline\s(?<line>\d+)|at\s(?<file>.+?):(?<line>\d+):(?<column>\d+)|at\s(?<file>.+?):(?<line>\d+)|in\s(?<file>.+?))}
+
+        PUPPET_LOGGER_PREFIX = %r{^(debug|info|notice|warning|error|alert|critical):\s.+?$}i
+        PUPPET_SYNTAX_PATTERN = %r{^
+          (?<severity>.+?):\s
+          (?<message>.+?)
+          (?:
+            \s\(#{ERROR_CONTEXT}(,\s#{ERROR_CONTEXT})*\)| # attempt to match the new localisation friendly location
+            \s#{ERROR_CONTEXT_LEGACY}| # attempt to match the old " at file:line:column" location
+            $                                               # handle cases where the output has no location
+          )
+        $}x
+
+        def name
+          'puppet-epp'
+        end
+
+        def cmd
+          'puppet'
+        end
+
+        def pattern
+          contextual_pattern('**/*.epp')
+        end
+
+        def spinner_text_for_targets(_targets)
+          _('Checking Puppet EPP syntax (%{pattern}).') % { pattern: pattern.join(' ') }
+        end
+
+        def parse_options(targets)
+          # Due to PDK-1266 we need to run `puppet parser validate` with an empty
+          # modulepath. On *nix, Ruby treats `/dev/null` as an empty directory
+          # however it doesn't do so with `NUL` on Windows. The workaround for
+          # this to ensure consistent behaviour is to create an empty temporary
+          # directory and use that as the modulepath.
+          ['epp', 'validate', '--config', null_file, '--modulepath', validate_tmpdir].concat(targets)
+        end
+
+        def invoke(report)
+          super
+        ensure
+          remove_validate_tmpdir
+        end
+
+        def validate_tmpdir
+          require 'tmpdir'
+
+          @validate_tmpdir ||= Dir.mktmpdir('puppet-epp-validate')
+        end
+
+        def remove_validate_tmpdir
+          return unless @validate_tmpdir
+          return unless PDK::Util::Filesystem.directory?(@validate_tmpdir)
+
+          PDK::Util::Filesystem.remove_entry_secure(@validate_tmpdir)
+          @validate_tmpdir = nil
+        end
+
+        def null_file
+          Gem.win_platform? ? 'NUL' : '/dev/null'
+        end
+
+        def parse_output(report, result, targets)
+          # Due to PUP-7504, we will have to programmatically construct the json
+          # object from the text output for now.
+          output = result[:stderr].split(%r{\r?\n}).reject { |entry| entry.empty? }
+
+          results_data = []
+          output.each do |offense|
+            offense_data = parse_offense(offense)
+            results_data << offense_data
+          end
+
+          # puppet parser validate does not include files without problems in its
+          # output, so we need to go through the list of targets and add passing
+          # events to the report for any target not listed in the output.
+          targets.reject { |target| results_data.any? { |j| j[:file] =~ %r{#{target}} } }.each do |target|
+            report.add_event(
+              file:     target,
+              source:   name,
+              severity: :ok,
+              state:    :passed,
+            )
+          end
+
+          results_data.each do |offense|
+            report.add_event(offense)
+          end
+        end
+
+        def parse_offense(offense)
+          sanitize_console_output(offense)
+
+          offense_data = {
+            source:  name,
+            state:  :failure,
+          }
+
+          if offense.match(PUPPET_LOGGER_PREFIX)
+            attributes = offense.match(PUPPET_SYNTAX_PATTERN)
+
+            unless attributes.nil?
+              attributes.names.each do |name|
+                offense_data[name.to_sym] = attributes[name] unless attributes[name].nil?
+              end
+            end
+          else
+            offense_data[:message] = offense
+          end
+
+          offense_data
+        end
+
+        def sanitize_console_output(line)
+          line.gsub!(%r{\e\[([;\d]+)?m}, '')
+        end
+      end
+    end
+  end
+end