pdk 1.9.0 → 3.2.0

data/lib/pdk/validate/metadata/metadata_syntax_validator.rb

@@ -0,0 +1,76 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Metadata
+      class MetadataSyntaxValidator < InternalRubyValidator
+        def name
+          'metadata-syntax'
+        end
+
+        def pattern
+          contextual_pattern(['metadata.json', 'tasks/*.json'])
+        end
+
+        def spinner_text
+          format('Checking metadata syntax (%{patterns}).', patterns: pattern.join(' '))
+        end
+
+        def invoke(report)
+          super
+        ensure
+          JSON.parser = JSON::Ext::Parser if defined?(JSON::Ext::Parser)
+        end
+
+        def before_validation
+          # The pure ruby JSON parser gives much nicer parse error messages than
+          # the C extension at the cost of slightly slower parsing. We require it
+          # here and restore the C extension at the end of the method (if it was
+          # being used).
+          require 'json/pure'
+          JSON.parser = JSON::Pure::Parser
+        end
+
+        def validate_target(report, target)
+          unless PDK::Util::Filesystem.readable?(target)
+            report.add_event(
+              file: target,
+              source: name,
+              state: :failure,
+              severity: 'error',
+              message: 'Could not be read.'
+            )
+            return 1
+          end
+
+          begin
+            JSON.parse(PDK::Util::Filesystem.read_file(target))
+
+            report.add_event(
+              file: target,
+              source: name,
+              state: :passed,
+              severity: 'ok'
+            )
+            0
+          rescue JSON::ParserError => e
+            # Because the message contains a raw segment of the file, we use
+            # String#dump here to unescape any escape characters like newlines.
+            # We then strip out the surrounding quotes and the exclaimation
+            # point that json_pure likes to put in exception messages.
+            sane_message = e.message.dump[/\A"(.+?)!?"\Z/, 1]
+
+            report.add_event(
+              file: target,
+              source: name,
+              state: :failure,
+              severity: 'error',
+              message: sane_message
+            )
+            1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/metadata/metadata_validator_group.rb

@@ -0,0 +1,20 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Metadata
+      class MetadataValidatorGroup < ValidatorGroup
+        def name
+          'metadata'
+        end
+
+        def validators
+          [
+            MetadataSyntaxValidator,
+            MetadataJSONLintValidator
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/puppet/puppet_epp_validator.rb

@@ -0,0 +1,131 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Puppet
+      class PuppetEPPValidator < ExternalCommandValidator
+        # In Puppet >= 5.3.4, the error context formatting was changed to facilitate localization
+        ERROR_CONTEXT = /(?:file:\s(?<file>.+?)|line:\s(?<line>.+?)|column:\s(?<column>.+?))/.freeze
+        # In Puppet < 5.3.3, the error context was formatted in these variations:
+        #   - "at file_path:line_num:col_num"
+        #   - "at file_path:line_num"
+        #   - "at line line_num"
+        #   - "in file_path"
+        ERROR_CONTEXT_LEGACY = /(?:at\sline\s(?<line>\d+)|at\s(?<file>.+?):(?<line>\d+):(?<column>\d+)|at\s(?<file>.+?):(?<line>\d+)|in\s(?<file>.+?))/.freeze
+
+        PUPPET_LOGGER_PREFIX = /^(debug|info|notice|warning|error|alert|critical):\s.+?$/i.freeze
+        PUPPET_SYNTAX_PATTERN = /^
+          (?<severity>.+?):\s
+          (?<message>.+?)
+          (?:
+            \s\(#{ERROR_CONTEXT}(,\s#{ERROR_CONTEXT})*\)| # attempt to match the new localisation friendly location
+            \s#{ERROR_CONTEXT_LEGACY}| # attempt to match the old " at file:line:column" location
+            $                                               # handle cases where the output has no location
+          )
+        $/x.freeze
+
+        def name
+          'puppet-epp'
+        end
+
+        def cmd
+          'puppet'
+        end
+
+        def pattern
+          contextual_pattern('**/*.epp')
+        end
+
+        def spinner_text_for_targets(_targets)
+          format('Checking Puppet EPP syntax (%{pattern}).', pattern: pattern.join(' '))
+        end
+
+        def parse_options(targets)
+          # Due to PDK-1266 we need to run `puppet parser validate` with an empty
+          # modulepath. On *nix, Ruby treats `/dev/null` as an empty directory
+          # however it doesn't do so with `NUL` on Windows. The workaround for
+          # this to ensure consistent behaviour is to create an empty temporary
+          # directory and use that as the modulepath.
+          ['epp', 'validate', '--config', null_file, '--modulepath', validate_tmpdir].concat(targets)
+        end
+
+        def invoke(report)
+          super
+        ensure
+          remove_validate_tmpdir
+        end
+
+        def validate_tmpdir
+          require 'tmpdir'
+
+          @validate_tmpdir ||= Dir.mktmpdir('puppet-epp-validate')
+        end
+
+        def remove_validate_tmpdir
+          return unless @validate_tmpdir
+          return unless PDK::Util::Filesystem.directory?(@validate_tmpdir)
+
+          PDK::Util::Filesystem.remove_entry_secure(@validate_tmpdir)
+          @validate_tmpdir = nil
+        end
+
+        def null_file
+          Gem.win_platform? ? 'NUL' : '/dev/null'
+        end
+
+        def parse_output(report, result, targets)
+          # Due to PUP-7504, we will have to programmatically construct the json
+          # object from the text output for now.
+          output = result[:stderr].split(/\r?\n/).reject(&:empty?)
+
+          results_data = []
+          output.each do |offense|
+            offense_data = parse_offense(offense)
+            results_data << offense_data
+          end
+
+          # puppet parser validate does not include files without problems in its
+          # output, so we need to go through the list of targets and add passing
+          # events to the report for any target not listed in the output.
+          targets.reject { |target| results_data.any? { |j| j[:file] =~ /#{target}/ } }.each do |target|
+            report.add_event(
+              file: target,
+              source: name,
+              severity: :ok,
+              state: :passed
+            )
+          end
+
+          results_data.each do |offense|
+            report.add_event(offense)
+          end
+        end
+
+        def parse_offense(offense)
+          sanitize_console_output(offense)
+
+          offense_data = {
+            source: name,
+            state: :failure
+          }
+
+          if offense.match(PUPPET_LOGGER_PREFIX)
+            attributes = offense.match(PUPPET_SYNTAX_PATTERN)
+
+            attributes&.names&.each do |name|
+              offense_data[name.to_sym] = attributes[name] unless attributes[name].nil?
+            end
+          else
+            offense_data[:message] = offense
+          end
+
+          offense_data
+        end
+
+        def sanitize_console_output(line)
+          line.gsub!(/\e\[([;\d]+)?m/, '')
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/puppet/puppet_lint_validator.rb

@@ -0,0 +1,66 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Puppet
+      class PuppetLintValidator < ExternalCommandValidator
+        def name
+          'puppet-lint'
+        end
+
+        def cmd
+          'puppet-lint'
+        end
+
+        def pattern
+          contextual_pattern('**/*.pp')
+        end
+
+        def spinner_text_for_targets(_targets)
+          format('Checking Puppet manifest style (%{pattern}).', pattern: pattern.join(' '))
+        end
+
+        def parse_options(targets)
+          cmd_options = ['--json', '--relative']
+
+          cmd_options << '--fix' if options[:auto_correct]
+
+          cmd_options.concat(targets)
+        end
+
+        def parse_output(report, result, targets)
+          begin
+            json_data = JSON.parse(result[:stdout]).flatten
+          rescue JSON::ParserError
+            raise PDK::Validate::ParseOutputError, result[:stdout]
+          end
+
+          # puppet-lint does not include files without problems in its JSON
+          # output, so we need to go through the list of targets and add passing
+          # events to the report for any target not listed in the JSON output.
+          targets.reject { |target| json_data.any? { |j| j['path'] == target } }.each do |target|
+            report.add_event(
+              file: target,
+              source: name,
+              severity: 'ok',
+              state: :passed
+            )
+          end
+
+          json_data.each do |offense|
+            report.add_event({
+                               file: offense['path'],
+                               source: name,
+                               line: offense['line'],
+                               column: offense['column'],
+                               message: offense['message'],
+                               test: offense['check'],
+                               severity: offense['kind'] == 'fixed' ? 'corrected' : offense['kind'],
+                               state: :failure
+                             })
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/puppet/puppet_plan_syntax_validator.rb

@@ -0,0 +1,38 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Puppet
+      class PuppetPlanSyntaxValidator < PuppetSyntaxValidator
+        def name
+          'puppet-plan-syntax'
+        end
+
+        def pattern
+          contextual_pattern('plans/**/*.pp')
+        end
+
+        def pattern_ignore; end
+
+        def spinner_text_for_targets(_targets)
+          format('Checking Puppet plan syntax (%{pattern}).', pattern: pattern.join(' '))
+        end
+
+        def parse_options(targets)
+          # Due to PDK-1266 we need to run `puppet parser validate` with an empty
+          # modulepath. On *nix, Ruby treats `/dev/null` as an empty directory
+          # however it doesn't do so with `NUL` on Windows. The workaround for
+          # this to ensure consistent behaviour is to create an empty temporary
+          # directory and use that as the modulepath.
+          ['parser', 'validate', '--tasks', '--config', null_file, '--modulepath', validate_tmpdir].concat(targets)
+        end
+
+        def validate_tmpdir
+          require 'tmpdir'
+
+          @validate_tmpdir ||= Dir.mktmpdir('puppet-plan-parser-validate')
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/puppet/puppet_syntax_validator.rb

@@ -0,0 +1,135 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Puppet
+      class PuppetSyntaxValidator < ExternalCommandValidator
+        # In Puppet >= 5.3.4, the error context formatting was changed to facilitate localization
+        ERROR_CONTEXT = /(?:file:\s(?<file>.+?)|line:\s(?<line>.+?)|column:\s(?<column>.+?))/.freeze
+        # In Puppet < 5.3.3, the error context was formatted in these variations:
+        #   - "at file_path:line_num:col_num"
+        #   - "at file_path:line_num"
+        #   - "at line line_num"
+        #   - "in file_path"
+        ERROR_CONTEXT_LEGACY = /(?:at\sline\s(?<line>\d+)|at\s(?<file>.+?):(?<line>\d+):(?<column>\d+)|at\s(?<file>.+?):(?<line>\d+)|in\s(?<file>.+?))/.freeze
+
+        PUPPET_LOGGER_PREFIX = /^(debug|info|notice|warning|error|alert|critical):\s.+?$/i.freeze
+        PUPPET_SYNTAX_PATTERN = /^
+          (?<severity>.+?):\s
+          (?<message>.+?)
+          (?:
+            \s\(#{ERROR_CONTEXT}(,\s#{ERROR_CONTEXT})*\)| # attempt to match the new localisation friendly location
+            \s#{ERROR_CONTEXT_LEGACY}| # attempt to match the old " at file:line:column" location
+            $                                               # handle cases where the output has no location
+          )
+        $/x.freeze
+
+        def name
+          'puppet-syntax'
+        end
+
+        def cmd
+          'puppet'
+        end
+
+        def pattern
+          contextual_pattern('**/*.pp')
+        end
+
+        def pattern_ignore
+          contextual_pattern('plans/**/*.pp')
+        end
+
+        def spinner_text_for_targets(_targets)
+          format('Checking Puppet manifest syntax (%{pattern}).', pattern: pattern.join(' '))
+        end
+
+        def parse_options(targets)
+          # Due to PDK-1266 we need to run `puppet parser validate` with an empty
+          # modulepath. On *nix, Ruby treats `/dev/null` as an empty directory
+          # however it doesn't do so with `NUL` on Windows. The workaround for
+          # this to ensure consistent behaviour is to create an empty temporary
+          # directory and use that as the modulepath.
+          ['parser', 'validate', '--config', null_file, '--modulepath', validate_tmpdir].concat(targets)
+        end
+
+        def invoke(report)
+          super
+        ensure
+          remove_validate_tmpdir
+        end
+
+        def validate_tmpdir
+          require 'tmpdir'
+
+          @validate_tmpdir ||= Dir.mktmpdir('puppet-parser-validate')
+        end
+
+        def remove_validate_tmpdir
+          return unless @validate_tmpdir
+          return unless PDK::Util::Filesystem.directory?(@validate_tmpdir)
+
+          PDK::Util::Filesystem.remove_entry_secure(@validate_tmpdir)
+          @validate_tmpdir = nil
+        end
+
+        def null_file
+          Gem.win_platform? ? 'NUL' : '/dev/null'
+        end
+
+        def parse_output(report, result, targets)
+          # Due to PUP-7504, we will have to programmatically construct the json
+          # object from the text output for now.
+          output = result[:stderr].split(/\r?\n/).reject(&:empty?)
+
+          results_data = []
+          output.each do |offense|
+            offense_data = parse_offense(offense)
+            results_data << offense_data
+          end
+
+          # puppet parser validate does not include files without problems in its
+          # output, so we need to go through the list of targets and add passing
+          # events to the report for any target not listed in the output.
+          targets.reject { |target| results_data.any? { |j| j[:file] =~ /#{target}/ } }.each do |target|
+            report.add_event(
+              file: target,
+              source: name,
+              severity: :ok,
+              state: :passed
+            )
+          end
+
+          results_data.each do |offense|
+            report.add_event(offense)
+          end
+        end
+
+        def parse_offense(offense)
+          sanitize_console_output(offense)
+
+          offense_data = {
+            source: name,
+            state: :failure
+          }
+
+          if offense.match(PUPPET_LOGGER_PREFIX)
+            attributes = offense.match(PUPPET_SYNTAX_PATTERN)
+
+            attributes&.names&.each do |name|
+              offense_data[name.to_sym] = attributes[name] unless attributes[name].nil?
+            end
+          else
+            offense_data[:message] = offense
+          end
+
+          offense_data
+        end
+
+        def sanitize_console_output(line)
+          line.gsub!(/\e\[([;\d]+)?m/, '')
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/puppet/puppet_validator_group.rb

@@ -0,0 +1,22 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Puppet
+      class PuppetValidatorGroup < ValidatorGroup
+        def name
+          'puppet'
+        end
+
+        def validators
+          [
+            PuppetSyntaxValidator,
+            PuppetPlanSyntaxValidator,
+            PuppetLintValidator,
+            PuppetEPPValidator
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/ruby/ruby_rubocop_validator.rb

@@ -0,0 +1,79 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Ruby
+      class RubyRubocopValidator < ExternalCommandValidator
+        def allow_empty_targets?
+          true
+        end
+
+        def name
+          'rubocop'
+        end
+
+        def cmd
+          'rubocop'
+        end
+
+        def pattern
+          if context.is_a?(PDK::Context::ControlRepo)
+            ['Puppetfile', '**/**.rb']
+          else
+            '**/**.rb'
+          end
+        end
+
+        def spinner_text_for_targets(_targets)
+          format('Checking Ruby code style (%{pattern}).', pattern: pattern)
+        end
+
+        def parse_options(targets)
+          cmd_options = ['--format', 'json']
+
+          cmd_options << '--auto-correct' if options[:auto_correct]
+
+          cmd_options.concat(targets)
+        end
+
+        def parse_output(report, result, _targets)
+          return if result[:stdout].empty?
+
+          begin
+            json_data = JSON.parse(result[:stdout])
+          rescue JSON::ParserError
+            raise PDK::Validate::ParseOutputError, result[:stdout]
+          end
+
+          return unless json_data.key?('files')
+
+          json_data['files'].each do |file_info|
+            next unless file_info.key?('offenses')
+
+            result = {
+              file: file_info['path'],
+              source: 'rubocop'
+            }
+
+            if file_info['offenses'].empty?
+              report.add_event(result.merge(state: :passed, severity: :ok))
+            else
+              file_info['offenses'].each do |offense|
+                report.add_event(
+                  result.merge(
+                    line: offense['location']['line'],
+                    column: offense['location']['column'],
+                    message: offense['message'],
+                    severity: offense['corrected'] ? 'corrected' : offense['severity'],
+                    test: offense['cop_name'],
+                    state: :failure
+                  )
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/ruby/ruby_validator_group.rb

@@ -0,0 +1,19 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Ruby
+      class RubyValidatorGroup < ValidatorGroup
+        def name
+          'ruby'
+        end
+
+        def validators
+          [
+            RubyRubocopValidator
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/tasks/tasks_metadata_lint_validator.rb

@@ -0,0 +1,83 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module Tasks
+      class TasksMetadataLintValidator < InternalRubyValidator
+        def name
+          'task-metadata-lint'
+        end
+
+        def pattern
+          contextual_pattern('tasks/*.json')
+        end
+
+        def spinner_text
+          format('Checking task metadata style (%{pattern}).', pattern: pattern.join(' '))
+        end
+
+        def schema_file
+          require 'pdk/config'
+
+          path = PDK::Config.json_schema('task')
+          schema = PDK::Util::Filesystem.read_file(path)
+
+          JSON.parse(schema)
+        rescue JSON::ParserError
+          raise PDK::CLI::FatalError, 'Failed to parse Task Metadata Schema file.'
+        end
+
+        def validate_target(report, target)
+          unless PDK::Util::Filesystem.readable?(target)
+            report.add_event(
+              file: target,
+              source: name,
+              state: :failure,
+              severity: 'error',
+              message: 'Could not be read.'
+            )
+            return 1
+          end
+
+          require 'json-schema'
+          begin
+            # Need to set the JSON Parser and State Generator to the Native one to be
+            # compatible with the multi_json adapter.
+            JSON.parser = JSON::Ext::Parser if defined?(JSON::Ext::Parser)
+            JSON.generator = JSON::Ext::Generator if defined?(JSON::Ext::Generator)
+
+            begin
+              errors = JSON::Validator.fully_validate(schema_file, PDK::Util::Filesystem.read_file(target)) || []
+            rescue JSON::Schema::SchemaError => e
+              raise PDK::CLI::FatalError, format('Unable to validate Task Metadata. %{error}.', error: e.message)
+            end
+
+            if errors.empty?
+              report.add_event(
+                file: target,
+                source: name,
+                state: :passed,
+                severity: 'ok'
+              )
+              0
+            else
+              errors.each do |error|
+                # strip off the trailing parts that aren't relevant
+                error = error.split('in schema').first if error.include? 'in schema'
+
+                report.add_event(
+                  file: target,
+                  source: name,
+                  state: :failure,
+                  severity: 'error',
+                  message: error
+                )
+              end
+              1
+            end
+          end
+        end
+      end
+    end
+  end
+end