puppetlabs-syntax 7.2.1

data/lib/puppetlabs-syntax/hiera.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'base64'
+
+module PuppetlabsSyntax
+  class Hiera
+    def check_hiera_key(key)
+      if key.is_a? Symbol
+        if key.to_s.start_with?(':')
+          "Puppet automatic lookup will not use leading '::'"
+        elsif !/^[a-z]+$/.match?(key) # we allow Hiera's own configuration
+          'Puppet automatic lookup will not look up symbols'
+        end
+      elsif !/^([a-z][a-z0-9_]+::)*[a-z0-9_][a-zA-Z0-9_]+$/.match?(key) # adapted from Puppet docs combining namespaced and un-namespaced regex
+        return 'Looks like a missing colon' if /[^:]:[^:]/.match?(key)
+
+        # be extra helpful
+
+        'Not a valid Puppet variable name for automatic lookup'
+
+      end
+    end
+
+    def check_hiera_data(_key, value)
+      # using filter_map to remove nil values
+      # there will be nil values if check_broken_function_call didn't return a string
+      # this is a shorthand for filter.compact
+      # https://blog.saeloun.com/2019/05/25/ruby-2-7-enumerable-filter-map/
+      keys_and_values(value).filter_map do |element|
+        check_broken_function_call(element)
+      end
+    end
+
+    # Recurse through complex data structures.  Return on first error.
+    def check_eyaml_data(name, val)
+      error = nil
+      if val.is_a? String
+        err = check_eyaml_blob(val)
+        error = "Key #{name} #{err}" if err
+      elsif val.is_a? Array
+        val.each_with_index do |v, idx|
+          error = check_eyaml_data("#{name}[#{idx}]", v)
+          break if error
+        end
+      elsif val.is_a? Hash
+        val.each do |k, v|
+          error = check_eyaml_data("#{name}['#{k}']", v)
+          break if error
+        end
+      end
+      error
+    end
+
+    def check_eyaml_blob(val)
+      # strip newlines and extra spaces
+      val.gsub!(/\s+/, '')
+
+      encodes_length = val.scan('ENC[').length
+
+      # Return if there's no encoded material
+      return if encodes_length == 0
+
+      found_encodes = val.scan(/ENC\[([^,\]]+,)?([^\]]+)?\]/)
+
+      return 'has unterminated eyaml value' unless found_encodes.length == encodes_length
+
+      known_methods = %w[PKCS7 GPG GKMS KMS TWOFAC SecretBox VAULT GCPKMS RSA SSHAGENT VAULT_RS cli]
+
+      found_encodes.each do |match|
+        # if no method is found we use the default PKCS7 method
+        method = match[0]
+        method = +'PKCS7' if method.nil?
+        method.delete_suffix!(',')
+        base64 = match[1]
+
+        return 'has invalid eyaml encoded format' if base64.nil?
+
+        return "has unknown eyaml method #{method}" unless known_methods.include? method
+
+        return 'has unpadded or truncated base64 data' unless base64.length % 4 == 0
+
+        return 'has corrupt base64 data' unless base64.match?(%r{^[a-zA-Z0-9+/=]+$})
+      end
+      # all good
+      nil
+    end
+
+    def check(filelist)
+      raise 'Expected an array of files' unless filelist.is_a?(Array)
+
+      errors = []
+
+      yamlargs = (Psych::VERSION >= '4.0') ? { aliases: true } : {}
+
+      filelist.each do |hiera_file|
+        begin
+          yamldata = YAML.load_file(hiera_file, **yamlargs)
+        rescue Exception => e
+          errors << "ERROR: Failed to parse #{hiera_file}: #{e}"
+          next
+        end
+        next unless yamldata
+
+        unless yamldata.is_a?(Hash)
+          errors << "ERROR: #{hiera_file} doesn't contain a valid Hash, datatype is #{yamldata.class}"
+          next
+        end
+
+        yamldata.each do |k, v|
+          if PuppetlabsSyntax.check_hiera_keys
+            key_msg = check_hiera_key(k)
+            errors << "WARNING: #{hiera_file}: Key :#{k}: #{key_msg}" if key_msg
+          end
+          if PuppetlabsSyntax.check_hiera_data
+            check_hiera_data(k, v).each do |value_msg|
+              errors << "WARNING: #{hiera_file}: Key :#{k}: #{value_msg}"
+            end
+          end
+          eyaml_msg = check_eyaml_data(k, v)
+          errors << "WARNING: #{hiera_file}: #{eyaml_msg}" if eyaml_msg
+        end
+      end
+
+      errors.map! { |e| e.to_s }
+
+      errors
+    end
+
+    private
+
+    # you can call functions in hiera, like this:
+    # "%{lookup('this_is_ok')}"
+    # you can do this in everywhere in a hiera value
+    # you cannot do string concatenation within {}:
+    # "%{lookup('this_is_ok'):3306}"
+    # You can do string concatenation outside of {}:
+    # "%{lookup('this_is_ok')}:3306"
+    def check_broken_function_call(element)
+      'string after a function call but before `}` in the value' if element.is_a?(String) && /%{[^}]+\('[^}]*'\)[^}\s]+}/.match?(element)
+    end
+
+    # gets a hash or array, returns all keys + values as array
+    def flatten_data(data, parent_key = [])
+      if data.is_a?(Hash)
+        data.flat_map do |key, value|
+          current_key = parent_key + [key.to_s]
+          if value.is_a?(Hash) || value.is_a?(Array)
+            flatten_data(value, current_key)
+          else
+            [current_key.join('.'), value]
+          end
+        end
+      elsif data.is_a?(Array) && !data.empty?
+        data.flat_map do |value|
+          if value.is_a?(Hash) || value.is_a?(Array)
+            flatten_data(value, parent_key)
+          else
+            [parent_key.join('.'), value]
+          end
+        end
+      else
+        [parent_key.join('.')]
+      end
+    end
+
+    # gets a string, hash or array, returns all keys + values as flattened + unique array
+    def keys_and_values(value)
+      if value.is_a?(Hash) || value.is_a?(Array)
+        flatten_data(value).flatten.uniq
+      else
+        [value]
+      end
+    end
+  end
+end

data/lib/puppetlabs-syntax/manifests.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module PuppetlabsSyntax
+  class Manifests
+    def check(filelist)
+      raise 'Expected an array of files' unless filelist.is_a?(Array)
+
+      require 'puppet'
+      require 'puppet/version'
+      require 'puppet/face'
+      require 'puppet/test/test_helper'
+
+      output = []
+
+      Puppet::Test::TestHelper.initialize
+      Puppet::Test::TestHelper.before_all_tests
+      called_before_all_tests = true
+
+      # Catch syntax warnings.
+      Puppet::Util::Log.newdestination(Puppet::Test::LogCollector.new(output))
+      Puppet::Util::Log.level = :warning
+
+      filelist.each do |puppet_file|
+        Puppet::Test::TestHelper.before_each_test
+        begin
+          error = validate_manifest(puppet_file)
+          output << error.values.first if error.is_a?(Hash) && !error.empty? # Puppet 6.5.0 onwards
+        rescue SystemExit
+          # Disregard exit(1) from face.
+          # This is how puppet < 6.5.0 `validate_manifest` worked.
+        rescue StandardError => e
+          output << e
+        ensure
+          Puppet::Test::TestHelper.after_each_test
+        end
+      end
+
+      Puppet::Util::Log.close_all
+      output.map! { |e| e.to_s }
+
+      # Exported resources will raise warnings when outside a puppetmaster.
+      output.reject! do |e|
+        e =~ /^You cannot collect( exported resources)? without storeconfigs being set/
+      end
+
+      # tag and schedule parameters in class raise warnings notice in output that prevent from succeed
+      output.reject! do |e|
+        e =~ /^(tag|schedule) is a metaparam; this value will inherit to all contained resources in the /
+      end
+
+      deprecations = output.select do |e|
+        e =~ /^Deprecation notice:|is deprecated/
+      end
+
+      # Errors exist if there is any output that isn't a deprecation notice.
+      has_errors = (output != deprecations)
+
+      [output, has_errors]
+    ensure
+      Puppet::Test::TestHelper.after_all_tests if called_before_all_tests
+    end
+
+    private
+
+    def validate_manifest(file)
+      Puppet[:tasks] = true if %r{.*plans/.*\.pp$}.match?(file)
+      Puppet::Face[:parser, :current].validate(file)
+    end
+  end
+end

data/lib/puppetlabs-syntax/tasks/puppetlabs-syntax.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'puppetlabs-syntax'
+require 'rake'
+require 'rake/tasklib'
+
+module PuppetlabsSyntax
+  class RakeTask < ::Rake::TaskLib
+    def filelist(paths)
+      excludes = PuppetlabsSyntax.exclude_paths
+      excludes.push('pkg/**/*')
+      excludes.push('vendor/**/*')
+      files = FileList[paths]
+      files.reject! { |f| File.directory?(f) }
+      files.exclude(*excludes)
+    end
+
+    def filelist_manifests
+      filelist(PuppetlabsSyntax.manifests_paths)
+    end
+
+    def filelist_templates
+      filelist(PuppetlabsSyntax.templates_paths)
+    end
+
+    def filelist_hiera_yaml
+      filelist(PuppetlabsSyntax.hieradata_paths)
+    end
+
+    def initialize(*_args)
+      desc 'Syntax check for Puppet manifests, templates and Hiera'
+      task syntax: [
+        'syntax:manifests',
+        'syntax:templates',
+        'syntax:hiera',
+      ]
+
+      namespace :syntax do
+        desc 'Syntax check Puppet manifests'
+        task :manifests do |t|
+          warn "---> #{t.name}"
+
+          c = PuppetlabsSyntax::Manifests.new
+          output, has_errors = c.check(filelist_manifests)
+          $stdout.puts "#{output.join("\n")}\n" unless output.empty?
+          exit 1 if has_errors || (output.any? && PuppetlabsSyntax.fail_on_deprecation_notices)
+        end
+
+        desc 'Syntax check Puppet templates'
+        task :templates do |t|
+          warn "---> #{t.name}"
+
+          c = PuppetlabsSyntax::Templates.new
+          result = c.check(filelist_templates)
+          unless result[:warnings].empty?
+            $stdout.puts 'WARNINGS:'
+            $stdout.puts result[:warnings].join("\n")
+          end
+          unless result[:errors].empty?
+            warn 'ERRORS:'
+            warn result[:errors].join("\n")
+            exit 1
+          end
+        end
+
+        desc 'Syntax check Hiera config files'
+        task hiera: [
+          'syntax:hiera:yaml',
+        ]
+
+        namespace :hiera do
+          task :yaml do |t|
+            warn "---> #{t.name}"
+            warn "#{t.name} was called, but PuppetlabsSyntax.check_hiera_keys is false. hiera syntax won't be checked" unless PuppetlabsSyntax.check_hiera_keys
+            c = PuppetlabsSyntax::Hiera.new
+            errors = c.check(filelist_hiera_yaml)
+            $stdout.puts "#{errors.join("\n")}\n" unless errors.empty?
+            exit 1 unless errors.empty?
+          end
+        end
+      end
+    end
+  end
+end
+
+PuppetlabsSyntax::RakeTask.new

data/lib/puppetlabs-syntax/templates.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'stringio'
+
+module PuppetlabsSyntax
+  class Templates
+    def check(filelist)
+      raise 'Expected an array of files' unless filelist.is_a?(Array)
+
+      # We now have to redirect STDERR in order to capture warnings.
+      $stderr = warnings = StringIO.new
+      result = { warnings: [], errors: [] }
+
+      filelist.each do |file|
+        if File.extname(file) == '.epp' or PuppetlabsSyntax.epp_only
+          tmp = validate_epp(file)
+        elsif File.extname(file) == '.erb'
+          tmp = validate_erb(file)
+        end
+        result.merge!(tmp) { |_k, a, b| a.concat(b) } unless tmp.nil?
+      end
+
+      $stderr = STDERR
+      result[:warnings] << warnings.string unless warnings.string.empty?
+
+      result[:errors].map! { |e| e.to_s }
+      result[:warnings].map! { |w| w.to_s }
+
+      result
+    end
+
+    def validate_epp(filename)
+      require 'puppet'
+      result = { warnings: [], errors: [] }
+      formatter = Puppet::Pops::Validation::DiagnosticFormatterPuppetStyle.new
+      evaluating_parser = Puppet::Pops::Parser::EvaluatingParser::EvaluatingEppParser.new
+      parser = evaluating_parser.parser
+      begin
+        parse_result = parser.parse_file(filename)
+        validation_result = evaluating_parser.validate(parse_result.model)
+
+        # print out any warnings
+        validation_result.warnings.each do |warn|
+          message = formatter.format_message(warn)
+          file = warn.file
+          line = warn.source_pos.line
+          column = warn.source_pos.pos
+          result[:warnings] << "#{file}:#{line}:#{column}: #{message}"
+        end
+
+        # collect errors and return them in order to indicate validation failure
+        validation_result.errors.each do |err|
+          message = formatter.format_message(err)
+          file = err.file
+          line = err.source_pos.line
+          column = err.source_pos.pos
+          result[:errors] << "#{file}:#{line}:#{column}: #{message}"
+        end
+      rescue Puppet::ParseError, SyntaxError => e
+        result[:errors] << e
+      rescue StandardError => e
+        result[:errors] << e
+      end
+
+      result
+    end
+
+    def validate_erb(filename)
+      result = { warnings: [], errors: [] }
+
+      begin
+        template = File.read(filename)
+        erb = ERB.new(template, trim_mode: '-')
+        erb.filename = filename
+        erb.result
+      rescue NameError
+        # This is normal because we don't have the variables that would
+        # ordinarily be bound by the parent Puppet manifest.
+      rescue TypeError
+        # This is normal because we don't have the variables that would
+        # ordinarily be bound by the parent Puppet manifest.
+      rescue SyntaxError => e
+        result[:errors] << e
+      end
+
+      result
+    end
+  end
+end

data/lib/puppetlabs-syntax/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module PuppetlabsSyntax
+  VERSION = '7.2.1'
+end

data/lib/puppetlabs-syntax.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'puppetlabs-syntax/version'
+
+module PuppetlabsSyntax
+  autoload :Hiera, 'puppetlabs-syntax/hiera'
+  autoload :Manifests, 'puppetlabs-syntax/manifests'
+  autoload :Templates, 'puppetlabs-syntax/templates'
+
+  @exclude_paths = [
+    'spec/fixtures/**/*',
+    'pkg/**/*',
+    'vendor/**/*',
+    '.vendor/**/*',
+  ]
+  @hieradata_paths = [
+    '**/data/**/*.*{yaml,yml}',
+    'hieradata/**/*.*{yaml,yml}',
+    'hiera*.*{yaml,yml}',
+  ]
+  @manifests_paths = [
+    '**/*.pp',
+  ]
+  @templates_paths = [
+    '**/templates/**/*.erb',
+    '**/templates/**/*.epp',
+  ]
+  @fail_on_deprecation_notices = true
+  @check_hiera_keys = true
+  @check_hiera_data = true
+
+  class << self
+    attr_accessor :exclude_paths,
+                  :hieradata_paths,
+                  :manifests_paths,
+                  :templates_paths,
+                  :fail_on_deprecation_notices,
+                  :epp_only,
+                  :check_hiera_keys,
+                  :check_hiera_data
+  end
+end

data/puppetlabs-syntax.gemspec

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'puppetlabs-syntax/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'puppetlabs-syntax'
+  spec.version       = PuppetlabsSyntax::VERSION
+  spec.email         = ['modules-team@puppet.com']
+  spec.authors       = ['Puppet, Inc.']
+  spec.summary       = 'Syntax checks for Puppet manifests, templates, and Hiera YAML'
+  spec.homepage      = 'https://github.com/puppetlabs/puppetlabs-syntax/'
+  spec.license       = 'MIT'
+  spec.description   = <<-DESC
+    Syntax checks for Puppet manifests and templates.
+  DESC
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 3.2'
+
+  spec.add_dependency 'puppet', '>= 8', '< 10'
+  spec.add_dependency 'rake', '~> 13.1'
+
+  spec.add_development_dependency 'voxpupuli-rubocop', '~> 5.2.0'
+end

data/spec/fixtures/hiera/hiera_bad.eyaml

@@ -0,0 +1,25 @@
+---
+acme::warning1: ENC[unknown-method,aGVsbG8sIHdvcmxk] # unknown method
+acme::warning2: ENC[PKCS7,aGVsbG8sIHdvcmxk # has unterminated eyaml value
+acme::warning3: ENC[PKCS7,aGVsbG8sIHdvcmxk==] # unpadded or truncated base64 data
+acme::warning4: ENC[PKCS7,aGVsbG8sf&IHdvcmxk==] # corrupt base64 data
+acme::warning5:
+  key1: foo
+  key2: ENC[PKCS7,aGVs!!!!bG8sIHdvcmxk] # corrupt base64 data
+acme::warning6:
+  hash_key:
+    - element1
+    - >
+      ENC[PKCS7,
+          aGVsbG8sIHdvcmxk
+      ]
+    - >
+      ENC[PKCS7,
+          aGVs!!!!bG8sIHdvcmxk
+      ]
+    # corrupt base64 data
+acme::warning7: ENC[] # has invalid eyaml encoded format
+acme::warning8: ENC[PKCS7,aGV&&&sbG8sIHdvcmxk] # unpadded or truncated base64 data
+acme::warning9: | # has unterminated eyaml value
+  ENC[aGVsbG8sIHdvcmxk
+  ENC[aGVsbG8sIHdvcmxk]

data/spec/fixtures/hiera/hiera_bad.yaml

@@ -0,0 +1 @@
+%YAMLBAD

data/spec/fixtures/hiera/hiera_badkey.yaml

@@ -0,0 +1,18 @@
+---
+this_is_ok: 0
+this_is_ok::too: 0
+th1s_is_ok::two3: 0
+:eventhis: 0
+good::value_with_CAPS: true
+rootValueWithCaps3:    true
+
+typical:typo::warning1:      true
+::notsotypical::warning2:    true
+noCamelCase::warning3:       true
+root::noCamelCase::warning4: true
+Leading_caps_warning5:       true
+namespace::Leading_warning6: true
+no-hyphens::warning7:        true
+:picky::warning8:            true
+this_is::warning9:
+  "%{lookup('foobar'):3306}": []

data/spec/fixtures/hiera/hiera_badvalue.yaml

@@ -0,0 +1,8 @@
+---
+this_is_ok::concat_func: "%{lookup('foo')}%{lookup('bar')}"
+
+this_is::warning1:
+  - "%{lookup('foobar'):3306}"
+this_is::warning2:
+  foo: "%{lookup('foobar'):3306}"
+this_is::warning3: "%{lookup('foo'):3306}%{lookup('bar')}"

data/spec/fixtures/hiera/hiera_good.eyaml

@@ -0,0 +1,13 @@
+---
+acme::good1: ENC[KMS,aGVsbG8sIdGHdvcmxk==]
+acme::good2: ENC[PKCS7,aGVsbG8sf3IHdvcmxk==]
+acme::good3: ENC[KMS,aGVsbG8sIdGHdvcmxk==]ENC[KMS,aGVsbG8sIdGHdvcmxk==]
+acme::good4: >
+   ENC[PKCS7,
+     aGVsbG8sIHdvcmxk]
+acme::good5: ENC[GPG,aGVsbG8sIHdvcmxkIQ==]
+acme::good6: ENC[GPG,aGVsbG8sIHdvcmxkISE=]
+acme::good7: |
+    text
+    ENC[GPG,aGVsbG8sIHdvcmxkISE=]
+acme::good8: ENC[aGVsbG8sf3IHdvcmxk==]

data/spec/fixtures/hiera/hiera_good.yaml

@@ -0,0 +1,7 @@
+---
+:backends:
+  - yaml
+:yaml:
+  :datadir: './hieradata/good'
+:hierarchy:
+  - common

data/spec/fixtures/hiera/hiera_key_no_value.yaml

@@ -0,0 +1,2 @@
+prometheus::initstyle:sysv
+prometheus::node_exporter::init_style:sysv

data/spec/fixtures/test_module/manifests/deprecation_notice.pp

@@ -0,0 +1,6 @@
+node test inherits default {
+  import 'pass.pp'
+  notify { 'this should give a deprecation notice':
+    message => 'inheritance is gone in the future parser',
+  }
+}

data/spec/fixtures/test_module/manifests/fail_error.pp

@@ -0,0 +1,3 @@
+class failing_puppet_module {
+  this is not a valid resource name
+}

data/spec/fixtures/test_module/manifests/fail_warning.pp

@@ -0,0 +1,5 @@
+class warning_puppet_module {
+  notify { 'this should raise a warning':
+    message => "because of \[\] escape characters",
+  }
+}

data/spec/fixtures/test_module/manifests/future_syntax.pp

@@ -0,0 +1,3 @@
+class module_with_future_syntax {
+	$a = $b = 10
+}

data/spec/fixtures/test_module/manifests/pass.pp

@@ -0,0 +1,5 @@
+class normal_puppet_module {
+  notify { 'should should pass':
+    message => 'with flying colours',
+  }
+}

data/spec/fixtures/test_module/manifests/pass_storeconfigs.pp

@@ -0,0 +1,3 @@
+class uses_storeconfigs {
+  @@notify { 'exported': }
+}

data/spec/fixtures/test_module/manifests/schedule_notice.pp

@@ -0,0 +1,5 @@
+class schedule_parameter_test ($schedule=undef){
+  notify { 'schedule_should pass':
+    message => 'with flying colours',
+  }
+}

data/spec/fixtures/test_module/manifests/tag_notice.pp

@@ -0,0 +1,5 @@
+class tag_parameter_test ($tag=undef){
+  notify { 'tag_should pass':
+    message => 'with flying colours',
+  }
+}

data/spec/fixtures/test_module/manifests/test_app.pp

@@ -0,0 +1,3 @@
+application test_app {
+
+}

data/spec/fixtures/test_module/templates/fail_error.epp

@@ -0,0 +1,3 @@
+This is plain text
+<% This is not valid EPP %>
+This is also plain text

data/spec/fixtures/test_module/templates/fail_error.erb

@@ -0,0 +1,3 @@
+This is plain text
+<% This is not valid Ruby( %>
+This is also plain text

data/spec/fixtures/test_module/templates/fail_error_also.epp

@@ -0,0 +1,3 @@
+This is plain text
+<% } %>
+This is also plain text

data/spec/fixtures/test_module/templates/fail_warning.erb

@@ -0,0 +1,2 @@
+Assignment in condition should warn.
+<%= "oh yes" if foo = true %>

data/spec/fixtures/test_module/templates/ignore.tpl

@@ -0,0 +1,3 @@
+This is plain text
+<% This is not valid Ruby %>
+This is also plain text