puppet-lint 2.3.6 → 2.5.0

data/README.md

@@ -120,6 +120,18 @@ Or to specify a whitelist of allowed checks, include a line like:
 --only-checks=trailing_whitespace,hard_tabs,duplicate_params,double_quoted_strings,unquoted_file_mode,only_variable_string,variables_not_enclosed,single_quote_string_with_variables,variable_contains_dash,ensure_not_symlink_target,unquoted_resource_title,relative_classname_inclusion,file_mode,resource_reference_without_title_capital,leading_zero,arrow_alignment,variable_is_lowercase,ensure_first_param,resource_reference_without_whitespace,file_ensure,trailing_comma,leading_zero
 ```
 
+Please note that there is an important difference between reading options from the command line and reading options from a configuration file: In the former case the shell interprets one level of quotes. That does not happen in the latter case. So, it would make sense to quote some configuration values on the command line, like so:
+
+```
+$ puppet-lint --ignore-paths 'modules/stdlib/*' modules/
+```
+
+When reading from a configuration file those quotes would be passed on to the option parser -- probably not giving the expected result. Instead the line should read
+
+```
+--ignore-paths=modules/stdlib/*
+```
+
 ## Testing with Puppet Lint as a Rake task
 
 To test your entire Puppet manifest directory, add `require 'puppet-lint/tasks/puppet-lint'` to your Rakefile and then run:
@@ -195,6 +207,12 @@ You can also disable checks when running Puppet Lint through the supplied Rake t
   PuppetLint.configuration.pattern = "modules"
   ```
 
+## Testing with Puppet Lint as a GitHub Action
+
+There is a GitHub Actions action available to get linter feedback in workflows:
+
+* [puppet-lint-action](https://github.com/marketplace/actions/puppet-lint-action)
+
 ## Options
 
 See `puppet-lint --help` for a full list of command line options and checks.

data/lib/puppet-lint.rb

@@ -100,9 +100,9 @@ class PuppetLint
   # Returns a format String to be used with String#%.
   def log_format
     if configuration.log_format.nil? || configuration.log_format.empty?
-      ## recreate previous old log format as far as thats possible.
       format = '%{KIND}: %{message} on line %{line}'
       format.prepend('%{path} - ') if configuration.with_filename
+      format.concat(' (check: %{check})')
       configuration.log_format = format
     end
 

data/lib/puppet-lint/data.rb

@@ -171,21 +171,21 @@ class PuppetLint::Data
         tokens.select { |t| t.type == :COLON }.each do |colon_token|
           next unless colon_token.next_code_token && colon_token.next_code_token.type != :LBRACE
 
-          start_idx = tokens.index(colon_token)
-          next if start_idx < marker
+          rel_start_idx = tokens[marker..-1].index(colon_token)
+          break if rel_start_idx.nil?
+          start_idx = rel_start_idx + marker
           end_token = colon_token.next_token_of([:SEMIC, :RBRACE])
-          end_idx = tokens.index(end_token)
-
-          raise PuppetLint::SyntaxError, colon_token if end_idx.nil?
+          rel_end_idx = tokens[start_idx..-1].index(end_token)
+          raise PuppetLint::SyntaxError, colon_token if rel_end_idx.nil?
+          marker = rel_end_idx + start_idx
 
           result << {
             :start        => start_idx + 1,
-            :end          => end_idx,
-            :tokens       => tokens[start_idx..end_idx],
+            :end          => marker,
+            :tokens       => tokens[start_idx..marker],
             :type         => find_resource_type_token(start_idx),
-            :param_tokens => find_resource_param_tokens(tokens[start_idx..end_idx]),
+            :param_tokens => find_resource_param_tokens(tokens[start_idx..marker]),
           }
-          marker = end_idx
         end
         result
       end
@@ -201,6 +201,9 @@ class PuppetLint::Data
       lbrace_idx = tokens[0..index].rindex do |token|
         token.type == :LBRACE && token.prev_code_token.type != :QMARK
       end
+
+      raise PuppetLint::SyntaxError, tokens[index] if lbrace_idx.nil?
+
       tokens[lbrace_idx].prev_code_token
     end
 
@@ -212,9 +215,21 @@ class PuppetLint::Data
     #
     # Returns an Array of Token objects.
     def find_resource_param_tokens(resource_tokens)
-      resource_tokens.select do |token|
-        token.type == :NAME && token.next_code_token.type == :FARROW
+      param_tokens = []
+
+      iter_token = resource_tokens.first.prev_token
+
+      until iter_token.nil?
+        iter_token = iter_token.next_token_of(:NAME)
+
+        break unless resource_tokens.include?(iter_token)
+
+        if iter_token && iter_token.next_code_token.type == :FARROW
+          param_tokens << iter_token
+        end
       end
+
+      param_tokens
     end
 
     # Internal: Calculate the positions of all class definitions within the

data/lib/puppet-lint/lexer.rb

@@ -2,8 +2,9 @@
 
 require 'pp'
 require 'strscan'
-require 'puppet-lint/lexer/token'
 require 'set'
+require 'puppet-lint/lexer/token'
+require 'puppet-lint/lexer/string_slurper'
 
 class PuppetLint
   # Internal: A generic error thrown by the lexer when it encounters something
@@ -28,11 +29,15 @@ class PuppetLint
       @column = column
       @reason = reason
     end
+
+    def to_s
+      "PuppetLint::LexerError: Line:#{line_no} Column: #{column} Reason: #{reason}"
+    end
   end
 
   # Internal: The puppet-lint lexer. Converts your manifest into its tokenised
   # form.
-  class Lexer # rubocop:disable Metrics/ClassLength
+  class Lexer
     def initialize
       @line_no = 1
       @column = 1
@@ -99,18 +104,32 @@ class PuppetLint
       :IF      => true,
       :ELSIF   => true,
       :LPAREN  => true,
+      :EQUALS  => true,
     }.freeze
 
+    # Internal: some commonly used regular expressions
+    # \t == tab
+    # \v == vertical tab
+    # \f == form feed
+    # \p{Zs} == ASCII + Unicode non-linebreaking whitespace
+    WHITESPACE_RE = RUBY_VERSION == '1.8.7' ? %r{[\t\v\f ]} : %r{[\t\v\f\p{Zs}]}
+
+    LINE_END_RE = %r{(?:\r\n|\r|\n)}
+
+    NAME_RE = %r{\A((?:(?:::)?[_a-z0-9][-\w]*)(?:::[a-z0-9][-\w]*)*)}
+
     # Internal: An Array of Arrays containing tokens that can be described by
     # a single regular expression.  Each sub-Array contains 2 elements, the
     # name of the token as a Symbol and a regular expression describing the
     # value of the token.
-    NAME_RE = %r{\A(((::)?[_a-z0-9][-\w]*)(::[a-z0-9][-\w]*)*)}
     KNOWN_TOKENS = [
-      [:TYPE, %r{\A(Integer|Float|Boolean|Regexp|String|Array|Hash|Resource|Class|Collection|Scalar|Numeric|CatalogEntry|Data|Tuple|Struct|Optional|NotUndef|Variant|Enum|Pattern|Any|Callable|Type|Runtime|Undef|Default)\b}],
+      [:WHITESPACE, %r{\A(#{WHITESPACE_RE}+)}],
+      # FIXME: Future breaking change, the following :TYPE tokens conflict with
+      #        the :TYPE keyword token.
+      [:TYPE, %r{\A(Integer|Float|Boolean|Regexp|String|Array|Hash|Resource|Class|Collection|Scalar|Numeric|CatalogEntry|Data|Tuple|Struct|Optional|NotUndef|Variant|Enum|Pattern|Any|Callable|Type|Runtime|Undef|Default|Sensitive)\b}], # rubocop:disable Metrics/LineLength
       [:CLASSREF, %r{\A(((::){0,1}[A-Z][-\w]*)+)}],
       [:NUMBER, %r{\A\b((?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?))\b}],
-      [:FUNCTION_NAME, %r{#{NAME_RE}\(}],
+      [:FUNCTION_NAME, %r{#{NAME_RE}(?=\()}],
       [:NAME, NAME_RE],
       [:LBRACK, %r{\A(\[)}],
       [:RBRACK, %r{\A(\])}],
@@ -166,14 +185,6 @@ class PuppetLint
       :INDENT        => true,
     }.freeze
 
-    # \t == tab
-    # \v == vertical tab
-    # \f == form feed
-    # \p{Zs} == ASCII + Unicode non-linebreaking whitespace
-    WHITESPACE_RE = RUBY_VERSION == '1.8.7' ? %r{[\t\v\f ]} : %r{[\t\v\f\p{Zs}]}
-
-    LINE_END_RE = %r{(?:\r\n|\r|\n)}
-
     # Internal: Access the internal token storage.
     #
     # Returns an Array of PuppetLint::Lexer::Toxen objects.
@@ -200,13 +211,12 @@ class PuppetLint
           value = chunk[regex, 1]
           next if value.nil?
 
-          length = value.size
+          i += value.size
           tokens << if type == :NAME && KEYWORDS.include?(value)
                       new_token(value.upcase.to_sym, value)
                     else
                       new_token(type, value)
                     end
-          i += length
           found = true
           break
         end
@@ -215,7 +225,12 @@ class PuppetLint
 
         if var_name = chunk[%r{\A\$((::)?(\w+(-\w+)*::)*\w+(-\w+)*(\[.+?\])*)}, 1]
           length = var_name.size + 1
-          tokens << new_token(:VARIABLE, var_name)
+          opts = if chunk.start_with?('$')
+                   { :raw => "$#{var_name}" }
+                 else
+                   {}
+                 end
+          tokens << new_token(:VARIABLE, var_name, opts)
 
         elsif chunk =~ %r{\A'.*?'}m
           str_content = StringScanner.new(code[i + 1..-1]).scan_until(%r{(\A|[^\\])(\\\\)*'}m)
@@ -223,12 +238,16 @@ class PuppetLint
           tokens << new_token(:SSTRING, str_content[0..-2])
 
         elsif chunk.start_with?('"')
-          str_contents = slurp_string(code[i + 1..-1])
-          lines_parsed = code[0..i].split(LINE_END_RE)
-          interpolate_string(str_contents, lines_parsed.count, lines_parsed.last.length)
-          length = str_contents.size + 1
+          slurper = PuppetLint::Lexer::StringSlurper.new(code[i + 1..-1])
+          begin
+            string_segments = slurper.parse
+            process_string_segments(string_segments)
+            length = slurper.consumed_chars + 1
+          rescue PuppetLint::Lexer::StringSlurper::UnterminatedStringError
+            raise PuppetLint::LexerError.new(@line_no, @column, 'unterminated string')
+          end
 
-        elsif heredoc_name = chunk[%r{\A@\(("?.+?"?(:.+?)?(/.*?)?)\)}, 1]
+        elsif heredoc_name = chunk[%r{\A@\(("?.+?"?(:.+?)?#{WHITESPACE_RE}*(/.*?)?)\)}, 1]
           heredoc_queue << heredoc_name
           tokens << new_token(:HEREDOC_OPEN, heredoc_name)
           length = heredoc_name.size + 3
@@ -251,7 +270,7 @@ class PuppetLint
           mlcomment.gsub!(%r{^ *\*}, '')
           tokens << new_token(:MLCOMMENT, mlcomment, :raw => mlcomment_raw)
 
-        elsif chunk.match(%r{\A/.*?/}) && possible_regex?
+        elsif chunk.match(%r{\A/.*?/}m) && possible_regex?
           str_content = StringScanner.new(code[i + 1..-1]).scan_until(%r{(\A|[^\\])(\\\\)*/}m)
           length = str_content.size + 1
           tokens << new_token(:REGEX, str_content[0..-2])
@@ -267,27 +286,22 @@ class PuppetLint
             length += indent.size
           else
             heredoc_tag = heredoc_queue.shift
-            heredoc_name = heredoc_tag[%r{\A"?(.+?)"?(:.+?)?(/.*)?\Z}, 1]
-            str_contents = StringScanner.new(code[(i + length)..-1]).scan_until(%r{\|?\s*-?\s*#{heredoc_name}})
-            interpolate_heredoc(str_contents, heredoc_tag)
-            length += str_contents.size
+            slurper = PuppetLint::Lexer::StringSlurper.new(code[i + length..-1])
+            heredoc_segments = slurper.parse_heredoc(heredoc_tag)
+            process_heredoc_segments(heredoc_segments)
+            length += slurper.consumed_chars
           end
 
-        elsif whitespace = chunk[%r{\A(#{WHITESPACE_RE}+)}, 1]
-          length = whitespace.size
-          tokens << new_token(:WHITESPACE, whitespace)
-
         elsif eol = chunk[%r{\A(#{LINE_END_RE})}, 1]
           length = eol.size
           tokens << new_token(:NEWLINE, eol)
 
           unless heredoc_queue.empty?
             heredoc_tag = heredoc_queue.shift
-            heredoc_name = heredoc_tag[%r{\A"?(.+?)"?(:.+?)?(/.*)?\Z}, 1]
-            str_contents = StringScanner.new(code[(i + length)..-1]).scan_until(%r{\|?\s*-?\s*#{heredoc_name}})
-            _ = code[0..(i + length)].split(LINE_END_RE)
-            interpolate_heredoc(str_contents, heredoc_tag)
-            length += str_contents.size
+            slurper = PuppetLint::Lexer::StringSlurper.new(code[i + length..-1])
+            heredoc_segments = slurper.parse_heredoc(heredoc_tag)
+            process_heredoc_segments(heredoc_segments)
+            length += slurper.consumed_chars
           end
 
         elsif chunk.start_with?('/')
@@ -308,22 +322,6 @@ class PuppetLint
       tokens
     end
 
-    def slurp_string(string)
-      dq_str_regexp = %r{(\$\{|(\A|[^\\])(\\\\)*")}m
-      scanner = StringScanner.new(string)
-      contents = scanner.scan_until(dq_str_regexp)
-
-      if scanner.matched.nil?
-        raise LexerError.new(@line_no, @column, 'Double quoted string missing closing quote')
-      end
-
-      until scanner.matched.end_with?('"')
-        contents += scanner.scan_until(%r{\}}m)
-        contents += scanner.scan_until(dq_str_regexp)
-      end
-      contents
-    end
-
     # Internal: Given the tokens already processed, determine if the next token
     # could be a regular expression.
     #
@@ -403,167 +401,66 @@ class PuppetLint
       token
     end
 
-    # Internal: Split a string on multiple terminators, excluding escaped
-    # terminators.
-    #
-    # string      - The String to be split.
-    # terminators - The String of terminators that the String should be split
-    #               on.
-    #
-    # Returns an Array consisting of two Strings, the String up to the first
-    # terminator and the terminator that was found.
-    def get_string_segment(string, terminators)
-      str = string.scan_until(%r{([^\\]|^|[^\\])([\\]{2})*[#{terminators}]+})
-      begin
-        [str[0..-2], str[-1, 1]]
-      rescue
-        [nil, nil]
+    def process_string_segments(segments)
+      return if segments.empty?
+
+      if segments.length == 1
+        tokens << new_token(:STRING, segments[0][1])
+        return
       end
-    end
 
-    # Internal: Tokenise the contents of a double quoted string.
-    #
-    # string - The String to be tokenised.
-    # line   - The Integer line number of the start of the passed string.
-    # column - The Integer column number of the start of the passed string.
-    #
-    # Returns nothing.
-    def interpolate_string(string, line, column)
-      ss = StringScanner.new(string)
-      first = true
-      value, terminator = get_string_segment(ss, '"$')
-      until value.nil?
-        if terminator == '"'
-          if first
-            tokens << new_token(:STRING, value, :line => line, :column => column)
-            first = false
-          else
-            token_column = column + (ss.pos - value.size)
-            tokens << new_token(:DQPOST, value, :line => line, :column => token_column)
-            line += value.scan(LINE_END_RE).size
-            @column = column + ss.pos + 1
-            @line_no = line
+      pre_segment = segments.delete_at(0)
+      post_segment = segments.delete_at(-1)
+
+      tokens << new_token(:DQPRE, pre_segment[1])
+      segments.each do |segment|
+        case segment[0]
+        when :INTERP
+          lexer = PuppetLint::Lexer.new
+          lexer.tokenise(segment[1])
+          lexer.tokens.each_with_index do |t, i|
+            type = i.zero? && t.interpolated_variable? ? :VARIABLE : t.type
+            tokens << new_token(type, t.value, :raw => t.raw)
           end
+        when :UNENC_VAR
+          tokens << new_token(:UNENC_VARIABLE, segment[1].gsub(%r{\A\$}, ''))
         else
-          if first
-            tokens << new_token(:DQPRE, value, :line => line, :column => column)
-            first = false
-          else
-            token_column = column + (ss.pos - value.size)
-            tokens << new_token(:DQMID, value, :line => line, :column => token_column)
-            line += value.scan(LINE_END_RE).size
-          end
-          if ss.scan(%r{\{}).nil?
-            var_name = ss.scan(%r{(::)?(\w+(-\w+)*::)*\w+(-\w+)*})
-            if var_name.nil?
-              token_column = column + ss.pos - 1
-              tokens << new_token(:DQMID, '$', :line => line, :column => token_column)
-            else
-              token_column = column + (ss.pos - var_name.size)
-              tokens << new_token(:UNENC_VARIABLE, var_name, :line => line, :column => token_column)
-            end
-          else
-            line += value.scan(LINE_END_RE).size
-            contents = ss.scan_until(%r{\}})[0..-2]
-            raw = contents.dup
-            if contents.match(%r{\A(::)?([\w-]+::)*[\w-]+(\[.+?\])*}) && !contents.match(%r{\A\w+\(})
-              contents = "$#{contents}"
-            end
-            lexer = PuppetLint::Lexer.new
-            lexer.tokenise(contents)
-            lexer.tokens.each do |token|
-              tok_col = column + token.column + (ss.pos - contents.size - 1)
-              tok_line = token.line + line - 1
-              tokens << new_token(token.type, token.value, :line => tok_line, :column => tok_col)
-            end
-            if lexer.tokens.length == 1 && lexer.tokens[0].type == :VARIABLE
-              tokens.last.raw = raw
-            end
-          end
+          tokens << new_token(:DQMID, segment[1])
         end
-        value, terminator = get_string_segment(ss, '"$')
       end
+      tokens << new_token(:DQPOST, post_segment[1])
     end
 
-    # Internal: Tokenise the contents of a heredoc.
-    #
-    # string - The String to be tokenised.
-    # name   - The String name/endtext of the heredoc.
-    #
-    # Returns nothing.
-    def interpolate_heredoc(string, name)
-      ss = StringScanner.new(string)
-      eos_text = name[%r{\A"?(.+?)"?(:.+?)?(/.*)?\Z}, 1]
-      first = true
-      interpolate = name.start_with?('"')
-      value, terminator = get_heredoc_segment(ss, eos_text, interpolate)
-      until value.nil?
-        if terminator =~ %r{\A\|?\s*-?\s*#{Regexp.escape(eos_text)}}
-          if first
-            tokens << new_token(:HEREDOC, value, :raw => "#{value}#{terminator}")
-            first = false
-          else
-            tokens << new_token(:HEREDOC_POST, value, :raw => "#{value}#{terminator}")
+    def process_heredoc_segments(segments)
+      return if segments.empty?
+
+      end_tag = segments.delete_at(-1)
+
+      if segments.length == 1
+        tokens << new_token(:HEREDOC, segments[0][1], :raw => "#{segments[0][1]}#{end_tag[1]}")
+        return
+      end
+
+      pre_segment = segments.delete_at(0)
+      post_segment = segments.delete_at(-1)
+
+      tokens << new_token(:HEREDOC_PRE, pre_segment[1])
+      segments.each do |segment|
+        case segment[0]
+        when :INTERP
+          lexer = PuppetLint::Lexer.new
+          lexer.tokenise(segment[1])
+          lexer.tokens.each_with_index do |t, i|
+            type = i.zero? && t.interpolated_variable? ? :VARIABLE : t.type
+            tokens << new_token(type, t.value, :raw => t.raw)
           end
+        when :UNENC_VAR
+          tokens << new_token(:UNENC_VARIABLE, segment[1].gsub(%r{\A\$}, ''))
         else
-          if first
-            tokens << new_token(:HEREDOC_PRE, value)
-            first = false
-          else
-            tokens << new_token(:HEREDOC_MID, value)
-          end
-          if ss.scan(%r{\{}).nil?
-            var_name = ss.scan(%r{(::)?(\w+(-\w+)*::)*\w+(-\w+)*})
-            tokens << if var_name.nil?
-                        new_token(:HEREDOC_MID, '$')
-                      else
-                        new_token(:UNENC_VARIABLE, var_name)
-                      end
-          else
-            contents = ss.scan_until(%r{\}})[0..-2]
-            raw = contents.dup
-            if contents.match(%r{\A(::)?([\w-]+::)*[\w-]|(\[.+?\])*}) && !contents.match(%r{\A\w+\(})
-              contents = "$#{contents}" unless contents.start_with?('$')
-            end
-
-            lexer = PuppetLint::Lexer.new
-            lexer.tokenise(contents)
-            lexer.tokens.each do |token|
-              tokens << new_token(token.type, token.value)
-            end
-            if lexer.tokens.length == 1 && lexer.tokens[0].type == :VARIABLE
-              tokens.last.raw = raw
-            end
-          end
+          tokens << new_token(:HEREDOC_MID, segment[1])
         end
-        value, terminator = get_heredoc_segment(ss, eos_text, interpolate)
-      end
-    end
-
-    # Internal: Splits a heredoc String into segments if it is to be
-    # interpolated.
-    #
-    # string      - The String heredoc.
-    # eos_text    - The String endtext for the heredoc.
-    # interpolate - A Boolean that specifies whether this heredoc can contain
-    #               interpolated values (defaults to True).
-    #
-    # Returns an Array consisting of two Strings, the String up to the first
-    # terminator and the terminator that was found.
-    def get_heredoc_segment(string, eos_text, interpolate = true)
-      regexp = if interpolate
-                 %r{(([^\\]|^|[^\\])([\\]{2})*[$]+|\|?\s*-?#{Regexp.escape(eos_text)})}
-               else
-                 %r{\|?\s*-?#{Regexp.escape(eos_text)}}
-               end
-
-      str = string.scan_until(regexp)
-      begin
-        str =~ %r{\A(.*?)([$]+|\|?\s*-?#{Regexp.escape(eos_text)})\Z}m
-        [Regexp.last_match(1), Regexp.last_match(2)]
-      rescue
-        [nil, nil]
       end
+      tokens << new_token(:HEREDOC_POST, post_segment[1], :raw => "#{post_segment[1]}#{end_tag[1]}")
     end
   end
 end