puppet 4.0.0 → 4.1.0

data/lib/puppet/pops/parser/epp_support.rb

@@ -33,7 +33,7 @@ module Puppet::Pops::Parser::EppSupport
     ctx   = @lexing_context
     queue = @token_queue
 
-    lex_error "Internal Error: No string or file given to lexer to process." unless scn
+    lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => 'No string or file given to lexer to process.') unless scn
 
     ctx[:epp_mode] = :text
     enqueue_completed([:EPP_START, nil, 0], 0)
@@ -47,7 +47,7 @@ module Puppet::Pops::Parser::EppSupport
       end
     end
     if ctx[:epp_open_position]
-      lex_error("Unbalanced epp tag, reached <eof> without closing tag.", ctx[:epp_position])
+      lex_error(Puppet::Pops::Issues::EPP_UNBALANCED_TAG, {}, ctx[:epp_position])
     end
 
     # Signals end of input
@@ -65,7 +65,9 @@ module Puppet::Pops::Parser::EppSupport
     case eppscanner.mode
     when :text
       # Should be at end of scan, or something is terribly wrong
-      lex_error("Internal error: template scanner returns text mode and is not and end of input") unless @scanner.eos?
+      unless @scanner.eos?
+        lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => 'template scanner returns text mode and is not and end of input')
+      end
       if s
         # s may be nil if scanned text ends with an epp tag (i.e. no trailing text).
         enqueue_completed([:RENDER_STRING, s, scn.pos - before], before)
@@ -74,7 +76,7 @@ module Puppet::Pops::Parser::EppSupport
       # do nothing else, scanner is at the end
 
     when :error
-      lex_error(eppscanner.message())
+      lex_error(eppscanner.issue)
 
     when :epp
       # It is meaningless to render empty string segments, and it is harmful to do this at
@@ -97,7 +99,7 @@ module Puppet::Pops::Parser::EppSupport
       ctx[:epp_mode] = :expr
       ctx[:epp_open_position] = scn.pos
     else
-      lex_error("Internal Error, Unknown mode #{eppscanner.mode} returned by template scanner")
+      lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => "Unknown mode #{eppscanner.mode} returned by template scanner")
     end
     nil
   end
@@ -144,8 +146,8 @@ module Puppet::Pops::Parser::EppSupport
     #
     attr_reader :mode
 
-    # An error message if `mode == :error`, `nil` otherwise.
-    attr_reader :message
+    # An error issue if `mode == :error`, `nil` otherwise.
+    attr_reader :issue
 
     # If the first scan should skip leading whitespace (typically detected by the pp lexer when the
     # pp mode end-token is found (i.e. `-%>`) and then passed on to the scanner.
@@ -159,6 +161,13 @@ module Puppet::Pops::Parser::EppSupport
       @scanner = scanner
     end
 
+    # Here for backwards compatibility.
+    # @deprecated Use issue instead
+    # @return [String] the issue message
+    def message
+      @issue.nil? ? nil : @issue.format
+    end
+
     # Scans from the current position in the configured scanner, advances this scanner's position until the end
     # of the input, or to the first position after a mode switching token (`<%`, `<%-` or `<%=`). Number of processed
     # lines and continuation mode can be obtained via {#lines}, and {#mode}.
@@ -189,7 +198,7 @@ module Puppet::Pops::Parser::EppSupport
           # if s ends with <% then this is an error (unbalanced <% %>)
           if s.end_with? "<%"
             @mode = :error
-            @message = "Unbalanced embedded expression - opening <% and reaching end of input"
+            @issue = Puppet::Pops::Issues::EPP_UNBALANCED_EXPRESSION
           else
             mode = :epp
           end
@@ -227,7 +236,7 @@ module Puppet::Pops::Parser::EppSupport
           # preceded by a % (i.e. skip %%>)
           part = scanner.scan_until(/[^%]%>/)
           unless part
-            @message = "Reaching end after opening <%# without seeing %>"
+            @issue = Puppet::Pops::Issues::EPP_UNBALANCED_COMMENT
             @mode = :error
             return s
           end

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -20,6 +20,8 @@ class Puppet::Pops::Parser::EvaluatingParser
     #
     begin
       assert_and_report(parser.parse_string(s))
+    rescue Puppet::ParseErrorWithIssue => e
+      raise e
     rescue Puppet::ParseError => e
       # TODO: This is not quite right, why does not the exception have the correct file?
       e.file = @file_source unless e.file.is_a?(String) && !e.file.empty?
@@ -63,12 +65,16 @@ class Puppet::Pops::Parser::EvaluatingParser
   end
 
   def evaluator
+    # Do not use the cached evaluator if this is a migration run
+    if (Puppet.lookup(:migration_checker) { nil })
+      return Puppet::Pops::Evaluator::EvaluatorImpl.new()
+    end
     @@evaluator ||= Puppet::Pops::Evaluator::EvaluatorImpl.new()
     @@evaluator
   end
 
   def convert_to_3x(object, scope)
-    val = @@evaluator.convert(object, scope, nil)
+    val = evaluator.convert(object, scope, nil)
   end
 
   def validate(parse_result)

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -1,4 +1,5 @@
 module Puppet::Pops::Parser::HeredocSupport
+  include Puppet::Pops::Parser::LexerSupport
 
   # Pattern for heredoc `@(endtag[:syntax][/escapes])
   # Produces groups for endtag (group 1), syntax (group 2), and escapes (group 3)
@@ -14,13 +15,12 @@ module Puppet::Pops::Parser::HeredocSupport
 
     # scanner is at position before @(
     # find end of the heredoc spec
-    str = scn.scan_until(/\)/) || lexer.lex_error("Unclosed parenthesis after '@(' followed by '#{followed_by}'")
+    str = scn.scan_until(/\)/) || lex_error(Puppet::Pops::Issues::HEREDOC_UNCLOSED_PARENTHESIS, :followed_by => followed_by)
     pos_after_heredoc = scn.pos
 
     # Note: allows '+' as separator in syntax, but this needs validation as empty segments are not allowed
-    unless md = str.match(PATTERN_HEREDOC)
-      lex_error("Invalid syntax in heredoc expected @(endtag[:syntax][/escapes])")
-    end
+    md = str.match(PATTERN_HEREDOC)
+    lex_error(Puppet::Pops::Issues::HEREDOC_INVALID_SYNTAX) unless md
     endtag = md[1]
     syntax = md[2] || ''
     escapes = md[3]
@@ -33,7 +33,7 @@ module Puppet::Pops::Parser::HeredocSupport
       endtag = $1.strip
     end
 
-    lexer.lex_error("Missing endtag in heredoc") unless endtag.length >= 1
+    lex_error(Puppet::Pops::Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
 
     resulting_escapes = []
     if escapes
@@ -41,7 +41,7 @@ module Puppet::Pops::Parser::HeredocSupport
 
       escapes = escapes.split('')
       unless escapes.length == escapes.uniq.length
-        lex_error("An escape char for @() may only appear once. Got '#{escapes.join(', ')}")
+        lex_error(Puppet::Pops::Issues::HEREDOC_MULTIPLE_AT_ESCAPES, :escapes => escapes)
       end
       resulting_escapes = ["\\"]
       escapes.each do |e|
@@ -51,7 +51,7 @@ module Puppet::Pops::Parser::HeredocSupport
         when "L"
           resulting_escapes += ["\n", "\r\n"]
         else
-          lex_error("Invalid heredoc escape char. Only t, r, n, s,  u, L, $ allowed. Got '#{e}'")
+          lex_error(Puppet::Pops::Issues::HEREDOC_INVALID_ESCAPE, :actual => e)
         end
       end
     end
@@ -66,14 +66,14 @@ module Puppet::Pops::Parser::HeredocSupport
     if ctx[:newline_jump]
       scn.pos = ctx[:newline_jump]
     else
-      scn.scan_until(/\n/) || lex_error("Heredoc without any following lines of text")
+      scn.scan_until(/\n/) || lex_error(Puppet::Pops::Issues::HEREDOC_WITHOUT_TEXT)
     end
     # offset 0 for the heredoc, and its line number
     heredoc_offset = scn.pos
     heredoc_line = locator.line_for_offset(heredoc_offset)-1
 
     # Compute message to emit if there is no end (to make it refer to the opening heredoc position).
-    eof_message = positioned_message("Heredoc without end-tagged line")
+    eof_error = create_lex_error(Puppet::Pops::Issues::HEREDOC_WITHOUT_END_TAGGED_LINE)
 
     # Text from this position (+ lexing contexts offset for any preceding heredoc) is heredoc until a line
     # that terminates the heredoc is found.
@@ -82,7 +82,8 @@ module Puppet::Pops::Parser::HeredocSupport
     endline_pattern = /([[:blank:]]*)(?:([|])[[:blank:]]*)?(?:(\-)[[:blank:]]*)?#{Regexp.escape(endtag)}[[:blank:]]*\r?(?:\n|\z)/
     lines = []
     while !scn.eos? do
-      one_line = scn.scan_until(/(?:\n|\z)/) || lexer.lex_error_without_pos(eof_message)
+      one_line = scn.scan_until(/(?:\n|\z)/)
+      raise eof_error unless one_line
       if md = one_line.match(endline_pattern)
         leading      = md[1]
         has_margin   = md[2] == '|'
@@ -116,7 +117,7 @@ module Puppet::Pops::Parser::HeredocSupport
         lines << one_line
       end
     end
-    lex_error_without_pos(eof_message)
+    raise eof_error
   end
 
   # Produces the heredoc text string given the individual (unprocessed) lines as an array.

data/lib/puppet/pops/parser/interpolation_support.rb

@@ -193,9 +193,15 @@ module Puppet::Pops::Parser::InterpolationSupport
         token_name = token[0]
         ctx[:after] = token_name
         if token_name == :RBRACE && ctx[:brace_count] == brace_count
-          if queue.size - queue_size == 1
+          qlength = queue.size - queue_size
+          if qlength == 1
             # Single token is subject to replacement
             queue[-1] = transform_to_variable(queue[-1])
+          elsif qlength > 1 && [:DOT, :LBRACK].include?(queue[queue_size + 1][0])
+            # A first word, number of name token followed by '[' or '.' is subject to replacement
+            # But not for other operators such as ?, +, - etc. where user must use a $ before the name
+            # to get a variable
+            queue[queue_size] = transform_to_variable(queue[queue_size])
           end
           return
         end

data/lib/puppet/pops/parser/lexer2.rb

@@ -214,7 +214,7 @@ class Puppet::Pops::Parser::Lexer2
     @scanner = StringScanner.new(string)
     @locator = locator || Puppet::Pops::Parser::Locator.locator(string, '')
     @lexing_context[:escapes] = escapes || UQ_ESCAPES
-    @lexing_context[:uq_slurp_pattern] = (interpolate || !escapes.empty?) ? SLURP_UQ_PATTERN : SLURP_ALL_PATTERN
+    @lexing_context[:uq_slurp_pattern] = interpolate ? (escapes.include?('$') ? SLURP_UQ_PATTERN : SLURP_UQNE_PATTERN) : SLURP_ALL_PATTERN
   end
 
   # Convenience method, and for compatibility with older lexer. Use the lex_file instead.
@@ -275,7 +275,7 @@ class Puppet::Pops::Parser::Lexer2
     ctx   = @lexing_context
     queue = @token_queue
 
-    lex_error_without_pos("Internal Error: No string or file given to lexer to process.") unless scn
+    lex_error_without_pos(Puppet::Pops::Issues::NO_INPUT_TO_LEXER) unless scn
 
     scn.skip(PATTERN_WS)
 
@@ -527,13 +527,13 @@ class Puppet::Pops::Parser::Lexer2
         if la2 >= 'A' && la2 <= 'Z'
           # CLASSREF or error
           value = scn.scan(PATTERN_CLASSREF)
-          if value
+          if value && scn.peek(2) != '::'
             after = scn.pos
             emit_completed([:CLASSREF, value.freeze, after-before], before)
           else
             # move to faulty position ('::<uc-letter>' was ok)
             scn.pos = scn.pos + 3
-            lex_error("Illegal fully qualified class reference")
+            lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_CLASS_REFERENCE)
           end
         else
           value = scn.scan(PATTERN_BARE_WORD)
@@ -546,7 +546,7 @@ class Puppet::Pops::Parser::Lexer2
           else
             # move to faulty position ('::' was ok)
             scn.pos = scn.pos + 2
-            lex_error("Illegal fully qualified name")
+            lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
           end
         end
       else
@@ -573,13 +573,30 @@ class Puppet::Pops::Parser::Lexer2
       value = scn.scan(PATTERN_NUMBER)
       if value
         length = scn.pos - before
-        assert_numeric(value, length)
+        assert_numeric(value, before)
         emit_completed([:NUMBER, value.freeze, length], before)
       else
-        # move to faulty position ([0-9] was ok)
         invalid_number = scn.scan_until(PATTERN_NON_WS)
+        if before > 1
+          after = scn.pos
+          scn.pos = before - 1
+          if scn.peek(1) == '.'
+            # preceded by a dot. Is this a bad decimal number then?
+            scn.pos = before - 2
+            while scn.peek(1) =~ /^\d$/
+              invalid_number = nil
+              before = scn.pos
+              break if before == 0
+              scn.pos = scn.pos - 1
+            end
+          end
+          scn.pos = before
+          invalid_number = scn.peek(after - before) unless invalid_number
+        end
+        length = scn.pos - before
+        assert_numeric(invalid_number, before)
         scn.pos = before + 1
-        lex_error("Illegal number '#{invalid_number}'")
+        lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER, {:value => invalid_number})
       end
 
     when 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
@@ -595,21 +612,21 @@ class Puppet::Pops::Parser::Lexer2
         scn.pos = scn.pos + 1
         fully_qualified = scn.match?(/::/)
         if fully_qualified
-          lex_error("Illegal fully qualified name")
+          lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
         else
-          lex_error("Illegal name or bare word")
+          lex_error(Puppet::Pops::Issues::ILLEGAL_NAME_OR_BARE_WORD)
         end
       end
 
     when 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
     'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'
       value = scn.scan(PATTERN_CLASSREF)
-      if value
+      if value && scn.peek(2) != '::'
         emit_completed([:CLASSREF, value.freeze, scn.pos - before], before)
       else
         # move to faulty position ([A-Z] was ok)
         scn.pos = scn.pos + 1
-        lex_error("Illegal class reference")
+        lex_error(Puppet::Pops::Issues::ILLEGAL_CLASS_REFERENCE)
       end
 
     when "\n"

data/lib/puppet/pops/parser/lexer_support.rb

@@ -3,18 +3,6 @@
 #
 module Puppet::Pops::Parser::LexerSupport
 
-  # Formats given message by appending file, line and position if available.
-  def positioned_message(msg, pos = nil)
-    result = [msg]
-    file = @locator.file
-    line = @locator.line_for_offset(pos || @scanner.pos)
-    pos =  @locator.pos_on_line(pos || @scanner.pos)
-
-    result << "in file #{file}" if file && file.is_a?(String) && !file.empty?
-    result << "at line #{line}:#{pos}"
-    result.join(" ")
-  end
-
   # Returns "<eof>" if at end of input, else the following 5 characters with \n \r \t escaped
   def followed_by
     return "<eof>" if @scanner.eos?
@@ -35,27 +23,68 @@ module Puppet::Pops::Parser::LexerSupport
   end
 
   # Raises a Puppet::LexError with the given message
-  def lex_error_without_pos msg
-    raise Puppet::LexError.new(msg)
+  def lex_error_without_pos(issue, args = {})
+    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code)
   end
 
-  # Raises a Puppet::LexError with the given message
-  def lex_error(msg, pos=nil)
-    raise Puppet::LexError.new(positioned_message(msg, pos))
+  # Raises a Puppet::ParserErrorWithIssue with the given issue and arguments
+  def lex_error(issue, args = {}, pos=nil)
+    raise create_lex_error(issue, args, pos)
+  end
+
+  def filename
+    file = @locator.file
+    file.is_a?(String) && !file.empty? ? file : nil
+  end
+
+  def line(pos)
+    @locator.line_for_offset(pos || @scanner.pos)
+  end
+
+  def position(pos)
+    @locator.pos_on_line(pos || @scanner.pos)
+  end
+
+  def lex_warning(issue, args = {}, pos=nil)
+    Puppet::Util::Log.create({
+        :level => :warning,
+        :message => issue.format(args),
+        :issue_code => issue.issue_code,
+        :file => filename,
+        :line => line(pos),
+        :pos => position(pos),
+      })
+  end
+
+  # @param issue [Puppet::Pops::Issues::Issue] the issue
+  # @param args [Hash<Symbol,String>] Issue arguments
+  # @param pos [Integer]
+  # @return [Puppet::ParseErrorWithIssue] the created error
+  def create_lex_error(issue, args = {}, pos = nil)
+    Puppet::ParseErrorWithIssue.new(
+        issue.format(args),
+        filename,
+        line(pos),
+        position(pos),
+        nil,
+        issue.issue_code)
   end
 
   # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.
   # An error is raised if the given value does not comply.
   #
-  def assert_numeric(value, length)
-    if value =~ /^0[xX].*$/
-      lex_error("Not a valid hex number #{value}", length)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+  def assert_numeric(value, pos)
+    if value =~ /^0[xX]/
+      lex_error(Puppet::Pops::Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+
+    elsif value =~ /^0[^.]/
+      lex_error(Puppet::Pops::Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/
 
-    elsif value =~ /^0[^.].*$/
-      lex_error("Not a valid octal number #{value}", length)   unless value =~ /^0[0-7]+$/
+    elsif value =~ /^\d+[eE.]/
+      lex_error(Puppet::Pops::Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/
 
     else
-      lex_error("Not a valid decimal number #{value}", length) unless value =~ /0?\d+(?:\.\d+)?(?:[eE]-?\d+)?/
+      lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
     end
   end
 

data/lib/puppet/pops/parser/parser_support.rb

@@ -93,11 +93,8 @@ class Puppet::Pops::Parser::Parser
     else
       value_at = "'#{value[:value]}'"
     end
-    if @yydebug
-      error = "Syntax error at #{value_at}, token: #{token}"
-    else
-      error = "Syntax error at #{value_at}"
-    end
+    error = Puppet::Pops::Issues::SYNTAX_ERROR.format(:where => value_at)
+    error = "#{error}, token: #{token}" if @yydebug
 
     # Note, old parser had processing of "expected token here" - do not try to reinstate:
     # The 'expected' is only of value at end of input, otherwise any parse error involving a
@@ -110,19 +107,19 @@ class Puppet::Pops::Parser::Parser
     # must be handled by the grammar. The lexer may have enqueued tokens far ahead - the lexer's opinion about this
     # is not trustworthy.
     #
-
-    except = Puppet::ParseError.new(error)
+    file = nil
+    line = nil
+    pos  = nil
     if token != 0
-      path        = value[:file]
-      except.line = value[:line]
-      except.pos  = value[:pos]
+      file = value[:file]
+      line = value[:line]
+      pos  = value[:pos]
     else
       # At end of input, use what the lexer thinks is the source file
-      path        = lexer.file
+      file = lexer.file
     end
-    except.file = path if path.is_a?(String) && !path.empty?
-
-    raise except
+    file = nil unless file.is_a?(String) && !file.empty?
+    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, issue_code = Puppet::Pops::Issues::SYNTAX_ERROR.issue_code)
   end
 
   # Parses a String of pp DSL code.