puppet 3.8.7 → 4.0.0.rc1

data/lib/puppet/pops/parser/epp_support.rb

@@ -33,7 +33,7 @@ module Puppet::Pops::Parser::EppSupport
     ctx   = @lexing_context
     queue = @token_queue
 
-    lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => 'No string or file given to lexer to process.') unless scn
+    lex_error "Internal Error: No string or file given to lexer to process." unless scn
 
     ctx[:epp_mode] = :text
     enqueue_completed([:EPP_START, nil, 0], 0)
@@ -47,7 +47,7 @@ module Puppet::Pops::Parser::EppSupport
       end
     end
     if ctx[:epp_open_position]
-      lex_error(Puppet::Pops::Issues::EPP_UNBALANCED_TAG, {}, ctx[:epp_position])
+      lex_error("Unbalanced epp tag, reached <eof> without closing tag.", ctx[:epp_position])
     end
 
     # Signals end of input
@@ -65,9 +65,7 @@ module Puppet::Pops::Parser::EppSupport
     case eppscanner.mode
     when :text
       # Should be at end of scan, or something is terribly wrong
-      unless @scanner.eos?
-        lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => 'template scanner returns text mode and is not and end of input')
-      end
+      lex_error("Internal error: template scanner returns text mode and is not and end of input") unless @scanner.eos?
       if s
         # s may be nil if scanned text ends with an epp tag (i.e. no trailing text).
         enqueue_completed([:RENDER_STRING, s, scn.pos - before], before)
@@ -76,7 +74,7 @@ module Puppet::Pops::Parser::EppSupport
       # do nothing else, scanner is at the end
 
     when :error
-      lex_error(eppscanner.issue)
+      lex_error(eppscanner.message())
 
     when :epp
       # It is meaningless to render empty string segments, and it is harmful to do this at
@@ -99,7 +97,7 @@ module Puppet::Pops::Parser::EppSupport
       ctx[:epp_mode] = :expr
       ctx[:epp_open_position] = scn.pos
     else
-      lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => "Unknown mode #{eppscanner.mode} returned by template scanner")
+      lex_error("Internal Error, Unknown mode #{eppscanner.mode} returned by template scanner")
     end
     nil
   end
@@ -146,8 +144,8 @@ module Puppet::Pops::Parser::EppSupport
     #
     attr_reader :mode
 
-    # An error issue if `mode == :error`, `nil` otherwise.
-    attr_reader :issue
+    # An error message if `mode == :error`, `nil` otherwise.
+    attr_reader :message
 
     # If the first scan should skip leading whitespace (typically detected by the pp lexer when the
     # pp mode end-token is found (i.e. `-%>`) and then passed on to the scanner.
@@ -161,13 +159,6 @@ module Puppet::Pops::Parser::EppSupport
       @scanner = scanner
     end
 
-    # Here for backwards compatibility.
-    # @deprecated Use issue instead
-    # @return [String] the issue message
-    def message
-      @issue.nil? ? nil : @issue.format
-    end
-
     # Scans from the current position in the configured scanner, advances this scanner's position until the end
     # of the input, or to the first position after a mode switching token (`<%`, `<%-` or `<%=`). Number of processed
     # lines and continuation mode can be obtained via {#lines}, and {#mode}.
@@ -183,7 +174,7 @@ module Puppet::Pops::Parser::EppSupport
       until scanner.eos?
         part = @scanner.scan_until(/(<%)|\z/)
         if @skip_leading
-          part.sub!(/^[ \t]*\r?(?:\n|\z)?/,'')
+          part.gsub!(/^[ \t]*\r?\n?/,'')
           @skip_leading = false
         end
         # The spec for %%> is to transform it into a literal %>. This is done here, as %%> otherwise would go
@@ -198,7 +189,7 @@ module Puppet::Pops::Parser::EppSupport
           # if s ends with <% then this is an error (unbalanced <% %>)
           if s.end_with? "<%"
             @mode = :error
-            @issue = Puppet::Pops::Issues::EPP_UNBALANCED_EXPRESSION
+            @message = "Unbalanced embedded expression - opening <% and reaching end of input"
           else
             mode = :epp
           end
@@ -208,7 +199,7 @@ module Puppet::Pops::Parser::EppSupport
           # trim trailing whitespace on same line from accumulated s
           # return text and signal switch to pp mode
           @scanner.getch # drop the -
-          s.sub!(/[ \t]*<%\z/, '')
+          s.gsub!(/\r?\n?[ \t]*<%\z/, '')
           @mode = :epp
           return s
 
@@ -236,12 +227,10 @@ module Puppet::Pops::Parser::EppSupport
           # preceded by a % (i.e. skip %%>)
           part = scanner.scan_until(/[^%]%>/)
           unless part
-            @issue = Puppet::Pops::Issues::EPP_UNBALANCED_COMMENT
+            @message = "Reaching end after opening <%# without seeing %>"
             @mode = :error
             return s
           end
-          # Always trim leading whitespace on the same line when there is a comment
-          s.sub!(/[ \t]*\z/, '')
           @skip_leading = true if part.end_with?("-%>")
           # Continue scanning for more text
 

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -20,8 +20,6 @@ class Puppet::Pops::Parser::EvaluatingParser
     #
     begin
       assert_and_report(parser.parse_string(s))
-    rescue Puppet::ParseErrorWithIssue => e
-      raise e
     rescue Puppet::ParseError => e
       # TODO: This is not quite right, why does not the exception have the correct file?
       e.file = @file_source unless e.file.is_a?(String) && !e.file.empty?
@@ -39,8 +37,8 @@ class Puppet::Pops::Parser::EvaluatingParser
     evaluate(scope, parse_string(s, file_source))
   end
 
-  def evaluate_file(file)
-    evaluate(parse_file(file))
+  def evaluate_file(scope, file)
+    evaluate(scope, parse_file(file))
   end
 
   def clear()
@@ -57,17 +55,20 @@ class Puppet::Pops::Parser::EvaluatingParser
     evaluator.evaluate(model, scope)
   end
 
+  # Evaluates the given expression in a local scope with the given variable bindings
+  # set in this local scope, returns what the expression returns.
+  #
+  def evaluate_expression_with_bindings(scope, variable_bindings, expression)
+    evaluator.evaluate_block_with_bindings(scope, variable_bindings, expression)
+  end
+
   def evaluator
-    # Do not use the cached evaluator if this is a migration run
-    if (Puppet.lookup(:migration_checker) { nil })
-      return Puppet::Pops::Evaluator::EvaluatorImpl.new()
-    end
     @@evaluator ||= Puppet::Pops::Evaluator::EvaluatorImpl.new()
     @@evaluator
   end
 
   def convert_to_3x(object, scope)
-    val = evaluator.convert(object, scope, nil)
+    val = @@evaluator.convert(object, scope, nil)
   end
 
   def validate(parse_result)
@@ -86,7 +87,7 @@ class Puppet::Pops::Parser::EvaluatingParser
 
   def assert_and_report(parse_result)
     return nil unless parse_result
-    if parse_result.source_ref.nil? or parse_result.source_ref == ''
+    if parse_result.source_ref.nil? || parse_result.source_ref == ''
       parse_result.source_ref = @file_source
     end
     validation_result = validate(parse_result)

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -1,5 +1,4 @@
 module Puppet::Pops::Parser::HeredocSupport
-  include Puppet::Pops::Parser::LexerSupport
 
   # Pattern for heredoc `@(endtag[:syntax][/escapes])
   # Produces groups for endtag (group 1), syntax (group 2), and escapes (group 3)
@@ -15,12 +14,13 @@ module Puppet::Pops::Parser::HeredocSupport
 
     # scanner is at position before @(
     # find end of the heredoc spec
-    str = scn.scan_until(/\)/) || lex_error(Puppet::Pops::Issues::HEREDOC_UNCLOSED_PARENTHESIS, :followed_by => followed_by)
+    str = scn.scan_until(/\)/) || lexer.lex_error("Unclosed parenthesis after '@(' followed by '#{followed_by}'")
     pos_after_heredoc = scn.pos
 
     # Note: allows '+' as separator in syntax, but this needs validation as empty segments are not allowed
-    md = str.match(PATTERN_HEREDOC)
-    lex_error(Puppet::Pops::Issues::HEREDOC_INVALID_SYNTAX) unless md
+    unless md = str.match(PATTERN_HEREDOC)
+      lex_error("Invalid syntax in heredoc expected @(endtag[:syntax][/escapes])")
+    end
     endtag = md[1]
     syntax = md[2] || ''
     escapes = md[3]
@@ -33,7 +33,7 @@ module Puppet::Pops::Parser::HeredocSupport
       endtag = $1.strip
     end
 
-    lex_error(Puppet::Pops::Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
+    lexer.lex_error("Missing endtag in heredoc") unless endtag.length >= 1
 
     resulting_escapes = []
     if escapes
@@ -41,7 +41,7 @@ module Puppet::Pops::Parser::HeredocSupport
 
       escapes = escapes.split('')
       unless escapes.length == escapes.uniq.length
-        lex_error(Puppet::Pops::Issues::HEREDOC_MULTIPLE_AT_ESCAPES, :escapes => escapes)
+        lex_error("An escape char for @() may only appear once. Got '#{escapes.join(', ')}")
       end
       resulting_escapes = ["\\"]
       escapes.each do |e|
@@ -51,7 +51,7 @@ module Puppet::Pops::Parser::HeredocSupport
         when "L"
           resulting_escapes += ["\n", "\r\n"]
         else
-          lex_error(Puppet::Pops::Issues::HEREDOC_INVALID_ESCAPE, :actual => e)
+          lex_error("Invalid heredoc escape char. Only t, r, n, s,  u, L, $ allowed. Got '#{e}'")
         end
       end
     end
@@ -64,16 +64,16 @@ module Puppet::Pops::Parser::HeredocSupport
     # and it should start scanning after the first found \n (or if not found == error).
 
     if ctx[:newline_jump]
-      scn.pos = ctx[:newline_jump]
+      scn.pos = lexing_context[:newline_jump]
     else
-      scn.scan_until(/\n/) || lex_error(Puppet::Pops::Issues::HEREDOC_WITHOUT_TEXT)
+      scn.scan_until(/\n/) || lex_error("Heredoc without any following lines of text")
     end
     # offset 0 for the heredoc, and its line number
     heredoc_offset = scn.pos
     heredoc_line = locator.line_for_offset(heredoc_offset)-1
 
     # Compute message to emit if there is no end (to make it refer to the opening heredoc position).
-    eof_error = create_lex_error(Puppet::Pops::Issues::HEREDOC_WITHOUT_END_TAGGED_LINE)
+    eof_message = positioned_message("Heredoc without end-tagged line")
 
     # Text from this position (+ lexing contexts offset for any preceding heredoc) is heredoc until a line
     # that terminates the heredoc is found.
@@ -82,8 +82,7 @@ module Puppet::Pops::Parser::HeredocSupport
     endline_pattern = /([[:blank:]]*)(?:([|])[[:blank:]]*)?(?:(\-)[[:blank:]]*)?#{Regexp.escape(endtag)}[[:blank:]]*\r?(?:\n|\z)/
     lines = []
     while !scn.eos? do
-      one_line = scn.scan_until(/(?:\n|\z)/)
-      raise eof_error unless one_line
+      one_line = scn.scan_until(/(?:\n|\z)/) || lexer.lex_error_without_pos(eof_message)
       if md = one_line.match(endline_pattern)
         leading      = md[1]
         has_margin   = md[2] == '|'
@@ -117,7 +116,7 @@ module Puppet::Pops::Parser::HeredocSupport
         lines << one_line
       end
     end
-    raise eof_error
+    lex_error_without_pos(eof_message)
   end
 
   # Produces the heredoc text string given the individual (unprocessed) lines as an array.

data/lib/puppet/pops/parser/interpolation_support.rb

@@ -193,15 +193,9 @@ module Puppet::Pops::Parser::InterpolationSupport
         token_name = token[0]
         ctx[:after] = token_name
         if token_name == :RBRACE && ctx[:brace_count] == brace_count
-          qlength = queue.size - queue_size
-          if qlength == 1
+          if queue.size - queue_size == 1
             # Single token is subject to replacement
             queue[-1] = transform_to_variable(queue[-1])
-          elsif qlength > 1 && [:DOT, :LBRACK].include?(queue[queue_size + 1][0])
-            # A first word, number of name token followed by '[' or '.' is subject to replacement
-            # But not for other operators such as ?, +, - etc. where user must use a $ before the name
-            # to get a variable
-            queue[queue_size] = transform_to_variable(queue[queue_size])
           end
           return
         end

data/lib/puppet/pops/parser/lexer2.rb

@@ -131,13 +131,7 @@ class Puppet::Pops::Parser::Lexer2
     "type"     => [:TYPE,     'type',     4],
     "attr"     => [:ATTR,     'attr',     4],
     "private"  => [:PRIVATE,  'private',  7],
-    # The following tokens exist in reserved form. Later they will be made
-    # live subject to a feature switch.
-    "application"  => [:APPLICATION_R,  'application',  11],
-    "consumes"     => [:CONSUMES_R,  'consumes',  8],
-    "produces"     => [:PRODUCES_R,  'produces',  8],
   }
-
   KEYWORDS.each {|k,v| v[1].freeze; v.freeze }
   KEYWORDS.freeze
 
@@ -147,6 +141,7 @@ class Puppet::Pops::Parser::Lexer2
   KEYWORD_NAMES.freeze
 
   PATTERN_WS        = %r{[[:blank:]\r]+}
+  PATTERN_NON_WS    = %r{\w+\b?}
 
   # The single line comment includes the line ending.
   PATTERN_COMMENT   = %r{#.*\r?}
@@ -166,9 +161,9 @@ class Puppet::Pops::Parser::Lexer2
   # a letter a-z and may not contain dashes (\w includes letters, digits and _).
   #
   PATTERN_CLASSREF       = %r{((::){0,1}[A-Z][\w]*)+}
-  PATTERN_NAME           = %r{^((::)?[a-z][\w]*)(::[a-z][\w]*)*$}
+  PATTERN_NAME           = %r{((::)?[a-z][\w]*)(::[a-z][\w]*)*}
 
-  PATTERN_BARE_WORD     = %r{((?:::){0,1}(?:[a-z_](?:[\w-]*[\w])?))+}
+  PATTERN_BARE_WORD      = %r{[a-z_](?:[\w-]*[\w])?}
 
   PATTERN_DOLLAR_VAR     = %r{\$(::)?(\w+::)*\w+}
   PATTERN_NUMBER         = %r{\b(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b}
@@ -219,7 +214,7 @@ class Puppet::Pops::Parser::Lexer2
     @scanner = StringScanner.new(string)
     @locator = locator || Puppet::Pops::Parser::Locator.locator(string, '')
     @lexing_context[:escapes] = escapes || UQ_ESCAPES
-    @lexing_context[:uq_slurp_pattern] = interpolate ? (escapes.include?('$') ? SLURP_UQ_PATTERN : SLURP_UQNE_PATTERN) : SLURP_ALL_PATTERN
+    @lexing_context[:uq_slurp_pattern] = (interpolate || !escapes.empty?) ? SLURP_UQ_PATTERN : SLURP_ALL_PATTERN
   end
 
   # Convenience method, and for compatibility with older lexer. Use the lex_file instead.
@@ -280,7 +275,7 @@ class Puppet::Pops::Parser::Lexer2
     ctx   = @lexing_context
     queue = @token_queue
 
-    lex_error_without_pos(Puppet::Pops::Issues::NO_INPUT_TO_LEXER) unless scn
+    lex_error_without_pos("Internal Error: No string or file given to lexer to process.") unless scn
 
     scn.skip(PATTERN_WS)
 
@@ -311,14 +306,13 @@ class Puppet::Pops::Parser::Lexer2
     la = scn.peek(3)
     return nil if la.empty?
 
-    # Ruby 1.8.7 requires using offset and length (or integers are returned.
     # PERFORMANCE NOTE.
     # It is slightly faster to use these local variables than accessing la[0], la[1] etc. in ruby 1.9.3
     # But not big enough to warrant two completely different implementations.
     #
-    la0 = la[0,1]
-    la1 = la[1,1]
-    la2 = la[2,1]
+    la0 = la[0]
+    la1 = la[1]
+    la2 = la[2]
 
     # PERFORMANCE NOTE:
     # A case when, where all the cases are literal values is the fastest way to map from data to code.
@@ -539,20 +533,17 @@ class Puppet::Pops::Parser::Lexer2
           else
             # move to faulty position ('::<uc-letter>' was ok)
             scn.pos = scn.pos + 3
-            lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_CLASS_REFERENCE)
+            lex_error("Illegal fully qualified class reference")
           end
         else
-          value = scn.scan(PATTERN_BARE_WORD)
+          # NAME or error
+          value = scn.scan(PATTERN_NAME)
           if value
-            if value =~ PATTERN_NAME
-              emit_completed([:NAME, value.freeze, scn.pos-before], before)
-            else
-              emit_completed([:WORD, value.freeze, scn.pos - before], before)
-            end
+            emit_completed([:NAME, value.freeze, scn.pos-before], before)
           else
             # move to faulty position ('::' was ok)
             scn.pos = scn.pos + 2
-            lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
+            lex_error("Illegal fully qualified name")
           end
         end
       else
@@ -583,26 +574,32 @@ class Puppet::Pops::Parser::Lexer2
         emit_completed([:NUMBER, value.freeze, length], before)
       else
         # move to faulty position ([0-9] was ok)
-        scn.pos = scn.pos + 1
-        lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER)
+        invalid_number = scn.scan_until(PATTERN_NON_WS)
+        scn.pos = before + 1
+        lex_error("Illegal number '#{invalid_number}'")
       end
 
     when 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
     'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_'
-
-      value = scn.scan(PATTERN_BARE_WORD)
-      if value && value =~ PATTERN_NAME
+      value = scn.scan(PATTERN_NAME)
+      # NAME or false start because followed by hyphen(s), underscore or word
+      if value && !scn.match?(/^-+\w/)
         emit_completed(KEYWORDS[value] || [:NAME, value.freeze, scn.pos - before], before)
-      elsif value
-        emit_completed([:WORD, value.freeze, scn.pos - before], before)
       else
-        # move to faulty position ([a-z_] was ok)
-        scn.pos = scn.pos + 1
-        fully_qualified = scn.match?(/::/)
-        if fully_qualified
-          lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
+        # Restart and check entire pattern (for ease of detecting non allowed trailing hyphen)
+        scn.pos = before
+        value = scn.scan(PATTERN_BARE_WORD)
+        # If the WORD continues with :: it must be a correct fully qualified name
+        if value && !(fully_qualified = scn.match?(/::/))
+          emit_completed([:WORD, value.freeze, scn.pos - before], before)
         else
-          lex_error(Puppet::Pops::Issues::ILLEGAL_NAME_OR_BARE_WORD)
+          # move to faulty position ([a-z_] was ok)
+          scn.pos = scn.pos + 1
+          if fully_qualified
+            lex_error("Illegal fully qualified name")
+          else
+            lex_error("Illegal name or bare word")
+          end
         end
       end
 
@@ -614,7 +611,7 @@ class Puppet::Pops::Parser::Lexer2
       else
         # move to faulty position ([A-Z] was ok)
         scn.pos = scn.pos + 1
-        lex_error(Puppet::Pops::Issues::ILLEGAL_CLASS_REFERENCE)
+        lex_error("Illegal class reference")
       end
 
     when "\n"

data/lib/puppet/pops/parser/lexer_support.rb

@@ -3,6 +3,18 @@
 #
 module Puppet::Pops::Parser::LexerSupport
 
+  # Formats given message by appending file, line and position if available.
+  def positioned_message(msg, pos = nil)
+    result = [msg]
+    file = @locator.file
+    line = @locator.line_for_offset(pos || @scanner.pos)
+    pos =  @locator.pos_on_line(pos || @scanner.pos)
+
+    result << "in file #{file}" if file && file.is_a?(String) && !file.empty?
+    result << "at line #{line}:#{pos}"
+    result.join(" ")
+  end
+
   # Returns "<eof>" if at end of input, else the following 5 characters with \n \r \t escaped
   def followed_by
     return "<eof>" if @scanner.eos?
@@ -23,51 +35,13 @@ module Puppet::Pops::Parser::LexerSupport
   end
 
   # Raises a Puppet::LexError with the given message
-  def lex_error_without_pos(issue, args = {})
-    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code)
-  end
-
-  # Raises a Puppet::ParserErrorWithIssue with the given issue and arguments
-  def lex_error(issue, args = {}, pos=nil)
-    raise create_lex_error(issue, args, pos)
-  end
-
-  def filename
-    file = @locator.file
-    file.is_a?(String) && !file.empty? ? file : nil
-  end
-
-  def line(pos)
-    @locator.line_for_offset(pos || @scanner.pos)
-  end
-
-  def position(pos)
-    @locator.pos_on_line(pos || @scanner.pos)
-  end
-
-  def lex_warning(issue, args = {}, pos=nil)
-    Puppet::Util::Log.create({
-        :level => :warning,
-        :message => issue.format(args),
-        :issue_code => issue.issue_code,
-        :file => filename,
-        :line => line(pos),
-        :pos => position(pos),
-      })
+  def lex_error_without_pos msg
+    raise Puppet::LexError.new(msg)
   end
 
-  # @param issue [Puppet::Pops::Issues::Issue] the issue
-  # @param args [Hash<Symbol,String>] Issue arguments
-  # @param pos [Integer]
-  # @return [Puppet::ParseErrorWithIssue] the created error
-  def create_lex_error(issue, args = {}, pos = nil)
-    Puppet::ParseErrorWithIssue.new(
-        issue.format(args),
-        filename,
-        line(pos),
-        position(pos),
-        nil,
-        issue.issue_code)
+  # Raises a Puppet::LexError with the given message
+  def lex_error(msg, pos=nil)
+    raise Puppet::LexError.new(positioned_message(msg, pos))
   end
 
   # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.
@@ -75,13 +49,13 @@ module Puppet::Pops::Parser::LexerSupport
   #
   def assert_numeric(value, length)
     if value =~ /^0[xX].*$/
-      lex_error(Puppet::Pops::Issues::INVALID_HEX_NUMBER, {:value => value}, length)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+      lex_error("Not a valid hex number #{value}", length)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
 
     elsif value =~ /^0[^.].*$/
-      lex_error(Puppet::Pops::Issues::INVALID_OCTAL_NUMBER, {:value => value}, length)   unless value =~ /^0[0-7]+$/
+      lex_error("Not a valid octal number #{value}", length)   unless value =~ /^0[0-7]+$/
 
     else
-      lex_error(Puppet::Pops::Issues::INVALID_DECIMAL_NUMBER, {:value => value}, length) unless value =~ /0?\d+(?:\.\d+)?(?:[eE]-?\d+)?/
+      lex_error("Not a valid decimal number #{value}", length) unless value =~ /0?\d+(?:\.\d+)?(?:[eE]-?\d+)?/
     end
   end