puppet 4.0.0 → 4.9.1

data/lib/puppet/pops/parser/epp_support.rb

@@ -1,8 +1,10 @@
+module Puppet::Pops
+module Parser
 # This module is an integral part of the Lexer.
 # It handles scanning of EPP (Embedded Puppet), a form of string/expression interpolation similar to ERB.
 #
 require 'strscan'
-module Puppet::Pops::Parser::EppSupport
+module EppSupport
 
   TOKEN_RENDER_STRING = [:RENDER_STRING, nil, 0]
   TOKEN_RENDER_EXPR   = [:RENDER_EXPR, nil, 0]
@@ -33,7 +35,7 @@ module Puppet::Pops::Parser::EppSupport
     ctx   = @lexing_context
     queue = @token_queue
 
-    lex_error "Internal Error: No string or file given to lexer to process." unless scn
+    lex_error(Issues::EPP_INTERNAL_ERROR, :error => 'No string or file given to lexer to process.') unless scn
 
     ctx[:epp_mode] = :text
     enqueue_completed([:EPP_START, nil, 0], 0)
@@ -47,7 +49,7 @@ module Puppet::Pops::Parser::EppSupport
       end
     end
     if ctx[:epp_open_position]
-      lex_error("Unbalanced epp tag, reached <eof> without closing tag.", ctx[:epp_position])
+      lex_error(Issues::EPP_UNBALANCED_TAG, {}, ctx[:epp_position])
     end
 
     # Signals end of input
@@ -65,7 +67,9 @@ module Puppet::Pops::Parser::EppSupport
     case eppscanner.mode
     when :text
       # Should be at end of scan, or something is terribly wrong
-      lex_error("Internal error: template scanner returns text mode and is not and end of input") unless @scanner.eos?
+      unless @scanner.eos?
+        lex_error(Issues::EPP_INTERNAL_ERROR, :error => 'template scanner returns text mode and is not and end of input')
+      end
       if s
         # s may be nil if scanned text ends with an epp tag (i.e. no trailing text).
         enqueue_completed([:RENDER_STRING, s, scn.pos - before], before)
@@ -74,7 +78,7 @@ module Puppet::Pops::Parser::EppSupport
       # do nothing else, scanner is at the end
 
     when :error
-      lex_error(eppscanner.message())
+      lex_error(eppscanner.issue)
 
     when :epp
       # It is meaningless to render empty string segments, and it is harmful to do this at
@@ -97,7 +101,7 @@ module Puppet::Pops::Parser::EppSupport
       ctx[:epp_mode] = :expr
       ctx[:epp_open_position] = scn.pos
     else
-      lex_error("Internal Error, Unknown mode #{eppscanner.mode} returned by template scanner")
+      lex_error(Issues::EPP_INTERNAL_ERROR, :error => "Unknown mode #{eppscanner.mode} returned by template scanner")
     end
     nil
   end
@@ -144,8 +148,8 @@ module Puppet::Pops::Parser::EppSupport
     #
     attr_reader :mode
 
-    # An error message if `mode == :error`, `nil` otherwise.
-    attr_reader :message
+    # An error issue if `mode == :error`, `nil` otherwise.
+    attr_reader :issue
 
     # If the first scan should skip leading whitespace (typically detected by the pp lexer when the
     # pp mode end-token is found (i.e. `-%>`) and then passed on to the scanner.
@@ -159,6 +163,13 @@ module Puppet::Pops::Parser::EppSupport
       @scanner = scanner
     end
 
+    # Here for backwards compatibility.
+    # @deprecated Use issue instead
+    # @return [String] the issue message
+    def message
+      @issue.nil? ? nil : @issue.format
+    end
+
     # Scans from the current position in the configured scanner, advances this scanner's position until the end
     # of the input, or to the first position after a mode switching token (`<%`, `<%-` or `<%=`). Number of processed
     # lines and continuation mode can be obtained via {#lines}, and {#mode}.
@@ -174,7 +185,7 @@ module Puppet::Pops::Parser::EppSupport
       until scanner.eos?
         part = @scanner.scan_until(/(<%)|\z/)
         if @skip_leading
-          part.gsub!(/^[ \t]*\r?\n?/,'')
+          part.sub!(/^[ \t]*\r?(?:\n|\z)?/,'')
           @skip_leading = false
         end
         # The spec for %%> is to transform it into a literal %>. This is done here, as %%> otherwise would go
@@ -189,7 +200,7 @@ module Puppet::Pops::Parser::EppSupport
           # if s ends with <% then this is an error (unbalanced <% %>)
           if s.end_with? "<%"
             @mode = :error
-            @message = "Unbalanced embedded expression - opening <% and reaching end of input"
+            @issue = Issues::EPP_UNBALANCED_EXPRESSION
           else
             mode = :epp
           end
@@ -199,7 +210,7 @@ module Puppet::Pops::Parser::EppSupport
           # trim trailing whitespace on same line from accumulated s
           # return text and signal switch to pp mode
           @scanner.getch # drop the -
-          s.gsub!(/\r?\n?[ \t]*<%\z/, '')
+          s.sub!(/[ \t]*<%\z/, '')
           @mode = :epp
           return s
 
@@ -227,10 +238,12 @@ module Puppet::Pops::Parser::EppSupport
           # preceded by a % (i.e. skip %%>)
           part = scanner.scan_until(/[^%]%>/)
           unless part
-            @message = "Reaching end after opening <%# without seeing %>"
+            @issue = Issues::EPP_UNBALANCED_COMMENT
             @mode = :error
             return s
           end
+          # Always trim leading whitespace on the same line when there is a comment
+          s.sub!(/[ \t]*\z/, '')
           @skip_leading = true if part.end_with?("-%>")
           # Continue scanning for more text
 
@@ -245,3 +258,5 @@ module Puppet::Pops::Parser::EppSupport
   end
 
 end
+end
+end

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -1,15 +1,21 @@
+module Puppet::Pops
+module Parser
 
 # Does not support "import" and parsing ruby files
 #
-class Puppet::Pops::Parser::EvaluatingParser
+class EvaluatingParser
 
   attr_reader :parser
 
   def initialize()
-    @parser = Puppet::Pops::Parser::Parser.new()
+    @parser = Parser.new()
   end
 
-  def parse_string(s, file_source = 'unknown')
+  def self.singleton
+    @instance ||= new
+  end
+
+  def parse_string(s, file_source = nil)
     @file_source = file_source
     clear()
     # Handling of syntax error can be much improved (in general), now it bails out of the parser
@@ -19,7 +25,9 @@ class Puppet::Pops::Parser::EvaluatingParser
     # Also a possible improvement (if the YAML parser returns positions) is to provide correct output of position.
     #
     begin
-      assert_and_report(parser.parse_string(s))
+      assert_and_report(parser.parse_string(s, file_source))
+    rescue Puppet::ParseErrorWithIssue => e
+      raise e
     rescue Puppet::ParseError => e
       # TODO: This is not quite right, why does not the exception have the correct file?
       e.file = @file_source unless e.file.is_a?(String) && !e.file.empty?
@@ -33,7 +41,7 @@ class Puppet::Pops::Parser::EvaluatingParser
     assert_and_report(parser.parse_file(file))
   end
 
-  def evaluate_string(scope, s, file_source='unknown')
+  def evaluate_string(scope, s, file_source = nil)
     evaluate(scope, parse_string(s, file_source))
   end
 
@@ -47,7 +55,7 @@ class Puppet::Pops::Parser::EvaluatingParser
 
   # Create a closure that can be called in the given scope
   def closure(model, scope)
-    Puppet::Pops::Evaluator::Closure.new(evaluator, model, scope)
+    Evaluator::Closure::Dynamic.new(evaluator, model, scope)
   end
 
   def evaluate(scope, model)
@@ -63,12 +71,16 @@ class Puppet::Pops::Parser::EvaluatingParser
   end
 
   def evaluator
-    @@evaluator ||= Puppet::Pops::Evaluator::EvaluatorImpl.new()
+    # Do not use the cached evaluator if this is a migration run
+    if (Puppet.lookup(:migration_checker) { nil })
+      return Evaluator::EvaluatorImpl.new()
+    end
+    @@evaluator ||= Evaluator::EvaluatorImpl.new()
     @@evaluator
   end
 
   def convert_to_3x(object, scope)
-    val = @@evaluator.convert(object, scope, nil)
+    val = evaluator.convert(object, scope, nil)
   end
 
   def validate(parse_result)
@@ -78,11 +90,11 @@ class Puppet::Pops::Parser::EvaluatingParser
   end
 
   def acceptor()
-    Puppet::Pops::Validation::Acceptor.new
+    Validation::Acceptor.new
   end
 
   def validator(acceptor)
-    Puppet::Pops::Validation::ValidatorFactory_4_0.new().validator(acceptor)
+    Validation::ValidatorFactory_4_0.new().validator(acceptor)
   end
 
   def assert_and_report(parse_result)
@@ -92,7 +104,7 @@ class Puppet::Pops::Parser::EvaluatingParser
     end
     validation_result = validate(parse_result)
 
-    Puppet::Pops::IssueReporter.assert_and_report(validation_result,
+    IssueReporter.assert_and_report(validation_result,
                                           :emit_warnings => true)
     parse_result
   end
@@ -143,9 +155,11 @@ class Puppet::Pops::Parser::EvaluatingParser
     escaped << '"'
   end
 
-  class EvaluatingEppParser < Puppet::Pops::Parser::EvaluatingParser
+  class EvaluatingEppParser < EvaluatingParser
     def initialize()
-      @parser = Puppet::Pops::Parser::EppParser.new()
+      @parser = EppParser.new()
     end
   end
 end
+end
+end

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -1,9 +1,12 @@
-module Puppet::Pops::Parser::HeredocSupport
+module Puppet::Pops
+module Parser
+module HeredocSupport
+  include LexerSupport
 
   # Pattern for heredoc `@(endtag[:syntax][/escapes])
   # Produces groups for endtag (group 1), syntax (group 2), and escapes (group 3)
   #
-  PATTERN_HEREDOC = %r{@\(([^:/\r\n\)]+)(?::[:blank:]*([a-z][a-zA-Z0-9_+]+)[:blank:]*)?(?:/((?:\w|[$])*)[:blank:]*)?\)}
+  PATTERN_HEREDOC = %r{@\(([^:/\r\n\)]+)(?::[[:blank:]]*([a-z][a-zA-Z0-9_+]+)[[:blank:]]*)?(?:/((?:\w|[$])*)[[:blank:]]*)?\)}
 
 
   def heredoc
@@ -14,13 +17,11 @@ module Puppet::Pops::Parser::HeredocSupport
 
     # scanner is at position before @(
     # find end of the heredoc spec
-    str = scn.scan_until(/\)/) || lexer.lex_error("Unclosed parenthesis after '@(' followed by '#{followed_by}'")
+    str = scn.scan_until(/\)/) || lex_error(Issues::HEREDOC_UNCLOSED_PARENTHESIS, :followed_by => followed_by)
     pos_after_heredoc = scn.pos
-
     # Note: allows '+' as separator in syntax, but this needs validation as empty segments are not allowed
-    unless md = str.match(PATTERN_HEREDOC)
-      lex_error("Invalid syntax in heredoc expected @(endtag[:syntax][/escapes])")
-    end
+    md = str.match(PATTERN_HEREDOC)
+    lex_error(Issues::HEREDOC_INVALID_SYNTAX) unless md
     endtag = md[1]
     syntax = md[2] || ''
     escapes = md[3]
@@ -33,7 +34,7 @@ module Puppet::Pops::Parser::HeredocSupport
       endtag = $1.strip
     end
 
-    lexer.lex_error("Missing endtag in heredoc") unless endtag.length >= 1
+    lex_error(Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
 
     resulting_escapes = []
     if escapes
@@ -41,7 +42,7 @@ module Puppet::Pops::Parser::HeredocSupport
 
       escapes = escapes.split('')
       unless escapes.length == escapes.uniq.length
-        lex_error("An escape char for @() may only appear once. Got '#{escapes.join(', ')}")
+        lex_error(Issues::HEREDOC_MULTIPLE_AT_ESCAPES, :escapes => escapes)
       end
       resulting_escapes = ["\\"]
       escapes.each do |e|
@@ -51,7 +52,7 @@ module Puppet::Pops::Parser::HeredocSupport
         when "L"
           resulting_escapes += ["\n", "\r\n"]
         else
-          lex_error("Invalid heredoc escape char. Only t, r, n, s,  u, L, $ allowed. Got '#{e}'")
+          lex_error(Issues::HEREDOC_INVALID_ESCAPE, :actual => e)
         end
       end
     end
@@ -66,14 +67,14 @@ module Puppet::Pops::Parser::HeredocSupport
     if ctx[:newline_jump]
       scn.pos = ctx[:newline_jump]
     else
-      scn.scan_until(/\n/) || lex_error("Heredoc without any following lines of text")
+      scn.scan_until(/\n/) || lex_error(Issues::HEREDOC_WITHOUT_TEXT)
     end
     # offset 0 for the heredoc, and its line number
     heredoc_offset = scn.pos
     heredoc_line = locator.line_for_offset(heredoc_offset)-1
 
     # Compute message to emit if there is no end (to make it refer to the opening heredoc position).
-    eof_message = positioned_message("Heredoc without end-tagged line")
+    eof_error = create_lex_error(Issues::HEREDOC_WITHOUT_END_TAGGED_LINE)
 
     # Text from this position (+ lexing contexts offset for any preceding heredoc) is heredoc until a line
     # that terminates the heredoc is found.
@@ -82,7 +83,8 @@ module Puppet::Pops::Parser::HeredocSupport
     endline_pattern = /([[:blank:]]*)(?:([|])[[:blank:]]*)?(?:(\-)[[:blank:]]*)?#{Regexp.escape(endtag)}[[:blank:]]*\r?(?:\n|\z)/
     lines = []
     while !scn.eos? do
-      one_line = scn.scan_until(/(?:\n|\z)/) || lexer.lex_error_without_pos(eof_message)
+      one_line = scn.scan_until(/(?:\n|\z)/)
+      raise eof_error unless one_line
       if md = one_line.match(endline_pattern)
         leading      = md[1]
         has_margin   = md[2] == '|'
@@ -96,13 +98,13 @@ module Puppet::Pops::Parser::HeredocSupport
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Puppet::Pops::Parser::Locator::SubLocator.sub_locator(str,
+        locator = Locator::SubLocator.sub_locator(str,
           locator.file, heredoc_line, heredoc_offset, leading.length())
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.
         enqueue([:SUBLOCATE,
-          Puppet::Pops::Parser::LexerSupport::TokenValue.new([:SUBLOCATE,
+          LexerSupport::TokenValue.new([:SUBLOCATE,
             lines, lines.reduce(0) {|size, s| size + s.length} ],
             heredoc_offset,
             locator)])
@@ -116,7 +118,7 @@ module Puppet::Pops::Parser::HeredocSupport
         lines << one_line
       end
     end
-    lex_error_without_pos(eof_message)
+    raise eof_error
   end
 
   # Produces the heredoc text string given the individual (unprocessed) lines as an array.
@@ -137,3 +139,5 @@ module Puppet::Pops::Parser::HeredocSupport
 
 
 end
+end
+end

data/lib/puppet/pops/parser/interpolation_support.rb

@@ -185,17 +185,30 @@ module Puppet::Pops::Parser::InterpolationSupport
     scn = @scanner
     ctx = @lexing_context
     queue = @token_queue
+    queue_base = @token_queue[0]
 
     scn.skip(self.class::PATTERN_WS)
     queue_size = queue.size
     until scn.eos? do
       if token = lex_token
+        if token.equal?(queue_base)
+          # A nested #interpolate_dq call shifted the queue_base token from the @token_queue. It must
+          # be put back since it is intended for the top level #interpolate_dq call only.
+          queue.insert(0, token)
+          next # all relevant tokens are already on the queue
+        end
         token_name = token[0]
         ctx[:after] = token_name
         if token_name == :RBRACE && ctx[:brace_count] == brace_count
-          if queue.size - queue_size == 1
+          qlength = queue.size - queue_size
+          if qlength == 1
             # Single token is subject to replacement
             queue[-1] = transform_to_variable(queue[-1])
+          elsif qlength > 1 && [:DOT, :LBRACK].include?(queue[queue_size + 1][0])
+            # A first word, number of name token followed by '[' or '.' is subject to replacement
+            # But not for other operators such as ?, +, - etc. where user must use a $ before the name
+            # to get a variable
+            queue[queue_size] = transform_to_variable(queue[queue_size])
           end
           return
         end