puppet 5.3.4 → 6.29.0

data/lib/puppet/pops/parser/epp_parser.rb

@@ -6,7 +6,7 @@ class Puppet::Pops::Parser::EppParser < Puppet::Pops::Parser::Parser
   # @return [void]
   #
   def initvars
-    self.lexer = Puppet::Pops::Parser::Lexer2.new()# {:mode => :epp})
+    self.lexer = Puppet::Pops::Parser::Lexer2.new()
   end
 
   # Parses a file expected to contain epp text/DSL logic.

data/lib/puppet/pops/parser/epp_support.rb

@@ -44,7 +44,8 @@ module EppSupport
 
     # This is the lexer's main loop
     until queue.empty? && scn.eos? do
-      if token = queue.shift || lex_token
+      token = queue.shift || lex_token
+      if token
         yield [ ctx[:after] = token[0], token[1] ]
       end
     end
@@ -123,7 +124,7 @@ module EppSupport
   # The scanner supports
   # * scanning text until <%, <%-, <%=
   # * while scanning text:
-  #   * tokens <%% and %%> are translated to <% and %> respetively and is returned as text.
+  #   * tokens <%% and %%> are translated to <% and %>, respectively, and is returned as text.
   #   * tokens <%# and %> (or ending with -%>) and the enclosed text is a comment and is not included in the returned text
   #   * text following a comment that ends with -%> gets trailing whitespace (up to and including a line break) trimmed
   #     and this whitespace is not included in the returned text.
@@ -201,8 +202,6 @@ module EppSupport
           if s.end_with? "<%"
             @mode = :error
             @issue = Issues::EPP_UNBALANCED_EXPRESSION
-          else
-            mode = :epp
           end
           return s
 
@@ -231,6 +230,7 @@ module EppSupport
 
         when "#"
           # template comment
+
           # drop the scanned <%, and skip past -%>, or %>, but also skip %%>
           s.slice!(-2..-1)
 
@@ -242,8 +242,11 @@ module EppSupport
             @mode = :error
             return s
           end
-          # Always trim leading whitespace on the same line when there is a comment
-          s.sub!(/[ \t]*\z/, '')
+          # Trim leading whitespace on the same line when start was <%#-
+          if part[1] == '-'
+            s.sub!(/[ \t]*\z/, '')
+          end
+
           @skip_leading = true if part.end_with?("-%>")
           # Continue scanning for more text
 

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -1,9 +1,12 @@
+require 'puppet/concurrent/thread_local_singleton'
+
 module Puppet::Pops
 module Parser
 
 # Does not support "import" and parsing ruby files
 #
 class EvaluatingParser
+  extend Puppet::Concurrent::ThreadLocalSingleton
 
   attr_reader :parser
 
@@ -11,12 +14,7 @@ class EvaluatingParser
     @parser = Parser.new()
   end
 
-  def self.singleton
-    @instance ||= new
-  end
-
   def parse_string(s, file_source = nil)
-    @file_source = file_source
     clear()
     # Handling of syntax error can be much improved (in general), now it bails out of the parser
     # and does not have as rich information (when parsing a string), need to update it with the file source
@@ -25,20 +23,19 @@ class EvaluatingParser
     # Also a possible improvement (if the YAML parser returns positions) is to provide correct output of position.
     #
     begin
-      assert_and_report(parser.parse_string(s, file_source)).model
+      assert_and_report(parser.parse_string(s, file_source), file_source).model
     rescue Puppet::ParseErrorWithIssue => e
       raise e
     rescue Puppet::ParseError => e
       # TODO: This is not quite right, why does not the exception have the correct file?
-      e.file = @file_source unless e.file.is_a?(String) && !e.file.empty?
+      e.file = file_source unless e.file.is_a?(String) && !e.file.empty?
       raise e
     end
   end
 
   def parse_file(file)
-    @file_source = file
     clear()
-    assert_and_report(parser.parse_file(file)).model
+    assert_and_report(parser.parse_file(file), file).model
   end
 
   def evaluate_string(scope, s, file_source = nil)
@@ -80,7 +77,7 @@ class EvaluatingParser
   end
 
   def convert_to_3x(object, scope)
-    val = evaluator.convert(object, scope, nil)
+    evaluator.convert(object, scope, nil)
   end
 
   def validate(parse_result)
@@ -97,10 +94,10 @@ class EvaluatingParser
     Validation::ValidatorFactory_4_0.new().validator(acceptor)
   end
 
-  def assert_and_report(parse_result)
+  def assert_and_report(parse_result, file_source)
     return nil unless parse_result
     if parse_result['source_ref'].nil? || parse_result['source_ref'] == ''
-      parse_result['source_ref'] = @file_source
+      parse_result['source_ref'] = file_source
     end
     validation_result = validate(parse_result.model)
 

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -34,7 +34,7 @@ module HeredocSupport
       endtag = $1.strip
     end
 
-    lex_error(Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
+    lex_error(Issues::HEREDOC_EMPTY_ENDTAG) unless endtag.length >= 1
 
     resulting_escapes = []
     if escapes
@@ -85,7 +85,8 @@ module HeredocSupport
     while !scn.eos? do
       one_line = scn.scan_until(/(?:\n|\z)/)
       raise eof_error unless one_line
-      if md = one_line.match(endline_pattern)
+      md = one_line.match(endline_pattern)
+      if md
         leading      = md[1]
         has_margin   = md[2] == '|'
         remove_break = md[3] == '-'
@@ -94,12 +95,12 @@ module HeredocSupport
 
 
         # Process captured lines - remove leading, and trailing newline
-        str = heredoc_text(lines, leading, has_margin, remove_break)
+        # get processed string and index of removed margin/leading size per line
+        str, margin_per_line = heredoc_text(lines, leading, has_margin, remove_break)
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Locator::SubLocator.sub_locator(str,
-          locator.file, heredoc_line, heredoc_offset, leading.length())
+        locator = Locator::SubLocator.new(locator, str, heredoc_line, heredoc_offset, has_margin, margin_per_line)
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.
@@ -121,23 +122,32 @@ module HeredocSupport
     raise eof_error
   end
 
-  # Produces the heredoc text string given the individual (unprocessed) lines as an array.
+  # Produces the heredoc text string given the individual (unprocessed) lines as an array and array with margin sizes per line
   # @param lines [Array<String>] unprocessed lines of text in the heredoc w/o terminating line
   # @param leading [String] the leading text up (up to pipe or other terminating char)
   # @param has_margin [Boolean] if the left margin should be adjusted as indicated by `leading`
   # @param remove_break [Boolean] if the line break (\r?\n) at the end of the last line should be removed or not
+  # @return [Array] - a tuple with resulting string, and an array with margin size per line
   #
   def heredoc_text(lines, leading, has_margin, remove_break)
-    if has_margin
+    if has_margin && leading.length > 0
       leading_pattern = /^#{Regexp.escape(leading)}/
-      lines = lines.collect {|s| s.gsub(leading_pattern, '') }
+      # TODO: This implementation is not according to the specification, but is kept to be bug compatible.
+      # The specification says that leading space up to the margin marker should be removed, but this implementation
+      # simply leaves lines that have text in the margin untouched.
+      #
+      processed_lines = lines.collect {|s| s.gsub(leading_pattern, '') }
+      margin_per_line = Array.new(processed_lines.length) {|x| lines[x].length - processed_lines[x].length }
+      lines = processed_lines
+    else
+      # Array with a 0 per line
+      margin_per_line = Array.new(lines.length, 0)
     end
     result = lines.join('')
     result.gsub!(/\r?\n\z/m, '') if remove_break
-    result
+    [result, margin_per_line]
   end
 
-
 end
 end
 end

data/lib/puppet/pops/parser/interpolation_support.rb

@@ -10,7 +10,7 @@ module Puppet::Pops::Parser::InterpolationSupport
 
   # This is the starting point for a double quoted string with possible interpolation
   # The structure mimics that of the grammar.
-  # The logic is explicit (where the former implementation used parameters/strucures) given to a
+  # The logic is explicit (where the former implementation used parameters/structures) given to a
   # generic handler.
   # (This is both easier to understand and faster).
   #
@@ -23,7 +23,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_dqstring()
     text = value
     after = scn.pos
-    while true
+    loop do
       case terminator
       when '"'
         # simple case, there was no interpolation, return directly
@@ -37,7 +37,8 @@ module Puppet::Pops::Parser::InterpolationSupport
         enqueue_until count
         break
       when '$'
-        if varname = scn.scan(PATTERN_VARIABLE)
+        varname = scn.scan(PATTERN_VARIABLE)
+        if varname
           # The $ is counted towards the variable
           enqueue_completed([:DQPRE, text, after-before-1], before)
           enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
@@ -63,7 +64,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_dqstring
     text = value
     after = scn.pos
-    while true
+    loop do
       case terminator
       when '"'
         # simple case, there was no further interpolation, return directly
@@ -78,7 +79,8 @@ module Puppet::Pops::Parser::InterpolationSupport
         enqueue_until count
         break
       when '$'
-        if varname = scn.scan(PATTERN_VARIABLE)
+        varname = scn.scan(PATTERN_VARIABLE)
+        if varname
           # The $ is counted towards the variable
           enqueue_completed([:DQMID, text, after-before-1], before)
           enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
@@ -107,7 +109,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_uqstring()
     text = value
     after = scn.pos
-    while true
+    loop do
       case terminator
       when ''
         # simple case, there was no interpolation, return directly
@@ -122,7 +124,8 @@ module Puppet::Pops::Parser::InterpolationSupport
         enqueue_until count
         break
       when '$'
-        if varname = scn.scan(PATTERN_VARIABLE)
+        varname = scn.scan(PATTERN_VARIABLE)
+        if varname
           # The $ is counted towards the variable
           enqueue_completed([:DQPRE, text, after-before-1], before)
           enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
@@ -147,7 +150,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_uqstring
     text = value
     after = scn.pos
-    while true
+    loop do
       case terminator
       when ''
         # simple case, there was no further interpolation, return directly
@@ -162,7 +165,8 @@ module Puppet::Pops::Parser::InterpolationSupport
         enqueue_until count
         break
       when '$'
-        if varname = scn.scan(PATTERN_VARIABLE)
+        varname = scn.scan(PATTERN_VARIABLE)
+        if varname
           # The $ is counted towards the variable
           enqueue_completed([:DQMID, text, after-before-1], before)
           enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
@@ -190,7 +194,8 @@ module Puppet::Pops::Parser::InterpolationSupport
     scn.skip(self.class::PATTERN_WS)
     queue_size = queue.size
     until scn.eos? do
-      if token = lex_token
+      token = lex_token
+      if token
         if token.equal?(queue_base)
           # A nested #interpolate_dq call shifted the queue_base token from the @token_queue. It must
           # be put back since it is intended for the top level #interpolate_dq call only.
@@ -221,7 +226,7 @@ module Puppet::Pops::Parser::InterpolationSupport
 
   def transform_to_variable(token)
     token_name = token[0]
-    if [:NUMBER, :NAME, :WORD].include?(token_name) || self.class::KEYWORD_NAMES[token_name]
+    if [:NUMBER, :NAME, :WORD].include?(token_name) || self.class::KEYWORD_NAMES[token_name] || @taskm_keywords[token_name]
       t = token[1]
       ta = t.token_array
       [:VARIABLE, self.class::TokenValue.new([:VARIABLE, ta[1], ta[2]], t.offset, t.locator)]

data/lib/puppet/pops/parser/lexer2.rb

@@ -1,4 +1,5 @@
-# The Lexer is responsbile for turning source text into tokens.
+# frozen_string_literal: true
+# The Lexer is responsible for turning source text into tokens.
 # This version is a performance enhanced lexer (in comparison to the 3.x and earlier "future parser" lexer.
 #
 # Old returns tokens [:KEY, value, { locator = }
@@ -30,114 +31,114 @@ class Lexer2
   # The length is pre-calculated for all singleton tokens. The length is used both to signal the length of
   # the token, and to advance the scanner position (without having to advance it with a scan(regexp)).
   #
-  TOKEN_LBRACK       = [:LBRACK,       '['.freeze,   1].freeze
-  TOKEN_LISTSTART    = [:LISTSTART,    '['.freeze,   1].freeze
-  TOKEN_RBRACK       = [:RBRACK,       ']'.freeze,   1].freeze
-  TOKEN_LBRACE       = [:LBRACE,       '{'.freeze,   1].freeze
-  TOKEN_RBRACE       = [:RBRACE,       '}'.freeze,   1].freeze
-  TOKEN_SELBRACE     = [:SELBRACE,     '{'.freeze,   1].freeze
-  TOKEN_LPAREN       = [:LPAREN,       '('.freeze,   1].freeze
-  TOKEN_WSLPAREN     = [:WSLPAREN,     '('.freeze,   1].freeze
-  TOKEN_RPAREN       = [:RPAREN,       ')'.freeze,   1].freeze
-
-  TOKEN_EQUALS       = [:EQUALS,       '='.freeze,   1].freeze
-  TOKEN_APPENDS      = [:APPENDS,      '+='.freeze,  2].freeze
-  TOKEN_DELETES      = [:DELETES,      '-='.freeze,  2].freeze
-
-  TOKEN_ISEQUAL      = [:ISEQUAL,      '=='.freeze,  2].freeze
-  TOKEN_NOTEQUAL     = [:NOTEQUAL,     '!='.freeze,  2].freeze
-  TOKEN_MATCH        = [:MATCH,        '=~'.freeze,  2].freeze
-  TOKEN_NOMATCH      = [:NOMATCH,      '!~'.freeze,  2].freeze
-  TOKEN_GREATEREQUAL = [:GREATEREQUAL, '>='.freeze,  2].freeze
-  TOKEN_GREATERTHAN  = [:GREATERTHAN,  '>'.freeze,   1].freeze
-  TOKEN_LESSEQUAL    = [:LESSEQUAL,    '<='.freeze,  2].freeze
-  TOKEN_LESSTHAN     = [:LESSTHAN,     '<'.freeze,   1].freeze
-
-  TOKEN_FARROW       = [:FARROW,       '=>'.freeze,  2].freeze
-  TOKEN_PARROW       = [:PARROW,       '+>'.freeze,  2].freeze
-
-  TOKEN_LSHIFT       = [:LSHIFT,       '<<'.freeze,  2].freeze
-  TOKEN_LLCOLLECT    = [:LLCOLLECT,    '<<|'.freeze, 3].freeze
-  TOKEN_LCOLLECT     = [:LCOLLECT,     '<|'.freeze,  2].freeze
-
-  TOKEN_RSHIFT       = [:RSHIFT,       '>>'.freeze,  2].freeze
-  TOKEN_RRCOLLECT    = [:RRCOLLECT,    '|>>'.freeze, 3].freeze
-  TOKEN_RCOLLECT     = [:RCOLLECT,     '|>'.freeze,  2].freeze
-
-  TOKEN_PLUS         = [:PLUS,         '+'.freeze,   1].freeze
-  TOKEN_MINUS        = [:MINUS,        '-'.freeze,   1].freeze
-  TOKEN_DIV          = [:DIV,          '/'.freeze,   1].freeze
-  TOKEN_TIMES        = [:TIMES,        '*'.freeze,   1].freeze
-  TOKEN_MODULO       = [:MODULO,       '%'.freeze,   1].freeze
-
-  TOKEN_NOT          = [:NOT,          '!'.freeze,   1].freeze
-  TOKEN_DOT          = [:DOT,          '.'.freeze,   1].freeze
-  TOKEN_PIPE         = [:PIPE,         '|'.freeze,   1].freeze
-  TOKEN_AT           = [:AT ,          '@'.freeze,   1].freeze
-  TOKEN_ATAT         = [:ATAT ,        '@@'.freeze,  2].freeze
-  TOKEN_COLON        = [:COLON,        ':'.freeze,   1].freeze
-  TOKEN_COMMA        = [:COMMA,        ','.freeze,   1].freeze
-  TOKEN_SEMIC        = [:SEMIC,        ';'.freeze,   1].freeze
-  TOKEN_QMARK        = [:QMARK,        '?'.freeze,   1].freeze
-  TOKEN_TILDE        = [:TILDE,        '~'.freeze,   1].freeze # lexed but not an operator in Puppet
+  TOKEN_LBRACK       = [:LBRACK,       '[',   1].freeze
+  TOKEN_LISTSTART    = [:LISTSTART,    '[',   1].freeze
+  TOKEN_RBRACK       = [:RBRACK,       ']',   1].freeze
+  TOKEN_LBRACE       = [:LBRACE,       '{',   1].freeze
+  TOKEN_RBRACE       = [:RBRACE,       '}',   1].freeze
+  TOKEN_SELBRACE     = [:SELBRACE,     '{',   1].freeze
+  TOKEN_LPAREN       = [:LPAREN,       '(',   1].freeze
+  TOKEN_WSLPAREN     = [:WSLPAREN,     '(',   1].freeze
+  TOKEN_RPAREN       = [:RPAREN,       ')',   1].freeze
+
+  TOKEN_EQUALS       = [:EQUALS,       '=',   1].freeze
+  TOKEN_APPENDS      = [:APPENDS,      '+=',  2].freeze
+  TOKEN_DELETES      = [:DELETES,      '-=',  2].freeze
+
+  TOKEN_ISEQUAL      = [:ISEQUAL,      '==',  2].freeze
+  TOKEN_NOTEQUAL     = [:NOTEQUAL,     '!=',  2].freeze
+  TOKEN_MATCH        = [:MATCH,        '=~',  2].freeze
+  TOKEN_NOMATCH      = [:NOMATCH,      '!~',  2].freeze
+  TOKEN_GREATEREQUAL = [:GREATEREQUAL, '>=',  2].freeze
+  TOKEN_GREATERTHAN  = [:GREATERTHAN,  '>',   1].freeze
+  TOKEN_LESSEQUAL    = [:LESSEQUAL,    '<=',  2].freeze
+  TOKEN_LESSTHAN     = [:LESSTHAN,     '<',   1].freeze
+
+  TOKEN_FARROW       = [:FARROW,       '=>',  2].freeze
+  TOKEN_PARROW       = [:PARROW,       '+>',  2].freeze
+
+  TOKEN_LSHIFT       = [:LSHIFT,       '<<',  2].freeze
+  TOKEN_LLCOLLECT    = [:LLCOLLECT,    '<<|', 3].freeze
+  TOKEN_LCOLLECT     = [:LCOLLECT,     '<|',  2].freeze
+
+  TOKEN_RSHIFT       = [:RSHIFT,       '>>',  2].freeze
+  TOKEN_RRCOLLECT    = [:RRCOLLECT,    '|>>', 3].freeze
+  TOKEN_RCOLLECT     = [:RCOLLECT,     '|>',  2].freeze
+
+  TOKEN_PLUS         = [:PLUS,         '+',   1].freeze
+  TOKEN_MINUS        = [:MINUS,        '-',   1].freeze
+  TOKEN_DIV          = [:DIV,          '/',   1].freeze
+  TOKEN_TIMES        = [:TIMES,        '*',   1].freeze
+  TOKEN_MODULO       = [:MODULO,       '%',   1].freeze
+
+  TOKEN_NOT          = [:NOT,          '!',   1].freeze
+  TOKEN_DOT          = [:DOT,          '.',   1].freeze
+  TOKEN_PIPE         = [:PIPE,         '|',   1].freeze
+  TOKEN_AT           = [:AT ,          '@',   1].freeze
+  TOKEN_ATAT         = [:ATAT ,        '@@',  2].freeze
+  TOKEN_COLON        = [:COLON,        ':',   1].freeze
+  TOKEN_COMMA        = [:COMMA,        ',',   1].freeze
+  TOKEN_SEMIC        = [:SEMIC,        ';',   1].freeze
+  TOKEN_QMARK        = [:QMARK,        '?',   1].freeze
+  TOKEN_TILDE        = [:TILDE,        '~',   1].freeze # lexed but not an operator in Puppet
 
   TOKEN_REGEXP       = [:REGEXP,       nil,   0].freeze
 
-  TOKEN_IN_EDGE      = [:IN_EDGE,      '->'.freeze,  2].freeze
-  TOKEN_IN_EDGE_SUB  = [:IN_EDGE_SUB,  '~>'.freeze,  2].freeze
-  TOKEN_OUT_EDGE     = [:OUT_EDGE,     '<-'.freeze,  2].freeze
-  TOKEN_OUT_EDGE_SUB = [:OUT_EDGE_SUB, '<~'.freeze,  2].freeze
+  TOKEN_IN_EDGE      = [:IN_EDGE,      '->',  2].freeze
+  TOKEN_IN_EDGE_SUB  = [:IN_EDGE_SUB,  '~>',  2].freeze
+  TOKEN_OUT_EDGE     = [:OUT_EDGE,     '<-',  2].freeze
+  TOKEN_OUT_EDGE_SUB = [:OUT_EDGE_SUB, '<~',  2].freeze
 
   # Tokens that are always unique to what has been lexed
-  TOKEN_STRING         =  [:STRING, nil,          0].freeze
-  TOKEN_WORD           =  [:WORD, nil,            0].freeze
-  TOKEN_DQPRE          =  [:DQPRE,  nil,          0].freeze
-  TOKEN_DQMID          =  [:DQPRE,  nil,          0].freeze
-  TOKEN_DQPOS          =  [:DQPRE,  nil,          0].freeze
-  TOKEN_NUMBER         =  [:NUMBER, nil,          0].freeze
-  TOKEN_VARIABLE       =  [:VARIABLE, nil,        1].freeze
-  TOKEN_VARIABLE_EMPTY =  [:VARIABLE, ''.freeze,  1].freeze
+  TOKEN_STRING         = [:STRING,      nil,  0].freeze
+  TOKEN_WORD           = [:WORD,        nil,  0].freeze
+  TOKEN_DQPRE          = [:DQPRE,       nil,  0].freeze
+  TOKEN_DQMID          = [:DQPRE,       nil,  0].freeze
+  TOKEN_DQPOS          = [:DQPRE,       nil,  0].freeze
+  TOKEN_NUMBER         = [:NUMBER,      nil,  0].freeze
+  TOKEN_VARIABLE       = [:VARIABLE,    nil,  1].freeze
+  TOKEN_VARIABLE_EMPTY = [:VARIABLE,    '',   1].freeze
 
   # HEREDOC has syntax as an argument.
-  TOKEN_HEREDOC        =  [:HEREDOC, nil, 0].freeze
+  TOKEN_HEREDOC        = [:HEREDOC,     nil,  0].freeze
 
   # EPP_START is currently a marker token, may later get syntax
-  TOKEN_EPPSTART       =  [:EPP_START, nil, 0].freeze
-  TOKEN_EPPEND         =  [:EPP_END, '%>', 2].freeze
-  TOKEN_EPPEND_TRIM    =  [:EPP_END_TRIM, '-%>', 3].freeze
+  TOKEN_EPPSTART    = [:EPP_START,      nil,  0].freeze
+  TOKEN_EPPEND      = [:EPP_END,       '%>',  2].freeze
+  TOKEN_EPPEND_TRIM = [:EPP_END_TRIM, '-%>',  3].freeze
 
   # This is used for unrecognized tokens, will always be a single character. This particular instance
   # is not used, but is kept here for documentation purposes.
-  TOKEN_OTHER        = [:OTHER,  nil,  0]
+  TOKEN_OTHER = [:OTHER, nil, 0]
 
   # Keywords are all singleton tokens with pre calculated lengths.
   # Booleans are pre-calculated (rather than evaluating the strings "false" "true" repeatedly.
   #
   KEYWORDS = {
-    'case'     => [:CASE,     'case',     4],
-    'class'    => [:CLASS,    'class',    5],
-    'default'  => [:DEFAULT,  'default',  7],
-    'define'   => [:DEFINE,   'define',   6],
-    'if'       => [:IF,       'if',       2],
-    'elsif'    => [:ELSIF,    'elsif',    5],
-    'else'     => [:ELSE,     'else',     4],
-    'inherits' => [:INHERITS, 'inherits', 8],
-    'node'     => [:NODE,     'node',     4],
-    'and'      => [:AND,      'and',      3],
-    'or'       => [:OR,       'or',       2],
-    'undef'    => [:UNDEF,    'undef',    5],
-    'false'    => [:BOOLEAN,  false,      5],
-    'true'     => [:BOOLEAN,  true,       4],
-    'in'       => [:IN,       'in',       2],
-    'unless'   => [:UNLESS,   'unless',   6],
-    'function' => [:FUNCTION, 'function', 8],
-    'type'     => [:TYPE,     'type',     4],
-    'attr'     => [:ATTR,     'attr',     4],
-    'private'  => [:PRIVATE,  'private',  7],
-    'application' => [:APPLICATION, 'application',  11],
-    'consumes'    => [:CONSUMES,    'consumes',  8],
-    'produces'    => [:PRODUCES,    'produces',  8],
-    'site'        => [:SITE,        'site',  4],
+    'case'        => [:CASE,        'case',         4],
+    'class'       => [:CLASS,       'class',        5],
+    'default'     => [:DEFAULT,     'default',      7],
+    'define'      => [:DEFINE,      'define',       6],
+    'if'          => [:IF,          'if',           2],
+    'elsif'       => [:ELSIF,       'elsif',        5],
+    'else'        => [:ELSE,        'else',         4],
+    'inherits'    => [:INHERITS,    'inherits',     8],
+    'node'        => [:NODE,        'node',         4],
+    'and'         => [:AND,         'and',          3],
+    'or'          => [:OR,          'or',           2],
+    'undef'       => [:UNDEF,       'undef',        5],
+    'false'       => [:BOOLEAN,     false,          5],
+    'true'        => [:BOOLEAN,     true,           4],
+    'in'          => [:IN,          'in',           2],
+    'unless'      => [:UNLESS,      'unless',       6],
+    'function'    => [:FUNCTION,    'function',     8],
+    'type'        => [:TYPE,        'type',         4],
+    'attr'        => [:ATTR,        'attr',         4],
+    'private'     => [:PRIVATE,     'private',      7],
+    'application' => [:APPLICATION, 'application', 11],
+    'consumes'    => [:CONSUMES,    'consumes',     8],
+    'produces'    => [:PRODUCES,    'produces',     8],
+    'site'        => [:SITE,        'site',         4],
   }
 
   KEYWORDS.each {|k,v| v[1].freeze; v.freeze }
@@ -189,7 +190,12 @@ class Lexer2
       ',' => lambda {  emit(TOKEN_COMMA, @scanner.pos) },
       '[' => lambda do
         before = @scanner.pos
-        if (before == 0 || @scanner.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
+        # Must check the preceding character to see if it is whitespace.
+        # The fastest thing to do is to simply byteslice to get the string ending at the offset before
+        # and then check what the last character is. (This is the same as  what an locator.char_offset needs
+        # to compute, but with less overhead of trying to find out the global offset from a local offset in the
+        # case when this is sublocated in a heredoc).
+        if before == 0 || @scanner.string.byteslice(0, before)[-1] =~ /[[:blank:]\r\n]+/
           emit(TOKEN_LISTSTART, before)
         else
           emit(TOKEN_LBRACK, before)
@@ -350,7 +356,8 @@ class Lexer2
         else
           before = scn.pos
           # regexp position is a regexp, else a div
-          if regexp_acceptable? && value = scn.scan(PATTERN_REGEX)
+          value = scn.scan(PATTERN_REGEX) if regexp_acceptable?
+          if value
             # Ensure an escaped / was not matched
             while escaped_end(value)
               more = scn.scan_until(PATTERN_REGEX_END)
@@ -447,7 +454,8 @@ class Lexer2
       '$' => lambda do
         scn = @scanner
         before = scn.pos
-        if value = scn.scan(PATTERN_DOLLAR_VAR)
+        value = scn.scan(PATTERN_DOLLAR_VAR)
+        if value
           emit_completed([:VARIABLE, value[1..-1].freeze, scn.pos - before], before)
         else
           # consume the $ and let higher layer complain about the error instead of getting a syntax error
@@ -513,7 +521,6 @@ class Lexer2
             scn.pos = before
             invalid_number = scn.peek(after - before) unless invalid_number
           end
-          length = scn.pos - before
           assert_numeric(invalid_number, before)
           scn.pos = before + 1
           lex_error(Issues::ILLEGAL_NUMBER, {:value => invalid_number})
@@ -523,12 +530,11 @@ class Lexer2
     ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
       'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_'].each do |c|
       @selector[c] = lambda do
-
         scn = @scanner
         before = scn.pos
         value = scn.scan(PATTERN_BARE_WORD)
         if value && value =~ PATTERN_NAME
-          emit_completed(KEYWORDS[value] || [:NAME, value.freeze, scn.pos - before], before)
+          emit_completed(KEYWORDS[value] || @taskm_keywords[value] || [:NAME, value.freeze, scn.pos - before], before)
         elsif value
           emit_completed([:WORD, value.freeze, scn.pos - before], before)
         else
@@ -649,7 +655,7 @@ class Lexer2
   #
   def lex_file(file)
     initvars
-    contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file, :encoding => 'utf-8') : ''
+    contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file, :mode => 'rb', :encoding => 'utf-8') : ''
     assert_not_bom(contents)
     @scanner = StringScanner.new(contents.freeze)
     @locator = Locator.locator(contents, file)
@@ -663,6 +669,8 @@ class Lexer2
       :after => nil,
       :line_lexical_start => 0
     }
+    # Use of --tasks introduces the new keyword 'plan'
+    @taskm_keywords = Puppet[:tasks] ? { 'plan' => [:PLAN, 'plan',  4], 'apply' => [:APPLY, 'apply', 5] }.freeze : EMPTY_HASH
   end
 
   # Scans all of the content and returns it in an array
@@ -698,7 +706,8 @@ class Lexer2
 
     # This is the lexer's main loop
     until queue.empty? && scn.eos? do
-      if token = queue.shift || selector[scn.peek(1)].call
+      token = queue.shift || selector[scn.peek(1)].call
+      if token
         ctx[:after] = token[0]
         yield token
       end

data/lib/puppet/pops/parser/lexer_support.rb

@@ -30,7 +30,7 @@ module LexerSupport
 
   # Raises a Puppet::LexError with the given message
   def lex_error_without_pos(issue, args = {})
-    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code)
+    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code, args)
   end
 
   # Raises a Puppet::ParserErrorWithIssue with the given issue and arguments
@@ -73,7 +73,8 @@ module LexerSupport
         line(pos),
         position(pos),
         nil,
-        issue.issue_code)
+        issue.issue_code,
+        args)
   end
 
   # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.
@@ -135,7 +136,7 @@ module LexerSupport
     end
 
     def to_s
-      # This format is very compact and is intended for debugging output from racc parsser in
+      # This format is very compact and is intended for debugging output from racc parser in
       # debug mode. If this is made more elaborate the output from a debug run becomes very hard to read.
       #
       "'#{self[:value]} #{@token_array[0]}'"
@@ -204,7 +205,7 @@ module LexerSupport
 
   def get_bom(content)
     # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
-    # illegal characters on most platforms, and there is no get_bytes(n). xplicit calls are faster than
+    # illegal characters on most platforms, and there is no get_bytes(n). Explicit calls are faster than
     # looping with a lambda. The get_byte returns nil if there are too few characters, and they
     # are changed to spaces
     MM.new(

data/lib/puppet/pops/parser/locatable.rb

@@ -1,4 +1,4 @@
-# Interface for something that is "locateable" (holds offset and length).
+# Interface for something that is "locatable" (holds offset and length).
 class Puppet::Pops::Parser::Locatable
 
   # The offset in the locator's content