puppet 5.0.0 → 6.0.0

data/lib/puppet/pops/parser/epp_parser.rb

@@ -6,7 +6,7 @@ class Puppet::Pops::Parser::EppParser < Puppet::Pops::Parser::Parser
   # @return [void]
   #
   def initvars
-    self.lexer = Puppet::Pops::Parser::Lexer2.new()# {:mode => :epp})
+    self.lexer = Puppet::Pops::Parser::Lexer2.new()
   end
 
   # Parses a file expected to contain epp text/DSL logic.

data/lib/puppet/pops/parser/epp_support.rb

@@ -123,7 +123,7 @@ module EppSupport
   # The scanner supports
   # * scanning text until <%, <%-, <%=
   # * while scanning text:
-  #   * tokens <%% and %%> are translated to <% and %> respetively and is returned as text.
+  #   * tokens <%% and %%> are translated to <% and %>, respectively, and is returned as text.
   #   * tokens <%# and %> (or ending with -%>) and the enclosed text is a comment and is not included in the returned text
   #   * text following a comment that ends with -%> gets trailing whitespace (up to and including a line break) trimmed
   #     and this whitespace is not included in the returned text.
@@ -201,8 +201,6 @@ module EppSupport
           if s.end_with? "<%"
             @mode = :error
             @issue = Issues::EPP_UNBALANCED_EXPRESSION
-          else
-            mode = :epp
           end
           return s
 
@@ -231,6 +229,7 @@ module EppSupport
 
         when "#"
           # template comment
+
           # drop the scanned <%, and skip past -%>, or %>, but also skip %%>
           s.slice!(-2..-1)
 
@@ -242,8 +241,11 @@ module EppSupport
             @mode = :error
             return s
           end
-          # Always trim leading whitespace on the same line when there is a comment
-          s.sub!(/[ \t]*\z/, '')
+          # Trim leading whitespace on the same line when start was <%#-
+          if part[1] == '-'
+            s.sub!(/[ \t]*\z/, '')
+          end
+
           @skip_leading = true if part.end_with?("-%>")
           # Continue scanning for more text
 

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -80,7 +80,7 @@ class EvaluatingParser
   end
 
   def convert_to_3x(object, scope)
-    val = evaluator.convert(object, scope, nil)
+    evaluator.convert(object, scope, nil)
   end
 
   def validate(parse_result)

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -34,7 +34,7 @@ module HeredocSupport
       endtag = $1.strip
     end
 
-    lex_error(Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
+    lex_error(Issues::HEREDOC_EMPTY_ENDTAG) unless endtag.length >= 1
 
     resulting_escapes = []
     if escapes
@@ -133,7 +133,7 @@ module HeredocSupport
       lines = lines.collect {|s| s.gsub(leading_pattern, '') }
     end
     result = lines.join('')
-    result.gsub!(/\r?\n$/, '') if remove_break
+    result.gsub!(/\r?\n\z/m, '') if remove_break
     result
   end
 

data/lib/puppet/pops/parser/interpolation_support.rb

@@ -10,7 +10,7 @@ module Puppet::Pops::Parser::InterpolationSupport
 
   # This is the starting point for a double quoted string with possible interpolation
   # The structure mimics that of the grammar.
-  # The logic is explicit (where the former implementation used parameters/strucures) given to a
+  # The logic is explicit (where the former implementation used parameters/structures) given to a
   # generic handler.
   # (This is both easier to understand and faster).
   #
@@ -221,7 +221,7 @@ module Puppet::Pops::Parser::InterpolationSupport
 
   def transform_to_variable(token)
     token_name = token[0]
-    if [:NUMBER, :NAME, :WORD].include?(token_name) || self.class::KEYWORD_NAMES[token_name]
+    if [:NUMBER, :NAME, :WORD].include?(token_name) || self.class::KEYWORD_NAMES[token_name] || @taskm_keywords[token_name]
       t = token[1]
       ta = t.token_array
       [:VARIABLE, self.class::TokenValue.new([:VARIABLE, ta[1], ta[2]], t.offset, t.locator)]

data/lib/puppet/pops/parser/lexer2.rb

@@ -1,4 +1,4 @@
-# The Lexer is responsbile for turning source text into tokens.
+# The Lexer is responsible for turning source text into tokens.
 # This version is a performance enhanced lexer (in comparison to the 3.x and earlier "future parser" lexer.
 #
 # Old returns tokens [:KEY, value, { locator = }
@@ -513,7 +513,6 @@ class Lexer2
             scn.pos = before
             invalid_number = scn.peek(after - before) unless invalid_number
           end
-          length = scn.pos - before
           assert_numeric(invalid_number, before)
           scn.pos = before + 1
           lex_error(Issues::ILLEGAL_NUMBER, {:value => invalid_number})
@@ -523,12 +522,11 @@ class Lexer2
     ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
       'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_'].each do |c|
       @selector[c] = lambda do
-
         scn = @scanner
         before = scn.pos
         value = scn.scan(PATTERN_BARE_WORD)
         if value && value =~ PATTERN_NAME
-          emit_completed(KEYWORDS[value] || [:NAME, value.freeze, scn.pos - before], before)
+          emit_completed(KEYWORDS[value] || @taskm_keywords[value] || [:NAME, value.freeze, scn.pos - before], before)
         elsif value
           emit_completed([:WORD, value.freeze, scn.pos - before], before)
         else
@@ -649,7 +647,7 @@ class Lexer2
   #
   def lex_file(file)
     initvars
-    contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file, :encoding => 'utf-8') : ''
+    contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file, :mode => 'rb', :encoding => 'utf-8') : ''
     assert_not_bom(contents)
     @scanner = StringScanner.new(contents.freeze)
     @locator = Locator.locator(contents, file)
@@ -663,6 +661,8 @@ class Lexer2
       :after => nil,
       :line_lexical_start => 0
     }
+    # Use of --tasks introduces the new keyword 'plan'
+    @taskm_keywords = Puppet[:tasks] ? { 'plan' => [:PLAN, 'plan',  4], 'apply' => [:APPLY, 'apply', 5] }.freeze : EMPTY_HASH
   end
 
   # Scans all of the content and returns it in an array

data/lib/puppet/pops/parser/lexer_support.rb

@@ -30,7 +30,7 @@ module LexerSupport
 
   # Raises a Puppet::LexError with the given message
   def lex_error_without_pos(issue, args = {})
-    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code)
+    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code, args)
   end
 
   # Raises a Puppet::ParserErrorWithIssue with the given issue and arguments
@@ -73,7 +73,8 @@ module LexerSupport
         line(pos),
         position(pos),
         nil,
-        issue.issue_code)
+        issue.issue_code,
+        args)
   end
 
   # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.
@@ -135,7 +136,7 @@ module LexerSupport
     end
 
     def to_s
-      # This format is very compact and is intended for debugging output from racc parsser in
+      # This format is very compact and is intended for debugging output from racc parser in
       # debug mode. If this is made more elaborate the output from a debug run becomes very hard to read.
       #
       "'#{self[:value]} #{@token_array[0]}'"
@@ -204,7 +205,7 @@ module LexerSupport
 
   def get_bom(content)
     # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
-    # illegal characters on most platforms, and there is no get_bytes(n). xplicit calls are faster than
+    # illegal characters on most platforms, and there is no get_bytes(n). Explicit calls are faster than
     # looping with a lambda. The get_byte returns nil if there are too few characters, and they
     # are changed to spaces
     MM.new(

data/lib/puppet/pops/parser/locatable.rb

@@ -1,4 +1,4 @@
-# Interface for something that is "locateable" (holds offset and length).
+# Interface for something that is "locatable" (holds offset and length).
 class Puppet::Pops::Parser::Locatable
 
   # The offset in the locator's content

data/lib/puppet/pops/parser/locator.rb

@@ -43,11 +43,7 @@ class Locator
   def char_offset(byte_offset)
   end
 
-  # Returns the length measured in number of characters from the given start and end reported offset
-  def char_length(offset, end_offset)
-  end
-
-  # Returns the length measured in number of characters from the given start and end byte offseta
+  # Returns the length measured in number of characters from the given start and end byte offset
   def char_length(offset, end_offset)
   end
 
@@ -159,13 +155,9 @@ class Locator
     end
   end
 
-  private
-
   class AbstractLocator < Locator
     attr_accessor :line_index
     attr_accessor :string
-    attr_accessor :prev_offset
-    attr_accessor :prev_line
     attr_reader   :string
     attr_reader   :file
 
@@ -255,19 +247,19 @@ class Locator
 
     # Returns the line number (first line is 1) for the given offset
     def line_for_offset(offset)
-      if prev_offset == offset
+      if @prev_offset == offset
         # use cache
-        return prev_line
+        return @prev_line
       end
       if line_nbr = ary_bsearch_i(line_index, offset)
         # cache
-        prev_offset = offset
-        prev_line = line_nbr
+        @prev_offset = offset
+        @prev_line = line_nbr
         return line_nbr
       end
       # If not found it is after last
       # clear cache
-      prev_offset = prev_line = nil
+      @prev_offset = @prev_line = nil
       return line_index.size
     end
   end
@@ -328,7 +320,7 @@ class Locator
       string.byteslice(0, byte_offset).length
     end
 
-    # Returns the length measured in number of characters from the given start and end byte offseta
+    # Returns the length measured in number of characters from the given start and end byte offset
     def char_length(offset, end_offset)
       string.byteslice(offset, end_offset - offset).length
     end

data/lib/puppet/pops/parser/parser_support.rb

@@ -91,7 +91,7 @@ class Parser
   #
   def on_error(token,value,stack)
     if token == 0 # denotes end of file
-      value_at = 'end of file'
+      value_at = 'end of input'
     else
       value_at = "'#{value[:value]}'"
     end
@@ -121,7 +121,7 @@ class Parser
       file = lexer.file
     end
     file = nil unless file.is_a?(String) && !file.empty?
-    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, issue_code = Issues::SYNTAX_ERROR.issue_code)
+    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, Issues::SYNTAX_ERROR.issue_code)
   end
 
   # Parses a String of pp DSL code.
@@ -219,7 +219,7 @@ class Parser
     # Create a synthetic NOOP token at EOF offset with 0 size. The lexer does not produce an EOF token that is
     # visible to the grammar rules. Creating this token is mainly to reuse the positioning logic as it
     # expects a token decorated with location information.
-    token_sym, token = @lexer.emit_completed([:NOOP,'',0], locator.string.bytesize)
+    _, token = @lexer.emit_completed([:NOOP,'',0], locator.string.bytesize)
     loc(no_op, token)
     # Program with a Noop
     program = Factory.PROGRAM(no_op, [], locator)

data/lib/puppet/pops/parser/pn_parser.rb

@@ -0,0 +1,316 @@
+module Puppet::Pops
+module Parser
+
+class PNParser
+  LIT_TRUE = 'true'.freeze
+  LIT_FALSE = 'false'.freeze
+  LIT_NIL = 'nil'.freeze
+
+  TOKEN_END = 0
+  TOKEN_BOOL = 1
+  TOKEN_NIL = 2
+  TOKEN_INT = 3
+  TOKEN_FLOAT = 4
+  TOKEN_IDENTIFIER = 5
+  TOKEN_WS = 0x20
+  TOKEN_STRING = 0x22
+  TOKEN_KEY = 0x3a
+  TOKEN_LP = 0x28
+  TOKEN_RP = 0x29
+  TOKEN_LB = 0x5b
+  TOKEN_RB = 0x5d
+  TOKEN_LC = 0x7b
+  TOKEN_RC = 0x7d
+
+  TYPE_END = 0
+  TYPE_WS = 1
+  TYPE_DELIM = 2
+  TYPE_KEY_START = 3
+  TYPE_STRING_START = 4
+  TYPE_IDENTIFIER = 5
+  TYPE_MINUS = 6
+  TYPE_DIGIT = 7
+  TYPE_ALPHA = 8
+
+  def initialize
+    @char_types = self.class.char_types
+  end
+
+  def parse(text, locator = nil, offset = nil)
+    @locator = locator
+    @offset = offset
+    @text = text
+    @codepoints = text.codepoints.to_a.freeze
+    @pos = 0
+    @token = TOKEN_END
+    @token_value = nil
+    next_token
+    parse_next
+  end
+
+  private
+
+  def parse_next
+    case @token
+    when TOKEN_LB
+      parse_array
+    when TOKEN_LC
+      parse_map
+    when TOKEN_LP
+      parse_call
+    when TOKEN_BOOL, TOKEN_INT, TOKEN_FLOAT, TOKEN_STRING, TOKEN_NIL
+      parse_literal
+    when TOKEN_END
+      parse_error(_('unexpected end of input'))
+    else
+      parse_error(_('unexpected %{value}' % { value: @token_value }))
+    end
+  end
+
+  def parse_error(message)
+    file = ''
+    line = 1
+    pos = 1
+    if @locator
+      file = @locator.file
+      line = @locator.line_for_offset(@offset)
+      pos  = @locator.pos_on_line(@offset)
+    end
+    @codepoints[0, @pos].each do |c|
+      if c == 0x09
+        line += 1
+        pos = 1
+      end
+    end
+    raise Puppet::ParseError.new(message, file, line, pos)
+  end
+
+  def parse_literal
+    pn = PN::Literal.new(@token_value)
+    next_token
+    pn
+  end
+
+  def parse_array
+    next_token
+    PN::List.new(parse_elements(TOKEN_RB))
+  end
+
+  def parse_map
+    next_token
+    entries = []
+    while @token != TOKEN_RC && @token != TOKEN_END
+      parse_error(_('map key expected')) unless @token == TOKEN_KEY
+      key = @token_value
+      next_token
+      entries << parse_next.with_name(key)
+    end
+    next_token
+    PN::Map.new(entries)
+  end
+
+  def parse_call
+    next_token
+    parse_error(_("expected identifier to follow '('")) unless @token == TOKEN_IDENTIFIER
+    name = @token_value
+    next_token
+    PN::Call.new(name, *parse_elements(TOKEN_RP))
+  end
+
+  def parse_elements(end_token)
+    elements = []
+    while @token != end_token && @token != TOKEN_END
+      elements << parse_next
+    end
+    parse_error(_("missing '%{token}' to end list") % { token: end_token.chr(Encoding::UTF_8) } ) unless @token == end_token
+    next_token
+    elements
+  end
+
+  # All methods below belong to the PN lexer
+  def self.char_types
+    unless instance_variable_defined?(:@char_types)
+      @char_types = Array.new(0x80, TYPE_IDENTIFIER)
+      @char_types[0] = TYPE_END
+      [0x09, 0x0d, 0x0a, 0x20].each { |n| @char_types[n] = TYPE_WS }
+      [TOKEN_LP, TOKEN_RP, TOKEN_LB, TOKEN_RB, TOKEN_LC, TOKEN_RC].each { |n| @char_types[n] = TYPE_DELIM }
+      @char_types[0x2d] = TYPE_MINUS
+      (0x30..0x39).each { |n| @char_types[n] = TYPE_DIGIT }
+      (0x41..0x5a).each { |n| @char_types[n] = TYPE_ALPHA }
+      (0x61..0x7a).each { |n| @char_types[n] = TYPE_ALPHA }
+      @char_types[TOKEN_KEY] = TYPE_KEY_START
+      @char_types[TOKEN_STRING] = TYPE_STRING_START
+      @char_types.freeze
+    end
+    @char_types
+  end
+
+  def next_token
+    skip_white
+    s = @pos
+    c = next_cp
+
+    case @char_types[c]
+    when TYPE_END
+      @token_value = nil
+      @token = TOKEN_END
+
+    when TYPE_MINUS
+      if @char_types[peek_cp] == TYPE_DIGIT
+        next_token # consume float or integer
+        @token_value = -@token_value
+      else
+        consume_identifier(s)
+      end
+
+    when TYPE_DIGIT
+      skip_decimal_digits
+      c = peek_cp
+      if c == 0x2e # '.'
+        @pos += 1
+        consume_float(s, c)
+      else
+        @token_value = @text[s..@pos].to_i
+        @token = TOKEN_INT
+      end
+
+    when TYPE_DELIM
+      @token_value = @text[s]
+      @token = c
+
+    when TYPE_KEY_START
+      if @char_types[peek_cp] == TYPE_ALPHA
+        next_token
+        @token = TOKEN_KEY
+      else
+        parse_error(_("expected identifier after ':'"))
+      end
+
+    when TYPE_STRING_START
+      consume_string
+    else
+      consume_identifier(s)
+    end
+  end
+
+  def consume_identifier(s)
+    while @char_types[peek_cp] >= TYPE_IDENTIFIER do
+      @pos += 1
+    end
+    id = @text[s...@pos]
+    case id
+    when LIT_TRUE
+      @token = TOKEN_BOOL
+      @token_value = true
+    when LIT_FALSE
+      @token = TOKEN_BOOL
+      @token_value = false
+    when LIT_NIL
+      @token = TOKEN_NIL
+      @token_value = nil
+    else
+      @token = TOKEN_IDENTIFIER
+      @token_value = id
+    end
+  end
+
+  def consume_string
+    s = @pos
+    b = ''
+    loop do
+      c = next_cp
+      case c
+      when TOKEN_END
+        @pos = s - 1
+        parse_error(_('unterminated quote'))
+      when TOKEN_STRING
+        @token_value = b
+        @token = TOKEN_STRING
+        break
+      when 0x5c # '\'
+        c = next_cp
+        case c
+        when 0x74 # 't'
+          b << "\t"
+        when 0x72 # 'r'
+          b << "\r"
+        when 0x6e # 'n'
+          b << "\n"
+        when TOKEN_STRING
+          b << '"'
+        when 0x5c # '\'
+          b << "\\"
+        when 0x6f # 'o'
+          c = 0
+          3.times do
+            n = next_cp
+            if 0x30 <= n && n <= 0x37c
+              c *= 8
+              c += n - 0x30
+            else
+              parse_error(_('malformed octal quote'))
+            end
+          end
+          b << c
+        else
+          b << "\\"
+          b << c
+        end
+      else
+        b << c
+      end
+    end
+  end
+
+  def consume_float(s, d)
+    parse_error(_('digit expected')) if skip_decimal_digits == 0
+    c = peek_cp
+    if d == 0x2e # '.'
+      if c == 0x45 || c == 0x65 # 'E' or 'e'
+        @pos += 1
+        parse_error(_('digit expected')) if skip_decimal_digits == 0
+        c = peek_cp
+      end
+    end
+    parse_error(_('digit expected')) if @char_types[c] == TYPE_ALPHA
+    @token_value = @text[s...@pos].to_f
+    @token = TOKEN_FLOAT
+  end
+
+  def skip_decimal_digits
+    count = 0
+    c = peek_cp
+    if c == 0x2d || c == 0x2b # '-' or '+'
+      @pos += 1
+      c = peek_cp
+    end
+
+    while @char_types[c] == TYPE_DIGIT do
+      @pos += 1
+      c = peek_cp
+      count += 1
+    end
+    count
+  end
+
+  def skip_white
+    while @char_types[peek_cp] == TYPE_WS do
+      @pos += 1
+    end
+  end
+
+  def next_cp
+    c = 0
+    if @pos < @codepoints.size
+      c = @codepoints[@pos]
+      @pos += 1
+    end
+    c
+  end
+
+  def peek_cp
+    @pos < @codepoints.size ? @codepoints[@pos] : 0
+  end
+end
+end
+end

data/lib/puppet/pops/parser/slurp_support.rb

@@ -45,9 +45,6 @@ module SlurpSupport
    # Copy from old lexer - can do much better
    def slurp_uqstring
      scn = @scanner
-     last = scn.matched
-     ignore = true
-
      str = slurp(scn, @lexing_context[:uq_slurp_pattern], @lexing_context[:escapes], :ignore_invalid_escapes)
 
      # Terminator may be a single char '$', two characters '${', or empty string '' at the end of intput.

data/lib/puppet/pops/patterns.rb

@@ -48,7 +48,8 @@ module Puppet::Pops::Patterns
 
   # VAR_NAME matches the name part of a variable (The $ character is not included)
   # Note, that only the final segment may start with an underscore.
-  VAR_NAME = %r{\A(?:(::)?[a-z]\w*)*(?:(::)?[a-z_]\w*)\z}
+  # Note, regexp sensitive to backtracking
+  VAR_NAME = %r{\A(?:::)?(?:[a-z]\w*::)*[a-z_]\w*\z}
 
   # PARAM_NAME matches the name part of a parameter (The $ character is not included)
   PARAM_NAME = %r{\A[a-z_]\w*\z}