puppet 5.4.0 → 5.5.0

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -34,7 +34,7 @@ module HeredocSupport
       endtag = $1.strip
     end
 
-    lex_error(Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
+    lex_error(Issues::HEREDOC_EMPTY_ENDTAG) unless endtag.length >= 1
 
     resulting_escapes = []
     if escapes

data/lib/puppet/pops/parser/lexer_support.rb

@@ -30,7 +30,7 @@ module LexerSupport
 
   # Raises a Puppet::LexError with the given message
   def lex_error_without_pos(issue, args = {})
-    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code)
+    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code, args)
   end
 
   # Raises a Puppet::ParserErrorWithIssue with the given issue and arguments
@@ -73,7 +73,8 @@ module LexerSupport
         line(pos),
         position(pos),
         nil,
-        issue.issue_code)
+        issue.issue_code,
+        args)
   end
 
   # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.

data/lib/puppet/pops/parser/locator.rb

@@ -155,8 +155,6 @@ class Locator
     end
   end
 
-  private
-
   class AbstractLocator < Locator
     attr_accessor :line_index
     attr_accessor :string

data/lib/puppet/pops/parser/pn_parser.rb

@@ -0,0 +1,316 @@
+module Puppet::Pops
+module Parser
+
+class PNParser
+  LIT_TRUE = 'true'.freeze
+  LIT_FALSE = 'false'.freeze
+  LIT_NIL = 'nil'.freeze
+
+  TOKEN_END = 0
+  TOKEN_BOOL = 1
+  TOKEN_NIL = 2
+  TOKEN_INT = 3
+  TOKEN_FLOAT = 4
+  TOKEN_IDENTIFIER = 5
+  TOKEN_WS = 0x20
+  TOKEN_STRING = 0x22
+  TOKEN_KEY = 0x3a
+  TOKEN_LP = 0x28
+  TOKEN_RP = 0x29
+  TOKEN_LB = 0x5b
+  TOKEN_RB = 0x5d
+  TOKEN_LC = 0x7b
+  TOKEN_RC = 0x7d
+
+  TYPE_END = 0
+  TYPE_WS = 1
+  TYPE_DELIM = 2
+  TYPE_KEY_START = 3
+  TYPE_STRING_START = 4
+  TYPE_IDENTIFIER = 5
+  TYPE_MINUS = 6
+  TYPE_DIGIT = 7
+  TYPE_ALPHA = 8
+
+  def initialize
+    @char_types = self.class.char_types
+  end
+
+  def parse(text, locator = nil, offset = nil)
+    @locator = locator
+    @offset = offset
+    @text = text
+    @codepoints = text.codepoints.to_a.freeze
+    @pos = 0
+    @token = TOKEN_END
+    @token_value = nil
+    next_token
+    parse_next
+  end
+
+  private
+
+  def parse_next
+    case @token
+    when TOKEN_LB
+      parse_array
+    when TOKEN_LC
+      parse_map
+    when TOKEN_LP
+      parse_call
+    when TOKEN_BOOL, TOKEN_INT, TOKEN_FLOAT, TOKEN_STRING, TOKEN_NIL
+      parse_literal
+    when TOKEN_END
+      parse_error(_('unexpected end of input'))
+    else
+      parse_error(_('unexpected %{value}' % { value: @token_value }))
+    end
+  end
+
+  def parse_error(message)
+    file = ''
+    line = 1
+    pos = 1
+    if @locator
+      file = @locator.file
+      line = @locator.line_for_offset(@offset)
+      pos  = @locator.pos_on_line(@offset)
+    end
+    @codepoints[0, @pos].each do |c|
+      if c == 0x09
+        line += 1
+        pos = 1
+      end
+    end
+    raise Puppet::ParseError.new(message, file, line, pos)
+  end
+
+  def parse_literal
+    pn = PN::Literal.new(@token_value)
+    next_token
+    pn
+  end
+
+  def parse_array
+    next_token
+    PN::List.new(parse_elements(TOKEN_RB))
+  end
+
+  def parse_map
+    next_token
+    entries = []
+    while @token != TOKEN_RC && @token != TOKEN_END
+      parse_error(_('map key expected')) unless @token == TOKEN_KEY
+      key = @token_value
+      next_token
+      entries << parse_next.with_name(key)
+    end
+    next_token
+    PN::Map.new(entries)
+  end
+
+  def parse_call
+    next_token
+    parse_error(_("expected identifier to follow '('")) unless @token == TOKEN_IDENTIFIER
+    name = @token_value
+    next_token
+    PN::Call.new(name, *parse_elements(TOKEN_RP))
+  end
+
+  def parse_elements(end_token)
+    elements = []
+    while @token != end_token && @token != TOKEN_END
+      elements << parse_next
+    end
+    parse_error(_("missing '%{token}' to end list") % { token: end_token.chr(Encoding::UTF_8) } ) unless @token == end_token
+    next_token
+    elements
+  end
+
+  # All methods below belong to the PN lexer
+  def self.char_types
+    unless instance_variable_defined?(:@char_types)
+      @char_types = Array.new(0x80, TYPE_IDENTIFIER)
+      @char_types[0] = TYPE_END
+      [0x09, 0x0d, 0x0a, 0x20].each { |n| @char_types[n] = TYPE_WS }
+      [TOKEN_LP, TOKEN_RP, TOKEN_LB, TOKEN_RB, TOKEN_LC, TOKEN_RC].each { |n| @char_types[n] = TYPE_DELIM }
+      @char_types[0x2d] = TYPE_MINUS
+      (0x30..0x39).each { |n| @char_types[n] = TYPE_DIGIT }
+      (0x41..0x5a).each { |n| @char_types[n] = TYPE_ALPHA }
+      (0x61..0x7a).each { |n| @char_types[n] = TYPE_ALPHA }
+      @char_types[TOKEN_KEY] = TYPE_KEY_START
+      @char_types[TOKEN_STRING] = TYPE_STRING_START
+      @char_types.freeze
+    end
+    @char_types
+  end
+
+  def next_token
+    skip_white
+    s = @pos
+    c = next_cp
+
+    case @char_types[c]
+    when TYPE_END
+      @token_value = nil
+      @token = TOKEN_END
+
+    when TYPE_MINUS
+      if @char_types[peek_cp] == TYPE_DIGIT
+        next_token # consume float or integer
+        @token_value = -@token_value
+      else
+        consume_identifier(s)
+      end
+
+    when TYPE_DIGIT
+      skip_decimal_digits
+      c = peek_cp
+      if c == 0x2e # '.'
+        @pos += 1
+        consume_float(s, c)
+      else
+        @token_value = @text[s..@pos].to_i
+        @token = TOKEN_INT
+      end
+
+    when TYPE_DELIM
+      @token_value = @text[s]
+      @token = c
+
+    when TYPE_KEY_START
+      if @char_types[peek_cp] == TYPE_ALPHA
+        next_token
+        @token = TOKEN_KEY
+      else
+        parse_error(_("expected identifier after ':'"))
+      end
+
+    when TYPE_STRING_START
+      consume_string
+    else
+      consume_identifier(s)
+    end
+  end
+
+  def consume_identifier(s)
+    while @char_types[peek_cp] >= TYPE_IDENTIFIER do
+      @pos += 1
+    end
+    id = @text[s...@pos]
+    case id
+    when LIT_TRUE
+      @token = TOKEN_BOOL
+      @token_value = true
+    when LIT_FALSE
+      @token = TOKEN_BOOL
+      @token_value = false
+    when LIT_NIL
+      @token = TOKEN_NIL
+      @token_value = nil
+    else
+      @token = TOKEN_IDENTIFIER
+      @token_value = id
+    end
+  end
+
+  def consume_string
+    s = @pos
+    b = ''
+    loop do
+      c = next_cp
+      case c
+      when TOKEN_END
+        @pos = s - 1
+        parse_error(_('unterminated quote'))
+      when TOKEN_STRING
+        @token_value = b
+        @token = TOKEN_STRING
+        break
+      when 0x5c # '\'
+        c = next_cp
+        case c
+        when 0x74 # 't'
+          b << "\t"
+        when 0x72 # 'r'
+          b << "\r"
+        when 0x6e # 'n'
+          b << "\n"
+        when TOKEN_STRING
+          b << '"'
+        when 0x5c # '\'
+          b << "\\"
+        when 0x6f # 'o'
+          c = 0
+          3.times do
+            n = next_cp
+            if 0x30 <= n && n <= 0x37c
+              c *= 8
+              c += n - 0x30
+            else
+              parse_error(_('malformed octal quote'))
+            end
+          end
+          b << c
+        else
+          b << "\\"
+          b << c
+        end
+      else
+        b << c
+      end
+    end
+  end
+
+  def consume_float(s, d)
+    parse_error(_('digit expected')) if skip_decimal_digits == 0
+    c = peek_cp
+    if d == 0x2e # '.'
+      if c == 0x45 || c == 0x65 # 'E' or 'e'
+        @pos += 1
+        parse_error(_('digit expected')) if skip_decimal_digits == 0
+        c = peek_cp
+      end
+    end
+    parse_error(_('digit expected')) if @char_types[c] == TYPE_ALPHA
+    @token_value = @text[s...@pos].to_f
+    @token = TOKEN_FLOAT
+  end
+
+  def skip_decimal_digits
+    count = 0
+    c = peek_cp
+    if c == 0x2d || c == 0x2b # '-' or '+'
+      @pos += 1
+      c = peek_cp
+    end
+
+    while @char_types[c] == TYPE_DIGIT do
+      @pos += 1
+      c = peek_cp
+      count += 1
+    end
+    count
+  end
+
+  def skip_white
+    while @char_types[peek_cp] == TYPE_WS do
+      @pos += 1
+    end
+  end
+
+  def next_cp
+    c = 0
+    if @pos < @codepoints.size
+      c = @codepoints[@pos]
+      @pos += 1
+    end
+    c
+  end
+
+  def peek_cp
+    @pos < @codepoints.size ? @codepoints[@pos] : 0
+  end
+end
+end
+end

data/lib/puppet/pops/pcore.rb

@@ -31,14 +31,7 @@ module Pcore
     instance
   end
 
-  def self.init(loader, ir, for_agent)
-    add_alias('Pcore::URI_RX', TYPE_URI_RX, loader)
-    add_type(TYPE_URI_ALIAS, loader)
-    add_alias('Pcore::SimpleTypeName', TYPE_SIMPLE_TYPE_NAME, loader)
-    add_alias('Pcore::MemberName', TYPE_MEMBER_NAME, loader)
-    add_alias('Pcore::TypeName', TYPE_QUALIFIED_REFERENCE, loader)
-    add_alias('Pcore::QRef', TYPE_QUALIFIED_REFERENCE, loader)
-
+  def self.init_env(loader)
     if Puppet[:tasks]
       add_object_type('Task', <<-PUPPET, loader)
         {
@@ -83,18 +76,25 @@ module Pcore
         }
       PUPPET
     end
+  end
+
+  def self.init(loader, ir)
+    add_alias('Pcore::URI_RX', TYPE_URI_RX, loader)
+    add_type(TYPE_URI_ALIAS, loader)
+    add_alias('Pcore::SimpleTypeName', TYPE_SIMPLE_TYPE_NAME, loader)
+    add_alias('Pcore::MemberName', TYPE_MEMBER_NAME, loader)
+    add_alias('Pcore::TypeName', TYPE_QUALIFIED_REFERENCE, loader)
+    add_alias('Pcore::QRef', TYPE_QUALIFIED_REFERENCE, loader)
     Types::TypedModelObject.register_ptypes(loader, ir)
 
     @type = create_object_type(loader, ir, Pcore, 'Pcore', nil)
 
-    ir.register_implementation_namespace('Pcore', 'Puppet::Pops::Pcore', loader)
-    ir.register_implementation_namespace('Puppet::AST', 'Puppet::Pops::Model', loader)
-    ir.register_implementation('Puppet::AST::Locator', 'Puppet::Pops::Parser::Locator::Locator19', loader)
-    unless for_agent
-      Resource.register_ptypes(loader, ir)
-      Lookup::Context.register_ptype(loader, ir);
-      Lookup::DataProvider.register_types(loader)
-    end
+    ir.register_implementation_namespace('Pcore', 'Puppet::Pops::Pcore')
+    ir.register_implementation_namespace('Puppet::AST', 'Puppet::Pops::Model')
+    ir.register_implementation('Puppet::AST::Locator', 'Puppet::Pops::Parser::Locator::Locator19')
+    Resource.register_ptypes(loader, ir)
+    Lookup::Context.register_ptype(loader, ir);
+    Lookup::DataProvider.register_types(loader)
   end
 
   # Create and register a new `Object` type in the Puppet Type System and map it to an implementation class
@@ -116,7 +116,7 @@ module Pcore
     init_hash[Types::KEY_ATTRIBUTES] = attributes_hash unless attributes_hash.empty?
     init_hash[Types::KEY_FUNCTIONS] = functions_hash unless functions_hash.empty?
     init_hash[Types::KEY_EQUALITY] = equality unless equality.nil?
-    ir.register_implementation(type_name, impl_class, loader)
+    ir.register_implementation(type_name, impl_class)
     add_type(Types::PObjectType.new(type_name, init_hash), loader)
   end
 

data/lib/puppet/pops/pn.rb

@@ -0,0 +1,236 @@
+module Puppet::Pops
+module PN
+  KEY_PATTERN = /^[A-Za-z_-][0-9A-Za-z_-]*$/
+
+  def pnError(message)
+    raise ArgumentError, message
+  end
+
+  def as_call(name)
+    Call.new(name, self)
+  end
+
+  def as_parameters
+    [self]
+  end
+
+  def ==(o)
+    eql?(o)
+  end
+
+  def to_s
+    s = ''
+    format(nil, s)
+    s
+  end
+
+  def with_name(name)
+    Entry.new(name, self)
+  end
+
+  def double_quote(str, bld)
+    bld << '"'
+    str.each_codepoint do |codepoint|
+      case codepoint
+      when 0x09
+        bld << '\\t'
+      when 0x0a
+        bld << '\\n'
+      when 0x0d
+        bld << '\\r'
+      when 0x22
+        bld << '\\"'
+      when 0x5c
+        bld << '\\\\'
+      else
+        if codepoint < 0x20
+          bld << sprintf('\\o%3.3o', codepoint)
+        elsif codepoint <= 0x7f
+          bld << codepoint
+        else
+          bld << [codepoint].pack('U')
+        end
+      end
+    end
+    bld << '"'
+  end
+
+  def format_elements(elements, indent, b)
+    elements.each_with_index do |e, i|
+      if indent
+        b << "\n" << indent.current
+      elsif i > 0
+        b << ' '
+      end
+      e.format(indent, b)
+    end
+  end
+
+  class Indent
+    attr_reader :current
+    def initialize(indent = '  ', current = '')
+      @indent = indent
+      @current = current
+    end
+
+    def increase
+      Indent.new(@indent, @current + @indent)
+    end
+  end
+
+  class Call
+    include PN
+    attr_reader :name, :elements
+
+    def initialize(name, *elements)
+      @name = name
+      @elements = elements
+    end
+
+    def [](idx)
+      @elements[idx]
+    end
+
+    def as_call(name)
+      Call.new(name, *@elements)
+    end
+
+    def as_parameters
+      List.new(@elements)
+    end
+
+    def eql?(o)
+      o.is_a?(Call) && @name == o.name && @elements == o.elements
+    end
+
+    def format(indent, b)
+      b << '(' << @name
+      if @elements.size > 0
+        b << ' ' unless indent
+        format_elements(@elements, indent ? indent.increase : nil, b)
+      end
+      b << ')'
+    end
+
+    def to_data
+      { '^' => [@name] + @elements.map { |e| e.to_data } }
+    end
+  end
+
+  class Entry
+    attr_reader :key, :value
+    def initialize(key, value)
+      @key = key
+      @value = value
+    end
+
+    def eql?(o)
+      o.is_a?(Entry) && @key == o.key && @value == o.value
+    end
+
+    alias == eql?
+  end
+
+  class List
+    include PN
+    attr_reader :elements
+
+    def initialize(elements)
+      @elements = elements
+    end
+
+    def [](idx)
+      @elements[idx]
+    end
+
+    def as_call(name)
+      Call.new(name, *@elements)
+    end
+
+    def as_parameters
+      @elements
+    end
+
+    def eql?(o)
+      o.is_a?(List) && @elements == o.elements
+    end
+
+    def format(indent, b)
+      b << '['
+      format_elements(@elements, indent ? indent.increase : nil, b) unless @elements.empty?
+      b << ']'
+    end
+
+    def to_data
+      @elements.map { |e| e.to_data }
+    end
+  end
+
+  class Literal
+    include PN
+    attr_reader :value
+
+    def initialize(value)
+      @value = value
+    end
+
+    def format(indent, b)
+      if @value.nil?
+        b << 'nil'
+      elsif value.is_a?(String)
+        double_quote(value, b)
+      else
+        b << value.to_s
+      end
+    end
+
+    def eql?(o)
+      o.is_a?(Literal) && @value == o.value
+    end
+
+    def to_data
+      @value
+    end
+  end
+
+  class Map
+    include PN
+    attr_reader :entries
+
+    def initialize(entries)
+      entries.each { |e| pnError("key #{e.key} does not conform to pattern /#{KEY_PATTERN.source}/)") unless e.key =~ KEY_PATTERN }
+      @entries = entries
+    end
+
+    def eql?(o)
+      o.is_a?(Map) && @entries == o.entries
+    end
+
+    def format(indent, b)
+      local_indent = indent ? indent.increase : nil
+      b << '{'
+      @entries.each_with_index do |e,i|
+        if indent
+          b << "\n" << local_indent.current
+        elsif i > 0
+          b << ' '
+        end
+        b << ':' << e.key
+        b << ' '
+        e.value.format(local_indent, b)
+      end
+      b << '}'
+    end
+
+    def to_data
+      r = []
+      @entries.each { |e| r << e.key << e.value.to_data }
+      { '#' => r }
+    end
+  end
+end
+end
+
+require_relative 'model/pn_transformer'
+require_relative 'parser/pn_parser'
+