openvox 7.37.1

data/lib/puppet/pops/parser/parser_support.rb

@@ -0,0 +1,251 @@
+require_relative '../../../puppet/parser/functions'
+require_relative '../../../puppet/parser/files'
+require_relative '../../../puppet/resource/type_collection'
+require_relative '../../../puppet/resource/type'
+require 'monitor'
+
+module Puppet::Pops
+module Parser
+# Supporting logic for the parser.
+# This supporting logic has slightly different responsibilities compared to the original Puppet::Parser::Parser.
+# It is only concerned with parsing.
+#
+class Parser
+  # Note that the name of the contained class and the file name (currently parser_support.rb)
+  # needs to be different as the class is generated by Racc, and this file (parser_support.rb) is included as a mix in
+  #
+
+  # Simplify access to the Model factory
+  # Note that the parser/parser support does not have direct knowledge about the Model.
+  # All model construction/manipulation is made by the Factory.
+  #
+  Factory = Model::Factory
+
+  attr_accessor :lexer
+  attr_reader :definitions
+
+  # Returns the token text of the given lexer token, or nil, if token is nil
+  def token_text t
+    return t if t.nil?
+    if t.is_a?(Factory) && t.model_class <= Model::QualifiedName
+      t['value']
+    elsif t.is_a?(Model::QualifiedName)
+      t.value
+    else
+      # else it is a lexer token
+      t[:value]
+    end
+  end
+
+  # Produces the fully qualified name, with the full (current) namespace for a given name.
+  #
+  # This is needed because class bodies are lazily evaluated and an inner class' container(s) may not
+  # have been evaluated before some external reference is made to the inner class; its must therefore know its complete name
+  # before evaluation-time.
+  #
+  def classname(name)
+    [namespace, name].join('::').sub(/^::/, '')
+  end
+
+  # Raises a Parse error with location information. Information about file is always obtained from the
+  # lexer. Line and position is produced if the given semantic is a Positioned object and have been given an offset.
+  #
+  def error(semantic, message)
+    except = Puppet::ParseError.new(message)
+    if semantic.is_a?(LexerSupport::TokenValue)
+      except.file = semantic[:file];
+      except.line = semantic[:line];
+      except.pos = semantic[:pos];
+    else
+      locator = @lexer.locator
+      except.file = locator.file
+      if semantic.is_a?(Factory)
+        offset = semantic['offset']
+        unless offset.nil?
+          except.line = locator.line_for_offset(offset)
+          except.pos = locator.pos_on_line(offset)
+        end
+      end
+    end
+    raise except
+  end
+
+  # Parses a file expected to contain pp DSL logic.
+  def parse_file(file)
+    unless Puppet::FileSystem.exist?(file)
+      unless file =~ /\.pp$/
+        file = file + ".pp"
+      end
+    end
+    @lexer.file = file
+    _parse
+  end
+
+  def initialize()
+    @lexer = Lexer2.new
+    @namestack = []
+    @definitions = []
+  end
+
+  # This is a callback from the generated parser (when an error occurs while parsing)
+  #
+  def on_error(token,value,stack)
+    if token == 0 # denotes end of file
+      value_at = 'end of input'
+    else
+      value_at = "'#{value[:value]}'"
+    end
+    error = Issues::SYNTAX_ERROR.format(:where => value_at)
+    error = "#{error}, token: #{token}" if @yydebug
+
+    # Note, old parser had processing of "expected token here" - do not try to reinstate:
+    # The 'expected' is only of value at end of input, otherwise any parse error involving a
+    # start of a pair will be reported as expecting the close of the pair - e.g. "$x.each |$x {|", would
+    # report that "seeing the '{', the '}' is expected. That would be wrong.
+    # Real "expected" tokens are very difficult to compute (would require parsing of racc output data). Output of the stack
+    # could help, but can require extensive backtracking and produce many options.
+    #
+    # The lexer should handle the "expected instead of end of file for strings, and interpolation", other expectancies
+    # must be handled by the grammar. The lexer may have enqueued tokens far ahead - the lexer's opinion about this
+    # is not trustworthy.
+    #
+    file = nil
+    line = nil
+    pos  = nil
+    if token != 0
+      file = value[:file]
+      locator = value.locator
+      if locator.is_a?(Puppet::Pops::Parser::Locator::SubLocator)
+        # The error occurs when doing sub-parsing and the token must be transformed
+        # Transpose the local offset, length to global "coordinates"
+        global_offset, _ = locator.to_global(value.offset, value.length)
+        line = locator.locator.line_for_offset(global_offset)
+        pos = locator.locator.pos_on_line(global_offset)
+      else
+        line = value[:line]
+        pos  = value[:pos]
+      end
+    else
+      # At end of input, use what the lexer thinks is the source file
+      file = lexer.file
+    end
+    file = nil unless file.is_a?(String) && !file.empty?
+    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, Issues::SYNTAX_ERROR.issue_code)
+  end
+
+  # Parses a String of pp DSL code.
+  #
+  def parse_string(code, path = nil)
+    @lexer.lex_string(code, path)
+    _parse()
+  end
+
+  # Mark the factory wrapped model object with location information
+  # @return [Factory] the given factory
+  # @api private
+  #
+  def loc(factory, start_locatable, end_locatable = nil)
+    factory.record_position(@lexer.locator, start_locatable, end_locatable)
+  end
+
+  # Mark the factory wrapped heredoc model object with location information
+  # @return [Factory] the given factory
+  # @api private
+  #
+  def heredoc_loc(factory, start_locatable, end_locatable = nil)
+    factory.record_heredoc_position(start_locatable, end_locatable)
+  end
+
+  def aryfy(o)
+    o = [o] unless o.is_a?(Array)
+    o
+  end
+
+  def namespace
+    @namestack.join('::')
+  end
+
+  def namestack(name)
+    @namestack << name
+  end
+
+  def namepop()
+    @namestack.pop
+  end
+
+  def add_definition(definition)
+    @definitions << definition.model
+    definition
+  end
+
+  # Transforms an array of expressions containing literal name expressions to calls if followed by an
+  # expression, or expression list
+  #
+  def transform_calls(expressions)
+    # Factory transform raises an error if a non qualified name is followed by an argument list
+    # since there is no way that that can be transformed back to sanity. This occurs in situations like this:
+    #
+    #  $a = 10, notice hello
+    #
+    # where the "10, notice" forms an argument list. The parser builds an Array with the expressions and includes
+    # the comma tokens to enable the error to be reported against the first comma.
+    #
+    begin
+      Factory.transform_calls(expressions)
+    rescue Factory::ArgsToNonCallError => e
+      # e.args[1] is the first comma token in the list
+      # e.name_expr is the function name expression
+      if e.name_expr.is_a?(Factory) && e.name_expr.model_class <= Model::QualifiedName
+        error(e.args[1], _("attempt to pass argument list to the function '%{name}' which cannot be called without parentheses") % { name: e.name_expr['value'] })
+      else
+        error(e.args[1], _("illegal comma separated argument list"))
+      end
+    end
+  end
+
+  # Transforms a LEFT followed by the result of attribute_operations, this may be a call or an invalid sequence
+  def transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
+    Factory.transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
+  end
+
+  # Creates a program with the given body.
+  #
+  def create_program(body)
+    locator = @lexer.locator
+    Factory.PROGRAM(body, definitions, locator)
+  end
+
+  # Creates an empty program with a single No-op at the input's EOF offset with 0 length.
+  #
+  def create_empty_program()
+    locator = @lexer.locator
+    no_op = Factory.literal(nil)
+    # Create a synthetic NOOP token at EOF offset with 0 size. The lexer does not produce an EOF token that is
+    # visible to the grammar rules. Creating this token is mainly to reuse the positioning logic as it
+    # expects a token decorated with location information.
+    _, token = @lexer.emit_completed([:NOOP,'',0], locator.string.bytesize)
+    loc(no_op, token)
+    # Program with a Noop
+    program = Factory.PROGRAM(no_op, [], locator)
+    program
+  end
+
+  # Performs the parsing and returns the resulting model.
+  # The lexer holds state, and this is setup with {#parse_string}, or {#parse_file}.
+  #
+  # @api private
+  #
+  def _parse()
+    begin
+      @yydebug = false
+      main = yyparse(@lexer,:scan)
+    end
+    return main
+  ensure
+    @lexer.clear
+    @namestack = []
+    @definitions = []
+  end
+end
+end
+end

data/lib/puppet/pops/parser/pn_parser.rb

@@ -0,0 +1,317 @@
+module Puppet::Pops
+module Parser
+
+class PNParser
+  LIT_TRUE = 'true'.freeze
+  LIT_FALSE = 'false'.freeze
+  LIT_NIL = 'nil'.freeze
+
+  TOKEN_END = 0
+  TOKEN_BOOL = 1
+  TOKEN_NIL = 2
+  TOKEN_INT = 3
+  TOKEN_FLOAT = 4
+  TOKEN_IDENTIFIER = 5
+  TOKEN_WS = 0x20
+  TOKEN_STRING = 0x22
+  TOKEN_KEY = 0x3a
+  TOKEN_LP = 0x28
+  TOKEN_RP = 0x29
+  TOKEN_LB = 0x5b
+  TOKEN_RB = 0x5d
+  TOKEN_LC = 0x7b
+  TOKEN_RC = 0x7d
+
+  TYPE_END = 0
+  TYPE_WS = 1
+  TYPE_DELIM = 2
+  TYPE_KEY_START = 3
+  TYPE_STRING_START = 4
+  TYPE_IDENTIFIER = 5
+  TYPE_MINUS = 6
+  TYPE_DIGIT = 7
+  TYPE_ALPHA = 8
+
+  def initialize
+    @char_types = self.class.char_types
+  end
+
+  def parse(text, locator = nil, offset = nil)
+    @locator = locator
+    @offset = offset
+    @text = text
+    @codepoints = text.codepoints.to_a.freeze
+    @pos = 0
+    @token = TOKEN_END
+    @token_value = nil
+    next_token
+    parse_next
+  end
+
+  def self.char_types
+    unless instance_variable_defined?(:@char_types)
+      @char_types = Array.new(0x80, TYPE_IDENTIFIER)
+      @char_types[0] = TYPE_END
+      [0x09, 0x0d, 0x0a, 0x20].each { |n| @char_types[n] = TYPE_WS }
+      [TOKEN_LP, TOKEN_RP, TOKEN_LB, TOKEN_RB, TOKEN_LC, TOKEN_RC].each { |n| @char_types[n] = TYPE_DELIM }
+      @char_types[0x2d] = TYPE_MINUS
+      (0x30..0x39).each { |n| @char_types[n] = TYPE_DIGIT }
+      (0x41..0x5a).each { |n| @char_types[n] = TYPE_ALPHA }
+      (0x61..0x7a).each { |n| @char_types[n] = TYPE_ALPHA }
+      @char_types[TOKEN_KEY] = TYPE_KEY_START
+      @char_types[TOKEN_STRING] = TYPE_STRING_START
+      @char_types.freeze
+    end
+    @char_types
+  end
+
+  private
+
+  def parse_next
+    case @token
+    when TOKEN_LB
+      parse_array
+    when TOKEN_LC
+      parse_map
+    when TOKEN_LP
+      parse_call
+    when TOKEN_BOOL, TOKEN_INT, TOKEN_FLOAT, TOKEN_STRING, TOKEN_NIL
+      parse_literal
+    when TOKEN_END
+      parse_error(_('unexpected end of input'))
+    else
+      parse_error(_('unexpected %{value}' % { value: @token_value }))
+    end
+  end
+
+  def parse_error(message)
+    file = ''
+    line = 1
+    pos = 1
+    if @locator
+      file = @locator.file
+      line = @locator.line_for_offset(@offset)
+      pos  = @locator.pos_on_line(@offset)
+    end
+    @codepoints[0, @pos].each do |c|
+      if c == 0x09
+        line += 1
+        pos = 1
+      end
+    end
+    raise Puppet::ParseError.new(message, file, line, pos)
+  end
+
+  def parse_literal
+    pn = PN::Literal.new(@token_value)
+    next_token
+    pn
+  end
+
+  def parse_array
+    next_token
+    PN::List.new(parse_elements(TOKEN_RB))
+  end
+
+  def parse_map
+    next_token
+    entries = []
+    while @token != TOKEN_RC && @token != TOKEN_END
+      parse_error(_('map key expected')) unless @token == TOKEN_KEY
+      key = @token_value
+      next_token
+      entries << parse_next.with_name(key)
+    end
+    next_token
+    PN::Map.new(entries)
+  end
+
+  def parse_call
+    next_token
+    parse_error(_("expected identifier to follow '('")) unless @token == TOKEN_IDENTIFIER
+    name = @token_value
+    next_token
+    PN::Call.new(name, *parse_elements(TOKEN_RP))
+  end
+
+  def parse_elements(end_token)
+    elements = []
+    while @token != end_token && @token != TOKEN_END
+      elements << parse_next
+    end
+    parse_error(_("missing '%{token}' to end list") % { token: end_token.chr(Encoding::UTF_8) } ) unless @token == end_token
+    next_token
+    elements
+  end
+
+  # All methods below belong to the PN lexer
+
+  def next_token
+    skip_white
+    s = @pos
+    c = next_cp
+
+    case @char_types[c]
+    when TYPE_END
+      @token_value = nil
+      @token = TOKEN_END
+
+    when TYPE_MINUS
+      if @char_types[peek_cp] == TYPE_DIGIT
+        next_token # consume float or integer
+        @token_value = -@token_value
+      else
+        consume_identifier(s)
+      end
+
+    when TYPE_DIGIT
+      skip_decimal_digits
+      c = peek_cp
+      if c == 0x2e # '.'
+        @pos += 1
+        consume_float(s, c)
+      else
+        @token_value = @text[s..@pos].to_i
+        @token = TOKEN_INT
+      end
+
+    when TYPE_DELIM
+      @token_value = @text[s]
+      @token = c
+
+    when TYPE_KEY_START
+      if @char_types[peek_cp] == TYPE_ALPHA
+        next_token
+        @token = TOKEN_KEY
+      else
+        parse_error(_("expected identifier after ':'"))
+      end
+
+    when TYPE_STRING_START
+      consume_string
+    else
+      consume_identifier(s)
+    end
+  end
+
+  def consume_identifier(s)
+    while @char_types[peek_cp] >= TYPE_IDENTIFIER do
+      @pos += 1
+    end
+    id = @text[s...@pos]
+    case id
+    when LIT_TRUE
+      @token = TOKEN_BOOL
+      @token_value = true
+    when LIT_FALSE
+      @token = TOKEN_BOOL
+      @token_value = false
+    when LIT_NIL
+      @token = TOKEN_NIL
+      @token_value = nil
+    else
+      @token = TOKEN_IDENTIFIER
+      @token_value = id
+    end
+  end
+
+  def consume_string
+    s = @pos
+    b = ''
+    loop do
+      c = next_cp
+      case c
+      when TOKEN_END
+        @pos = s - 1
+        parse_error(_('unterminated quote'))
+      when TOKEN_STRING
+        @token_value = b
+        @token = TOKEN_STRING
+        break
+      when 0x5c # '\'
+        c = next_cp
+        case c
+        when 0x74 # 't'
+          b << "\t"
+        when 0x72 # 'r'
+          b << "\r"
+        when 0x6e # 'n'
+          b << "\n"
+        when TOKEN_STRING
+          b << '"'
+        when 0x5c # '\'
+          b << "\\"
+        when 0x6f # 'o'
+          c = 0
+          3.times do
+            n = next_cp
+            if 0x30 <= n && n <= 0x37c
+              c *= 8
+              c += n - 0x30
+            else
+              parse_error(_('malformed octal quote'))
+            end
+          end
+          b << c
+        else
+          b << "\\"
+          b << c
+        end
+      else
+        b << c
+      end
+    end
+  end
+
+  def consume_float(s, d)
+    parse_error(_('digit expected')) if skip_decimal_digits == 0
+    c = peek_cp
+    if d == 0x2e # '.'
+      if c == 0x45 || c == 0x65 # 'E' or 'e'
+        @pos += 1
+        parse_error(_('digit expected')) if skip_decimal_digits == 0
+        c = peek_cp
+      end
+    end
+    parse_error(_('digit expected')) if @char_types[c] == TYPE_ALPHA
+    @token_value = @text[s...@pos].to_f
+    @token = TOKEN_FLOAT
+  end
+
+  def skip_decimal_digits
+    count = 0
+    c = peek_cp
+    if c == 0x2d || c == 0x2b # '-' or '+'
+      @pos += 1
+      c = peek_cp
+    end
+
+    while @char_types[c] == TYPE_DIGIT do
+      @pos += 1
+      c = peek_cp
+      count += 1
+    end
+    count
+  end
+
+  def skip_white
+    while @char_types[peek_cp] == TYPE_WS do
+      @pos += 1
+    end
+  end
+
+  def next_cp
+    c = 0
+    if @pos < @codepoints.size
+      c = @codepoints[@pos]
+      @pos += 1
+    end
+    c
+  end
+
+  def peek_cp
+    @pos < @codepoints.size ? @codepoints[@pos] : 0
+  end
+end
+end
+end

data/lib/puppet/pops/parser/slurp_support.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+module Puppet::Pops
+module Parser
+# This module is an integral part of the Lexer.
+# It defines the string slurping behavior - finding the string and non string parts in interpolated
+# strings, translating escape sequences in strings to their single character equivalence.
+#
+# PERFORMANCE NOTE: The various kinds of slurping could be made even more generic, but requires
+# additional parameter passing and evaluation of conditional logic.
+# TODO: More detailed performance analysis of excessive character escaping and interpolation.
+#
+module SlurpSupport
+  include LexerSupport
+
+  SLURP_SQ_PATTERN  = /(?:[^\\]|^)(?:[\\]{2})*[']/
+  SLURP_DQ_PATTERN  = /(?:[^\\]|^)(?:[\\]{2})*(["]|[$]\{?)/
+  SLURP_UQ_PATTERN  = /(?:[^\\]|^)(?:[\\]{2})*([$]\{?|\z)/
+  # unquoted, no escapes
+  SLURP_UQNE_PATTERN  = /(\$\{?|\z)/m
+  SLURP_ALL_PATTERN = /.*(\z)/m
+  SQ_ESCAPES = %w{ \\ ' }
+  DQ_ESCAPES = %w{ \\  $ ' " r n t s u}+["\r\n", "\n"]
+  UQ_ESCAPES = %w{ \\  $ r n t s u}+["\r\n", "\n"]
+
+  def slurp_sqstring
+    # skip the leading '
+    @scanner.pos += 1
+    str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes)
+    lex_error(Issues::UNCLOSED_QUOTE, :after => "\"'\"", :followed_by => followed_by) unless str
+    str[0..-2] # strip closing "'" from result
+  end
+
+  def slurp_dqstring
+    scn = @scanner
+    last = scn.matched
+    str = slurp(scn, SLURP_DQ_PATTERN, DQ_ESCAPES, false)
+    unless str
+      lex_error(Issues::UNCLOSED_QUOTE, :after => format_quote(last), :followed_by => followed_by)
+    end
+
+    # Terminator may be a single char '"', '$', or two characters '${' group match 1 (scn[1]) from the last slurp holds this
+    terminator = scn[1]
+    [str[0..(-1 - terminator.length)], terminator]
+  end
+
+   # Copy from old lexer - can do much better
+   def slurp_uqstring
+     scn = @scanner
+     str = slurp(scn, @lexing_context[:uq_slurp_pattern], @lexing_context[:escapes], :ignore_invalid_escapes)
+
+     # Terminator may be a single char '$', two characters '${', or empty string '' at the end of intput.
+     # Group match 1 holds this.
+     # The exceptional case is found by looking at the subgroup 1 of the most recent match made by the scanner (i.e. @scanner[1]).
+     # This is the last match made by the slurp method (having called scan_until on the scanner).
+     # If there is a terminating character is must be stripped and returned separately.
+     #
+     terminator = scn[1]
+     [str[0..(-1 - terminator.length)], terminator]
+   end
+
+  # Slurps a string from the given scanner until the given pattern and then replaces any escaped
+  # characters given by escapes into their control-character equivalent or in case of line breaks, replaces the
+  # pattern \r?\n with an empty string.
+  # The returned string contains the terminating character. Returns nil if the scanner can not scan until the given
+  # pattern.
+  #
+  def slurp(scanner, pattern, escapes, ignore_invalid_escapes)
+    str = scanner.scan_until(pattern) || return
+
+    return str unless str.include?('\\')
+
+    return str.gsub!(/\\(\\|')/m, '\1') || str if escapes.equal?(SQ_ESCAPES)
+
+    # Process unicode escapes first as they require getting 4 hex digits
+    # If later a \u is found it is warned not to be a unicode escape
+    if escapes.include?('u')
+      # gsub must be repeated to cater for adjacent escapes
+      while(str.gsub!(/((?:[^\\]|^)(?:[\\]{2})*)\\u(?:([\da-fA-F]{4})|\{([\da-fA-F]{1,6})\})/m) { $1 + [($2 || $3).hex].pack("U") })
+        # empty block. Everything happens in the gsub block
+      end
+    end
+
+    begin
+    str.gsub!(/\\([^\r\n]|(?:\r?\n))/m) {
+      ch = $1
+      if escapes.include? ch
+        case ch
+        when 'r'   ; "\r"
+        when 'n'   ; "\n"
+        when 't'   ; "\t"
+        when 's'   ; ' '
+        when 'u'
+          lex_warning(Issues::ILLEGAL_UNICODE_ESCAPE)
+          "\\u"
+        when "\n"  ; ''
+        when "\r\n"; ''
+        else      ch
+        end
+      else
+        lex_warning(Issues::UNRECOGNIZED_ESCAPE, :ch => ch) unless ignore_invalid_escapes
+        "\\#{ch}"
+      end
+    }
+    rescue ArgumentError => e
+      # A invalid byte sequence may be the result of faulty input as well, but that could not possibly
+      # have reached this far... Unfortunately there is no more specific error and a match on message is
+      # required to differentiate from other internal problems.
+      if e.message =~ /invalid byte sequence/
+        lex_error(Issues::ILLEGAL_UNICODE_ESCAPE)
+      else
+        raise e
+      end
+    end
+    str
+  end
+end
+end
+end