puppet 4.0.0 → 4.9.1

data/lib/puppet/pops/parser/lexer_support.rb

@@ -1,19 +1,13 @@
+
+module Puppet::Pops
+module Parser
+
+require 'puppet/util/multi_match'
+
 # This is an integral part of the Lexer. It is broken out into a separate module
 # for maintainability of the code, and making the various parts of the lexer focused.
 #
-module Puppet::Pops::Parser::LexerSupport
-
-  # Formats given message by appending file, line and position if available.
-  def positioned_message(msg, pos = nil)
-    result = [msg]
-    file = @locator.file
-    line = @locator.line_for_offset(pos || @scanner.pos)
-    pos =  @locator.pos_on_line(pos || @scanner.pos)
-
-    result << "in file #{file}" if file && file.is_a?(String) && !file.empty?
-    result << "at line #{line}:#{pos}"
-    result.join(" ")
-  end
+module LexerSupport
 
   # Returns "<eof>" if at end of input, else the following 5 characters with \n \r \t escaped
   def followed_by
@@ -35,27 +29,68 @@ module Puppet::Pops::Parser::LexerSupport
   end
 
   # Raises a Puppet::LexError with the given message
-  def lex_error_without_pos msg
-    raise Puppet::LexError.new(msg)
+  def lex_error_without_pos(issue, args = {})
+    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code)
   end
 
-  # Raises a Puppet::LexError with the given message
-  def lex_error(msg, pos=nil)
-    raise Puppet::LexError.new(positioned_message(msg, pos))
+  # Raises a Puppet::ParserErrorWithIssue with the given issue and arguments
+  def lex_error(issue, args = {}, pos=nil)
+    raise create_lex_error(issue, args, pos)
+  end
+
+  def filename
+    file = @locator.file
+    file.is_a?(String) && !file.empty? ? file : nil
+  end
+
+  def line(pos)
+    @locator.line_for_offset(pos || @scanner.pos)
+  end
+
+  def position(pos)
+    @locator.pos_on_line(pos || @scanner.pos)
+  end
+
+  def lex_warning(issue, args = {}, pos=nil)
+    Puppet::Util::Log.create({
+        :level => :warning,
+        :message => issue.format(args),
+        :issue_code => issue.issue_code,
+        :file => filename,
+        :line => line(pos),
+        :pos => position(pos),
+      })
+  end
+
+  # @param issue [Issues::Issue] the issue
+  # @param args [Hash<Symbol,String>] Issue arguments
+  # @param pos [Integer]
+  # @return [Puppet::ParseErrorWithIssue] the created error
+  def create_lex_error(issue, args = {}, pos = nil)
+    Puppet::ParseErrorWithIssue.new(
+        issue.format(args),
+        filename,
+        line(pos),
+        position(pos),
+        nil,
+        issue.issue_code)
   end
 
   # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.
   # An error is raised if the given value does not comply.
   #
-  def assert_numeric(value, length)
-    if value =~ /^0[xX].*$/
-      lex_error("Not a valid hex number #{value}", length)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+  def assert_numeric(value, pos)
+    if value =~ /^0[xX]/
+      lex_error(Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+
+    elsif value =~ /^0[^.]/
+      lex_error(Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/
 
-    elsif value =~ /^0[^.].*$/
-      lex_error("Not a valid octal number #{value}", length)   unless value =~ /^0[0-7]+$/
+    elsif value =~ /^\d+[eE.]/
+      lex_error(Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/
 
     else
-      lex_error("Not a valid decimal number #{value}", length) unless value =~ /0?\d+(?:\.\d+)?(?:[eE]-?\d+)?/
+      lex_error(Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
     end
   end
 
@@ -63,7 +98,7 @@ module Puppet::Pops::Parser::LexerSupport
   # and its position in its source container. There is a cost associated with computing the
   # line and position on line information.
   #
-  class TokenValue < Puppet::Pops::Parser::Locatable
+  class TokenValue < Locatable
     attr_reader :token_array
     attr_reader :offset
     attr_reader :locator
@@ -110,4 +145,76 @@ module Puppet::Pops::Parser::LexerSupport
     #
   end
 
+
+  MM = Puppet::Util::MultiMatch
+  MM_ANY = MM::NOT_NIL
+
+  BOM_UTF_8      = MM.new(0xEF, 0xBB, 0xBF,        MM_ANY)
+  BOM_UTF_16_1   = MM.new(0xFE, 0xFF,              MM_ANY, MM_ANY)
+  BOM_UTF_16_2   = MM.new(0xFF, 0xFE,              MM_ANY, MM_ANY)
+  BOM_UTF_32_1   = MM.new(0x00, 0x00, 0xFE, 0xFF   )
+  BOM_UTF_32_2   = MM.new(0xFF, 0xFE, 0x00, 0x00   )
+
+  BOM_UTF_1      = MM.new(0xF7, 0x64, 0x4C,        MM_ANY)
+  BOM_UTF_EBCDIC = MM.new(0xDD, 0x73, 0x66, 0x73   )
+  BOM_SCSU       = MM.new(0x0E, 0xFE, 0xFF,        MM_ANY)
+  BOM_BOCU       = MM.new(0xFB, 0xEE, 0x28,        MM_ANY)
+  BOM_GB_18030   = MM.new(0x84, 0x31, 0x95, 0x33   )
+
+  LONGEST_BOM    = 4
+
+  def assert_not_bom(content)
+    name, size =
+    case bom = get_bom(content)
+
+    when BOM_UTF_32_1, BOM_UTF_32_2
+      ['UTF-32', 4]
+
+    when BOM_GB_18030
+      ['GB-18030', 4]
+
+    when BOM_UTF_EBCDIC
+      ['UTF-EBCDIC', 4]
+
+    when BOM_SCSU
+      ['SCSU', 3]
+
+    when BOM_UTF_8
+      ['UTF-8', 3]
+
+    when BOM_UTF_1
+      ['UTF-1', 3]
+
+    when BOM_BOCU
+      ['BOCU', 3]
+
+    when BOM_UTF_16_1, BOM_UTF_16_2
+      ['UTF-16', 2]
+
+    else
+      return
+    end
+
+    lex_error_without_pos(
+      Puppet::Pops::Issues::ILLEGAL_BOM,
+      { :format_name   => name,
+        :bytes  => "[#{bom.values[0,size].map {|b| "%X" % b}.join(" ")}]"
+      })
+  end
+
+  def get_bom(content)
+    # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
+    # illegal characters on most platforms, and there is no get_bytes(n). xplicit calls are faster than
+    # looping with a lambda. The get_byte returns nil if there are too few characters, and they
+    # are changed to spaces
+    MM.new(
+      (content.getbyte(0) || ' '),
+      (content.getbyte(1) || ' '),
+      (content.getbyte(2) || ' '),
+      (content.getbyte(3) || ' ')
+      )
+  end
+
+end
+end
 end

data/lib/puppet/pops/parser/locator.rb

@@ -1,12 +1,14 @@
+module Puppet::Pops
+module Parser
 # Helper class that keeps track of where line breaks are located and can answer questions about positions.
 #
-class Puppet::Pops::Parser::Locator
+class Locator
   # Creates, or recreates a Locator. A Locator is created if index is not given (a scan is then
   # performed of the given source string.
   #
   def self.locator(string, file, index = nil, char_offsets = false)
-    if(char_offsets)
-      LocatorForChars.new(string, file, index);
+    if char_offsets
+      LocatorForChars.new(string, file, index)
     else
       Locator19.new(string, file, index)
     end
@@ -41,6 +43,15 @@ class Puppet::Pops::Parser::Locator
   def char_length(offset, end_offset)
   end
 
+  # Returns the length measured in number of characters from the given start and end byte offseta
+  def char_length(offset, end_offset)
+  end
+
+  # Extracts the text from offset with given length (measured in what the locator uses for offset)
+  # @returns String - the extracted text
+  def extract_text(offset, length)
+  end
+
   # Returns the line index - an array of line offsets for the start position of each line, starting at 0 for
   # the first line.
   #
@@ -64,14 +75,14 @@ class Puppet::Pops::Parser::Locator
   #
   # Note that leading_offset must be the same for all lines and measured in characters.
   #
-  class SubLocator < Puppet::Pops::Parser::Locator
+  class SubLocator < Locator
     attr_reader :locator
     attr_reader :leading_line_count
     attr_reader :leading_offset
     attr_reader :leading_line_offset
 
     def self.sub_locator(string, file, leading_line_count, leading_offset, leading_line_offset)
-      self.new(Puppet::Pops::Parser::Locator.locator(string, file),
+      self.new(Locator.locator(string, file),
         leading_line_count,
         leading_offset,
         leading_line_offset)
@@ -121,7 +132,7 @@ class Puppet::Pops::Parser::Locator
 
   private
 
-  class AbstractLocator < Puppet::Pops::Parser::Locator
+  class AbstractLocator < Locator
     attr_accessor :line_index
     attr_accessor :string
     attr_accessor :prev_offset
@@ -132,13 +143,13 @@ class Puppet::Pops::Parser::Locator
     # Create a locator based on a content string, and a boolean indicating if ruby version support multi-byte strings
     # or not.
     #
-    def initialize(string, file, index = nil)
+    def initialize(string, file, line_index = nil)
       @string = string.freeze
       @file = file.freeze
       @prev_offset = nil
       @prev_line = nil
-      @line_index = index
-      compute_line_index if index.nil?
+      @line_index = line_index
+      compute_line_index if line_index.nil?
     end
 
     # Returns the position on line (first position on a line is 1)
@@ -194,6 +205,15 @@ class Puppet::Pops::Parser::Locator
       return low
     end
 
+    def hash
+      [string, file, line_index].hash
+    end
+
+    # Equal method needed by serializer to perform tabulation
+    def eql?(o)
+      self.class == o.class && string == o.string && file == o.file && line_index == o.line_index
+    end
+
     # Common impl for 18 and 19 since scanner is byte based
     def compute_line_index
       scanner = StringScanner.new(string)
@@ -238,6 +258,12 @@ class Puppet::Pops::Parser::Locator
       end_offset - offset
     end
 
+    # Extracts the text from char offset with given byte length
+    # @returns String - the extracted text
+    def extract_text(offset, length)
+      string.slice(offset, length)
+    end
+
   end
 
   # This implementation is for Ruby19 and Ruby20. It uses byteslice to get strings from byte based offsets.
@@ -245,6 +271,26 @@ class Puppet::Pops::Parser::Locator
   # strings are frozen).
   #
   class Locator19 < AbstractLocator
+    include Types::PuppetObject
+
+    def self._ptype
+      @type
+    end
+
+    # The Locator is not part of the Ecore model so no ObjectType is automatically inferred. Instead the
+    # type is explicitly added here.
+    # TODO: LocatorForChars is never added. It looks like it could be removed (remnant from Ruby 1.8 compatibility?)
+    # @api private
+    def self.register_ptype(loader, ir)
+      @type = Pcore::create_object_type(loader, ir, self, 'Puppet::AST::Locator', 'Any',
+        'string' => Types::PStringType::DEFAULT,
+        'file' => Types::PStringType::DEFAULT,
+        'line_index' => {
+          Types::KEY_TYPE => Types::POptionalType.new(Types::PArrayType.new(Types::PIntegerType::DEFAULT)),
+          Types::KEY_VALUE => nil
+        }
+      ).resolve(Types::TypeParser.singleton, loader)
+    end
 
     # Returns the offset on line (first offset on a line is 0).
     # Ruby 19 is multibyte but has no character position methods, must use byteslice
@@ -263,5 +309,13 @@ class Puppet::Pops::Parser::Locator
     def char_length(offset, end_offset)
       string.byteslice(offset, end_offset - offset).length
     end
+
+    # Extracts the text from byte offset with given byte length
+    # @returns String - the extracted text
+    def extract_text(offset, length)
+      string.byteslice(offset, length)
+    end
   end
 end
+end
+end

data/lib/puppet/pops/parser/parser_support.rb

@@ -1,15 +1,16 @@
 require 'puppet/parser/functions'
 require 'puppet/parser/files'
 require 'puppet/resource/type_collection'
-require 'puppet/resource/type_collection_helper'
 require 'puppet/resource/type'
 require 'monitor'
 
+module Puppet::Pops
+module Parser
 # Supporting logic for the parser.
 # This supporting logic has slightly different responsibilities compared to the original Puppet::Parser::Parser.
 # It is only concerned with parsing.
 #
-class Puppet::Pops::Parser::Parser
+class Parser
   # Note that the name of the contained class and the file name (currently parser_support.rb)
   # needs to be different as the class is generated by Racc, and this file (parser_support.rb) is included as a mix in
   #
@@ -18,10 +19,7 @@ class Puppet::Pops::Parser::Parser
   # Note that the parser/parser support does not have direct knowledge about the Model.
   # All model construction/manipulation is made by the Factory.
   #
-  Factory = Puppet::Pops::Model::Factory
-  Model = Puppet::Pops::Model
-
-  include Puppet::Resource::TypeCollectionHelper
+  Factory = Model::Factory
 
   attr_accessor :lexer
   attr_reader :definitions
@@ -50,21 +48,28 @@ class Puppet::Pops::Parser::Parser
   # lexer. Line and position is produced if the given semantic is a Positioned object and have been given an offset.
   #
   def error(semantic, message)
-    semantic = semantic.current() if semantic.is_a?(Puppet::Pops::Model::Factory)
-    # Adapt the model so it is possible to get location information.
-    # The model may not have been added to the source tree, so give it the lexer's locator
-    # directly instead of searching for the root Program where the locator is normally stored.
-    #
-    if semantic.is_a?(Puppet::Pops::Model::Positioned)
-      adapter = Puppet::Pops::Adapters::SourcePosAdapter.adapt(semantic)
-      adapter.locator = @lexer.locator
+    except = Puppet::ParseError.new(message)
+    if semantic.is_a?(LexerSupport::TokenValue)
+      except.file = semantic[:file];
+      except.line = semantic[:line];
+      except.pos = semantic[:pos];
     else
-      adapter = nil
+      semantic = semantic.current() if semantic.is_a?(Factory)
+
+      # Adapt the model so it is possible to get location information.
+      # The model may not have been added to the source tree, so give it the lexer's locator
+      # directly instead of searching for the root Program where the locator is normally stored.
+      #
+      if semantic.is_a?(Model::Positioned)
+        adapter = Adapters::SourcePosAdapter.adapt(semantic)
+        adapter.locator = @lexer.locator
+      else
+        adapter = nil
+      end
+      except.file = @lexer.locator.file
+      except.line = adapter.line if adapter
+      except.pos = adapter.pos if adapter
     end
-    except = Puppet::ParseError.new(message)
-    except.file = @lexer.locator.file
-    except.line = adapter.line if adapter
-    except.pos = adapter.pos if adapter
     raise except
   end
 
@@ -80,7 +85,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   def initialize()
-    @lexer = Puppet::Pops::Parser::Lexer2.new
+    @lexer = Lexer2.new
     @namestack = []
     @definitions = []
   end
@@ -93,11 +98,8 @@ class Puppet::Pops::Parser::Parser
     else
       value_at = "'#{value[:value]}'"
     end
-    if @yydebug
-      error = "Syntax error at #{value_at}, token: #{token}"
-    else
-      error = "Syntax error at #{value_at}"
-    end
+    error = Issues::SYNTAX_ERROR.format(:where => value_at)
+    error = "#{error}, token: #{token}" if @yydebug
 
     # Note, old parser had processing of "expected token here" - do not try to reinstate:
     # The 'expected' is only of value at end of input, otherwise any parse error involving a
@@ -110,19 +112,19 @@ class Puppet::Pops::Parser::Parser
     # must be handled by the grammar. The lexer may have enqueued tokens far ahead - the lexer's opinion about this
     # is not trustworthy.
     #
-
-    except = Puppet::ParseError.new(error)
+    file = nil
+    line = nil
+    pos  = nil
     if token != 0
-      path        = value[:file]
-      except.line = value[:line]
-      except.pos  = value[:pos]
+      file = value[:file]
+      line = value[:line]
+      pos  = value[:pos]
     else
       # At end of input, use what the lexer thinks is the source file
-      path        = lexer.file
+      file = lexer.file
     end
-    except.file = path if path.is_a?(String) && !path.empty?
-
-    raise except
+    file = nil unless file.is_a?(String) && !file.empty?
+    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, issue_code = Issues::SYNTAX_ERROR.issue_code)
   end
 
   # Parses a String of pp DSL code.
@@ -133,7 +135,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Mark the factory wrapped model object with location information
-  # @return [Puppet::Pops::Model::Factory] the given factory
+  # @return [Factory] the given factory
   # @api private
   #
   def loc(factory, start_locateable, end_locateable = nil)
@@ -141,7 +143,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Mark the factory wrapped heredoc model object with location information
-  # @return [Puppet::Pops::Model::Factory] the given factory
+  # @return [Factory] the given factory
   # @api private
   #
   def heredoc_loc(factory, start_locateabke, end_locateable = nil)
@@ -170,6 +172,11 @@ class Puppet::Pops::Parser::Parser
     definition
   end
 
+  def add_mapping(produces)
+    # The actual handling of mappings happens in PopsBridge
+    add_definition(produces)
+  end
+
   # Transforms an array of expressions containing literal name expressions to calls if followed by an
   # expression, or expression list
   #
@@ -184,10 +191,10 @@ class Puppet::Pops::Parser::Parser
     #
     begin
       Factory.transform_calls(expressions)
-    rescue Puppet::Pops::Model::Factory::ArgsToNonCallError => e
+    rescue Factory::ArgsToNonCallError => e
       # e.args[1] is the first comma token in the list
       # e.name_expr is the function name expression
-      if e.name_expr.is_a?(Puppet::Pops::Model::QualifiedName)
+      if e.name_expr.is_a?(Model::QualifiedName)
         error(e.args[1], "attempt to pass argument list to the function '#{e.name_expr.value}' which cannot be called without parentheses")
       else
         error(e.args[1], "illegal comma separated argument list")
@@ -196,8 +203,8 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Transforms a LEFT followed by the result of attribute_operations, this may be a call or an invalid sequence
-  def transform_resource_wo_title(left, resource)
-    Factory.transform_resource_wo_title(left, resource)
+  def transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
+    Factory.transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
   end
 
   # Creates a program with the given body.
@@ -240,3 +247,5 @@ class Puppet::Pops::Parser::Parser
   end
 
 end
+end
+end

data/lib/puppet/pops/parser/slurp_support.rb

@@ -1,3 +1,5 @@
+module Puppet::Pops
+module Parser
 # This module is an integral part of the Lexer.
 # It defines the string slurping behavior - finding the string and non string parts in interpolated
 # strings, translating escape sequences in strings to their single character equivalence.
@@ -6,11 +8,14 @@
 # additional parameter passing and evaluation of conditional logic.
 # TODO: More detailed performance analysis of excessive character escaping and interpolation.
 #
-module Puppet::Pops::Parser::SlurpSupport
+module SlurpSupport
+  include LexerSupport
 
-  SLURP_SQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*[']/
-  SLURP_DQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*(["]|[$]\{?)/
-  SLURP_UQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*([$]\{?|\z)/
+  SLURP_SQ_PATTERN  = /(?:[^\\]|^)(?:[\\]{2})*[']/
+  SLURP_DQ_PATTERN  = /(?:[^\\]|^)(?:[\\]{2})*(["]|[$]\{?)/
+  SLURP_UQ_PATTERN  = /(?:[^\\]|^)(?:[\\]{2})*([$]\{?|\z)/
+  # unquoted, no escapes
+  SLURP_UQNE_PATTERN  = /(\$\{?|\z)/m
   SLURP_ALL_PATTERN = /.*(\z)/m
   SQ_ESCAPES = %w{ \\ ' }
   DQ_ESCAPES = %w{ \\  $ ' " r n t s u}+["\r\n", "\n"]
@@ -19,7 +24,8 @@ module Puppet::Pops::Parser::SlurpSupport
   def slurp_sqstring
     # skip the leading '
     @scanner.pos += 1
-    str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes) || lex_error("Unclosed quote after \"'\" followed by '#{followed_by}'")
+    str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes)
+    lex_error(Issues::UNCLOSED_QUOTE, :after => "\"'\"", :followed_by => followed_by) unless str
     str[0..-2] # strip closing "'" from result
   end
 
@@ -28,7 +34,7 @@ module Puppet::Pops::Parser::SlurpSupport
     last = scn.matched
     str = slurp(scn, SLURP_DQ_PATTERN, DQ_ESCAPES, false)
     unless str
-      lex_error("Unclosed quote after #{format_quote(last)} followed by '#{followed_by}'")
+      lex_error(Issues::UNCLOSED_QUOTE, :after => format_quote(last), :followed_by => followed_by)
     end
 
     # Terminator may be a single char '"', '$', or two characters '${' group match 1 (scn[1]) from the last slurp holds this
@@ -41,6 +47,7 @@ module Puppet::Pops::Parser::SlurpSupport
      scn = @scanner
      last = scn.matched
      ignore = true
+
      str = slurp(scn, @lexing_context[:uq_slurp_pattern], @lexing_context[:escapes], :ignore_invalid_escapes)
 
      # Terminator may be a single char '$', two characters '${', or empty string '' at the end of intput.
@@ -62,14 +69,20 @@ module Puppet::Pops::Parser::SlurpSupport
   def slurp(scanner, pattern, escapes, ignore_invalid_escapes)
     str = scanner.scan_until(pattern) || return
 
+    return str unless str.include?('\\')
+
+    return str.gsub!(/\\(\\|')/m, '\1') || str if escapes.equal?(SQ_ESCAPES)
+
     # Process unicode escapes first as they require getting 4 hex digits
     # If later a \u is found it is warned not to be a unicode escape
     if escapes.include?('u')
-      str.gsub!(/\\u([\da-fA-F]{4})/m) {
-        [$1.hex].pack("U")
-      }
+      # gsub must be repeated to cater for adjacent escapes
+      while(str.gsub!(/((?:[^\\]|^)(?:[\\]{2})*)\\u(?:([\da-fA-F]{4})|\{([\da-fA-F]{1,6})\})/m) { $1 + [($2 || $3).hex].pack("U") })
+        # empty block. Everything happens in the gsub block
+      end
     end
 
+    begin
     str.gsub!(/\\([^\r\n]|(?:\r?\n))/m) {
       ch = $1
       if escapes.include? ch
@@ -77,19 +90,31 @@ module Puppet::Pops::Parser::SlurpSupport
         when 'r'   ; "\r"
         when 'n'   ; "\n"
         when 't'   ; "\t"
-        when 's'   ; " "
+        when 's'   ; ' '
         when 'u'
-          Puppet.warning(positioned_message("Unicode escape '\\u' was not followed by 4 hex digits"))
+          lex_warning(Issues::ILLEGAL_UNICODE_ESCAPE)
           "\\u"
         when "\n"  ; ''
         when "\r\n"; ''
         else      ch
         end
       else
-        Puppet.warning(positioned_message("Unrecognized escape sequence '\\#{ch}'")) unless ignore_invalid_escapes
+        lex_warning(Issues::UNRECOGNIZED_ESCAPE, :ch => ch) unless ignore_invalid_escapes
         "\\#{ch}"
       end
     }
+    rescue ArgumentError => e
+      # A invalid byte sequence may be the result of faulty input as well, but that could not possibly
+      # have reached this far... Unfortunately there is no more specific error and a match on message is
+      # required to differentiate from other internal problems.
+      if e.message =~ /invalid byte sequence/
+        lex_error(Issues::ILLEGAL_UNICODE_ESCAPE)
+      else
+        raise e
+      end
+    end
     str
   end
 end
+end
+end

data/lib/puppet/pops/patterns.rb

@@ -14,6 +14,10 @@ module Puppet::Pops::Patterns
   #
   NUMERIC = %r{\A[[:blank:]]*([-+]?)[[:blank:]]*((0[xX][0-9A-Fa-f]+)|(0?\d+)((?:\.\d+)?(?:[eE]-?\d+)?))[[:blank:]]*\z}
 
+  # Special expression that tests if there is whitespace between sign and number. The expression is used
+  # to strip such whitespace when normal Float or Integer conversion fails.
+  WS_BETWEEN_SIGN_AND_NUMBER = %r{\A([+-])[[:blank:]]+(.*)\z}
+
   # ILLEGAL_P3_1_HOSTNAME matches if a hostname contains illegal characters.
   # This check does not prevent pathological names like 'a....b', '.....', "---". etc.
   ILLEGAL_HOSTNAME_CHARS = %r{[^-\w.]}
@@ -26,17 +30,28 @@ module Puppet::Pops::Patterns
   #
   CLASSREF_EXT = %r{\A((::){0,1}[A-Z][\w]*)+\z}
 
+  # Same as CLASSREF_EXT but cannot start with '::'
+  #
+  CLASSREF_EXT_DECL = %r{\A[A-Z][\w]*(?:::[A-Z][\w]*)*\z}
+
   # CLASSREF matches a class reference the way it is represented internally in the
   # model (i.e. in lower case).
   #
   CLASSREF = %r{\A((::){0,1}[a-z][\w]*)+\z}
 
+  # Same as CLASSREF but cannot start with '::'
+  #
+  CLASSREF_DECL = %r{\A[a-z][\w]*(?:::[a-z][\w]*)*\z}
+
   # DOLLAR_VAR matches a variable name including the initial $ character
   DOLLAR_VAR     = %r{\$(::)?(\w+::)*\w+}
 
   # VAR_NAME matches the name part of a variable (The $ character is not included)
   # Note, that only the final segment may start with an underscore.
-  VAR_NAME = %r{\A(:?(::)?[a-z]\w*)*(:?(::)?[a-z_]\w*)\z}
+  VAR_NAME = %r{\A(?:(::)?[a-z]\w*)*(?:(::)?[a-z_]\w*)\z}
+
+  # PARAM_NAME matches the name part of a parameter (The $ character is not included)
+  PARAM_NAME = %r{\A[a-z_]\w*\z}
 
   # A Numeric var name must be the decimal number 0, or a decimal number not starting with 0
   NUMERIC_VAR_NAME = %r{\A(?:0|(?:[1-9][0-9]*))\z}