puppet 4.0.0 → 4.5.0

data/lib/puppet/pops/parser/lexer_support.rb

@@ -1,19 +1,13 @@
+
+module Puppet::Pops
+module Parser
+
+require 'puppet/util/multi_match'
+
 # This is an integral part of the Lexer. It is broken out into a separate module
 # for maintainability of the code, and making the various parts of the lexer focused.
 #
-module Puppet::Pops::Parser::LexerSupport
-
-  # Formats given message by appending file, line and position if available.
-  def positioned_message(msg, pos = nil)
-    result = [msg]
-    file = @locator.file
-    line = @locator.line_for_offset(pos || @scanner.pos)
-    pos =  @locator.pos_on_line(pos || @scanner.pos)
-
-    result << "in file #{file}" if file && file.is_a?(String) && !file.empty?
-    result << "at line #{line}:#{pos}"
-    result.join(" ")
-  end
+module LexerSupport
 
   # Returns "<eof>" if at end of input, else the following 5 characters with \n \r \t escaped
   def followed_by
@@ -35,27 +29,68 @@ module Puppet::Pops::Parser::LexerSupport
   end
 
   # Raises a Puppet::LexError with the given message
-  def lex_error_without_pos msg
-    raise Puppet::LexError.new(msg)
+  def lex_error_without_pos(issue, args = {})
+    raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code)
   end
 
-  # Raises a Puppet::LexError with the given message
-  def lex_error(msg, pos=nil)
-    raise Puppet::LexError.new(positioned_message(msg, pos))
+  # Raises a Puppet::ParserErrorWithIssue with the given issue and arguments
+  def lex_error(issue, args = {}, pos=nil)
+    raise create_lex_error(issue, args, pos)
+  end
+
+  def filename
+    file = @locator.file
+    file.is_a?(String) && !file.empty? ? file : nil
+  end
+
+  def line(pos)
+    @locator.line_for_offset(pos || @scanner.pos)
+  end
+
+  def position(pos)
+    @locator.pos_on_line(pos || @scanner.pos)
+  end
+
+  def lex_warning(issue, args = {}, pos=nil)
+    Puppet::Util::Log.create({
+        :level => :warning,
+        :message => issue.format(args),
+        :issue_code => issue.issue_code,
+        :file => filename,
+        :line => line(pos),
+        :pos => position(pos),
+      })
+  end
+
+  # @param issue [Issues::Issue] the issue
+  # @param args [Hash<Symbol,String>] Issue arguments
+  # @param pos [Integer]
+  # @return [Puppet::ParseErrorWithIssue] the created error
+  def create_lex_error(issue, args = {}, pos = nil)
+    Puppet::ParseErrorWithIssue.new(
+        issue.format(args),
+        filename,
+        line(pos),
+        position(pos),
+        nil,
+        issue.issue_code)
   end
 
   # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.
   # An error is raised if the given value does not comply.
   #
-  def assert_numeric(value, length)
-    if value =~ /^0[xX].*$/
-      lex_error("Not a valid hex number #{value}", length)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+  def assert_numeric(value, pos)
+    if value =~ /^0[xX]/
+      lex_error(Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+
+    elsif value =~ /^0[^.]/
+      lex_error(Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/
 
-    elsif value =~ /^0[^.].*$/
-      lex_error("Not a valid octal number #{value}", length)   unless value =~ /^0[0-7]+$/
+    elsif value =~ /^\d+[eE.]/
+      lex_error(Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/
 
     else
-      lex_error("Not a valid decimal number #{value}", length) unless value =~ /0?\d+(?:\.\d+)?(?:[eE]-?\d+)?/
+      lex_error(Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
     end
   end
 
@@ -63,7 +98,7 @@ module Puppet::Pops::Parser::LexerSupport
   # and its position in its source container. There is a cost associated with computing the
   # line and position on line information.
   #
-  class TokenValue < Puppet::Pops::Parser::Locatable
+  class TokenValue < Locatable
     attr_reader :token_array
     attr_reader :offset
     attr_reader :locator
@@ -110,4 +145,76 @@ module Puppet::Pops::Parser::LexerSupport
     #
   end
 
+
+  MM = Puppet::Util::MultiMatch
+  MM_ANY = MM::NOT_NIL
+
+  BOM_UTF_8      = MM.new(0xEF, 0xBB, 0xBF,        MM_ANY)
+  BOM_UTF_16_1   = MM.new(0xFE, 0xFF,              MM_ANY, MM_ANY)
+  BOM_UTF_16_2   = MM.new(0xFF, 0xFE,              MM_ANY, MM_ANY)
+  BOM_UTF_32_1   = MM.new(0x00, 0x00, 0xFE, 0xFF   )
+  BOM_UTF_32_2   = MM.new(0xFF, 0xFE, 0x00, 0x00   )
+
+  BOM_UTF_1      = MM.new(0xF7, 0x64, 0x4C,        MM_ANY)
+  BOM_UTF_EBCDIC = MM.new(0xDD, 0x73, 0x66, 0x73   )
+  BOM_SCSU       = MM.new(0x0E, 0xFE, 0xFF,        MM_ANY)
+  BOM_BOCU       = MM.new(0xFB, 0xEE, 0x28,        MM_ANY)
+  BOM_GB_18030   = MM.new(0x84, 0x31, 0x95, 0x33   )
+
+  LONGEST_BOM    = 4
+
+  def assert_not_bom(content)
+    name, size =
+    case bom = get_bom(content)
+
+    when BOM_UTF_32_1, BOM_UTF_32_2
+      ['UTF-32', 4]
+
+    when BOM_GB_18030
+      ['GB-18030', 4]
+
+    when BOM_UTF_EBCDIC
+      ['UTF-EBCDIC', 4]
+
+    when BOM_SCSU
+      ['SCSU', 3]
+
+    when BOM_UTF_8
+      ['UTF-8', 3]
+
+    when BOM_UTF_1
+      ['UTF-1', 3]
+
+    when BOM_BOCU
+      ['BOCU', 3]
+
+    when BOM_UTF_16_1, BOM_UTF_16_2
+      ['UTF-16', 2]
+
+    else
+      return
+    end
+
+    lex_error_without_pos(
+      Puppet::Pops::Issues::ILLEGAL_BOM,
+      { :format_name   => name,
+        :bytes  => "[#{bom.values[0,size].map {|b| "%X" % b}.join(" ")}]"
+      })
+  end
+
+  def get_bom(content)
+    # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
+    # illegal characters on most platforms, and there is no get_bytes(n). xplicit calls are faster than
+    # looping with a lambda. The get_byte returns nil if there are too few characters, and they
+    # are changed to spaces
+    MM.new(
+      (content.getbyte(0) || ' '),
+      (content.getbyte(1) || ' '),
+      (content.getbyte(2) || ' '),
+      (content.getbyte(3) || ' ')
+      )
+  end
+
+end
+end
 end

data/lib/puppet/pops/parser/locator.rb

@@ -1,11 +1,13 @@
+module Puppet::Pops
+module Parser
 # Helper class that keeps track of where line breaks are located and can answer questions about positions.
 #
-class Puppet::Pops::Parser::Locator
+class Locator
   # Creates, or recreates a Locator. A Locator is created if index is not given (a scan is then
   # performed of the given source string.
   #
   def self.locator(string, file, index = nil, char_offsets = false)
-    if(char_offsets)
+    if char_offsets
       LocatorForChars.new(string, file, index);
     else
       Locator19.new(string, file, index)
@@ -41,6 +43,15 @@ class Puppet::Pops::Parser::Locator
   def char_length(offset, end_offset)
   end
 
+  # Returns the length measured in number of characters from the given start and end byte offseta
+  def char_length(offset, end_offset)
+  end
+
+  # Extracts the text from offset with given length (measured in what the locator uses for offset)
+  # @returns String - the extracted text
+  def extract_text(offset, length)
+  end
+
   # Returns the line index - an array of line offsets for the start position of each line, starting at 0 for
   # the first line.
   #
@@ -64,14 +75,14 @@ class Puppet::Pops::Parser::Locator
   #
   # Note that leading_offset must be the same for all lines and measured in characters.
   #
-  class SubLocator < Puppet::Pops::Parser::Locator
+  class SubLocator < Locator
     attr_reader :locator
     attr_reader :leading_line_count
     attr_reader :leading_offset
     attr_reader :leading_line_offset
 
     def self.sub_locator(string, file, leading_line_count, leading_offset, leading_line_offset)
-      self.new(Puppet::Pops::Parser::Locator.locator(string, file),
+      self.new(Locator.locator(string, file),
         leading_line_count,
         leading_offset,
         leading_line_offset)
@@ -121,7 +132,7 @@ class Puppet::Pops::Parser::Locator
 
   private
 
-  class AbstractLocator < Puppet::Pops::Parser::Locator
+  class AbstractLocator < Locator
     attr_accessor :line_index
     attr_accessor :string
     attr_accessor :prev_offset
@@ -238,6 +249,12 @@ class Puppet::Pops::Parser::Locator
       end_offset - offset
     end
 
+    # Extracts the text from char offset with given byte length
+    # @returns String - the extracted text
+    def extract_text(offset, length)
+      string.slice(offset, length)
+    end
+
   end
 
   # This implementation is for Ruby19 and Ruby20. It uses byteslice to get strings from byte based offsets.
@@ -263,5 +280,13 @@ class Puppet::Pops::Parser::Locator
     def char_length(offset, end_offset)
       string.byteslice(offset, end_offset - offset).length
     end
+
+    # Extracts the text from byte offset with given byte length
+    # @returns String - the extracted text
+    def extract_text(offset, length)
+      string.byteslice(offset, length)
+    end
   end
 end
+end
+end

data/lib/puppet/pops/parser/parser_support.rb

@@ -5,11 +5,13 @@ require 'puppet/resource/type_collection_helper'
 require 'puppet/resource/type'
 require 'monitor'
 
+module Puppet::Pops
+module Parser
 # Supporting logic for the parser.
 # This supporting logic has slightly different responsibilities compared to the original Puppet::Parser::Parser.
 # It is only concerned with parsing.
 #
-class Puppet::Pops::Parser::Parser
+class Parser
   # Note that the name of the contained class and the file name (currently parser_support.rb)
   # needs to be different as the class is generated by Racc, and this file (parser_support.rb) is included as a mix in
   #
@@ -18,8 +20,7 @@ class Puppet::Pops::Parser::Parser
   # Note that the parser/parser support does not have direct knowledge about the Model.
   # All model construction/manipulation is made by the Factory.
   #
-  Factory = Puppet::Pops::Model::Factory
-  Model = Puppet::Pops::Model
+  Factory = Model::Factory
 
   include Puppet::Resource::TypeCollectionHelper
 
@@ -50,21 +51,28 @@ class Puppet::Pops::Parser::Parser
   # lexer. Line and position is produced if the given semantic is a Positioned object and have been given an offset.
   #
   def error(semantic, message)
-    semantic = semantic.current() if semantic.is_a?(Puppet::Pops::Model::Factory)
-    # Adapt the model so it is possible to get location information.
-    # The model may not have been added to the source tree, so give it the lexer's locator
-    # directly instead of searching for the root Program where the locator is normally stored.
-    #
-    if semantic.is_a?(Puppet::Pops::Model::Positioned)
-      adapter = Puppet::Pops::Adapters::SourcePosAdapter.adapt(semantic)
-      adapter.locator = @lexer.locator
+    except = Puppet::ParseError.new(message)
+    if semantic.is_a?(LexerSupport::TokenValue)
+      except.file = semantic[:file];
+      except.line = semantic[:line];
+      except.pos = semantic[:pos];
     else
-      adapter = nil
+      semantic = semantic.current() if semantic.is_a?(Factory)
+
+      # Adapt the model so it is possible to get location information.
+      # The model may not have been added to the source tree, so give it the lexer's locator
+      # directly instead of searching for the root Program where the locator is normally stored.
+      #
+      if semantic.is_a?(Model::Positioned)
+        adapter = Adapters::SourcePosAdapter.adapt(semantic)
+        adapter.locator = @lexer.locator
+      else
+        adapter = nil
+      end
+      except.file = @lexer.locator.file
+      except.line = adapter.line if adapter
+      except.pos = adapter.pos if adapter
     end
-    except = Puppet::ParseError.new(message)
-    except.file = @lexer.locator.file
-    except.line = adapter.line if adapter
-    except.pos = adapter.pos if adapter
     raise except
   end
 
@@ -80,7 +88,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   def initialize()
-    @lexer = Puppet::Pops::Parser::Lexer2.new
+    @lexer = Lexer2.new
     @namestack = []
     @definitions = []
   end
@@ -93,11 +101,8 @@ class Puppet::Pops::Parser::Parser
     else
       value_at = "'#{value[:value]}'"
     end
-    if @yydebug
-      error = "Syntax error at #{value_at}, token: #{token}"
-    else
-      error = "Syntax error at #{value_at}"
-    end
+    error = Issues::SYNTAX_ERROR.format(:where => value_at)
+    error = "#{error}, token: #{token}" if @yydebug
 
     # Note, old parser had processing of "expected token here" - do not try to reinstate:
     # The 'expected' is only of value at end of input, otherwise any parse error involving a
@@ -110,19 +115,19 @@ class Puppet::Pops::Parser::Parser
     # must be handled by the grammar. The lexer may have enqueued tokens far ahead - the lexer's opinion about this
     # is not trustworthy.
     #
-
-    except = Puppet::ParseError.new(error)
+    file = nil
+    line = nil
+    pos  = nil
     if token != 0
-      path        = value[:file]
-      except.line = value[:line]
-      except.pos  = value[:pos]
+      file = value[:file]
+      line = value[:line]
+      pos  = value[:pos]
     else
       # At end of input, use what the lexer thinks is the source file
-      path        = lexer.file
+      file = lexer.file
     end
-    except.file = path if path.is_a?(String) && !path.empty?
-
-    raise except
+    file = nil unless file.is_a?(String) && !file.empty?
+    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, issue_code = Issues::SYNTAX_ERROR.issue_code)
   end
 
   # Parses a String of pp DSL code.
@@ -133,7 +138,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Mark the factory wrapped model object with location information
-  # @return [Puppet::Pops::Model::Factory] the given factory
+  # @return [Factory] the given factory
   # @api private
   #
   def loc(factory, start_locateable, end_locateable = nil)
@@ -141,7 +146,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Mark the factory wrapped heredoc model object with location information
-  # @return [Puppet::Pops::Model::Factory] the given factory
+  # @return [Factory] the given factory
   # @api private
   #
   def heredoc_loc(factory, start_locateabke, end_locateable = nil)
@@ -170,6 +175,11 @@ class Puppet::Pops::Parser::Parser
     definition
   end
 
+  def add_mapping(produces)
+    # The actual handling of mappings happens in PopsBridge
+    add_definition(produces)
+  end
+
   # Transforms an array of expressions containing literal name expressions to calls if followed by an
   # expression, or expression list
   #
@@ -184,10 +194,10 @@ class Puppet::Pops::Parser::Parser
     #
     begin
       Factory.transform_calls(expressions)
-    rescue Puppet::Pops::Model::Factory::ArgsToNonCallError => e
+    rescue Factory::ArgsToNonCallError => e
       # e.args[1] is the first comma token in the list
       # e.name_expr is the function name expression
-      if e.name_expr.is_a?(Puppet::Pops::Model::QualifiedName)
+      if e.name_expr.is_a?(Model::QualifiedName)
         error(e.args[1], "attempt to pass argument list to the function '#{e.name_expr.value}' which cannot be called without parentheses")
       else
         error(e.args[1], "illegal comma separated argument list")
@@ -196,8 +206,8 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Transforms a LEFT followed by the result of attribute_operations, this may be a call or an invalid sequence
-  def transform_resource_wo_title(left, resource)
-    Factory.transform_resource_wo_title(left, resource)
+  def transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
+    Factory.transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
   end
 
   # Creates a program with the given body.
@@ -240,3 +250,5 @@ class Puppet::Pops::Parser::Parser
   end
 
 end
+end
+end

data/lib/puppet/pops/parser/slurp_support.rb

@@ -1,3 +1,5 @@
+module Puppet::Pops
+module Parser
 # This module is an integral part of the Lexer.
 # It defines the string slurping behavior - finding the string and non string parts in interpolated
 # strings, translating escape sequences in strings to their single character equivalence.
@@ -6,11 +8,14 @@
 # additional parameter passing and evaluation of conditional logic.
 # TODO: More detailed performance analysis of excessive character escaping and interpolation.
 #
-module Puppet::Pops::Parser::SlurpSupport
+module SlurpSupport
+  include LexerSupport
 
   SLURP_SQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*[']/
   SLURP_DQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*(["]|[$]\{?)/
   SLURP_UQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*([$]\{?|\z)/
+  # unquoted, no escapes
+  SLURP_UQNE_PATTERN  = /(\$\{?|\z)/m
   SLURP_ALL_PATTERN = /.*(\z)/m
   SQ_ESCAPES = %w{ \\ ' }
   DQ_ESCAPES = %w{ \\  $ ' " r n t s u}+["\r\n", "\n"]
@@ -19,7 +24,8 @@ module Puppet::Pops::Parser::SlurpSupport
   def slurp_sqstring
     # skip the leading '
     @scanner.pos += 1
-    str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes) || lex_error("Unclosed quote after \"'\" followed by '#{followed_by}'")
+    str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes)
+    lex_error(Issues::UNCLOSED_QUOTE, :after => "\"'\"", :followed_by => followed_by) unless str
     str[0..-2] # strip closing "'" from result
   end
 
@@ -28,7 +34,7 @@ module Puppet::Pops::Parser::SlurpSupport
     last = scn.matched
     str = slurp(scn, SLURP_DQ_PATTERN, DQ_ESCAPES, false)
     unless str
-      lex_error("Unclosed quote after #{format_quote(last)} followed by '#{followed_by}'")
+      lex_error(Issues::UNCLOSED_QUOTE, :after => format_quote(last), :followed_by => followed_by)
     end
 
     # Terminator may be a single char '"', '$', or two characters '${' group match 1 (scn[1]) from the last slurp holds this
@@ -41,6 +47,7 @@ module Puppet::Pops::Parser::SlurpSupport
      scn = @scanner
      last = scn.matched
      ignore = true
+
      str = slurp(scn, @lexing_context[:uq_slurp_pattern], @lexing_context[:escapes], :ignore_invalid_escapes)
 
      # Terminator may be a single char '$', two characters '${', or empty string '' at the end of intput.
@@ -62,14 +69,19 @@ module Puppet::Pops::Parser::SlurpSupport
   def slurp(scanner, pattern, escapes, ignore_invalid_escapes)
     str = scanner.scan_until(pattern) || return
 
+    return str unless str.include?('\\')
+
+    return str.gsub!(/\\(\\|')/m, '\1') || str if escapes.equal?(SQ_ESCAPES)
+
     # Process unicode escapes first as they require getting 4 hex digits
     # If later a \u is found it is warned not to be a unicode escape
     if escapes.include?('u')
-      str.gsub!(/\\u([\da-fA-F]{4})/m) {
-        [$1.hex].pack("U")
+      str.gsub!(/\\u(?:([\da-fA-F]{4})|\{([\da-fA-F]{1,6})\})/m) {
+        [($1 || $2).hex].pack("U")
       }
     end
 
+    begin
     str.gsub!(/\\([^\r\n]|(?:\r?\n))/m) {
       ch = $1
       if escapes.include? ch
@@ -77,19 +89,31 @@ module Puppet::Pops::Parser::SlurpSupport
         when 'r'   ; "\r"
         when 'n'   ; "\n"
         when 't'   ; "\t"
-        when 's'   ; " "
+        when 's'   ; ' '
         when 'u'
-          Puppet.warning(positioned_message("Unicode escape '\\u' was not followed by 4 hex digits"))
+          lex_warning(Issues::ILLEGAL_UNICODE_ESCAPE)
           "\\u"
         when "\n"  ; ''
         when "\r\n"; ''
         else      ch
         end
       else
-        Puppet.warning(positioned_message("Unrecognized escape sequence '\\#{ch}'")) unless ignore_invalid_escapes
+        lex_warning(Issues::UNRECOGNIZED_ESCAPE, :ch => ch) unless ignore_invalid_escapes
         "\\#{ch}"
       end
     }
+    rescue ArgumentError => e
+      # A invalid byte sequence may be the result of faulty input as well, but that could not possibly
+      # have reached this far... Unfortunately there is no more specific error and a match on message is
+      # required to differentiate from other internal problems.
+      if e.message =~ /invalid byte sequence/
+        lex_error(Issues::ILLEGAL_UNICODE_ESCAPE)
+      else
+        raise e
+      end
+    end
     str
   end
 end
+end
+end

data/lib/puppet/pops/patterns.rb

@@ -38,6 +38,9 @@ module Puppet::Pops::Patterns
   # Note, that only the final segment may start with an underscore.
   VAR_NAME = %r{\A(:?(::)?[a-z]\w*)*(:?(::)?[a-z_]\w*)\z}
 
+  # PARAM_NAME matches the name part of a parameter (The $ character is not included)
+  PARAM_NAME = %r{\A[a-z_]\w*\z}
+
   # A Numeric var name must be the decimal number 0, or a decimal number not starting with 0
   NUMERIC_VAR_NAME = %r{\A(?:0|(?:[1-9][0-9]*))\z}
 

data/lib/puppet/pops/pcore.rb

@@ -0,0 +1,30 @@
+require 'uri'
+
+module Puppet::Pops
+module Pcore
+  TYPE_QUALIFIED_REFERENCE = Types::TypeFactory.pattern(Types::TypeFactory.regexp(Patterns::CLASSREF_EXT))
+
+  def self.init(loader, ir)
+    add_alias('Puppet::Pcore::QualifiedReference', TYPE_QUALIFIED_REFERENCE, loader)
+
+    ir.register_implementation_namespace('Puppet::Pcore', 'Puppet::Pops::Pcore', loader)
+  end
+
+  def self.add_alias(name, type, loader)
+    add_type(Types::PTypeAliasType.new(name, nil, type), loader)
+  end
+
+  def self.add_type(type, loader)
+    loader.set_entry(Loader::Loader::TypedName.new(:type, type.name.downcase), type)
+  end
+
+  def self.register_implementations(*impls)
+    Loaders.loaders.register_implementations(*impls)
+  end
+
+  def self.register_aliases(aliases)
+    loader = Loaders.loaders.private_environment_loader
+    aliases.each { |name, type_string| add_type(Types::PTypeAliasType.new(name, Types::TypeFactory.type_reference(type_string), nil), loader) }
+  end
+end
+end