puppet 4.3.0 → 4.4.0

data/lib/puppet/pops/parser/lexer_support.rb

@@ -1,7 +1,13 @@
+
+module Puppet::Pops
+module Parser
+
+require 'puppet/util/multi_match'
+
 # This is an integral part of the Lexer. It is broken out into a separate module
 # for maintainability of the code, and making the various parts of the lexer focused.
 #
-module Puppet::Pops::Parser::LexerSupport
+module LexerSupport
 
   # Returns "<eof>" if at end of input, else the following 5 characters with \n \r \t escaped
   def followed_by
@@ -56,7 +62,7 @@ module Puppet::Pops::Parser::LexerSupport
       })
   end
 
-  # @param issue [Puppet::Pops::Issues::Issue] the issue
+  # @param issue [Issues::Issue] the issue
   # @param args [Hash<Symbol,String>] Issue arguments
   # @param pos [Integer]
   # @return [Puppet::ParseErrorWithIssue] the created error
@@ -75,16 +81,16 @@ module Puppet::Pops::Parser::LexerSupport
   #
   def assert_numeric(value, pos)
     if value =~ /^0[xX]/
-      lex_error(Puppet::Pops::Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+      lex_error(Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
 
     elsif value =~ /^0[^.]/
-      lex_error(Puppet::Pops::Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/
+      lex_error(Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/
 
     elsif value =~ /^\d+[eE.]/
-      lex_error(Puppet::Pops::Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/
+      lex_error(Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/
 
     else
-      lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
+      lex_error(Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
     end
   end
 
@@ -92,7 +98,7 @@ module Puppet::Pops::Parser::LexerSupport
   # and its position in its source container. There is a cost associated with computing the
   # line and position on line information.
   #
-  class TokenValue < Puppet::Pops::Parser::Locatable
+  class TokenValue < Locatable
     attr_reader :token_array
     attr_reader :offset
     attr_reader :locator
@@ -139,4 +145,76 @@ module Puppet::Pops::Parser::LexerSupport
     #
   end
 
+
+  MM = Puppet::Util::MultiMatch
+  MM_ANY = MM::NOT_NIL
+
+  BOM_UTF_8      = MM.new(0xEF, 0xBB, 0xBF,        MM_ANY)
+  BOM_UTF_16_1   = MM.new(0xFE, 0xFF,              MM_ANY, MM_ANY)
+  BOM_UTF_16_2   = MM.new(0xFF, 0xFE,              MM_ANY, MM_ANY)
+  BOM_UTF_32_1   = MM.new(0x00, 0x00, 0xFE, 0xFF   )
+  BOM_UTF_32_2   = MM.new(0xFF, 0xFE, 0x00, 0x00   )
+
+  BOM_UTF_1      = MM.new(0xF7, 0x64, 0x4C,        MM_ANY)
+  BOM_UTF_EBCDIC = MM.new(0xDD, 0x73, 0x66, 0x73   )
+  BOM_SCSU       = MM.new(0x0E, 0xFE, 0xFF,        MM_ANY)
+  BOM_BOCU       = MM.new(0xFB, 0xEE, 0x28,        MM_ANY)
+  BOM_GB_18030   = MM.new(0x84, 0x31, 0x95, 0x33   )
+
+  LONGEST_BOM    = 4
+
+  def assert_not_bom(content)
+    name, size =
+    case bom = get_bom(content)
+
+    when BOM_UTF_32_1, BOM_UTF_32_2
+      ['UTF-32', 4]
+
+    when BOM_GB_18030
+      ['GB-18030', 4]
+
+    when BOM_UTF_EBCDIC
+      ['UTF-EBCDIC', 4]
+
+    when BOM_SCSU
+      ['SCSU', 3]
+
+    when BOM_UTF_8
+      ['UTF-8', 3]
+
+    when BOM_UTF_1
+      ['UTF-1', 3]
+
+    when BOM_BOCU
+      ['BOCU', 3]
+
+    when BOM_UTF_16_1, BOM_UTF_16_2
+      ['UTF-16', 2]
+
+    else
+      return
+    end
+
+    lex_error_without_pos(
+      Puppet::Pops::Issues::ILLEGAL_BOM,
+      { :format_name   => name,
+        :bytes  => "[#{bom.values[0,size].map {|b| "%X" % b}.join(" ")}]"
+      })
+  end
+
+  def get_bom(content)
+    # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
+    # illegal characters on most platforms, and there is no get_bytes(n). xplicit calls are faster than
+    # looping with a lambda. The get_byte returns nil if there are too few characters, and they
+    # are changed to spaces
+    MM.new(
+      (content.getbyte(0) || ' '),
+      (content.getbyte(1) || ' '),
+      (content.getbyte(2) || ' '),
+      (content.getbyte(3) || ' ')
+      )
+  end
+
+end
+end
 end

data/lib/puppet/pops/parser/locator.rb

@@ -41,6 +41,15 @@ class Puppet::Pops::Parser::Locator
   def char_length(offset, end_offset)
   end
 
+  # Returns the length measured in number of characters from the given start and end byte offseta
+  def char_length(offset, end_offset)
+  end
+
+  # Extracts the text from offset with given length (measured in what the locator uses for offset)
+  # @returns String - the extracted text
+  def extract_text(offset, length)
+  end
+
   # Returns the line index - an array of line offsets for the start position of each line, starting at 0 for
   # the first line.
   #
@@ -238,6 +247,12 @@ class Puppet::Pops::Parser::Locator
       end_offset - offset
     end
 
+    # Extracts the text from char offset with given byte length
+    # @returns String - the extracted text
+    def extract_text(offset, length)
+      string.slice(offset, length)
+    end
+
   end
 
   # This implementation is for Ruby19 and Ruby20. It uses byteslice to get strings from byte based offsets.
@@ -263,5 +278,11 @@ class Puppet::Pops::Parser::Locator
     def char_length(offset, end_offset)
       string.byteslice(offset, end_offset - offset).length
     end
+
+    # Extracts the text from byte offset with given byte length
+    # @returns String - the extracted text
+    def extract_text(offset, length)
+      string.byteslice(offset, length)
+    end
   end
 end

data/lib/puppet/pops/parser/parser_support.rb

@@ -5,11 +5,13 @@ require 'puppet/resource/type_collection_helper'
 require 'puppet/resource/type'
 require 'monitor'
 
+module Puppet::Pops
+module Parser
 # Supporting logic for the parser.
 # This supporting logic has slightly different responsibilities compared to the original Puppet::Parser::Parser.
 # It is only concerned with parsing.
 #
-class Puppet::Pops::Parser::Parser
+class Parser
   # Note that the name of the contained class and the file name (currently parser_support.rb)
   # needs to be different as the class is generated by Racc, and this file (parser_support.rb) is included as a mix in
   #
@@ -18,8 +20,7 @@ class Puppet::Pops::Parser::Parser
   # Note that the parser/parser support does not have direct knowledge about the Model.
   # All model construction/manipulation is made by the Factory.
   #
-  Factory = Puppet::Pops::Model::Factory
-  Model = Puppet::Pops::Model
+  Factory = Model::Factory
 
   include Puppet::Resource::TypeCollectionHelper
 
@@ -51,19 +52,19 @@ class Puppet::Pops::Parser::Parser
   #
   def error(semantic, message)
     except = Puppet::ParseError.new(message)
-    if semantic.is_a?(Puppet::Pops::Parser::LexerSupport::TokenValue)
+    if semantic.is_a?(LexerSupport::TokenValue)
       except.file = semantic[:file];
       except.line = semantic[:line];
       except.pos = semantic[:pos];
     else
-      semantic = semantic.current() if semantic.is_a?(Puppet::Pops::Model::Factory)
+      semantic = semantic.current() if semantic.is_a?(Factory)
 
       # Adapt the model so it is possible to get location information.
       # The model may not have been added to the source tree, so give it the lexer's locator
       # directly instead of searching for the root Program where the locator is normally stored.
       #
-      if semantic.is_a?(Puppet::Pops::Model::Positioned)
-        adapter = Puppet::Pops::Adapters::SourcePosAdapter.adapt(semantic)
+      if semantic.is_a?(Model::Positioned)
+        adapter = Adapters::SourcePosAdapter.adapt(semantic)
         adapter.locator = @lexer.locator
       else
         adapter = nil
@@ -87,7 +88,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   def initialize()
-    @lexer = Puppet::Pops::Parser::Lexer2.new
+    @lexer = Lexer2.new
     @namestack = []
     @definitions = []
   end
@@ -100,7 +101,7 @@ class Puppet::Pops::Parser::Parser
     else
       value_at = "'#{value[:value]}'"
     end
-    error = Puppet::Pops::Issues::SYNTAX_ERROR.format(:where => value_at)
+    error = Issues::SYNTAX_ERROR.format(:where => value_at)
     error = "#{error}, token: #{token}" if @yydebug
 
     # Note, old parser had processing of "expected token here" - do not try to reinstate:
@@ -126,7 +127,7 @@ class Puppet::Pops::Parser::Parser
       file = lexer.file
     end
     file = nil unless file.is_a?(String) && !file.empty?
-    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, issue_code = Puppet::Pops::Issues::SYNTAX_ERROR.issue_code)
+    raise Puppet::ParseErrorWithIssue.new(error, file, line, pos, nil, issue_code = Issues::SYNTAX_ERROR.issue_code)
   end
 
   # Parses a String of pp DSL code.
@@ -137,7 +138,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Mark the factory wrapped model object with location information
-  # @return [Puppet::Pops::Model::Factory] the given factory
+  # @return [Factory] the given factory
   # @api private
   #
   def loc(factory, start_locateable, end_locateable = nil)
@@ -145,7 +146,7 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Mark the factory wrapped heredoc model object with location information
-  # @return [Puppet::Pops::Model::Factory] the given factory
+  # @return [Factory] the given factory
   # @api private
   #
   def heredoc_loc(factory, start_locateabke, end_locateable = nil)
@@ -193,10 +194,10 @@ class Puppet::Pops::Parser::Parser
     #
     begin
       Factory.transform_calls(expressions)
-    rescue Puppet::Pops::Model::Factory::ArgsToNonCallError => e
+    rescue Factory::ArgsToNonCallError => e
       # e.args[1] is the first comma token in the list
       # e.name_expr is the function name expression
-      if e.name_expr.is_a?(Puppet::Pops::Model::QualifiedName)
+      if e.name_expr.is_a?(Model::QualifiedName)
         error(e.args[1], "attempt to pass argument list to the function '#{e.name_expr.value}' which cannot be called without parentheses")
       else
         error(e.args[1], "illegal comma separated argument list")
@@ -205,8 +206,8 @@ class Puppet::Pops::Parser::Parser
   end
 
   # Transforms a LEFT followed by the result of attribute_operations, this may be a call or an invalid sequence
-  def transform_resource_wo_title(left, resource)
-    Factory.transform_resource_wo_title(left, resource)
+  def transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
+    Factory.transform_resource_wo_title(left, resource, lbrace_token, rbrace_token)
   end
 
   # Creates a program with the given body.
@@ -249,3 +250,5 @@ class Puppet::Pops::Parser::Parser
   end
 
 end
+end
+end

data/lib/puppet/pops/parser/slurp_support.rb

@@ -1,3 +1,5 @@
+module Puppet::Pops
+module Parser
 # This module is an integral part of the Lexer.
 # It defines the string slurping behavior - finding the string and non string parts in interpolated
 # strings, translating escape sequences in strings to their single character equivalence.
@@ -6,8 +8,8 @@
 # additional parameter passing and evaluation of conditional logic.
 # TODO: More detailed performance analysis of excessive character escaping and interpolation.
 #
-module Puppet::Pops::Parser::SlurpSupport
-  include Puppet::Pops::Parser::LexerSupport
+module SlurpSupport
+  include LexerSupport
 
   SLURP_SQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*[']/
   SLURP_DQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*(["]|[$]\{?)/
@@ -23,7 +25,7 @@ module Puppet::Pops::Parser::SlurpSupport
     # skip the leading '
     @scanner.pos += 1
     str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes)
-    lex_error(Puppet::Pops::Issues::UNCLOSED_QUOTE, :after => "\"'\"", :followed_by => followed_by) unless str
+    lex_error(Issues::UNCLOSED_QUOTE, :after => "\"'\"", :followed_by => followed_by) unless str
     str[0..-2] # strip closing "'" from result
   end
 
@@ -32,7 +34,7 @@ module Puppet::Pops::Parser::SlurpSupport
     last = scn.matched
     str = slurp(scn, SLURP_DQ_PATTERN, DQ_ESCAPES, false)
     unless str
-      lex_error(Puppet::Pops::Issues::UNCLOSED_QUOTE, :after => format_quote(last), :followed_by => followed_by)
+      lex_error(Issues::UNCLOSED_QUOTE, :after => format_quote(last), :followed_by => followed_by)
     end
 
     # Terminator may be a single char '"', '$', or two characters '${' group match 1 (scn[1]) from the last slurp holds this
@@ -67,6 +69,10 @@ module Puppet::Pops::Parser::SlurpSupport
   def slurp(scanner, pattern, escapes, ignore_invalid_escapes)
     str = scanner.scan_until(pattern) || return
 
+    return str unless str.include?('\\')
+
+    return str.gsub!(/\\(\\|')/m, '\1') || str if escapes.equal?(SQ_ESCAPES)
+
     # Process unicode escapes first as they require getting 4 hex digits
     # If later a \u is found it is warned not to be a unicode escape
     if escapes.include?('u')
@@ -83,16 +89,16 @@ module Puppet::Pops::Parser::SlurpSupport
         when 'r'   ; "\r"
         when 'n'   ; "\n"
         when 't'   ; "\t"
-        when 's'   ; " "
+        when 's'   ; ' '
         when 'u'
-          lex_warning(Puppet::Pops::Issues::ILLEGAL_UNICODE_ESCAPE)
+          lex_warning(Issues::ILLEGAL_UNICODE_ESCAPE)
           "\\u"
         when "\n"  ; ''
         when "\r\n"; ''
         else      ch
         end
       else
-        lex_warning(Puppet::Pops::Issues::UNRECOGNIZED_ESCAPE, :ch => ch) unless ignore_invalid_escapes
+        lex_warning(Issues::UNRECOGNIZED_ESCAPE, :ch => ch) unless ignore_invalid_escapes
         "\\#{ch}"
       end
     }
@@ -101,7 +107,7 @@ module Puppet::Pops::Parser::SlurpSupport
       # have reached this far... Unfortunately there is no more specific error and a match on message is
       # required to differentiate from other internal problems.
       if e.message =~ /invalid byte sequence/
-        lex_error(Puppet::Pops::Issues::ILLEGAL_UNICODE_ESCAPE)
+        lex_error(Issues::ILLEGAL_UNICODE_ESCAPE)
       else
         raise e
       end
@@ -109,3 +115,5 @@ module Puppet::Pops::Parser::SlurpSupport
     str
   end
 end
+end
+end

data/lib/puppet/pops/types/class_loader.rb

@@ -1,15 +1,17 @@
 require 'rgen/metamodel_builder'
 
+module Puppet::Pops
+module Types
 # The ClassLoader provides a Class instance given a class name or a meta-type.
 # If the class is not already loaded, it is loaded using the Puppet Autoloader.
 # This means it can load a class from a gem, or from puppet modules.
 #
-class Puppet::Pops::Types::ClassLoader
+class ClassLoader
   @autoloader = Puppet::Util::Autoload.new("ClassLoader", "")
 
   # Returns a Class given a fully qualified class name.
   # Lookup of class is never relative to the calling namespace.
-  # @param name [String, Array<String>, Array<Symbol>, Puppet::Pops::Types::PAnyType] A fully qualified
+  # @param name [String, Array<String>, Array<Symbol>, PAnyType] A fully qualified
   #   class name String (e.g. '::Foo::Bar', 'Foo::Bar'), a PAnyType, or a fully qualified name in Array form where each part
   #   is either a String or a Symbol, e.g. `%w{Puppetx Puppetlabs SomeExtension}`.
   # @return [Class, nil] the looked up class or nil if no such class is loaded
@@ -24,7 +26,7 @@ class Puppet::Pops::Types::ClassLoader
     when Array
       provide_from_name_path(name.join('::'), name)
 
-    when Puppet::Pops::Types::PAnyType, Puppet::Pops::Types::PType
+    when PAnyType, PType
       provide_from_type(name)
 
     else
@@ -36,38 +38,38 @@ class Puppet::Pops::Types::ClassLoader
 
   def self.provide_from_type(type)
     case type
-    when Puppet::Pops::Types::PRuntimeType
+    when PRuntimeType
       raise ArgumentError.new("Only Runtime type 'ruby' is supported, got #{type.runtime}") unless type.runtime == :ruby
       provide_from_string(type.runtime_type_name)
 
-    when Puppet::Pops::Types::PBooleanType
+    when PBooleanType
       # There is no other thing to load except this Enum meta type
       RGen::MetamodelBuilder::MMBase::Boolean
 
-    when Puppet::Pops::Types::PType
+    when PType
       # TODO: PType should has a type argument (a PAnyType) so the Class' class could be returned
       #       (but this only matters in special circumstances when meta programming has been used).
       Class
 
-    when Puppet::Pops::Type::POptionalType
+    when POptionalType
       # cannot make a distinction between optional and its type
       provide_from_type(type.optional_type)
 
     # Although not expected to be the first choice for getting a concrete class for these
     # types, these are of value if the calling logic just has a reference to type.
     #
-    when Puppet::Pops::Types::PArrayType    ; Array
-    when Puppet::Pops::Types::PTupleType    ; Array
-    when Puppet::Pops::Types::PHashType     ; Hash
-    when Puppet::Pops::Types::PStructType   ; Hash
-    when Puppet::Pops::Types::PRegexpType   ; Regexp
-    when Puppet::Pops::Types::PIntegerType  ; Integer
-    when Puppet::Pops::Types::PStringType   ; String
-    when Puppet::Pops::Types::PPatternType  ; String
-    when Puppet::Pops::Types::PEnumType     ; String
-    when Puppet::Pops::Types::PFloatType    ; Float
-    when Puppet::Pops::Types::PUndefType      ; NilClass
-    when Puppet::Pops::Types::PCallableType ; Proc
+    when PArrayType    ; Array
+    when PTupleType    ; Array
+    when PHashType     ; Hash
+    when PStructType   ; Hash
+    when PRegexpType   ; Regexp
+    when PIntegerType  ; Integer
+    when PStringType   ; String
+    when PPatternType  ; String
+    when PEnumType     ; String
+    when PFloatType    ; Float
+    when PUndefType      ; NilClass
+    when PCallableType ; Proc
     else
       nil
     end
@@ -127,3 +129,5 @@ class Puppet::Pops::Types::ClassLoader
   end
 
 end
+end
+end

data/lib/puppet/pops/types/enumeration.rb

@@ -1,33 +1,16 @@
 # The Enumeration class provides default Enumerable::Enumerator creation for Puppet Programming Language
 # runtime objects that supports the concept of enumeration.
 #
-class Puppet::Pops::Types::Enumeration
-  # Produces an Enumerable::Enumerator for Array, Hash, Integer, Integer Range, and String.
-  #
-  def self.enumerator(o)
-    @@singleton ||= new
-    @@singleton.enumerator(o)
-  end
+module Puppet::Pops::Types
+  class Enumeration
+    def self.enumerator(o)
+      Puppet.deprecation_warning('Enumeration.enumerator is deprecated. Use Iterable.on instead')
+      Iterable.on(o)
+    end
 
-  # Produces an Enumerator for Array, Hash, Integer, Integer Range, and String.
-  #
-  def enumerator(o)
-    case o
-    when String
-      x = o.chars
-      # Ruby 1.9.3 returns an Enumerator, and 2.0.0 an Array
-      x.is_a?(Array) ? x.each : x
-    when Integer
-      o.times
-    when Array
-      o.each
-    when Hash
-      o.each
-    when Puppet::Pops::Types::PIntegerType
-      # Not enumerable if representing an infinite range
-      o.enumerable? ? o.each : nil
-    else
-      nil
+    def enumerator(o)
+      Puppet.deprecation_warning('Enumeration.enumerator is deprecated. Use Iterable.on instead')
+      Iterable.on(o)
     end
   end
 end