puppet 4.3.2 → 4.4.0

data/lib/puppet/pops/parser/epp_support.rb

@@ -1,8 +1,10 @@
+module Puppet::Pops
+module Parser
 # This module is an integral part of the Lexer.
 # It handles scanning of EPP (Embedded Puppet), a form of string/expression interpolation similar to ERB.
 #
 require 'strscan'
-module Puppet::Pops::Parser::EppSupport
+module EppSupport
 
   TOKEN_RENDER_STRING = [:RENDER_STRING, nil, 0]
   TOKEN_RENDER_EXPR   = [:RENDER_EXPR, nil, 0]
@@ -33,7 +35,7 @@ module Puppet::Pops::Parser::EppSupport
     ctx   = @lexing_context
     queue = @token_queue
 
-    lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => 'No string or file given to lexer to process.') unless scn
+    lex_error(Issues::EPP_INTERNAL_ERROR, :error => 'No string or file given to lexer to process.') unless scn
 
     ctx[:epp_mode] = :text
     enqueue_completed([:EPP_START, nil, 0], 0)
@@ -47,7 +49,7 @@ module Puppet::Pops::Parser::EppSupport
       end
     end
     if ctx[:epp_open_position]
-      lex_error(Puppet::Pops::Issues::EPP_UNBALANCED_TAG, {}, ctx[:epp_position])
+      lex_error(Issues::EPP_UNBALANCED_TAG, {}, ctx[:epp_position])
     end
 
     # Signals end of input
@@ -66,7 +68,7 @@ module Puppet::Pops::Parser::EppSupport
     when :text
       # Should be at end of scan, or something is terribly wrong
       unless @scanner.eos?
-        lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => 'template scanner returns text mode and is not and end of input')
+        lex_error(Issues::EPP_INTERNAL_ERROR, :error => 'template scanner returns text mode and is not and end of input')
       end
       if s
         # s may be nil if scanned text ends with an epp tag (i.e. no trailing text).
@@ -99,7 +101,7 @@ module Puppet::Pops::Parser::EppSupport
       ctx[:epp_mode] = :expr
       ctx[:epp_open_position] = scn.pos
     else
-      lex_error(Puppet::Pops::Issues::EPP_INTERNAL_ERROR, :error => "Unknown mode #{eppscanner.mode} returned by template scanner")
+      lex_error(Issues::EPP_INTERNAL_ERROR, :error => "Unknown mode #{eppscanner.mode} returned by template scanner")
     end
     nil
   end
@@ -198,7 +200,7 @@ module Puppet::Pops::Parser::EppSupport
           # if s ends with <% then this is an error (unbalanced <% %>)
           if s.end_with? "<%"
             @mode = :error
-            @issue = Puppet::Pops::Issues::EPP_UNBALANCED_EXPRESSION
+            @issue = Issues::EPP_UNBALANCED_EXPRESSION
           else
             mode = :epp
           end
@@ -236,7 +238,7 @@ module Puppet::Pops::Parser::EppSupport
           # preceded by a % (i.e. skip %%>)
           part = scanner.scan_until(/[^%]%>/)
           unless part
-            @issue = Puppet::Pops::Issues::EPP_UNBALANCED_COMMENT
+            @issue = Issues::EPP_UNBALANCED_COMMENT
             @mode = :error
             return s
           end
@@ -256,3 +258,5 @@ module Puppet::Pops::Parser::EppSupport
   end
 
 end
+end
+end

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -1,12 +1,14 @@
+module Puppet::Pops
+module Parser
 
 # Does not support "import" and parsing ruby files
 #
-class Puppet::Pops::Parser::EvaluatingParser
+class EvaluatingParser
 
   attr_reader :parser
 
   def initialize()
-    @parser = Puppet::Pops::Parser::Parser.new()
+    @parser = Parser.new()
   end
 
   def parse_string(s, file_source = nil)
@@ -49,7 +51,7 @@ class Puppet::Pops::Parser::EvaluatingParser
 
   # Create a closure that can be called in the given scope
   def closure(model, scope)
-    Puppet::Pops::Evaluator::Closure.new(evaluator, model, scope)
+    Evaluator::Closure.new(evaluator, model, scope)
   end
 
   def evaluate(scope, model)
@@ -67,9 +69,9 @@ class Puppet::Pops::Parser::EvaluatingParser
   def evaluator
     # Do not use the cached evaluator if this is a migration run
     if (Puppet.lookup(:migration_checker) { nil })
-      return Puppet::Pops::Evaluator::EvaluatorImpl.new()
+      return Evaluator::EvaluatorImpl.new()
     end
-    @@evaluator ||= Puppet::Pops::Evaluator::EvaluatorImpl.new()
+    @@evaluator ||= Evaluator::EvaluatorImpl.new()
     @@evaluator
   end
 
@@ -84,11 +86,11 @@ class Puppet::Pops::Parser::EvaluatingParser
   end
 
   def acceptor()
-    Puppet::Pops::Validation::Acceptor.new
+    Validation::Acceptor.new
   end
 
   def validator(acceptor)
-    Puppet::Pops::Validation::ValidatorFactory_4_0.new().validator(acceptor)
+    Validation::ValidatorFactory_4_0.new().validator(acceptor)
   end
 
   def assert_and_report(parse_result)
@@ -98,7 +100,7 @@ class Puppet::Pops::Parser::EvaluatingParser
     end
     validation_result = validate(parse_result)
 
-    Puppet::Pops::IssueReporter.assert_and_report(validation_result,
+    IssueReporter.assert_and_report(validation_result,
                                           :emit_warnings => true)
     parse_result
   end
@@ -149,9 +151,11 @@ class Puppet::Pops::Parser::EvaluatingParser
     escaped << '"'
   end
 
-  class EvaluatingEppParser < Puppet::Pops::Parser::EvaluatingParser
+  class EvaluatingEppParser < EvaluatingParser
     def initialize()
-      @parser = Puppet::Pops::Parser::EppParser.new()
+      @parser = EppParser.new()
     end
   end
 end
+end
+end

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -1,5 +1,7 @@
-module Puppet::Pops::Parser::HeredocSupport
-  include Puppet::Pops::Parser::LexerSupport
+module Puppet::Pops
+module Parser
+module HeredocSupport
+  include LexerSupport
 
   # Pattern for heredoc `@(endtag[:syntax][/escapes])
   # Produces groups for endtag (group 1), syntax (group 2), and escapes (group 3)
@@ -15,12 +17,12 @@ module Puppet::Pops::Parser::HeredocSupport
 
     # scanner is at position before @(
     # find end of the heredoc spec
-    str = scn.scan_until(/\)/) || lex_error(Puppet::Pops::Issues::HEREDOC_UNCLOSED_PARENTHESIS, :followed_by => followed_by)
+    str = scn.scan_until(/\)/) || lex_error(Issues::HEREDOC_UNCLOSED_PARENTHESIS, :followed_by => followed_by)
     pos_after_heredoc = scn.pos
 
     # Note: allows '+' as separator in syntax, but this needs validation as empty segments are not allowed
     md = str.match(PATTERN_HEREDOC)
-    lex_error(Puppet::Pops::Issues::HEREDOC_INVALID_SYNTAX) unless md
+    lex_error(Issues::HEREDOC_INVALID_SYNTAX) unless md
     endtag = md[1]
     syntax = md[2] || ''
     escapes = md[3]
@@ -33,7 +35,7 @@ module Puppet::Pops::Parser::HeredocSupport
       endtag = $1.strip
     end
 
-    lex_error(Puppet::Pops::Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
+    lex_error(Issues::HEREDOC_MISSING_ENDTAG) unless endtag.length >= 1
 
     resulting_escapes = []
     if escapes
@@ -41,7 +43,7 @@ module Puppet::Pops::Parser::HeredocSupport
 
       escapes = escapes.split('')
       unless escapes.length == escapes.uniq.length
-        lex_error(Puppet::Pops::Issues::HEREDOC_MULTIPLE_AT_ESCAPES, :escapes => escapes)
+        lex_error(Issues::HEREDOC_MULTIPLE_AT_ESCAPES, :escapes => escapes)
       end
       resulting_escapes = ["\\"]
       escapes.each do |e|
@@ -51,7 +53,7 @@ module Puppet::Pops::Parser::HeredocSupport
         when "L"
           resulting_escapes += ["\n", "\r\n"]
         else
-          lex_error(Puppet::Pops::Issues::HEREDOC_INVALID_ESCAPE, :actual => e)
+          lex_error(Issues::HEREDOC_INVALID_ESCAPE, :actual => e)
         end
       end
     end
@@ -66,14 +68,14 @@ module Puppet::Pops::Parser::HeredocSupport
     if ctx[:newline_jump]
       scn.pos = ctx[:newline_jump]
     else
-      scn.scan_until(/\n/) || lex_error(Puppet::Pops::Issues::HEREDOC_WITHOUT_TEXT)
+      scn.scan_until(/\n/) || lex_error(Issues::HEREDOC_WITHOUT_TEXT)
     end
     # offset 0 for the heredoc, and its line number
     heredoc_offset = scn.pos
     heredoc_line = locator.line_for_offset(heredoc_offset)-1
 
     # Compute message to emit if there is no end (to make it refer to the opening heredoc position).
-    eof_error = create_lex_error(Puppet::Pops::Issues::HEREDOC_WITHOUT_END_TAGGED_LINE)
+    eof_error = create_lex_error(Issues::HEREDOC_WITHOUT_END_TAGGED_LINE)
 
     # Text from this position (+ lexing contexts offset for any preceding heredoc) is heredoc until a line
     # that terminates the heredoc is found.
@@ -97,13 +99,13 @@ module Puppet::Pops::Parser::HeredocSupport
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Puppet::Pops::Parser::Locator::SubLocator.sub_locator(str,
+        locator = Locator::SubLocator.sub_locator(str,
           locator.file, heredoc_line, heredoc_offset, leading.length())
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.
         enqueue([:SUBLOCATE,
-          Puppet::Pops::Parser::LexerSupport::TokenValue.new([:SUBLOCATE,
+          LexerSupport::TokenValue.new([:SUBLOCATE,
             lines, lines.reduce(0) {|size, s| size + s.length} ],
             heredoc_offset,
             locator)])
@@ -138,3 +140,5 @@ module Puppet::Pops::Parser::HeredocSupport
 
 
 end
+end
+end

data/lib/puppet/pops/parser/lexer2.rb

@@ -12,12 +12,14 @@ require 'puppet/pops/parser/interpolation_support'
 require 'puppet/pops/parser/epp_support'
 require 'puppet/pops/parser/slurp_support'
 
-class Puppet::Pops::Parser::Lexer2
-  include Puppet::Pops::Parser::LexerSupport
-  include Puppet::Pops::Parser::HeredocSupport
-  include Puppet::Pops::Parser::InterpolationSupport
-  include Puppet::Pops::Parser::SlurpSupport
-  include Puppet::Pops::Parser::EppSupport
+module Puppet::Pops
+module Parser
+class Lexer2
+  include LexerSupport
+  include HeredocSupport
+  include InterpolationSupport
+  include SlurpSupport
+  include EppSupport
 
   # ALl tokens have three slots, the token name (a Symbol), the token text (String), and a token text length.
   # All operator and punctuation tokens reuse singleton arrays Tokens that require unique values create
@@ -356,7 +358,7 @@ class Puppet::Pops::Parser::Lexer2
         scn = @scanner
         la = scn.peek(2)
         if la[1] == '*'
-          lex_error(Puppet::Pops::Issues::UNCLOSED_MLCOMMENT) if scn.skip(PATTERN_MLCOMMENT).nil?
+          lex_error(Issues::UNCLOSED_MLCOMMENT) if scn.skip(PATTERN_MLCOMMENT).nil?
           nil
         else
           before = scn.pos
@@ -432,7 +434,7 @@ class Puppet::Pops::Parser::Lexer2
             else
               # move to faulty position ('::<uc-letter>' was ok)
               scn.pos = scn.pos + 3
-              lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_CLASS_REFERENCE)
+              lex_error(Issues::ILLEGAL_FULLY_QUALIFIED_CLASS_REFERENCE)
             end
           else
             value = scn.scan(PATTERN_BARE_WORD)
@@ -445,7 +447,7 @@ class Puppet::Pops::Parser::Lexer2
             else
               # move to faulty position ('::' was ok)
               scn.pos = scn.pos + 2
-              lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
+              lex_error(Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
             end
           end
         else
@@ -524,7 +526,7 @@ class Puppet::Pops::Parser::Lexer2
           length = scn.pos - before
           assert_numeric(invalid_number, before)
           scn.pos = before + 1
-          lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER, {:value => invalid_number})
+          lex_error(Issues::ILLEGAL_NUMBER, {:value => invalid_number})
         end
       end
     end
@@ -544,9 +546,9 @@ class Puppet::Pops::Parser::Lexer2
           scn.pos = scn.pos + 1
           fully_qualified = scn.match?(/::/)
           if fully_qualified
-            lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
+            lex_error(Issues::ILLEGAL_FULLY_QUALIFIED_NAME)
           else
-            lex_error(Puppet::Pops::Issues::ILLEGAL_NAME_OR_BARE_WORD)
+            lex_error(Issues::ILLEGAL_NAME_OR_BARE_WORD)
           end
         end
       end
@@ -563,7 +565,7 @@ class Puppet::Pops::Parser::Lexer2
         else
           # move to faulty position ([A-Z] was ok)
           scn.pos = scn.pos + 1
-          lex_error(Puppet::Pops::Issues::ILLEGAL_CLASS_REFERENCE)
+          lex_error(Issues::ILLEGAL_CLASS_REFERENCE)
         end
       end
     end
@@ -604,20 +606,22 @@ class Puppet::Pops::Parser::Lexer2
 
   def lex_string(string, path='')
     initvars
+    assert_not_bom(string)
     @scanner = StringScanner.new(string)
-    @locator = Puppet::Pops::Parser::Locator.locator(string, path)
+    @locator = Locator.locator(string, path)
   end
 
   # Lexes an unquoted string.
   # @param string [String] the string to lex
-  # @param locator [Puppet::Pops::Parser::Locator] the locator to use (a default is used if nil is given)
+  # @param locator [Locator] the locator to use (a default is used if nil is given)
   # @param escapes [Array<String>] array of character strings representing the escape sequences to transform
   # @param interpolate [Boolean] whether interpolation of expressions should be made or not.
   #
   def lex_unquoted_string(string, locator, escapes, interpolate)
     initvars
+    assert_not_bom(string)
     @scanner = StringScanner.new(string)
-    @locator = locator || Puppet::Pops::Parser::Locator.locator(string, '')
+    @locator = locator || Locator.locator(string, '')
     @lexing_context[:escapes] = escapes || UQ_ESCAPES
     @lexing_context[:uq_slurp_pattern] = interpolate ? (escapes.include?('$') ? SLURP_UQ_PATTERN : SLURP_UQNE_PATTERN) : SLURP_ALL_PATTERN
   end
@@ -640,9 +644,10 @@ class Puppet::Pops::Parser::Lexer2
   #
   def lex_file(file)
     initvars
-    contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file) : ''
+    contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file, :encoding => 'utf-8') : ''
+    assert_not_bom(contents)
     @scanner = StringScanner.new(contents.freeze)
-    @locator = Puppet::Pops::Parser::Locator.locator(contents, file)
+    @locator = Locator.locator(contents, file)
   end
 
   def initvars
@@ -679,7 +684,7 @@ class Puppet::Pops::Parser::Lexer2
     # every token in the lexed content.
     #
     scn   = @scanner
-    lex_error_without_pos(Puppet::Pops::Issues::NO_INPUT_TO_LEXER) unless scn
+    lex_error_without_pos(Issues::NO_INPUT_TO_LEXER) unless scn
 
     ctx   = @lexing_context
     queue = @token_queue
@@ -759,3 +764,5 @@ class Puppet::Pops::Parser::Lexer2
   end
 
 end
+end
+end

data/lib/puppet/pops/parser/lexer_support.rb

@@ -1,7 +1,13 @@
+
+module Puppet::Pops
+module Parser
+
+require 'puppet/util/multi_match'
+
 # This is an integral part of the Lexer. It is broken out into a separate module
 # for maintainability of the code, and making the various parts of the lexer focused.
 #
-module Puppet::Pops::Parser::LexerSupport
+module LexerSupport
 
   # Returns "<eof>" if at end of input, else the following 5 characters with \n \r \t escaped
   def followed_by
@@ -56,7 +62,7 @@ module Puppet::Pops::Parser::LexerSupport
       })
   end
 
-  # @param issue [Puppet::Pops::Issues::Issue] the issue
+  # @param issue [Issues::Issue] the issue
   # @param args [Hash<Symbol,String>] Issue arguments
   # @param pos [Integer]
   # @return [Puppet::ParseErrorWithIssue] the created error
@@ -75,16 +81,16 @@ module Puppet::Pops::Parser::LexerSupport
   #
   def assert_numeric(value, pos)
     if value =~ /^0[xX]/
-      lex_error(Puppet::Pops::Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+      lex_error(Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
 
     elsif value =~ /^0[^.]/
-      lex_error(Puppet::Pops::Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/
+      lex_error(Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/
 
     elsif value =~ /^\d+[eE.]/
-      lex_error(Puppet::Pops::Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/
+      lex_error(Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/
 
     else
-      lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
+      lex_error(Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
     end
   end
 
@@ -92,7 +98,7 @@ module Puppet::Pops::Parser::LexerSupport
   # and its position in its source container. There is a cost associated with computing the
   # line and position on line information.
   #
-  class TokenValue < Puppet::Pops::Parser::Locatable
+  class TokenValue < Locatable
     attr_reader :token_array
     attr_reader :offset
     attr_reader :locator
@@ -139,4 +145,76 @@ module Puppet::Pops::Parser::LexerSupport
     #
   end
 
+
+  MM = Puppet::Util::MultiMatch
+  MM_ANY = MM::NOT_NIL
+
+  BOM_UTF_8      = MM.new(0xEF, 0xBB, 0xBF,        MM_ANY)
+  BOM_UTF_16_1   = MM.new(0xFE, 0xFF,              MM_ANY, MM_ANY)
+  BOM_UTF_16_2   = MM.new(0xFF, 0xFE,              MM_ANY, MM_ANY)
+  BOM_UTF_32_1   = MM.new(0x00, 0x00, 0xFE, 0xFF   )
+  BOM_UTF_32_2   = MM.new(0xFF, 0xFE, 0x00, 0x00   )
+
+  BOM_UTF_1      = MM.new(0xF7, 0x64, 0x4C,        MM_ANY)
+  BOM_UTF_EBCDIC = MM.new(0xDD, 0x73, 0x66, 0x73   )
+  BOM_SCSU       = MM.new(0x0E, 0xFE, 0xFF,        MM_ANY)
+  BOM_BOCU       = MM.new(0xFB, 0xEE, 0x28,        MM_ANY)
+  BOM_GB_18030   = MM.new(0x84, 0x31, 0x95, 0x33   )
+
+  LONGEST_BOM    = 4
+
+  def assert_not_bom(content)
+    name, size =
+    case bom = get_bom(content)
+
+    when BOM_UTF_32_1, BOM_UTF_32_2
+      ['UTF-32', 4]
+
+    when BOM_GB_18030
+      ['GB-18030', 4]
+
+    when BOM_UTF_EBCDIC
+      ['UTF-EBCDIC', 4]
+
+    when BOM_SCSU
+      ['SCSU', 3]
+
+    when BOM_UTF_8
+      ['UTF-8', 3]
+
+    when BOM_UTF_1
+      ['UTF-1', 3]
+
+    when BOM_BOCU
+      ['BOCU', 3]
+
+    when BOM_UTF_16_1, BOM_UTF_16_2
+      ['UTF-16', 2]
+
+    else
+      return
+    end
+
+    lex_error_without_pos(
+      Puppet::Pops::Issues::ILLEGAL_BOM,
+      { :format_name   => name,
+        :bytes  => "[#{bom.values[0,size].map {|b| "%X" % b}.join(" ")}]"
+      })
+  end
+
+  def get_bom(content)
+    # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
+    # illegal characters on most platforms, and there is no get_bytes(n). xplicit calls are faster than
+    # looping with a lambda. The get_byte returns nil if there are too few characters, and they
+    # are changed to spaces
+    MM.new(
+      (content.getbyte(0) || ' '),
+      (content.getbyte(1) || ' '),
+      (content.getbyte(2) || ' '),
+      (content.getbyte(3) || ' ')
+      )
+  end
+
+end
+end
 end