puppet 3.4.3 → 3.5.0.rc1

data/lib/puppet/pops/parser/epp_parser.rb

@@ -0,0 +1,51 @@
+# The EppParser is a specialized Puppet Parser that starts parsing in Epp Text mode
+class Puppet::Pops::Parser::EppParser < Puppet::Pops::Parser::Parser
+
+  # Initializes the epp parser support by creating a new instance of {Puppet::Pops::Parser::Lexer}
+  # configured to start in Epp Lexing mode.
+  # @return [void]
+  #
+  def initvars
+    self.lexer = Puppet::Pops::Parser::Lexer2.new()# {:mode => :epp})
+  end
+
+  # Parses a file expected to contain epp text/DSL logic.
+  def parse_file(file)
+    unless FileTest.exist?(file)
+      unless file =~ /\.epp$/
+        file = file + ".epp"
+      end
+    end
+    @lexer.file = file
+    _parse()
+  end
+
+  # Performs the parsing and returns the resulting model.
+  # The lexer holds state, and this is setup with {#parse_string}, or {#parse_file}.
+  #
+  # TODO: deal with options containing origin (i.e. parsing a string from externally known location).
+  # TODO: should return the model, not a Hostclass
+  #
+  # @api private
+  #
+  def _parse()
+    begin
+      @yydebug = false
+      main = yyparse(@lexer,:scan_epp)
+      # #Commented out now because this hides problems in the racc grammar while developing
+      # # TODO include this when test coverage is good enough.
+      #      rescue Puppet::ParseError => except
+      #        except.line ||= @lexer.line
+      #        except.file ||= @lexer.file
+      #        except.pos  ||= @lexer.pos
+      #        raise except
+      #      rescue => except
+      #        raise Puppet::ParseError.new(except.message, @lexer.file, @lexer.line, @lexer.pos, except)
+    end
+    return main
+  ensure
+    @lexer.clear
+    @namestack = []
+    @definitions = []
+  end
+end

data/lib/puppet/pops/parser/epp_support.rb

@@ -0,0 +1,247 @@
+# This module is an integral part of the Lexer.
+# It handles scanning of EPP (Embedded Puppet), a form of string/expression interpolation similar to ERB.
+#
+require 'strscan'
+module Puppet::Pops::Parser::EppSupport
+
+  TOKEN_RENDER_STRING = [:RENDER_STRING, nil, 0]
+  TOKEN_RENDER_EXPR   = [:RENDER_EXPR, nil, 0]
+
+  # Scans all of the content and returns it in an array
+  # Note that the terminating [false, false] token is included in the result.
+  #
+  def fullscan_epp
+    result = []
+    scan_epp {|token, value| result.push([token, value]) }
+    result
+  end
+
+  # A block must be passed to scan. It will be called with two arguments, a symbol for the token,
+  # and an instance of LexerSupport::TokenValue
+  # PERFORMANCE NOTE: The TokenValue is designed to reduce the amount of garbage / temporary data
+  # and to only convert the lexer's internal tokens on demand. It is slightly more costly to create an
+  # instance of a class defined in Ruby than an Array or Hash, but the gain is much bigger since transformation
+  # logic is avoided for many of its members (most are never used (e.g. line/pos information which is only of
+  # value in general for error messages, and for some expressions (which the lexer does not know about).
+  #
+  def scan_epp
+    # PERFORMANCE note: it is faster to access local variables than instance variables.
+    # This makes a small but notable difference since instance member access is avoided for
+    # every token in the lexed content.
+    #
+    scn   = @scanner
+    ctx   = @lexing_context
+    queue = @token_queue
+
+    lex_error "Internal Error: No string or file given to lexer to process." unless scn
+
+    ctx[:epp_mode] = :text
+    enqueue_completed([:EPP_START, nil, 0], 0)
+
+    interpolate_epp
+
+    # This is the lexer's main loop
+    until queue.empty? && scn.eos? do
+      if token = queue.shift || lex_token
+        yield [ ctx[:after] = token[0], token[1] ]
+      end
+    end
+    if ctx[:epp_open_position]
+      lex_error("Unbalanced epp tag, reached <eof> without closing tag.", ctx[:epp_position])
+    end
+
+    # Signals end of input
+    yield [false, false]
+  end
+
+  def interpolate_epp(skip_leading=false)
+    scn = @scanner
+    ctx = @lexing_context
+    eppscanner = EppScanner.new(scn)
+    before = scn.pos
+
+    s = eppscanner.scan(skip_leading)
+
+    case eppscanner.mode
+    when :text
+      # Should be at end of scan, or something is terribly wrong
+      lex_error("Internal error: template scanner returns text mode and is not and end of input") unless @scanner.eos?
+      if s
+        # s may be nil if scanned text ends with an epp tag (i.e. no trailing text).
+        enqueue_completed([:RENDER_STRING, s, scn.pos - before], before)
+      end
+      ctx[:epp_open_position] = nil
+      # do nothing else, scanner is at the end
+
+    when :error
+      lex_error(eppscanner.message())
+
+    when :epp
+      # It is meaningless to render empty string segments, and it is harmful to do this at
+      # the start of the scan as it prevents specification of parameters with <%- ($x, $y) -%>
+      #
+      if s && s.length > 0
+        enqueue_completed([:RENDER_STRING, s, scn.pos - before], before)
+      end
+      # switch epp_mode to general (embedded) pp logic (non rendered result)
+      ctx[:epp_mode] = :epp
+      ctx[:epp_open_position] = scn.pos
+
+    when :expr
+      # It is meaningless to render an empty string segment
+      if s && s.length > 0
+        enqueue_completed([:RENDER_STRING, s, scn.pos - before], before)
+      end
+      enqueue_completed(TOKEN_RENDER_EXPR, before)
+      # switch mode to "epp expr interpolation"
+      ctx[:epp_mode] = :expr
+      ctx[:epp_open_position] = scn.pos
+    else
+      lex_error("Internal Error, Unknown mode #{eppscanner.mode} returned by template scanner")
+    end
+    nil
+  end
+
+  # A scanner specialized in processing text with embedded EPP (Embedded Puppet) tags.
+  # The scanner is initialized with a StringScanner which it mutates as scanning takes place.
+  # The intent is to use one instance of EppScanner per wanted scan, and this instance represents
+  # the state after the scan.
+  #
+  # @example Sample usage
+  #   a = "some text <% pp code %> some more text"
+  #   scan = StringScanner.new(a)
+  #   eppscan = EppScanner.new(scan)
+  #   str = eppscan.scan
+  #   eppscan.mode # => :epp
+  #   eppscan.lines # => 0
+  #   eppscan
+  #
+  # The scanner supports
+  # * scanning text until <%, <%-, <%=
+  # * while scanning text:
+  #   * tokens <%% and %%> are translated to <% and %> respetively and is returned as text.
+  #   * tokens <%# and %> (or ending with -%>) and the enclosed text is a comment and is not included in the returned text
+  #   * text following a comment that ends with -%> gets trailing whitespace (up to and including a line break) trimmed
+  #     and this whitespace is not included in the returned text.
+  # * The continuation {#mode} is set to one of:
+  #   * `:epp` - for a <% token
+  #   * `:expr` - for a <%= token
+  #   * `:text` - when there was no continuation mode (e.g. when input ends with text)
+  #   * ':error` - if the tokens are unbalanced (reaching the end without a closing matching token). An error message
+  #     is then also available via the method {#message}.
+  #
+  # Note that the intent is to use this specialized scanner to scan the text parts, when continuation mode is `:epp` or `:expr`
+  # the pp lexer should advance scanning (using the string scanner) until it reaches and consumes a `-%>` or '%>´ token. If it
+  # finds a `-%> token it should pass this on as a `skip_leading` parameter when it performs the next {#scan}.
+  #
+  class EppScanner
+    # The original scanner used by the lexer/container using EppScanner
+    attr_reader :scanner
+
+    # The resulting mode after the scan.
+    # The mode is one of `:text` (the initial mode), `:epp` embedded code (no output), `:expr` (embedded
+    # expression), or `:error`
+    #
+    attr_reader :mode
+
+    # An error message if `mode == :error`, `nil` otherwise.
+    attr_reader :message
+
+    # If the first scan should skip leading whitespace (typically detected by the pp lexer when the
+    # pp mode end-token is found (i.e. `-%>`) and then passed on to the scanner.
+    #
+    attr_reader :skip_leading
+
+    # Creates an EppScanner based on a StringScanner that represents the state where EppScanner should start scanning.
+    # The given scanner will be mutated (i.e. position moved) to reflect the EppScanner's end state after a scan.
+    #
+    def initialize(scanner)
+      @scanner = scanner
+    end
+
+    # Scans from the current position in the configured scanner, advances this scanner's position until the end
+    # of the input, or to the first position after a mode switching token (`<%`, `<%-` or `<%=`). Number of processed
+    # lines and continuation mode can be obtained via {#lines}, and {#mode}.
+    #
+    # @return [String, nil] the scanned and processed text, or nil if at the end of the input.
+    #
+    def scan(skip_leading=false)
+      @mode = :text
+      @skip_leading = skip_leading
+
+      return nil if scanner.eos?
+      s = ""
+      until scanner.eos?
+        part = @scanner.scan_until(/(<%)|\z/)
+        if @skip_leading
+          part.gsub!(/^[ \t]*\r?\n?/,'')
+          @skip_leading = false
+        end
+        # The spec for %%> is to transform it into a literal %>. This is done here, as %%> otherwise would go
+        # undetected in text mode. (i.e. it is not really necessary to escape %> with %%> in text mode unless
+        # adding checks stating that a literal %> is illegal in text (unbalanced).
+        #
+        part.gsub!(/%%>/, '%>')
+        s += part
+        case @scanner.peek(1)
+        when ""
+          # at the end
+          # if s ends with <% then this is an error (unbalanced <% %>)
+          if s.end_with? "<%"
+            @mode = :error
+            @message = "Unbalanced embedded expression - opening <% and reaching end of input"
+          else
+            mode = :epp
+          end
+          return s
+
+        when "-"
+          # trim trailing whitespace on same line from accumulated s
+          # return text and signal switch to pp mode
+          @scanner.getch # drop the -
+          s.gsub!(/\r?\n?[ \t]*<%\z/, '')
+          @mode = :epp
+          return s
+
+        when "%"
+          # verbatim text
+          # keep the scanned <%, and continue scanning after skipping one %
+          # (i.e. do nothing here)
+          @scanner.getch # drop the % to get a literal <% in the output
+
+        when "="
+          # expression
+          # return text and signal switch to expression mode
+          # drop the scanned <%, and skip past -%>, or %>, but also skip %%>
+          @scanner.getch # drop the =
+          s.slice!(-2..-1)
+          @mode = :expr
+          return s
+
+        when "#"
+          # template comment
+          # drop the scanned <%, and skip past -%>, or %>, but also skip %%>
+          s.slice!(-2..-1)
+
+          # unless there is an immediate termination i.e. <%#%> scan for the next %> that is not
+          # preceded by a % (i.e. skip %%>)
+          part = scanner.scan_until(/[^%]%>/)
+          unless part
+            @message = "Reaching end after opening <%# without seeing %>"
+            @mode = :error
+            return s
+          end
+          @skip_leading = true if part.end_with?("-%>")
+          # Continue scanning for more text
+
+        else
+          # Switch to pp after having removed the <%
+          s.slice!(-2..-1)
+          @mode = :epp
+          return s
+        end
+      end
+    end
+  end
+
+end

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -3,6 +3,8 @@
 #
 class Puppet::Pops::Parser::EvaluatingParser
 
+  attr_reader :parser
+
   def initialize()
     @parser = Puppet::Pops::Parser::Parser.new()
   end
@@ -17,9 +19,10 @@ class Puppet::Pops::Parser::EvaluatingParser
     # Also a possible improvement (if the YAML parser returns positions) is to provide correct output of position.
     #
     begin
-      assert_and_report(@parser.parse_string(s))
+      assert_and_report(parser.parse_string(s))
     rescue Puppet::ParseError => e
-      e.file = @file_source unless e.file
+      # TODO: This is not quite right, why does not the exception have the correct file?
+      e.file = @file_source unless e.file.is_a?(String) && !e.file.empty?
       raise e
     end
   end
@@ -27,7 +30,7 @@ class Puppet::Pops::Parser::EvaluatingParser
   def parse_file(file)
     @file_source = file
     clear()
-    assert_and_report(@parser.parse_file(file))
+    assert_and_report(parser.parse_file(file))
   end
 
   def evaluate_string(scope, s, file_source='unknown')
@@ -49,34 +52,38 @@ class Puppet::Pops::Parser::EvaluatingParser
     ast.safeevaluate(scope)
   end
 
+  def validate(parse_result)
+    resulting_acceptor = acceptor()
+    validator(resulting_acceptor).validate(parse_result)
+    resulting_acceptor
+  end
+
   def acceptor()
-    @acceptor ||= Puppet::Pops::Validation::Acceptor.new
-    @acceptor
+    Puppet::Pops::Validation::Acceptor.new
   end
 
-  def validator()
-    @validator ||= Puppet::Pops::Validation::ValidatorFactory_3_1.new().validator(acceptor)
+  def validator(acceptor)
+    Puppet::Pops::Validation::ValidatorFactory_3_1.new().validator(acceptor)
   end
 
   def assert_and_report(parse_result)
     return nil unless parse_result
-    # make sure the result has an origin (if parsed from a string)
-    unless Puppet::Pops::Adapters::OriginAdapter.get(parse_result.model)
-      Puppet::Pops::Adapters::OriginAdapter.adapt(parse_result.model).origin = @file_source
+    if parse_result.source_ref.nil? or parse_result.source_ref == ''
+      parse_result.source_ref = @file_source
     end
-    validator.validate(parse_result)
+    validation_result = validate(parse_result)
 
     max_errors = Puppet[:max_errors]
     max_warnings = Puppet[:max_warnings] + 1
     max_deprecations = Puppet[:max_deprecations] + 1
 
     # If there are warnings output them
-    warnings = acceptor.warnings
+    warnings = validation_result.warnings
     if warnings.size > 0
       formatter = Puppet::Pops::Validation::DiagnosticFormatterPuppetStyle.new
       emitted_w = 0
       emitted_dw = 0
-      acceptor.warnings.each {|w|
+      validation_result.warnings.each {|w|
         if w.severity == :deprecation
           # Do *not* call Puppet.deprecation_warning it is for internal deprecation, not
           # deprecation of constructs in manifests! (It is not designed for that purpose even if
@@ -93,12 +100,11 @@ class Puppet::Pops::Parser::EvaluatingParser
     end
 
     # If there were errors, report the first found. Use a puppet style formatter.
-    errors = acceptor.errors
+    errors = validation_result.errors
     if errors.size > 0
       formatter = Puppet::Pops::Validation::DiagnosticFormatterPuppetStyle.new
       if errors.size == 1 || max_errors <= 1
         # raise immediately
-        require 'debugger'; debugger
         raise Puppet::ParseError.new(formatter.format(errors[0]))
       end
       emitted = 0
@@ -128,6 +134,8 @@ class Puppet::Pops::Parser::EvaluatingParser
   # The method makes an exception for the two character sequences \$ and \s. They
   # will not be escaped since they have a special meaning in puppet syntax.
   #
+  # TODO: Handle \uXXXX characters ??
+  #
   # @param x [String] The string to quote and "unparse"
   # @return [String] The quoted string
   #
@@ -159,4 +167,34 @@ class Puppet::Pops::Parser::EvaluatingParser
     escaped << p unless p.nil?
     escaped << '"'
   end
+
+  # This is a temporary solution to making it possible to use the new evaluator. The main class
+  # will eventually have this behavior instead of using transformation to Puppet 3.x AST
+  class Transitional < Puppet::Pops::Parser::EvaluatingParser
+
+    def evaluator
+      @@evaluator ||= Puppet::Pops::Evaluator::EvaluatorImpl.new()
+      @@evaluator
+    end
+
+    def evaluate(scope, model)
+      return nil unless model
+      evaluator.evaluate(model, scope)
+    end
+
+    def validator(acceptor)
+      Puppet::Pops::Validation::ValidatorFactory_4_0.new().validator(acceptor)
+    end
+
+    # Create a closure that can be called in the given scope
+    def closure(model, scope)
+      Puppet::Pops::Evaluator::Closure.new(evaluator, model, scope)
+    end
+  end
+
+  class EvaluatingEppParser < Transitional
+    def initialize()
+      @parser = Puppet::Pops::Parser::EppParser.new()
+    end
+  end
 end

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -0,0 +1,139 @@
+module Puppet::Pops::Parser::HeredocSupport
+
+  # Pattern for heredoc `@(endtag[:syntax][/escapes])
+  # Produces groups for endtag (group 1), syntax (group 2), and escapes (group 3)
+  #
+  PATTERN_HEREDOC = %r{@\(([^:/\r\n\)]+)(?::[:blank:]*([a-z][a-zA-Z0-9_+]+)[:blank:]*)?(?:/((?:\w|[$])*)[:blank:]*)?\)}
+
+
+  def heredoc
+    scn = @scanner
+    ctx = @lexing_context
+    locator = @locator
+    before = scn.pos
+
+    # scanner is at position before @(
+    # find end of the heredoc spec
+    str = scn.scan_until(/\)/) || lexer.lex_error("Unclosed parenthesis after '@(' followed by '#{followed_by}'")
+    pos_after_heredoc = scn.pos
+
+    # Note: allows '+' as separator in syntax, but this needs validation as empty segments are not allowed
+    unless md = str.match(PATTERN_HEREDOC)
+      lex_error("Invalid syntax in heredoc expected @(endtag[:syntax][/escapes])")
+    end
+    endtag = md[1]
+    syntax = md[2] || ''
+    escapes = md[3]
+
+    endtag.strip!
+
+    # Is this a dq string style heredoc? (endtag enclosed in "")
+    if endtag =~ /^"(.*)"$/
+      dqstring_style = true
+      endtag = $1.strip
+    end
+
+    lexer.lex_error("Missing endtag in heredoc") unless endtag.length >= 1
+
+    resulting_escapes = []
+    if escapes
+      escapes = "trnsuL$" if escapes.length < 1
+
+      escapes = escapes.split('')
+      unless escapes.length == escapes.uniq.length
+        lex_error("An escape char for @() may only appear once. Got '#{escapes.join(', ')}")
+      end
+      resulting_escapes = ["\\"]
+      escapes.each do |e|
+        case e
+        when "t", "r", "n", "s", "u", "$"
+          resulting_escapes << e
+        when "L"
+          resulting_escapes += ["\n", "\r\n"]
+        else
+          lex_error("Invalid heredoc escape char. Only t, r, n, s,  u, L, $ allowed. Got '#{e}'")
+        end
+      end
+    end
+
+    # Produce a heredoc token to make the syntax available to the grammar
+    enqueue_completed([:HEREDOC, syntax, pos_after_heredoc - before], before)
+
+    # If this is the second or subsequent heredoc on the line, the lexing context's :newline_jump contains
+    # the position after the \n where the next heredoc text should scan. If not set, this is the first
+    # and it should start scanning after the first found \n (or if not found == error).
+
+    if ctx[:newline_jump]
+      scn.pos = lexing_context[:newline_jump]
+    else
+      scn.scan_until(/\n/) || lex_error("Heredoc without any following lines of text")
+    end
+    # offset 0 for the heredoc, and its line number
+    heredoc_offset = scn.pos
+    heredoc_line = locator.line_for_offset(heredoc_offset)-1
+
+    # Compute message to emit if there is no end (to make it refer to the opening heredoc position).
+    eof_message = positioned_message("Heredoc without end-tagged line")
+
+    # Text from this position (+ lexing contexts offset for any preceding heredoc) is heredoc until a line
+    # that terminates the heredoc is found.
+
+    # (Endline in EBNF form): WS* ('|' WS*)? ('-' WS*)? endtag WS* \r? (\n|$)
+    endline_pattern = /([[:blank:]]*)(?:([|])[[:blank:]]*)?(?:(\-)[[:blank:]]*)?#{Regexp.escape(endtag)}[[:blank:]]*\r?(?:\n|\z)/
+    lines = []
+    while !scn.eos? do
+      one_line = scn.scan_until(/(?:\n|\z)/) || lexer.lex_error_without_pos(eof_message)
+      if md = one_line.match(endline_pattern)
+        leading      = md[1]
+        has_margin   = md[2] == '|'
+        remove_break = md[3] == '-'
+        # Record position where next heredoc (from same line as current @()) should start scanning for content
+        ctx[:newline_jump] = scn.pos
+
+
+        # Process captured lines - remove leading, and trailing newline
+        str = heredoc_text(lines, leading, has_margin, remove_break)
+
+        # Use a new lexer instance configured with a sub-locator to enable correct positioning
+        sublexer = self.class.new()
+        locator = Puppet::Pops::Parser::Locator::SubLocator.sub_locator(str,
+          locator.file, heredoc_line, heredoc_offset, leading.length())
+
+        # Emit a token that provides the grammar with location information about the lines on which the heredoc
+        # content is based.
+        enqueue([:SUBLOCATE,
+          Puppet::Pops::Parser::LexerSupport::TokenValue.new([:SUBLOCATE,
+            lines, lines.reduce(0) {|size, s| size + s.length} ],
+            heredoc_offset,
+            locator)])
+
+        sublexer.lex_unquoted_string(str, locator, resulting_escapes, dqstring_style)
+        sublexer.interpolate_uq_to(self)
+        # Continue scan after @(...)
+        scn.pos = pos_after_heredoc
+        return
+      else
+        lines << one_line
+      end
+    end
+    lex_error_without_pos(eof_message)
+  end
+
+  # Produces the heredoc text string given the individual (unprocessed) lines as an array.
+  # @param lines [Array<String>] unprocessed lines of text in the heredoc w/o terminating line
+  # @param leading [String] the leading text up (up to pipe or other terminating char)
+  # @param has_margin [Boolean] if the left margin should be adjusted as indicated by `leading`
+  # @param remove_break [Boolean] if the line break (\r?\n) at the end of the last line should be removed or not
+  #
+  def heredoc_text(lines, leading, has_margin, remove_break)
+    if has_margin
+      leading_pattern = /^#{Regexp.escape(leading)}/
+      lines = lines.collect {|s| s.gsub(leading_pattern, '') }
+    end
+    result = lines.join('')
+    result.gsub!(/\r?\n$/, '') if remove_break
+    result
+  end
+
+
+end