puppet 5.5.22 → 6.0.0

data/lib/puppet/pops/parser/epp_support.rb

@@ -229,6 +229,7 @@ module EppSupport
 
         when "#"
           # template comment
+
           # drop the scanned <%, and skip past -%>, or %>, but also skip %%>
           s.slice!(-2..-1)
 
@@ -240,8 +241,11 @@ module EppSupport
             @mode = :error
             return s
           end
-          # Always trim leading whitespace on the same line when there is a comment
-          s.sub!(/[ \t]*\z/, '')
+          # Trim leading whitespace on the same line when start was <%#-
+          if part[1] == '-'
+            s.sub!(/[ \t]*\z/, '')
+          end
+
           @skip_leading = true if part.end_with?("-%>")
           # Continue scanning for more text
 

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -94,12 +94,12 @@ module HeredocSupport
 
 
         # Process captured lines - remove leading, and trailing newline
-        # get processed string and index of removed margin/leading size per line
-        str, margin_per_line = heredoc_text(lines, leading, has_margin, remove_break)
+        str = heredoc_text(lines, leading, has_margin, remove_break)
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Locator::SubLocator.new(locator, str, heredoc_line, heredoc_offset, has_margin, margin_per_line)
+        locator = Locator::SubLocator.sub_locator(str,
+          locator.file, heredoc_line, heredoc_offset, leading.length())
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.
@@ -121,32 +121,23 @@ module HeredocSupport
     raise eof_error
   end
 
-  # Produces the heredoc text string given the individual (unprocessed) lines as an array and array with margin sizes per line
+  # Produces the heredoc text string given the individual (unprocessed) lines as an array.
   # @param lines [Array<String>] unprocessed lines of text in the heredoc w/o terminating line
   # @param leading [String] the leading text up (up to pipe or other terminating char)
   # @param has_margin [Boolean] if the left margin should be adjusted as indicated by `leading`
   # @param remove_break [Boolean] if the line break (\r?\n) at the end of the last line should be removed or not
-  # @return [Array] - a tuple with resulting string, and an array with margin size per line
   #
   def heredoc_text(lines, leading, has_margin, remove_break)
-    if has_margin && leading.length > 0
+    if has_margin
       leading_pattern = /^#{Regexp.escape(leading)}/
-      # TODO: This implementation is not according to the specification, but is kept to be bug compatible.
-      # The specification says that leading space up to the margin marker should be removed, but this implementation
-      # simply leaves lines that have text in the margin untouched.
-      #
-      processed_lines = lines.collect {|s| s.gsub(leading_pattern, '') }
-      margin_per_line = Array.new(processed_lines.length) {|x| lines[x].length - processed_lines[x].length }
-      lines = processed_lines
-    else
-      # Array with a 0 per line
-      margin_per_line = Array.new(lines.length, 0)
+      lines = lines.collect {|s| s.gsub(leading_pattern, '') }
     end
     result = lines.join('')
     result.gsub!(/\r?\n\z/m, '') if remove_break
-    [result, margin_per_line]
+    result
   end
 
+
 end
 end
 end

data/lib/puppet/pops/parser/interpolation_support.rb

@@ -23,7 +23,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_dqstring()
     text = value
     after = scn.pos
-    loop do
+    while true
       case terminator
       when '"'
         # simple case, there was no interpolation, return directly
@@ -63,7 +63,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_dqstring
     text = value
     after = scn.pos
-    loop do
+    while true
       case terminator
       when '"'
         # simple case, there was no further interpolation, return directly
@@ -107,7 +107,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_uqstring()
     text = value
     after = scn.pos
-    loop do
+    while true
       case terminator
       when ''
         # simple case, there was no interpolation, return directly
@@ -147,7 +147,7 @@ module Puppet::Pops::Parser::InterpolationSupport
     value,terminator = slurp_uqstring
     text = value
     after = scn.pos
-    loop do
+    while true
       case terminator
       when ''
         # simple case, there was no further interpolation, return directly

data/lib/puppet/pops/parser/lexer2.rb

@@ -189,12 +189,7 @@ class Lexer2
       ',' => lambda {  emit(TOKEN_COMMA, @scanner.pos) },
       '[' => lambda do
         before = @scanner.pos
-        # Must check the preceding character to see if it is whitespace.
-        # The fastest thing to do is to simply byteslice to get the string ending at the offset before
-        # and then check what the last character is. (This is the same as  what an locator.char_offset needs
-        # to compute, but with less overhead of trying to find out the global offset from a local offset in the
-        # case when this is sublocated in a heredoc).
-        if before == 0 || @scanner.string.byteslice(0, before)[-1] =~ /[[:blank:]\r\n]+/
+        if (before == 0 || @scanner.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
           emit(TOKEN_LISTSTART, before)
         else
           emit(TOKEN_LBRACK, before)
@@ -667,7 +662,7 @@ class Lexer2
       :line_lexical_start => 0
     }
     # Use of --tasks introduces the new keyword 'plan'
-    @taskm_keywords = Puppet[:tasks] ? { 'plan' => [:PLAN, 'plan',  4] }.freeze : EMPTY_HASH
+    @taskm_keywords = Puppet[:tasks] ? { 'plan' => [:PLAN, 'plan',  4], 'apply' => [:APPLY, 'apply', 5] }.freeze : EMPTY_HASH
   end
 
   # Scans all of the content and returns it in an array

data/lib/puppet/pops/parser/locator.rb

@@ -71,16 +71,6 @@ class Locator
   def line_index()
   end
 
-  # Common byte based impl that works for all rubies (stringscanner is byte based
-  def self.compute_line_index(string)
-    scanner = StringScanner.new(string)
-    result = [0] # first line starts at 0
-    while scanner.scan_until(/\n/)
-      result << scanner.pos
-    end
-    result.freeze
-  end
-
   # Produces an URI with path?line=n&pos=n. If origin is unknown the URI is string:?line=n&pos=n
   def to_uri(ast)
     f = file
@@ -90,11 +80,84 @@ class Locator
       f = Puppet::Util.path_to_uri(f).to_s
     end
     offset = ast.offset
-    URI("#{f}?line=#{line_for_offset(offset)}&pos=#{pos_on_line(offset)}")
+    URI("#{f}?line=#{line_for_offset(offset).to_s}&pos=#{pos_on_line(offset).to_s}")
+  end
+
+  # A Sublocator locates a concrete locator (subspace) in a virtual space.
+  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
+  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
+  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
+  #
+  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
+  #
+  #      1:XXXXXXXX
+  #      2:XXXX.... .. ... ..
+  #      3:XXXX. . .... ..
+  #      4:XXXX............
+  #
+  # This sublocator would be configured with leading_line_count = 1,
+  # leading_offset=8, and leading_line_offset=4
+  #
+  # Note that leading_offset must be the same for all lines and measured in characters.
+  #
+  class SubLocator < Locator
+    attr_reader :locator
+    attr_reader :leading_line_count
+    attr_reader :leading_offset
+    attr_reader :leading_line_offset
+
+    def self.sub_locator(string, file, leading_line_count, leading_offset, leading_line_offset)
+      self.new(Locator.locator(string, file),
+        leading_line_count,
+        leading_offset,
+        leading_line_offset)
+    end
+
+    def initialize(locator, leading_line_count, leading_offset, leading_line_offset)
+      @locator = locator
+      @leading_line_count = leading_line_count
+      @leading_offset = leading_offset
+      @leading_line_offset = leading_line_offset
+    end
+
+    def file
+      @locator.file
+    end
+
+    def string
+      @locator.string
+    end
+
+    # Given offset is offset in the subspace
+    def line_for_offset(offset)
+      @locator.line_for_offset(offset) + @leading_line_count
+    end
+
+    # Given offset is offset in the subspace
+    def offset_on_line(offset)
+      @locator.offset_on_line(offset) + @leading_line_offset
+    end
+
+    # Given offset is offset in the subspace
+    def char_offset(offset)
+      effective_line = @locator.line_for_offset(offset)
+      locator.char_offset(offset) + (effective_line * @leading_line_offset) + @leading_offset
+    end
+
+    # Given offsets are offsets in the subspace
+    def char_length(offset, end_offset)
+      effective_line = @locator.line_for_offset(end_offset) - @locator.line_for_offset(offset)
+      locator.char_length(offset, end_offset) + (effective_line * @leading_line_offset)
+    end
+
+    def pos_on_line(offset)
+      offset_on_line(offset) +1
+    end
   end
 
   class AbstractLocator < Locator
     attr_accessor :line_index
+    attr_accessor :string
     attr_reader   :string
     attr_reader   :file
 
@@ -106,7 +169,8 @@ class Locator
       @file = file.freeze
       @prev_offset = nil
       @prev_line = nil
-      @line_index = line_index.nil? ? Locator.compute_line_index(@string) : line_index
+      @line_index = line_index
+      compute_line_index if line_index.nil?
     end
 
     # Returns the position on line (first position on a line is 1)
@@ -171,6 +235,16 @@ class Locator
       self.class == o.class && string == o.string && file == o.file && line_index == o.line_index
     end
 
+    # Common impl for 18 and 19 since scanner is byte based
+    def compute_line_index
+      scanner = StringScanner.new(string)
+      result = [0] # first line starts at 0
+      while scanner.scan_until(/\n/)
+        result << scanner.pos
+      end
+      self.line_index = result.freeze
+    end
+
     # Returns the line number (first line is 1) for the given offset
     def line_for_offset(offset)
       if @prev_offset == offset
@@ -190,100 +264,6 @@ class Locator
     end
   end
 
-  # A Sublocator locates a concrete locator (subspace) in a virtual space.
-  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
-  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
-  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
-  #
-  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
-  #
-  #      1:XXXXXXXX
-  #      2:XXXX.... .. ... ..
-  #      3:XXXX. . .... ..
-  #      4:XXXX............
-  #
-  # This sublocator would be configured with leading_line_count = 1,
-  # leading_offset=8, and leading_line_offset=4
-  #
-  # Note that leading_offset must be the same for all lines and measured in characters.
-  #
-  # A SubLocator is only used during parsing as the parser will translate the local offsets/lengths to
-  # the parent locator when a sublocated expression is reduced. Do not call the methods
-  # `char_offset` or `char_length` as those methods will raise an error.
-  #
-  class SubLocator < AbstractLocator
-    attr_reader :locator
-    attr_reader :leading_line_count
-    attr_reader :leading_offset
-    attr_reader :has_margin
-    attr_reader :margin_per_line
-
-    def initialize(locator, str, leading_line_count, leading_offset, has_margin, margin_per_line)
-      super(str, locator.file)
-      @locator = locator
-      @leading_line_count = leading_line_count
-      @leading_offset = leading_offset
-      @has_margin = has_margin
-      @margin_per_line = margin_per_line
-
-      # Since lines can have different margin - accumulated margin per line must be computed
-      # and since this accumulated margin adjustment is needed more than once; both for start offset,
-      # and for end offset (to compute global length) it is computed up front here.
-      # The accumulated_offset holds the sum of all removed margins before a position on line n (line index is 1-n,
-      # and (unused) position 0 is always 0).
-      # The last entry is duplicated since there will be  the line "after last line" that would otherwise require
-      # conditional logic.
-      #
-      @accumulated_margin = margin_per_line.reduce([0]) {|memo, val| memo << memo[-1] + val; memo }
-      @accumulated_margin << @accumulated_margin[-1]
-    end
-
-    def file
-      @locator.file
-    end
-
-    # Returns array with transposed (local) offset and (local) length. The transposed values
-    # take the margin into account such that it is added to the content to the right
-    # 
-    # Using X to denote margin and where end of line is explicitly shown as \n:
-    # ```
-    # XXXXabc\n
-    # XXXXdef\n
-    # ```
-    # A local offset of 0 is translated to the start of the first heredoc line, and a length of 1 is adjusted to
-    # 5 - i.e to cover "XXXXa". A local offset of 1, with length 1 would cover "b".
-    # A local offset of 4 and length 1 would cover "XXXXd"
-    #
-    # It is possible that lines have different margin and that is taken into account.
-    #
-    def to_global(offset, length)
-      # simple case, no margin
-      return [offset + @leading_offset, length] unless @has_margin
-
-      # compute local start and end line
-      start_line = line_for_offset(offset)
-      end_line = line_for_offset(offset+length)
-
-      # complex case when there is a margin
-      transposed_offset = offset == 0 ? @leading_offset : offset + @leading_offset + @accumulated_margin[start_line]
-      transposed_length = length +
-        @accumulated_margin[end_line] - @accumulated_margin[start_line] +    # the margins between start and end (0 is line 1)
-        (offset_on_line(offset) == 0 ? margin_per_line[start_line - 1] : 0)  # include start's margin in position 0
-      [transposed_offset, transposed_length]
-    end
-
-    # Do not call this method
-    def char_offset(offset)
-      raise "Should not be called"
-    end
-
-    # Do not call this method
-    def char_length(offset, end_offset)
-      raise "Should not be called"
-    end
-
-  end
-
   class LocatorForChars < AbstractLocator
 
     def offset_on_line(offset)
@@ -331,7 +311,7 @@ class Locator
     # Ruby 19 is multibyte but has no character position methods, must use byteslice
     def offset_on_line(offset)
       line_offset = line_index[ line_for_offset(offset)-1 ]
-      @string.byteslice(line_offset, offset-line_offset).length
+      string.byteslice(line_offset, offset-line_offset).length
     end
 
     # Returns the character offset for a given byte offset

data/lib/puppet/pops/parser/parser_support.rb

@@ -114,17 +114,8 @@ class Parser
     pos  = nil
     if token != 0
       file = value[:file]
-      locator = value.locator
-      if locator.is_a?(Puppet::Pops::Parser::Locator::SubLocator)
-        # The error occurs when doing sub-parsing and the token must be transformed
-        # Transpose the local offset, length to global "coordinates"
-        global_offset, _ = locator.to_global(value.offset, value.length)
-        line = locator.locator.line_for_offset(global_offset)
-        pos = locator.locator.pos_on_line(global_offset)
-      else
-        line = value[:line]
-        pos  = value[:pos]
-      end
+      line = value[:line]
+      pos  = value[:pos]
     else
       # At end of input, use what the lexer thinks is the source file
       file = lexer.file

data/lib/puppet/pops/parser/pn_parser.rb

@@ -48,23 +48,6 @@ class PNParser
     parse_next
   end
 
-  def self.char_types
-    unless instance_variable_defined?(:@char_types)
-      @char_types = Array.new(0x80, TYPE_IDENTIFIER)
-      @char_types[0] = TYPE_END
-      [0x09, 0x0d, 0x0a, 0x20].each { |n| @char_types[n] = TYPE_WS }
-      [TOKEN_LP, TOKEN_RP, TOKEN_LB, TOKEN_RB, TOKEN_LC, TOKEN_RC].each { |n| @char_types[n] = TYPE_DELIM }
-      @char_types[0x2d] = TYPE_MINUS
-      (0x30..0x39).each { |n| @char_types[n] = TYPE_DIGIT }
-      (0x41..0x5a).each { |n| @char_types[n] = TYPE_ALPHA }
-      (0x61..0x7a).each { |n| @char_types[n] = TYPE_ALPHA }
-      @char_types[TOKEN_KEY] = TYPE_KEY_START
-      @char_types[TOKEN_STRING] = TYPE_STRING_START
-      @char_types.freeze
-    end
-    @char_types
-  end
-
   private
 
   def parse_next
@@ -145,6 +128,22 @@ class PNParser
   end
 
   # All methods below belong to the PN lexer
+  def self.char_types
+    unless instance_variable_defined?(:@char_types)
+      @char_types = Array.new(0x80, TYPE_IDENTIFIER)
+      @char_types[0] = TYPE_END
+      [0x09, 0x0d, 0x0a, 0x20].each { |n| @char_types[n] = TYPE_WS }
+      [TOKEN_LP, TOKEN_RP, TOKEN_LB, TOKEN_RB, TOKEN_LC, TOKEN_RC].each { |n| @char_types[n] = TYPE_DELIM }
+      @char_types[0x2d] = TYPE_MINUS
+      (0x30..0x39).each { |n| @char_types[n] = TYPE_DIGIT }
+      (0x41..0x5a).each { |n| @char_types[n] = TYPE_ALPHA }
+      (0x61..0x7a).each { |n| @char_types[n] = TYPE_ALPHA }
+      @char_types[TOKEN_KEY] = TYPE_KEY_START
+      @char_types[TOKEN_STRING] = TYPE_STRING_START
+      @char_types.freeze
+    end
+    @char_types
+  end
 
   def next_token
     skip_white

data/lib/puppet/pops/pcore.rb

@@ -35,20 +35,20 @@ module Pcore
     if Puppet[:tasks]
       add_object_type('Task', <<-PUPPET, loader)
         {
-          attributes => {   
+          attributes => {
             # Fully qualified name of the task
             name => { type => Pattern[/\\A[a-z][a-z0-9_]*(?:::[a-z][a-z0-9_]*)*\\z/] },
 
-            # Full path to executable
-            executable => { type => String },
+            # List of implementations with requirements
+            implementations => { type => Array[Struct[name => String, path => String, Optional[requirements] => Array[String]], 1] },
 
             # Task description
             description => { type => Optional[String], value => undef },
 
             # Puppet Task version
             puppet_task_version => { type => Integer, value => 1 },
-  
-            # Type, description, and sensitive property of each parameter 
+
+            # Type, description, and sensitive property of each parameter
             parameters => {
               type => Optional[Hash[
                 Pattern[/\\A[a-z][a-z0-9_]*\\z/],
@@ -59,7 +59,7 @@ module Pcore
               value => undef
             },
 
-             # Type, description, and sensitive property of each output 
+             # Type, description, and sensitive property of each output
             output => {
               type => Optional[Hash[
                 Pattern[/\\A[a-z][a-z0-9_]*\\z/],
@@ -69,9 +69,9 @@ module Pcore
                   type => Type]]],
               value => undef
             },
- 
+
             supports_noop => { type => Boolean, value => false },
-            input_method => { type => String, value => 'both' },
+            input_method => { type => Optional[String] },
           }
         }
       PUPPET
@@ -95,6 +95,17 @@ module Pcore
     Resource.register_ptypes(loader, ir)
     Lookup::Context.register_ptype(loader, ir);
     Lookup::DataProvider.register_types(loader)
+
+    add_object_type('Deferred', <<-PUPPET, loader)
+      {
+        attributes => {
+          # Fully qualified name of the function
+          name  => { type => Pattern[/\\A[$]?[a-z][a-z0-9_]*(?:::[a-z][a-z0-9_]*)*\\z/] },
+          arguments => { type => Optional[Array[Any]], value => undef},
+        }
+      }
+    PUPPET
+
   end
 
   # Create and register a new `Object` type in the Puppet Type System and map it to an implementation class