puppet 6.2.0-x64-mingw32 → 6.3.0-x64-mingw32

data/lib/puppet/functions/partition.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# Returns two arrays, the first containing the elements of enum for which the block evaluates to true,
+# the second containing the rest.
+Puppet::Functions.create_function(:partition) do
+  # @param collection A collection of things to partition.
+  # @example Partition array of empty strings, results in e.g. [[''], [b, c]]
+  #   ['', b, c].partition |$s| { $s.empty }
+  # @example Partition array of strings using index, results in e.g. [['', 'ab'], ['b']]
+  #   ['', b, ab].partition |$i, $s| { $i == 2 or $s.empty }
+  # @example Partition hash of strings by key-value pair, results in e.g. [[['b', []]], [['a', [1, 2]]]]
+  #   { a => [1, 2], b => [] }.partition |$kv| { $kv[1].empty }
+  # @example Partition hash of strings by key and value, results in e.g. [[['b', []]], [['a', [1, 2]]]]
+  #   { a => [1, 2], b => [] }.partition |$k, $v| { $v.empty }
+  dispatch :partition_1 do
+    required_param 'Collection', :collection
+    block_param 'Callable[1,1]', :block
+    return_type 'Tuple[Array, Array]'
+  end
+
+  dispatch :partition_2a do
+    required_param 'Array', :array
+    block_param 'Callable[2,2]', :block
+    return_type 'Tuple[Array, Array]'
+  end
+
+  dispatch :partition_2 do
+    required_param 'Collection', :collection
+    block_param 'Callable[2,2]', :block
+    return_type 'Tuple[Array, Array]'
+  end
+
+  def partition_1(collection)
+    collection.partition do |item|
+      yield(item)
+    end.freeze
+  end
+
+  def partition_2a(array)
+    partitioned = array.size.times.zip(array).partition do |k, v|
+      yield(k, v)
+    end
+
+    partitioned.map do |part|
+      part.map { |item| item[1] }
+    end.freeze
+  end
+
+  def partition_2(collection)
+    collection.partition do |k, v|
+      yield(k, v)
+    end.freeze
+  end
+end

data/lib/puppet/pops/evaluator/evaluator_impl.rb

@@ -1028,7 +1028,7 @@ class EvaluatorImpl
 
   # Evaluates Puppet DSL Heredoc
   def eval_HeredocExpression o, scope
-    expr = o.text_expr.expr # Always a SublocatedExpression with an expression
+    expr = o.text_expr
     result = evaluate(o.text_expr, scope)
     unless expr.is_a?(Model::LiteralString)
       # When expr is a LiteralString, validation has already validated this

data/lib/puppet/pops/issues.rb

@@ -759,6 +759,10 @@ module Issues
     _("An escape char for @() may only appear once. Got '%{escapes}'") % { escapes: escapes.join(', ') }
   end
 
+  HEREDOC_DIRTY_MARGIN = hard_issue :HEREDOC_DIRTY_MARGIN, :heredoc_line do
+    _("Heredoc with text in the margin is not allowed (line %{heredoc_line} in this heredoc)") % { heredoc_line: heredoc_line }
+  end
+
   ILLEGAL_BOM = hard_issue :ILLEGAL_BOM, :format_name, :bytes do
     _("Illegal %{format} Byte Order mark at beginning of input: %{bom} - remove these from the puppet source") % { format: format_name, bom: bytes }
   end

data/lib/puppet/pops/model/factory.rb

@@ -28,6 +28,7 @@ class Factory
   BUILD_VISITOR = Visitor.new(self, 'build')
   INFER_VISITOR = Visitor.new(self, 'infer')
   INTERPOLATION_VISITOR = Visitor.new(self, 'interpolate')
+  MAPOFFSET_VISITOR = Visitor.new(self, 'map_offset')
 
   def self.infer(o)
     if o.instance_of?(Factory)
@@ -89,6 +90,29 @@ class Factory
     end
   end
 
+  def map_offset(model, locator)
+    MAPOFFSET_VISITOR.visit_this_1(self, model, locator)
+  end
+
+  def map_offset_Object(o, locator)
+    o
+  end
+
+  def map_offset_Factory(o, locator)
+    map_offset(o.model, locator)
+  end
+
+  def map_offset_Positioned(o, locator)
+    # Transpose the local offset, length to global "coordinates"
+    global_offset, global_length = locator.to_global(o.offset, o.length)
+
+    # mutate
+    o.instance_variable_set(:'@offset', global_offset)
+    o.instance_variable_set(:'@length', global_length)
+    # Change locator since the positions were transposed to the global coordinates
+    o.instance_variable_set(:'@locator', locator.locator) if locator.is_a? Puppet::Pops::Parser::Locator::SubLocator
+  end
+
   # Polymorphic interpolate
   def interpolate()
     INTERPOLATION_VISITOR.visit_this_class(self, @model_class, EMPTY_ARRAY)
@@ -433,8 +457,7 @@ class Factory
     @init_hash[KEY_LOCATOR] = locator
     @init_hash['leading_line_count'] = locator.leading_line_count
     @init_hash['leading_line_offset'] = locator.leading_line_offset
-    # Index is held in sublocator's parent locator - needed to be able to reconstruct
-    @init_hash['line_offsets'] = locator.locator.line_index
+    @init_hash['line_offsets'] = locator.line_index # index of lines in sublocated
   end
 
   def build_SelectorEntry(o, matching, value)
@@ -729,8 +752,6 @@ class Factory
 
   def self.STRING(*args);                new(ConcatenatedString, args);                  end
 
-  def self.SUBLOCATE(token, expr)        new(SubLocatedExpression, token, expr);         end
-
   def self.LIST(entries);                new(LiteralList, entries);                      end
 
   def self.PARAM(name, expr=nil);        new(Parameter, name, expr);                     end
@@ -765,6 +786,19 @@ class Factory
     o.instance_of?(Factory) && o.model_class <= QualifiedReference ? self : new(QualifiedReference, o)
   end
 
+  def self.SUBLOCATE(token, expr_factory)
+    # expr is a Factory wrapped LiteralString, or ConcatenatedString
+    # The token is SUBLOCATED token which has a SubLocator as the token's locator
+    # Use the SubLocator to recalculate the offsets and lengths.
+    model = expr_factory.model
+    locator = token.locator
+    expr_factory.map_offset(model, locator)
+    model._pcore_all_contents([]) { |element| expr_factory.map_offset(element, locator) }
+
+    # Returned the factory wrapping the now offset/length transformed expression(s)
+    expr_factory
+  end
+
   def self.TEXT(expr)
     new(TextExpression, infer(expr).interpolate)
   end

data/lib/puppet/pops/parser/egrammar.ra

@@ -857,8 +857,8 @@ heredoc
   : HEREDOC sublocated_text  { result = Factory.HEREDOC(val[0][:value], val[1]); loc result, val[0] }
 
 sublocated_text
-  : SUBLOCATE string    { result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] }
-  | SUBLOCATE dq_string { result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] }
+  : SUBLOCATE string    { result = Factory.SUBLOCATE(val[0], val[1]); }
+  | SUBLOCATE dq_string { result = Factory.SUBLOCATE(val[0], val[1]); }
 
 epp_expression
   : EPP_START epp_parameters_list optional_statements { result = Factory.EPP(val[1], val[2]); loc result, val[0] }

data/lib/puppet/pops/parser/eparser.rb

@@ -3204,14 +3204,14 @@ module_eval(<<'.,.,', 'egrammar.ra', 856)
 
 module_eval(<<'.,.,', 'egrammar.ra', 859)
   def _reduce_259(val, _values, result)
-     result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] 
+     result = Factory.SUBLOCATE(val[0], val[1]); 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'egrammar.ra', 860)
   def _reduce_260(val, _values, result)
-     result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] 
+     result = Factory.SUBLOCATE(val[0], val[1]); 
     result
   end
 .,.,

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -94,11 +94,12 @@ module HeredocSupport
 
 
         # Process captured lines - remove leading, and trailing newline
-        str = heredoc_text(lines, leading, has_margin, remove_break)
+        # get processed string and index of removed margin/leading size per line
+        str, margin_per_line = heredoc_text(lines, leading, has_margin, remove_break)
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Locator::SubLocator.new(locator, heredoc_line, heredoc_offset, leading.length())
+        locator = Locator::SubLocator.new(locator, str, heredoc_line, heredoc_offset, has_margin, margin_per_line)
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.
@@ -120,23 +121,32 @@ module HeredocSupport
     raise eof_error
   end
 
-  # Produces the heredoc text string given the individual (unprocessed) lines as an array.
+  # Produces the heredoc text string given the individual (unprocessed) lines as an array and array with margin sizes per line
   # @param lines [Array<String>] unprocessed lines of text in the heredoc w/o terminating line
   # @param leading [String] the leading text up (up to pipe or other terminating char)
   # @param has_margin [Boolean] if the left margin should be adjusted as indicated by `leading`
   # @param remove_break [Boolean] if the line break (\r?\n) at the end of the last line should be removed or not
+  # @return [Array] - a tuple with resulting string, and an array with margin size per line
   #
   def heredoc_text(lines, leading, has_margin, remove_break)
-    if has_margin
+    if has_margin && leading.length > 0
       leading_pattern = /^#{Regexp.escape(leading)}/
-      lines = lines.collect {|s| s.gsub(leading_pattern, '') }
+      # TODO: This implementation is not according to the specification, but is kept to be bug compatible.
+      # The specification says that leading space up to the margin marker should be removed, but this implementation
+      # simply leaves lines that have text in the margin untouched.
+      #
+      processed_lines = lines.collect {|s| s.gsub(leading_pattern, '') }
+      margin_per_line = processed_lines.length.times.map {|x| lines[x].length - processed_lines[x].length }
+      lines = processed_lines
+    else
+      # Array with a 0 per line
+      margin_per_line = Array.new(lines.length, 0)
     end
     result = lines.join('')
     result.gsub!(/\r?\n\z/m, '') if remove_break
-    result
+    [result, margin_per_line]
   end
 
-
 end
 end
 end

data/lib/puppet/pops/parser/lexer2.rb

@@ -189,7 +189,12 @@ class Lexer2
       ',' => lambda {  emit(TOKEN_COMMA, @scanner.pos) },
       '[' => lambda do
         before = @scanner.pos
-        if (before == 0 || locator.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
+        # Must check the preceding character to see if it is whitespace.
+        # The fastest thing to do is to simply byteslice to get the string ending at the offset before
+        # and then check what the last character is. (This is the same as  what an locator.char_offset needs
+        # to compute, but with less overhead of trying to find out the global offset from a local offset in the
+        # case when this is sublocated in a heredoc).
+        if before == 0 || @scanner.string.byteslice(0, before)[-1] =~ /[[:blank:]\r\n]+/
           emit(TOKEN_LISTSTART, before)
         else
           emit(TOKEN_LBRACK, before)

data/lib/puppet/pops/parser/locator.rb

@@ -71,6 +71,16 @@ class Locator
   def line_index()
   end
 
+  # Common byte based impl that works for all rubies (stringscanner is byte based
+  def self.compute_line_index(string)
+    scanner = StringScanner.new(string)
+    result = [0] # first line starts at 0
+    while scanner.scan_until(/\n/)
+      result << scanner.pos
+    end
+    result.freeze
+  end
+
   # Produces an URI with path?line=n&pos=n. If origin is unknown the URI is string:?line=n&pos=n
   def to_uri(ast)
     f = file
@@ -83,81 +93,8 @@ class Locator
     URI("#{f}?line=#{line_for_offset(offset).to_s}&pos=#{pos_on_line(offset).to_s}")
   end
 
-  # A Sublocator locates a concrete locator (subspace) in a virtual space.
-  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
-  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
-  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
-  #
-  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
-  #
-  #      1:XXXXXXXX
-  #      2:XXXX.... .. ... ..
-  #      3:XXXX. . .... ..
-  #      4:XXXX............
-  #
-  # This sublocator would be configured with leading_line_count = 1,
-  # leading_offset=8, and leading_line_offset=4
-  #
-  # Note that leading_offset must be the same for all lines and measured in characters.
-  #
-  class SubLocator < Locator
-    attr_reader :locator
-    attr_reader :leading_line_count
-    attr_reader :leading_offset
-    attr_reader :leading_line_offset
-
-    def self.sub_locator(string, file, leading_line_count, leading_offset, leading_line_offset)
-      self.new(Locator.locator(string, file),
-        leading_line_count,
-        leading_offset,
-        leading_line_offset)
-    end
-
-    def initialize(locator, leading_line_count, leading_offset, leading_line_offset)
-      @locator = locator
-      @leading_line_count = leading_line_count
-      @leading_offset = leading_offset
-      @leading_line_offset = leading_line_offset
-    end
-
-    def file
-      @locator.file
-    end
-
-    def string
-      @locator.string
-    end
-
-    # Given offset is offset in the subspace
-    def line_for_offset(offset)
-      @locator.line_for_offset(offset) + @leading_line_count
-    end
-
-    # Given offset is offset in the subspace
-    def offset_on_line(offset)
-      @locator.offset_on_line(offset) + @leading_line_offset
-    end
-
-    # Given offset is offset in the subspace
-    def char_offset(offset)
-      effective_line = @locator.line_for_offset(offset)
-      locator.char_offset(offset) + (effective_line * @leading_line_offset) + @leading_offset
-    end
-
-    # Given offsets are offsets in the subspace
-    def char_length(offset, end_offset)
-      effective_line = @locator.line_for_offset(end_offset) - @locator.line_for_offset(offset)
-      locator.char_length(offset, end_offset) + (effective_line * @leading_line_offset)
-    end
-
-    def pos_on_line(offset)
-      offset_on_line(offset) +1
-    end
-  end
-
   class AbstractLocator < Locator
     attr_accessor :line_index
-    attr_accessor :string
     attr_reader   :string
     attr_reader   :file
 
@@ -169,8 +106,7 @@ class Locator
       @file = file.freeze
       @prev_offset = nil
       @prev_line = nil
-      @line_index = line_index
-      compute_line_index if line_index.nil?
+      @line_index = line_index.nil? ? Locator.compute_line_index(@string) : line_index
     end
 
     # Returns the position on line (first position on a line is 1)
@@ -235,16 +171,6 @@ class Locator
       self.class == o.class && string == o.string && file == o.file && line_index == o.line_index
     end
 
-    # Common impl for 18 and 19 since scanner is byte based
-    def compute_line_index
-      scanner = StringScanner.new(string)
-      result = [0] # first line starts at 0
-      while scanner.scan_until(/\n/)
-        result << scanner.pos
-      end
-      self.line_index = result.freeze
-    end
-
     # Returns the line number (first line is 1) for the given offset
     def line_for_offset(offset)
       if @prev_offset == offset
@@ -264,6 +190,100 @@ class Locator
     end
   end
 
+  # A Sublocator locates a concrete locator (subspace) in a virtual space.
+  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
+  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
+  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
+  #
+  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
+  #
+  #      1:XXXXXXXX
+  #      2:XXXX.... .. ... ..
+  #      3:XXXX. . .... ..
+  #      4:XXXX............
+  #
+  # This sublocator would be configured with leading_line_count = 1,
+  # leading_offset=8, and leading_line_offset=4
+  #
+  # Note that leading_offset must be the same for all lines and measured in characters.
+  #
+  # A SubLocator is only used during parsing as the parser will translate the local offsets/lengths to
+  # the parent locator when a sublocated expression is reduced. Do not call the methods
+  # `char_offset` or `char_length` as those methods will raise an error.
+  #
+  class SubLocator < AbstractLocator
+    attr_reader :locator
+    attr_reader :leading_line_count
+    attr_reader :leading_offset
+    attr_reader :has_margin
+    attr_reader :margin_per_line
+
+    def initialize(locator, str, leading_line_count, leading_offset, has_margin, margin_per_line)
+      super(str, locator.file)
+      @locator = locator
+      @leading_line_count = leading_line_count
+      @leading_offset = leading_offset
+      @has_margin = has_margin
+      @margin_per_line = margin_per_line
+
+      # Since lines can have different margin - accumulated margin per line must be computed
+      # and since this accumulated margin adjustment is needed more than once; both for start offset,
+      # and for end offset (to compute global length) it is computed up front here.
+      # The accumulated_offset holds the sum of all removed margins before a position on line n (line index is 1-n,
+      # and (unused) position 0 is always 0).
+      # The last entry is duplicated since there will be  the line "after last line" that would otherwise require
+      # conditional logic.
+      #
+      @accumulated_margin = margin_per_line.reduce([0]) {|memo, val| memo << memo[-1] + val; memo }
+      @accumulated_margin << @accumulated_margin[-1]
+    end
+
+    def file
+      @locator.file
+    end
+
+    # Returns array with transposed (local) offset and (local) length. The transposed values
+    # take the margin into account such that it is added to the content to the right
+    # 
+    # Using X to denote margin and where end of line is explicitly shown as \n:
+    # ```
+    # XXXXabc\n
+    # XXXXdef\n
+    # ```
+    # A local offset of 0 is translated to the start of the first heredoc line, and a length of 1 is adjusted to
+    # 5 - i.e to cover "XXXXa". A local offset of 1, with length 1 would cover "b".
+    # A local offset of 4 and length 1 would cover "XXXXd"
+    #
+    # It is possible that lines have different margin and that is taken into account.
+    #
+    def to_global(offset, length)
+      # simple case, no margin
+      return [offset + @leading_offset, length] unless @has_margin
+
+      # compute local start and end line
+      start_line = line_for_offset(offset)
+      end_line = line_for_offset(offset+length)
+
+      # complex case when there is a margin
+      transposed_offset = offset == 0 ? @leading_offset : offset + @leading_offset + @accumulated_margin[start_line]
+      transposed_length = length +
+        @accumulated_margin[end_line] - @accumulated_margin[start_line] +    # the margins between start and end (0 is line 1)
+        (offset_on_line(offset) == 0 ? margin_per_line[start_line - 1] : 0)  # include start's margin in position 0
+      [transposed_offset, transposed_length]
+    end
+
+    # Do not call this method
+    def char_offset(offset)
+      raise "Should not be called"
+    end
+
+    # Do not call this method
+    def char_length(offset, end_offset)
+      raise "Should not be called"
+    end
+
+  end
+
   class LocatorForChars < AbstractLocator
 
     def offset_on_line(offset)
@@ -311,7 +331,7 @@ class Locator
     # Ruby 19 is multibyte but has no character position methods, must use byteslice
     def offset_on_line(offset)
       line_offset = line_index[ line_for_offset(offset)-1 ]
-      string.byteslice(line_offset, offset-line_offset).length
+      @string.byteslice(line_offset, offset-line_offset).length
     end
 
     # Returns the character offset for a given byte offset