puppet 6.0.5 → 6.0.7

data/lib/puppet/pops/model/factory.rb

@@ -28,6 +28,7 @@ class Factory
   BUILD_VISITOR = Visitor.new(self, 'build')
   INFER_VISITOR = Visitor.new(self, 'infer')
   INTERPOLATION_VISITOR = Visitor.new(self, 'interpolate')
+  MAPOFFSET_VISITOR = Visitor.new(self, 'map_offset')
 
   def self.infer(o)
     if o.instance_of?(Factory)
@@ -89,6 +90,29 @@ class Factory
     end
   end
 
+  def map_offset(model, locator)
+    MAPOFFSET_VISITOR.visit_this_1(self, model, locator)
+  end
+
+  def map_offset_Object(o, locator)
+    o
+  end
+
+  def map_offset_Factory(o, locator)
+    map_offset(o.model, locator)
+  end
+
+  def map_offset_Positioned(o, locator)
+    # Transpose the local offset, length to global "coordinates"
+    global_offset, global_length = locator.to_global(o.offset, o.length)
+
+    # mutate
+    o.instance_variable_set(:'@offset', global_offset)
+    o.instance_variable_set(:'@length', global_length)
+    # Change locator since the positions were transposed to the global coordinates
+    o.instance_variable_set(:'@locator', locator.locator) if locator.is_a? Puppet::Pops::Parser::Locator::SubLocator
+  end
+
   # Polymorphic interpolate
   def interpolate()
     INTERPOLATION_VISITOR.visit_this_class(self, @model_class, EMPTY_ARRAY)
@@ -433,8 +457,7 @@ class Factory
     @init_hash[KEY_LOCATOR] = locator
     @init_hash['leading_line_count'] = locator.leading_line_count
     @init_hash['leading_line_offset'] = locator.leading_line_offset
-    # Index is held in sublocator's parent locator - needed to be able to reconstruct
-    @init_hash['line_offsets'] = locator.locator.line_index
+    @init_hash['line_offsets'] = locator.line_index # index of lines in sublocated
   end
 
   def build_SelectorEntry(o, matching, value)
@@ -729,8 +752,6 @@ class Factory
 
   def self.STRING(*args);                new(ConcatenatedString, args);                  end
 
-  def self.SUBLOCATE(token, expr)        new(SubLocatedExpression, token, expr);         end
-
   def self.LIST(entries);                new(LiteralList, entries);                      end
 
   def self.PARAM(name, expr=nil);        new(Parameter, name, expr);                     end
@@ -765,6 +786,19 @@ class Factory
     o.instance_of?(Factory) && o.model_class <= QualifiedReference ? self : new(QualifiedReference, o)
   end
 
+  def self.SUBLOCATE(token, expr_factory)
+    # expr is a Factory wrapped LiteralString, or ConcatenatedString
+    # The token is SUBLOCATED token which has a SubLocator as the token's locator
+    # Use the SubLocator to recalculate the offsets and lengths.
+    model = expr_factory.model
+    locator = token.locator
+    expr_factory.map_offset(model, locator)
+    model._pcore_all_contents([]) { |element| expr_factory.map_offset(element, locator) }
+
+    # Returned the factory wrapping the now offset/length transformed expression(s)
+    expr_factory
+  end
+
   def self.TEXT(expr)
     new(TextExpression, infer(expr).interpolate)
   end

data/lib/puppet/pops/parser/egrammar.ra

@@ -857,8 +857,8 @@ heredoc
   : HEREDOC sublocated_text  { result = Factory.HEREDOC(val[0][:value], val[1]); loc result, val[0] }
 
 sublocated_text
-  : SUBLOCATE string    { result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] }
-  | SUBLOCATE dq_string { result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] }
+  : SUBLOCATE string    { result = Factory.SUBLOCATE(val[0], val[1]); }
+  | SUBLOCATE dq_string { result = Factory.SUBLOCATE(val[0], val[1]); }
 
 epp_expression
   : EPP_START epp_parameters_list optional_statements { result = Factory.EPP(val[1], val[2]); loc result, val[0] }

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -94,11 +94,12 @@ module HeredocSupport
 
 
         # Process captured lines - remove leading, and trailing newline
-        str = heredoc_text(lines, leading, has_margin, remove_break)
+        # get processed string and index of removed margin/leading size per line
+        str, margin_per_line = heredoc_text(lines, leading, has_margin, remove_break)
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Locator::SubLocator.new(locator, heredoc_line, heredoc_offset, leading.length())
+        locator = Locator::SubLocator.new(locator, str, heredoc_line, heredoc_offset, has_margin, margin_per_line)
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.
@@ -120,23 +121,32 @@ module HeredocSupport
     raise eof_error
   end
 
-  # Produces the heredoc text string given the individual (unprocessed) lines as an array.
+  # Produces the heredoc text string given the individual (unprocessed) lines as an array and array with margin sizes per line
   # @param lines [Array<String>] unprocessed lines of text in the heredoc w/o terminating line
   # @param leading [String] the leading text up (up to pipe or other terminating char)
   # @param has_margin [Boolean] if the left margin should be adjusted as indicated by `leading`
   # @param remove_break [Boolean] if the line break (\r?\n) at the end of the last line should be removed or not
+  # @return [Array] - a tuple with resulting string, and an array with margin size per line
   #
   def heredoc_text(lines, leading, has_margin, remove_break)
-    if has_margin
+    if has_margin && leading.length > 0
       leading_pattern = /^#{Regexp.escape(leading)}/
-      lines = lines.collect {|s| s.gsub(leading_pattern, '') }
+      # TODO: This implementation is not according to the specification, but is kept to be bug compatible.
+      # The specification says that leading space up to the margin marker should be removed, but this implementation
+      # simply leaves lines that have text in the margin untouched.
+      #
+      processed_lines = lines.collect {|s| s.gsub(leading_pattern, '') }
+      margin_per_line = processed_lines.length.times.map {|x| lines[x].length - processed_lines[x].length }
+      lines = processed_lines
+    else
+      # Array with a 0 per line
+      margin_per_line = Array.new(lines.length, 0)
     end
     result = lines.join('')
     result.gsub!(/\r?\n\z/m, '') if remove_break
-    result
+    [result, margin_per_line]
   end
 
-
 end
 end
 end

data/lib/puppet/pops/parser/lexer2.rb

@@ -189,7 +189,12 @@ class Lexer2
       ',' => lambda {  emit(TOKEN_COMMA, @scanner.pos) },
       '[' => lambda do
         before = @scanner.pos
-        if (before == 0 || locator.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
+        # Must check the preceding character to see if it is whitespace.
+        # The fastest thing to do is to simply byteslice to get the string ending at the offset before
+        # and then check what the last character is. (This is the same as  what an locator.char_offset needs
+        # to compute, but with less overhead of trying to find out the global offset from a local offset in the
+        # case when this is sublocated in a heredoc).
+        if before == 0 || @scanner.string.byteslice(0, before)[-1] =~ /[[:blank:]\r\n]+/
           emit(TOKEN_LISTSTART, before)
         else
           emit(TOKEN_LBRACK, before)

data/lib/puppet/pops/parser/locator.rb

@@ -71,6 +71,16 @@ class Locator
   def line_index()
   end
 
+  # Common byte based impl that works for all rubies (stringscanner is byte based
+  def self.compute_line_index(string)
+    scanner = StringScanner.new(string)
+    result = [0] # first line starts at 0
+    while scanner.scan_until(/\n/)
+      result << scanner.pos
+    end
+    result.freeze
+  end
+
   # Produces an URI with path?line=n&pos=n. If origin is unknown the URI is string:?line=n&pos=n
   def to_uri(ast)
     f = file
@@ -83,81 +93,8 @@ class Locator
     URI("#{f}?line=#{line_for_offset(offset).to_s}&pos=#{pos_on_line(offset).to_s}")
   end
 
-  # A Sublocator locates a concrete locator (subspace) in a virtual space.
-  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
-  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
-  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
-  #
-  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
-  #
-  #      1:XXXXXXXX
-  #      2:XXXX.... .. ... ..
-  #      3:XXXX. . .... ..
-  #      4:XXXX............
-  #
-  # This sublocator would be configured with leading_line_count = 1,
-  # leading_offset=8, and leading_line_offset=4
-  #
-  # Note that leading_offset must be the same for all lines and measured in characters.
-  #
-  class SubLocator < Locator
-    attr_reader :locator
-    attr_reader :leading_line_count
-    attr_reader :leading_offset
-    attr_reader :leading_line_offset
-
-    def self.sub_locator(string, file, leading_line_count, leading_offset, leading_line_offset)
-      self.new(Locator.locator(string, file),
-        leading_line_count,
-        leading_offset,
-        leading_line_offset)
-    end
-
-    def initialize(locator, leading_line_count, leading_offset, leading_line_offset)
-      @locator = locator
-      @leading_line_count = leading_line_count
-      @leading_offset = leading_offset
-      @leading_line_offset = leading_line_offset
-    end
-
-    def file
-      @locator.file
-    end
-
-    def string
-      @locator.string
-    end
-
-    # Given offset is offset in the subspace
-    def line_for_offset(offset)
-      @locator.line_for_offset(offset) + @leading_line_count
-    end
-
-    # Given offset is offset in the subspace
-    def offset_on_line(offset)
-      @locator.offset_on_line(offset) + @leading_line_offset
-    end
-
-    # Given offset is offset in the subspace
-    def char_offset(offset)
-      effective_line = @locator.line_for_offset(offset)
-      locator.char_offset(offset) + (effective_line * @leading_line_offset) + @leading_offset
-    end
-
-    # Given offsets are offsets in the subspace
-    def char_length(offset, end_offset)
-      effective_line = @locator.line_for_offset(end_offset) - @locator.line_for_offset(offset)
-      locator.char_length(offset, end_offset) + (effective_line * @leading_line_offset)
-    end
-
-    def pos_on_line(offset)
-      offset_on_line(offset) +1
-    end
-  end
-
   class AbstractLocator < Locator
     attr_accessor :line_index
-    attr_accessor :string
     attr_reader   :string
     attr_reader   :file
 
@@ -169,8 +106,7 @@ class Locator
       @file = file.freeze
       @prev_offset = nil
       @prev_line = nil
-      @line_index = line_index
-      compute_line_index if line_index.nil?
+      @line_index = line_index.nil? ? Locator.compute_line_index(@string) : line_index
     end
 
     # Returns the position on line (first position on a line is 1)
@@ -235,16 +171,6 @@ class Locator
       self.class == o.class && string == o.string && file == o.file && line_index == o.line_index
     end
 
-    # Common impl for 18 and 19 since scanner is byte based
-    def compute_line_index
-      scanner = StringScanner.new(string)
-      result = [0] # first line starts at 0
-      while scanner.scan_until(/\n/)
-        result << scanner.pos
-      end
-      self.line_index = result.freeze
-    end
-
     # Returns the line number (first line is 1) for the given offset
     def line_for_offset(offset)
       if @prev_offset == offset
@@ -264,6 +190,100 @@ class Locator
     end
   end
 
+  # A Sublocator locates a concrete locator (subspace) in a virtual space.
+  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
+  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
+  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
+  #
+  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
+  #
+  #      1:XXXXXXXX
+  #      2:XXXX.... .. ... ..
+  #      3:XXXX. . .... ..
+  #      4:XXXX............
+  #
+  # This sublocator would be configured with leading_line_count = 1,
+  # leading_offset=8, and leading_line_offset=4
+  #
+  # Note that leading_offset must be the same for all lines and measured in characters.
+  #
+  # A SubLocator is only used during parsing as the parser will translate the local offsets/lengths to
+  # the parent locator when a sublocated expression is reduced. Do not call the methods
+  # `char_offset` or `char_length` as those methods will raise an error.
+  #
+  class SubLocator < AbstractLocator
+    attr_reader :locator
+    attr_reader :leading_line_count
+    attr_reader :leading_offset
+    attr_reader :has_margin
+    attr_reader :margin_per_line
+
+    def initialize(locator, str, leading_line_count, leading_offset, has_margin, margin_per_line)
+      super(str, locator.file)
+      @locator = locator
+      @leading_line_count = leading_line_count
+      @leading_offset = leading_offset
+      @has_margin = has_margin
+      @margin_per_line = margin_per_line
+
+      # Since lines can have different margin - accumulated margin per line must be computed
+      # and since this accumulated margin adjustment is needed more than once; both for start offset,
+      # and for end offset (to compute global length) it is computed up front here.
+      # The accumulated_offset holds the sum of all removed margins before a position on line n (line index is 1-n,
+      # and (unused) position 0 is always 0).
+      # The last entry is duplicated since there will be  the line "after last line" that would otherwise require
+      # conditional logic.
+      #
+      @accumulated_margin = margin_per_line.reduce([0]) {|memo, val| memo << memo[-1] + val; memo }
+      @accumulated_margin << @accumulated_margin[-1]
+    end
+
+    def file
+      @locator.file
+    end
+
+    # Returns array with transposed (local) offset and (local) length. The transposed values
+    # take the margin into account such that it is added to the content to the right
+    # 
+    # Using X to denote margin and where end of line is explicitly shown as \n:
+    # ```
+    # XXXXabc\n
+    # XXXXdef\n
+    # ```
+    # A local offset of 0 is translated to the start of the first heredoc line, and a length of 1 is adjusted to
+    # 5 - i.e to cover "XXXXa". A local offset of 1, with length 1 would cover "b".
+    # A local offset of 4 and length 1 would cover "XXXXd"
+    #
+    # It is possible that lines have different margin and that is taken into account.
+    #
+    def to_global(offset, length)
+      # simple case, no margin
+      return [offset + @leading_offset, length] unless @has_margin
+
+      # compute local start and end line
+      start_line = line_for_offset(offset)
+      end_line = line_for_offset(offset+length)
+
+      # complex case when there is a margin
+      transposed_offset = offset == 0 ? @leading_offset : offset + @leading_offset + @accumulated_margin[start_line]
+      transposed_length = length +
+        @accumulated_margin[end_line] - @accumulated_margin[start_line] +    # the margins between start and end (0 is line 1)
+        (offset_on_line(offset) == 0 ? margin_per_line[start_line - 1] : 0)  # include start's margin in position 0
+      [transposed_offset, transposed_length]
+    end
+
+    # Do not call this method
+    def char_offset(offset)
+      raise "Should not be called"
+    end
+
+    # Do not call this method
+    def char_length(offset, end_offset)
+      raise "Should not be called"
+    end
+
+  end
+
   class LocatorForChars < AbstractLocator
 
     def offset_on_line(offset)
@@ -311,7 +331,7 @@ class Locator
     # Ruby 19 is multibyte but has no character position methods, must use byteslice
     def offset_on_line(offset)
       line_offset = line_index[ line_for_offset(offset)-1 ]
-      string.byteslice(line_offset, offset-line_offset).length
+      @string.byteslice(line_offset, offset-line_offset).length
     end
 
     # Returns the character offset for a given byte offset

data/lib/puppet/pops/parser/parser_support.rb

@@ -114,8 +114,17 @@ class Parser
     pos  = nil
     if token != 0
       file = value[:file]
-      line = value[:line]
-      pos  = value[:pos]
+      locator = value.locator
+      if locator.is_a?(Puppet::Pops::Parser::Locator::SubLocator)
+        # The error occurs when doing sub-parsing and the token must be transformed
+        # Transpose the local offset, length to global "coordinates"
+        global_offset, _ = locator.to_global(value.offset, value.length)
+        line = locator.locator.line_for_offset(global_offset)
+        pos = locator.locator.pos_on_line(global_offset)
+      else
+        line = value[:line]
+        pos  = value[:pos]
+      end
     else
       # At end of input, use what the lexer thinks is the source file
       file = lexer.file

data/lib/puppet/pops/types/type_mismatch_describer.rb

@@ -753,7 +753,7 @@ module Types
     end
 
     def describe_POptionalType(expected, original, actual, path)
-      return EMPTY_ARRAY if actual.is_a?(PUndefType)
+      return EMPTY_ARRAY if actual.is_a?(PUndefType) || expected.optional_type.nil?
       internal_describe(expected.optional_type, original.is_a?(PTypeAliasType) ? original : expected, actual, path)
     end
 

data/lib/puppet/provider/file/windows.rb

@@ -71,7 +71,9 @@ Puppet::Type.type(:file).provide :windows do
 
   def mode=(value)
     begin
-      set_mode(value.to_i(8), resource[:path])
+      managing_owner = !resource[:owner].nil?
+      managing_group = !resource[:group].nil?
+      set_mode(value.to_i(8), resource[:path], true, managing_owner, managing_group)
     rescue => detail
       error = Puppet::Error.new(_("failed to set mode %{mode} on %{path}: %{message}") % { mode: mode, path: resource[:path], message: detail.message })
       error.set_backtrace detail.backtrace
@@ -86,6 +88,52 @@ Puppet::Type.type(:file).provide :windows do
     end
   end
 
+  # munge the windows group permissions if the user or group are set to SYSTEM
+  #
+  # when SYSTEM user is the group or user and the resoure is not managing them then treat
+  # the resource as insync if System has FullControl access.
+  #
+  # @param [String] current - the current mode returned by the resource
+  # @param [String] should  - what the mode should be
+  #
+  # @return [String, nil] munged mode or nil if the resource should be out of sync
+  def munge_windows_system_group(current, should)
+    [
+      {
+        'type'        => 'group',
+        'resource'    => resource[:group],
+        'set_to_user' => group,
+        'fullcontrol' => "070".to_i(8),
+        'remove_mask' => "707".to_i(8),
+        'should_mask' => (should[0].to_i(8) & "070".to_i(8)),
+      },
+      {
+        'type'        => 'owner',
+        'resource'    => resource[:owner],
+        'set_to_user' => owner,
+        'fullcontrol' => "700".to_i(8),
+        'remove_mask' => "077".to_i(8),
+        'should_mask' => (should[0].to_i(8) & "700".to_i(8)),
+      }
+    ].each do |mode_part|
+      if mode_part['resource'].nil? && (mode_part['set_to_user'] == Puppet::Util::Windows::SID::LocalSystem)
+        if (current.to_i(8) & mode_part['fullcontrol']) == mode_part['fullcontrol']
+          # Since the group is LocalSystem, and the permissions are FullControl,
+          # replace the value returned with the value expected. This will treat
+          # this specific situation as "insync"
+          current = ( (current.to_i(8) & mode_part['remove_mask']) | mode_part['should_mask'] ).to_s(8).rjust(4, '0')
+        else
+          # If the SYSTEM account does _not_ have FullControl in this scenario, we should
+          # force the resource out of sync no matter what.
+          #TRANSLATORS 'SYSTEM' is a Windows name and should not be translated
+          Puppet.debug _("%{resource_name}: %{mode_part_type} set to SYSTEM. SYSTEM permissions cannot be set below FullControl ('7')") % { resource_name: resource[:name], mode_part_type: mode_part['type']}
+          return nil
+        end
+      end
+    end
+    current
+  end
+
   attr_reader :file
   private
   def file