puppet 6.0.10 → 6.1.0

data/lib/puppet/pops/model/pn_transformer.rb

@@ -200,11 +200,6 @@ class PNTransformer
     PN::Call.new('nop')
   end
 
-  # Some elements may have a nil element instead of a Nop Expression
-  def transform_NilClass(e)
-    PN::Call.new('nop')
-  end
-
   def transform_NotExpression(e)
     PN::Call.new('!', transform(e.expr))
   end

data/lib/puppet/pops/parser/egrammar.ra

@@ -857,8 +857,8 @@ heredoc
   : HEREDOC sublocated_text  { result = Factory.HEREDOC(val[0][:value], val[1]); loc result, val[0] }
 
 sublocated_text
-  : SUBLOCATE string    { result = Factory.SUBLOCATE(val[0], val[1]); }
-  | SUBLOCATE dq_string { result = Factory.SUBLOCATE(val[0], val[1]); }
+  : SUBLOCATE string    { result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] }
+  | SUBLOCATE dq_string { result = Factory.SUBLOCATE(val[0], val[1]); loc result, val[0] }
 
 epp_expression
   : EPP_START epp_parameters_list optional_statements { result = Factory.EPP(val[1], val[2]); loc result, val[0] }

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -94,12 +94,11 @@ module HeredocSupport
 
 
         # Process captured lines - remove leading, and trailing newline
-        # get processed string and index of removed margin/leading size per line
-        str, margin_per_line = heredoc_text(lines, leading, has_margin, remove_break)
+        str = heredoc_text(lines, leading, has_margin, remove_break)
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Locator::SubLocator.new(locator, str, heredoc_line, heredoc_offset, has_margin, margin_per_line)
+        locator = Locator::SubLocator.new(locator, heredoc_line, heredoc_offset, leading.length())
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.
@@ -121,32 +120,23 @@ module HeredocSupport
     raise eof_error
   end
 
-  # Produces the heredoc text string given the individual (unprocessed) lines as an array and array with margin sizes per line
+  # Produces the heredoc text string given the individual (unprocessed) lines as an array.
   # @param lines [Array<String>] unprocessed lines of text in the heredoc w/o terminating line
   # @param leading [String] the leading text up (up to pipe or other terminating char)
   # @param has_margin [Boolean] if the left margin should be adjusted as indicated by `leading`
   # @param remove_break [Boolean] if the line break (\r?\n) at the end of the last line should be removed or not
-  # @return [Array] - a tuple with resulting string, and an array with margin size per line
   #
   def heredoc_text(lines, leading, has_margin, remove_break)
-    if has_margin && leading.length > 0
+    if has_margin
       leading_pattern = /^#{Regexp.escape(leading)}/
-      # TODO: This implementation is not according to the specification, but is kept to be bug compatible.
-      # The specification says that leading space up to the margin marker should be removed, but this implementation
-      # simply leaves lines that have text in the margin untouched.
-      #
-      processed_lines = lines.collect {|s| s.gsub(leading_pattern, '') }
-      margin_per_line = processed_lines.length.times.map {|x| lines[x].length - processed_lines[x].length }
-      lines = processed_lines
-    else
-      # Array with a 0 per line
-      margin_per_line = Array.new(lines.length, 0)
+      lines = lines.collect {|s| s.gsub(leading_pattern, '') }
     end
     result = lines.join('')
     result.gsub!(/\r?\n\z/m, '') if remove_break
-    [result, margin_per_line]
+    result
   end
 
+
 end
 end
 end

data/lib/puppet/pops/parser/lexer2.rb

@@ -189,12 +189,7 @@ class Lexer2
       ',' => lambda {  emit(TOKEN_COMMA, @scanner.pos) },
       '[' => lambda do
         before = @scanner.pos
-        # Must check the preceding character to see if it is whitespace.
-        # The fastest thing to do is to simply byteslice to get the string ending at the offset before
-        # and then check what the last character is. (This is the same as  what an locator.char_offset needs
-        # to compute, but with less overhead of trying to find out the global offset from a local offset in the
-        # case when this is sublocated in a heredoc).
-        if before == 0 || @scanner.string.byteslice(0, before)[-1] =~ /[[:blank:]\r\n]+/
+        if (before == 0 || locator.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
           emit(TOKEN_LISTSTART, before)
         else
           emit(TOKEN_LBRACK, before)

data/lib/puppet/pops/parser/locator.rb

@@ -71,16 +71,6 @@ class Locator
   def line_index()
   end
 
-  # Common byte based impl that works for all rubies (stringscanner is byte based
-  def self.compute_line_index(string)
-    scanner = StringScanner.new(string)
-    result = [0] # first line starts at 0
-    while scanner.scan_until(/\n/)
-      result << scanner.pos
-    end
-    result.freeze
-  end
-
   # Produces an URI with path?line=n&pos=n. If origin is unknown the URI is string:?line=n&pos=n
   def to_uri(ast)
     f = file
@@ -93,8 +83,81 @@ class Locator
     URI("#{f}?line=#{line_for_offset(offset).to_s}&pos=#{pos_on_line(offset).to_s}")
   end
 
+  # A Sublocator locates a concrete locator (subspace) in a virtual space.
+  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
+  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
+  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
+  #
+  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
+  #
+  #      1:XXXXXXXX
+  #      2:XXXX.... .. ... ..
+  #      3:XXXX. . .... ..
+  #      4:XXXX............
+  #
+  # This sublocator would be configured with leading_line_count = 1,
+  # leading_offset=8, and leading_line_offset=4
+  #
+  # Note that leading_offset must be the same for all lines and measured in characters.
+  #
+  class SubLocator < Locator
+    attr_reader :locator
+    attr_reader :leading_line_count
+    attr_reader :leading_offset
+    attr_reader :leading_line_offset
+
+    def self.sub_locator(string, file, leading_line_count, leading_offset, leading_line_offset)
+      self.new(Locator.locator(string, file),
+        leading_line_count,
+        leading_offset,
+        leading_line_offset)
+    end
+
+    def initialize(locator, leading_line_count, leading_offset, leading_line_offset)
+      @locator = locator
+      @leading_line_count = leading_line_count
+      @leading_offset = leading_offset
+      @leading_line_offset = leading_line_offset
+    end
+
+    def file
+      @locator.file
+    end
+
+    def string
+      @locator.string
+    end
+
+    # Given offset is offset in the subspace
+    def line_for_offset(offset)
+      @locator.line_for_offset(offset) + @leading_line_count
+    end
+
+    # Given offset is offset in the subspace
+    def offset_on_line(offset)
+      @locator.offset_on_line(offset) + @leading_line_offset
+    end
+
+    # Given offset is offset in the subspace
+    def char_offset(offset)
+      effective_line = @locator.line_for_offset(offset)
+      locator.char_offset(offset) + (effective_line * @leading_line_offset) + @leading_offset
+    end
+
+    # Given offsets are offsets in the subspace
+    def char_length(offset, end_offset)
+      effective_line = @locator.line_for_offset(end_offset) - @locator.line_for_offset(offset)
+      locator.char_length(offset, end_offset) + (effective_line * @leading_line_offset)
+    end
+
+    def pos_on_line(offset)
+      offset_on_line(offset) +1
+    end
+  end
+
   class AbstractLocator < Locator
     attr_accessor :line_index
+    attr_accessor :string
     attr_reader   :string
     attr_reader   :file
 
@@ -106,7 +169,8 @@ class Locator
       @file = file.freeze
       @prev_offset = nil
       @prev_line = nil
-      @line_index = line_index.nil? ? Locator.compute_line_index(@string) : line_index
+      @line_index = line_index
+      compute_line_index if line_index.nil?
     end
 
     # Returns the position on line (first position on a line is 1)
@@ -171,6 +235,16 @@ class Locator
       self.class == o.class && string == o.string && file == o.file && line_index == o.line_index
     end
 
+    # Common impl for 18 and 19 since scanner is byte based
+    def compute_line_index
+      scanner = StringScanner.new(string)
+      result = [0] # first line starts at 0
+      while scanner.scan_until(/\n/)
+        result << scanner.pos
+      end
+      self.line_index = result.freeze
+    end
+
     # Returns the line number (first line is 1) for the given offset
     def line_for_offset(offset)
       if @prev_offset == offset
@@ -190,100 +264,6 @@ class Locator
     end
   end
 
-  # A Sublocator locates a concrete locator (subspace) in a virtual space.
-  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
-  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
-  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
-  #
-  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
-  #
-  #      1:XXXXXXXX
-  #      2:XXXX.... .. ... ..
-  #      3:XXXX. . .... ..
-  #      4:XXXX............
-  #
-  # This sublocator would be configured with leading_line_count = 1,
-  # leading_offset=8, and leading_line_offset=4
-  #
-  # Note that leading_offset must be the same for all lines and measured in characters.
-  #
-  # A SubLocator is only used during parsing as the parser will translate the local offsets/lengths to
-  # the parent locator when a sublocated expression is reduced. Do not call the methods
-  # `char_offset` or `char_length` as those methods will raise an error.
-  #
-  class SubLocator < AbstractLocator
-    attr_reader :locator
-    attr_reader :leading_line_count
-    attr_reader :leading_offset
-    attr_reader :has_margin
-    attr_reader :margin_per_line
-
-    def initialize(locator, str, leading_line_count, leading_offset, has_margin, margin_per_line)
-      super(str, locator.file)
-      @locator = locator
-      @leading_line_count = leading_line_count
-      @leading_offset = leading_offset
-      @has_margin = has_margin
-      @margin_per_line = margin_per_line
-
-      # Since lines can have different margin - accumulated margin per line must be computed
-      # and since this accumulated margin adjustment is needed more than once; both for start offset,
-      # and for end offset (to compute global length) it is computed up front here.
-      # The accumulated_offset holds the sum of all removed margins before a position on line n (line index is 1-n,
-      # and (unused) position 0 is always 0).
-      # The last entry is duplicated since there will be  the line "after last line" that would otherwise require
-      # conditional logic.
-      #
-      @accumulated_margin = margin_per_line.reduce([0]) {|memo, val| memo << memo[-1] + val; memo }
-      @accumulated_margin << @accumulated_margin[-1]
-    end
-
-    def file
-      @locator.file
-    end
-
-    # Returns array with transposed (local) offset and (local) length. The transposed values
-    # take the margin into account such that it is added to the content to the right
-    # 
-    # Using X to denote margin and where end of line is explicitly shown as \n:
-    # ```
-    # XXXXabc\n
-    # XXXXdef\n
-    # ```
-    # A local offset of 0 is translated to the start of the first heredoc line, and a length of 1 is adjusted to
-    # 5 - i.e to cover "XXXXa". A local offset of 1, with length 1 would cover "b".
-    # A local offset of 4 and length 1 would cover "XXXXd"
-    #
-    # It is possible that lines have different margin and that is taken into account.
-    #
-    def to_global(offset, length)
-      # simple case, no margin
-      return [offset + @leading_offset, length] unless @has_margin
-
-      # compute local start and end line
-      start_line = line_for_offset(offset)
-      end_line = line_for_offset(offset+length)
-
-      # complex case when there is a margin
-      transposed_offset = offset == 0 ? @leading_offset : offset + @leading_offset + @accumulated_margin[start_line]
-      transposed_length = length +
-        @accumulated_margin[end_line] - @accumulated_margin[start_line] +    # the margins between start and end (0 is line 1)
-        (offset_on_line(offset) == 0 ? margin_per_line[start_line - 1] : 0)  # include start's margin in position 0
-      [transposed_offset, transposed_length]
-    end
-
-    # Do not call this method
-    def char_offset(offset)
-      raise "Should not be called"
-    end
-
-    # Do not call this method
-    def char_length(offset, end_offset)
-      raise "Should not be called"
-    end
-
-  end
-
   class LocatorForChars < AbstractLocator
 
     def offset_on_line(offset)
@@ -331,7 +311,7 @@ class Locator
     # Ruby 19 is multibyte but has no character position methods, must use byteslice
     def offset_on_line(offset)
       line_offset = line_index[ line_for_offset(offset)-1 ]
-      @string.byteslice(line_offset, offset-line_offset).length
+      string.byteslice(line_offset, offset-line_offset).length
     end
 
     # Returns the character offset for a given byte offset

data/lib/puppet/pops/parser/parser_support.rb

@@ -114,17 +114,8 @@ class Parser
     pos  = nil
     if token != 0
       file = value[:file]
-      locator = value.locator
-      if locator.is_a?(Puppet::Pops::Parser::Locator::SubLocator)
-        # The error occurs when doing sub-parsing and the token must be transformed
-        # Transpose the local offset, length to global "coordinates"
-        global_offset, _ = locator.to_global(value.offset, value.length)
-        line = locator.locator.line_for_offset(global_offset)
-        pos = locator.locator.pos_on_line(global_offset)
-      else
-        line = value[:line]
-        pos  = value[:pos]
-      end
+      line = value[:line]
+      pos  = value[:pos]
     else
       # At end of input, use what the lexer thinks is the source file
       file = lexer.file

data/lib/puppet/pops/types/string_converter.rb

@@ -813,7 +813,7 @@ class StringConverter
       Kernel.format(f.orig_fmt, val)
 
     when :p
-      apply_string_flags(f, puppet_quote(val))
+      apply_string_flags(f, puppet_quote(val, f.alt?))
 
     when :c
       c_val = val.capitalize
@@ -845,10 +845,15 @@ class StringConverter
   # strings will be quoted using double quotes.
   #
   # @param [String] str the string that should be formatted
+  # @param [Boolean] enforce_double_quotes if true the result will be double quoted (even if single quotes would be possible)
   # @return [String] the formatted string
   #
   # @api public
-  def puppet_quote(str)
+  def puppet_quote(str, enforce_double_quotes = false)
+    if enforce_double_quotes
+      return puppet_double_quote(str)
+    end
+
     # Assume that the string can be single quoted
     bld = '\''
     bld.force_encoding(str.encoding)

data/lib/puppet/pops/types/type_mismatch_describer.rb

@@ -753,7 +753,7 @@ module Types
     end
 
     def describe_POptionalType(expected, original, actual, path)
-      return EMPTY_ARRAY if actual.is_a?(PUndefType) || expected.optional_type.nil?
+      return EMPTY_ARRAY if actual.is_a?(PUndefType)
       internal_describe(expected.optional_type, original.is_a?(PTypeAliasType) ? original : expected, actual, path)
     end
 

data/lib/puppet/pops/types/types.rb

@@ -843,16 +843,14 @@ INTEGER_HEX = '(?:0[xX][0-9A-Fa-f]+)'
 INTEGER_OCT = '(?:0[0-7]+)'
 INTEGER_BIN = '(?:0[bB][01]+)'
 INTEGER_DEC = '(?:0|[1-9]\d*)'
-INTEGER_DEC_OR_OCT = '(?:\d+)'
 SIGN_PREFIX = '[+-]?\s*'
 
 OPTIONAL_FRACTION = '(?:\.\d+)?'
 OPTIONAL_EXPONENT = '(?:[eE]-?\d+)?'
 FLOAT_DEC = '(?:' + INTEGER_DEC + OPTIONAL_FRACTION + OPTIONAL_EXPONENT + ')'
 
-INTEGER_PATTERN          = '\A' + SIGN_PREFIX + '(?:' + INTEGER_DEC + '|' + INTEGER_HEX + '|' + INTEGER_OCT + '|' + INTEGER_BIN + ')\z'
-INTEGER_PATTERN_LENIENT = '\A' + SIGN_PREFIX + '(?:' + INTEGER_DEC_OR_OCT + '|' + INTEGER_HEX + '|' + INTEGER_BIN + ')\z'
-FLOAT_PATTERN            = '\A' + SIGN_PREFIX + '(?:' + FLOAT_DEC + '|' + INTEGER_HEX + '|' + INTEGER_OCT + '|' + INTEGER_BIN + ')\z'
+INTEGER_PATTERN = '\A' + SIGN_PREFIX + '(?:' + INTEGER_DEC + '|' + INTEGER_HEX + '|' + INTEGER_OCT + '|' + INTEGER_BIN + ')\z'
+FLOAT_PATTERN = '\A' + SIGN_PREFIX + '(?:' + FLOAT_DEC + '|' + INTEGER_HEX + '|' + INTEGER_OCT + '|' + INTEGER_BIN + ')\z'
 
 # @api public
 #
@@ -1091,7 +1089,7 @@ class PIntegerType < PNumericType
     @@new_function ||= Puppet::Functions.create_loaded_function(:new, loader) do
       local_types do
         type 'Radix       = Variant[Default, Integer[2,2], Integer[8,8], Integer[10,10], Integer[16,16]]'
-        type "Convertible = Variant[Numeric, Boolean, Pattern[/#{INTEGER_PATTERN_LENIENT}/], Timespan, Timestamp]"
+        type "Convertible = Variant[Numeric, Boolean, Pattern[/#{INTEGER_PATTERN}/], Timespan, Timestamp]"
         type 'NamedArgs   = Struct[{from => Convertible, Optional[radix] => Radix, Optional[abs] => Boolean}]'
       end
 

data/lib/puppet/pops/validation/checker4_0.rb

@@ -1,5 +1,8 @@
+require 'puppet/pops/evaluator/external_syntax_support'
+
 module Puppet::Pops
 module Validation
+
 # A Validator validates a model.
 #
 # Validation is performed on each model element in isolation. Each method should validate the model element's state
@@ -12,6 +15,8 @@ module Validation
 #       This is however mostly valuable when validating model to model transformations, and is therefore T.B.D
 #
 class Checker4_0 < Evaluator::LiteralEvaluator
+  include Puppet::Pops::Evaluator::ExternalSyntaxSupport
+
   attr_reader :acceptor
   attr_reader :migration_checker
 
@@ -324,6 +329,14 @@ class Checker4_0 < Evaluator::LiteralEvaluator
     end
   end
 
+  def check_HeredocExpression(o)
+    # Only check static text in heredoc
+    expr = o.text_expr.expr # Always a SublocatedExpression with an expression
+    if expr.is_a?(Model::LiteralString)
+      assert_external_syntax(nil, expr.value, o.syntax, o.text_expr)
+    end
+  end
+
   def check_MethodCallExpression(o)
     unless o.functor_expr.is_a? Model::QualifiedName
       acceptor.accept(Issues::ILLEGAL_EXPRESSION, o.functor_expr, :feature => 'function name', :container => o)

data/lib/puppet/provider.rb

@@ -395,7 +395,7 @@ class Puppet::Provider
   # @raise [Puppet::DevError] Error indicating that the method should have been implemented by subclass.
   # @see prefetch
   def self.instances
-    raise Puppet::DevError, _("To support listing resources of this type the '%{provider}' provider needs to implement an 'instances' class method returning the current set of resources. We recommend porting your module to the simpler Resource API instead: https://puppet.com/search/docs?keys=resource+api") % { provider: self.name }
+    raise Puppet::DevError, _("Provider %{provider} has not defined the 'instances' class method") % { provider: self.name }
   end
 
   # Creates getter- and setter- methods for each property supported by the resource type.
@@ -621,3 +621,4 @@ class Puppet::Provider
   # @return [void]
   # @api public
 end
+