puppet 4.8.2 → 4.9.0

data/lib/puppet/pops/parser/locator.rb

@@ -143,13 +143,13 @@ class Locator
     # Create a locator based on a content string, and a boolean indicating if ruby version support multi-byte strings
     # or not.
     #
-    def initialize(string, file, index = nil)
+    def initialize(string, file, line_index = nil)
       @string = string.freeze
       @file = file.freeze
       @prev_offset = nil
       @prev_line = nil
-      @line_index = index
-      compute_line_index if index.nil?
+      @line_index = line_index
+      compute_line_index if line_index.nil?
     end
 
     # Returns the position on line (first position on a line is 1)

data/lib/puppet/pops/parser/slurp_support.rb

@@ -76,9 +76,10 @@ module SlurpSupport
     # Process unicode escapes first as they require getting 4 hex digits
     # If later a \u is found it is warned not to be a unicode escape
     if escapes.include?('u')
-      str.gsub!(/\\u(?:([\da-fA-F]{4})|\{([\da-fA-F]{1,6})\})/m) {
-        [($1 || $2).hex].pack("U")
-      }
+      # gsub must be repeated to cater for adjacent escapes
+      while(str.gsub!(/((?:[^\\]|^)(?:[\\]{2})*)\\u(?:([\da-fA-F]{4})|\{([\da-fA-F]{1,6})\})/m) { $1 + [($2 || $3).hex].pack("U") })
+        # empty block. Everything happens in the gsub block
+      end
     end
 
     begin

data/lib/puppet/pops/pcore.rb

@@ -4,6 +4,7 @@ module Puppet::Pops
 module Pcore
   TYPE_URI_RX = Types::TypeFactory.regexp(URI.regexp)
   TYPE_URI = Types::TypeFactory.pattern(TYPE_URI_RX)
+  TYPE_URI_ALIAS = Types::PTypeAliasType.new('Pcore::URI', nil, TYPE_URI)
   TYPE_SIMPLE_TYPE_NAME = Types::TypeFactory.pattern(/\A[A-Z]\w*\z/)
   TYPE_QUALIFIED_REFERENCE = Types::TypeFactory.pattern(/\A[A-Z][\w]*(?:::[A-Z][\w]*)*\z/)
 
@@ -11,30 +12,36 @@ module Pcore
   KEY_PCORE_VERSION = 'pcore_version'.freeze
 
   PCORE_URI = 'http://puppet.com/2016.1/pcore'
-  PCORE_VERSION = Semantic::Version.new(1,0,0)
-  PARSABLE_PCORE_VERSIONS = Semantic::VersionRange.parse('1.x')
+  PCORE_VERSION = SemanticPuppet::Version.new(1,0,0)
+  PARSABLE_PCORE_VERSIONS = SemanticPuppet::VersionRange.parse('1.x')
 
   RUNTIME_NAME_AUTHORITY = 'http://puppet.com/2016.1/runtime'
 
-  def self.init(loader, ir)
+  def self.init(loader, ir, for_agent)
     add_alias('Pcore::URI_RX', TYPE_URI_RX, loader)
-    add_alias('Pcore::URI', TYPE_URI, loader)
+    add_type(TYPE_URI_ALIAS, loader)
     add_alias('Pcore::SimpleTypeName', TYPE_SIMPLE_TYPE_NAME, loader)
     add_alias('Pcore::TypeName', TYPE_QUALIFIED_REFERENCE, loader)
     add_alias('Pcore::QRef', TYPE_QUALIFIED_REFERENCE, loader)
+    begin
     Types::TypedModelObject.register_ptypes(loader, ir)
+    rescue Exception => e
+      puts e.message
+    end
 
     ir.register_implementation_namespace('Pcore', 'Puppet::Pops::Pcore', loader)
-    ir.register_implementation_namespace('Puppet::AST', 'Puppet::Pops::Model', loader)
-    ast_type_set = Serialization::RGen::TypeGenerator.new.generate_type_set('Puppet::AST', Puppet::Pops::Model, loader)
+    unless for_agent
+      ir.register_implementation_namespace('Puppet::AST', 'Puppet::Pops::Model', loader)
+      ast_type_set = Serialization::RGen::TypeGenerator.new.generate_type_set('Puppet::AST', Puppet::Pops::Model, loader)
 
-    # Extend the Puppet::AST type set with the Locator (it's not an RGen class, but nevertheless, used in the model)
-    ast_ts_i12n = ast_type_set.i12n_hash
-    ast_ts_i12n['types'] = ast_ts_i12n['types'].merge('Locator' => Parser::Locator::Locator19.register_ptype(loader, ir))
-    add_type(Types::PTypeSetType.new(ast_ts_i12n), loader)
+      # Extend the Puppet::AST type set with the Locator (it's not an RGen class, but nevertheless, used in the model)
+      ast_ts_i12n = ast_type_set.i12n_hash
+      ast_ts_i12n['types'] = ast_ts_i12n['types'].merge('Locator' => Parser::Locator::Locator19.register_ptype(loader, ir))
+      add_type(Types::PTypeSetType.new(ast_ts_i12n), loader)
 
-    Resource.register_ptypes(loader, ir)
-    Lookup::Context.register_ptype(loader, ir);
+      Resource.register_ptypes(loader, ir)
+      Lookup::Context.register_ptype(loader, ir);
+    end
   end
 
   # Create and register a new `Object` type in the Puppet Type System and map it to an implementation class
@@ -82,6 +89,8 @@ module Pcore
     aliases.each do |name, type_string|
       add_type(Types::PTypeAliasType.new(name, Types::TypeFactory.type_reference(type_string), nil), loader, name_authority)
     end
+    parser = Types::TypeParser.singleton
+    aliases.each_key.map { |name| loader.load(:type, name.downcase).resolve(parser, loader) }
   end
 end
 end

data/lib/puppet/pops/serialization/abstract_reader.rb

@@ -105,8 +105,12 @@ class AbstractReader
       read_payload(data) { |ep| Extension::MapStart.new(ep.read) }
     end
 
+    register_type(Extension::PCORE_OBJECT_START) do |data|
+      read_payload(data) { |ep| type_name = read_tpl_qname(ep); Extension::PcoreObjectStart.new(type_name, ep.read) }
+    end
+
     register_type(Extension::OBJECT_START) do |data|
-      read_payload(data) { |ep| type_name = read_tpl_qname(ep); Extension::ObjectStart.new(type_name, ep.read) }
+      read_payload(data) { |ep| Extension::ObjectStart.new(ep.read) }
     end
 
     register_type(Extension::DEFAULT) do |data|
@@ -117,12 +121,16 @@ class AbstractReader
       read_payload(data) { |ep| Extension::Comment.new(ep.read) }
     end
 
+    register_type(Extension::SENSITIVE_START) do |data|
+      read_payload(data) { |ep| Extension::SensitiveStart::INSTANCE }
+    end
+
     register_type(Extension::REGEXP) do |data|
       read_payload(data) { |ep| Regexp.new(ep.read) }
     end
 
     register_type(Extension::TYPE_REFERENCE) do |data|
-      read_payload(data) { |ep| Types::PTypeReferenceType.new(read_tpl_qname(ep)) }
+      read_payload(data) { |ep| Types::PTypeReferenceType.new(ep.read) }
     end
 
     register_type(Extension::SYMBOL) do |data|
@@ -146,19 +154,21 @@ class AbstractReader
     end
 
     register_type(Extension::VERSION) do |data|
-      read_payload(data) { |ep| Semantic::Version.parse(ep.read) }
+      read_payload(data) { |ep| SemanticPuppet::Version.parse(ep.read) }
     end
 
     register_type(Extension::VERSION_RANGE) do |data|
-      read_payload(data) { |ep| Semantic::VersionRange.parse(ep.read) }
+      read_payload(data) { |ep| SemanticPuppet::VersionRange.parse(ep.read) }
     end
 
-    register_type(Extension::SENSITIVE) do |data|
-      read_payload(data) { |ep| Types::PSensitiveType::Sensitive.new(ep.read) }
+    register_type(Extension::BASE64) do |data|
+      read_payload(data) { |ep| Types::PBinaryType::Binary.from_base64_strict(ep.read) }
     end
 
     register_type(Extension::BINARY) do |data|
-      read_payload(data) { |ep| Types::PBinaryType::Binary.from_base64_strict(ep.read) }
+      # The Ruby MessagePack implementation have special treatment for "ASCII-8BIT" strings. They
+      # are written as binary data.
+      read_payload(data) { |ep| Types::PBinaryType::Binary.new(ep.read) }
     end
   end
 end

data/lib/puppet/pops/serialization/abstract_writer.rb

@@ -45,6 +45,10 @@ class AbstractWriter
     @packer.flush
   end
 
+  def supports_binary?
+    false
+  end
+
   # Write a value on the underlying stream
   # @api public
   def write(value)
@@ -53,7 +57,7 @@ class AbstractWriter
     when Integer
       # not tabulated, but integers larger than 64-bit cannot be allowed.
       raise SerializationError, 'Integer out of bounds' if value > MAX_INTEGER || value < MIN_INTEGER
-    when Numeric, Symbol, Types::PSensitiveType::Sensitive, Extension::NotTabulated, true, false, nil
+    when Numeric, Symbol, Extension::NotTabulated, true, false, nil
       # not tabulated
     else
       if @tabulate
@@ -134,10 +138,14 @@ class AbstractWriter
       build_payload { |ep| ep.write(o.size) }
     end
 
-    register_type(Extension::OBJECT_START, Extension::ObjectStart) do |o|
+    register_type(Extension::PCORE_OBJECT_START, Extension::PcoreObjectStart) do |o|
       build_payload { |ep| write_tpl_qname(ep, o.type_name); ep.write(o.attribute_count) }
     end
 
+    register_type(Extension::OBJECT_START, Extension::ObjectStart) do |o|
+      build_payload { |ep| ep.write(o.attribute_count) }
+    end
+
     # 0x20 - 0x2f reserved for special extension objects
 
     register_type(Extension::DEFAULT, Extension::Default) do |o|
@@ -148,6 +156,10 @@ class AbstractWriter
       build_payload { |ep| ep.write(o.comment) }
     end
 
+    register_type(Extension::SENSITIVE_START, Extension::SensitiveStart) do |o|
+      build_payload { |ep| }
+    end
+
     # 0x30 - 0x7f reserved for mapping of specific runtime classes
 
     register_type(Extension::REGEXP, Regexp) do |o|
@@ -155,7 +167,7 @@ class AbstractWriter
     end
 
     register_type(Extension::TYPE_REFERENCE, Types::PTypeReferenceType) do |o|
-      build_payload { |ep| write_tpl_qname(ep, o.type_string) }
+      build_payload { |ep| ep.write(o.type_string) }
     end
 
     register_type(Extension::SYMBOL, Symbol) do |o|
@@ -170,23 +182,26 @@ class AbstractWriter
       build_payload { |ep| nsecs = o.nsecs; ep.write(nsecs / 1000000000); ep.write(nsecs % 1000000000) }
     end
 
-    register_type(Extension::VERSION, Semantic::Version) do |o|
+    register_type(Extension::VERSION, SemanticPuppet::Version) do |o|
       build_payload { |ep| ep.write(o.to_s) }
     end
 
-    register_type(Extension::VERSION_RANGE, Semantic::VersionRange) do |o|
+    register_type(Extension::VERSION_RANGE, SemanticPuppet::VersionRange) do |o|
       build_payload { |ep| ep.write(o.to_s) }
     end
 
-    register_type(Extension::SENSITIVE, Types::PSensitiveType::Sensitive) do |o|
-      build_payload { |ep| ep.write(o.unwrap) }
-    end
-
-    register_type(Extension::BINARY, Types::PBinaryType::Binary) do |o|
-      build_payload { |ep| ep.write(o.to_s) }
+    if supports_binary?
+      register_type(Extension::BINARY, Types::PBinaryType::Binary) do |o|
+        # The Ruby MessagePack implementation has special treatment for "ASCII-8BIT" strings. They
+        # are written as binary data.
+        build_payload { |ep| ep.write(o.binary_buffer) }
+      end
+    else
+      register_type(Extension::BASE64, Types::PBinaryType::Binary) do |o|
+        build_payload { |ep| ep.write(o.to_s) }
+      end
     end
   end
 end
 end
 end
-

data/lib/puppet/pops/serialization/deserializer.rb

@@ -8,7 +8,7 @@ module Serialization
   class Deserializer
     # Provides access to the reader.
     # @api private
-    attr_reader :reader
+    attr_reader :reader, :loader
 
     # @param [AbstractReader] reader the reader used when reading primitive objects from a stream
     # @param [Loader::Loader] loader the loader used when resolving names of types
@@ -38,12 +38,25 @@ module Serialization
         result = remember({})
         val.size.times { key = read; result[key] = read }
         result
-      when Extension::ObjectStart
+      when Extension::SensitiveStart
+        Types::PSensitiveType::Sensitive.new(read)
+      when Extension::PcoreObjectStart
         type_name = val.type_name
         type = Types::TypeParser.singleton.parse(type_name, @loader)
         raise SerializationError, "No implementation mapping found for Puppet Type #{type_name}" if type.is_a?(Types::PTypeReferenceType)
-        type.read(val.attribute_count, self)
-      when Numeric, String, true, false, nil, Time
+        result = type.read(val.attribute_count, self)
+        if result.is_a?(Types::PObjectType)
+          existing_type = loader.load(:type, result.name)
+
+          # Add result to the loader unless it is the exact same instance as the existing_type. The add
+          # will succeed when the existing_type is nil.
+          loader.add_entry(:type, result.name, result, nil) unless result.equal?(existing_type)
+        end
+        result
+      when Extension::ObjectStart
+        type = read
+        type.read(val.attribute_count - 1, self)
+      when Numeric, String, true, false, nil
         val
       else
         remember(val)

data/lib/puppet/pops/serialization/extension.rb

@@ -12,7 +12,9 @@ module Extension
   # 0x10 - 0x1F are reserved for structural extensions
   ARRAY_START = 0x10
   MAP_START = 0x11
-  OBJECT_START = 0x12
+  PCORE_OBJECT_START = 0x12
+  OBJECT_START = 0x13
+  SENSITIVE_START = 0x14
 
   # 0x20 - 0x2f reserved for special extension objects
   DEFAULT = 0x20
@@ -26,13 +28,13 @@ module Extension
   TIMESPAN = 0x34
   VERSION = 0x35
   VERSION_RANGE = 0x36
-  SENSITIVE = 0x37
-  BINARY = 0x38
+  BINARY = 0x37
+  BASE64 = 0x38
 
   # Marker module indicating whether or not an instance is tabulated or not
   module NotTabulated; end
 
-  # Marker module for objects that starts a sequence, i.e. ArrayStart, MapStart, and ObjectStart
+  # Marker module for objects that starts a sequence, i.e. ArrayStart, MapStart, and PcoreObjectStart
   module SequenceStart; end
 
   # The class that triggers the use of the DEFAULT extension. It doesn't have any payload
@@ -83,9 +85,15 @@ module Extension
     end
   end
 
-  # The class that triggers the use of the OBJECT_START extension. The payload is the name of the object type and the
-  # number of attributes in the instance.
-  class ObjectStart
+  # The class that triggers the use of the SENSITIVE_START extension. It has no payload
+  class SensitiveStart
+    include NotTabulated
+    INSTANCE = SensitiveStart.new
+  end
+
+  # The class that triggers the use of the PCORE_OBJECT_START extension. The payload is the name of the object type
+  # and the number of attributes in the instance.
+  class PcoreObjectStart
     include SequenceStart
     attr_reader :type_name, :attribute_count
     def initialize(type_name, attribute_count)
@@ -98,7 +106,28 @@ module Extension
     end
 
     def eql?(o)
-      o.is_a?(ObjectStart) && o.type_name == @type_name && o.attribute_count == @attribute_count
+      o.is_a?(PcoreObjectStart) && o.type_name == @type_name && o.attribute_count == @attribute_count
+    end
+    alias == eql?
+
+    def sequence_size
+      @attribute_count
+    end
+  end
+
+  class ObjectStart
+    include SequenceStart
+    attr_reader :attribute_count
+    def initialize(attribute_count)
+      @attribute_count = attribute_count
+    end
+
+    def hash
+      attribute_count.hash
+    end
+
+    def eql?(o)
+      o.is_a?(ObjectStart) && o.attribute_count == @attribute_count
     end
     alias == eql?
 

data/lib/puppet/pops/serialization/object.rb

@@ -11,10 +11,10 @@ class ObjectReader
 
   def read(impl_class, value_count, deserializer)
     type = impl_class._ptype
-    (names, types, required_count) = type.parameter_info
+    (names, types, required_count) = type.parameter_info(impl_class)
     max = names.size
     unless value_count >= required_count && value_count <= max
-      raise Serialization::SerializationError, "Feature count mismatch for #{impl_class.name}. Expected #{min} - #{max}, actual #{value_count}"
+      raise Serialization::SerializationError, "Feature count mismatch for #{impl_class.name}. Expected #{required_count} - #{max}, actual #{value_count}"
     end
     # Deserializer must know about this instance before we read its attributes
     val = deserializer.remember(impl_class.allocate)
@@ -43,16 +43,24 @@ class ObjectWriter
 
   def write(type, value, serializer)
     impl_class = value.class
-    (names, types, required_count) = type.parameter_info(true)
+    (names, types, required_count) = type.parameter_info(impl_class, true)
     args = names.map { |name| value.send(name) }
 
-    # Pop optional arguments that are nil
+    # Pop optional arguments that are default
     while args.size > required_count
-      break unless args.last.nil?
+      break unless args.last == type[names[args.size-1]].value
       args.pop
     end
 
-    serializer.start_object(type.name, args.size)
+    if type.name.start_with?('Pcore::')
+      serializer.push_written(value)
+      serializer.start_pcore_object(type.name, args.size)
+    else
+      serializer.start_object(args.size + 1)
+      serializer.write(type)
+      serializer.push_written(value)
+    end
+
     args.each { |arg| serializer.write(arg) }
   end
 

data/lib/puppet/pops/serialization/rgen.rb

@@ -37,7 +37,8 @@ module RGen
     def write(type, value, serializer)
       impl_class = value.class
       features = features(impl_class)
-      serializer.start_object(type.name, features.size)
+      serializer.push_written(value)
+      serializer.start_pcore_object(type.name, features.size)
       features.each { |feature| serializer.write(value.getGeneric(feature.name)) }
     end
 

data/lib/puppet/pops/serialization/serializer.rb

@@ -28,7 +28,7 @@ module Serialization
     # @api public
     def write(value)
       case value
-      when Integer, Float, String, true, false, nil, Time
+      when Integer, Float, String, true, false, nil
         @writer.write(value)
       when :default
         @writer.write(Extension::Default::INSTANCE)
@@ -37,7 +37,7 @@ module Serialization
         if index.nil?
           write_tabulated_first_time(value)
         else
-          @writer.write(Extension::Tabulation.new(index)) unless index.nil?
+          @writer.write(Extension::Tabulation.new(index))
         end
       end
     end
@@ -56,12 +56,29 @@ module Serialization
       @writer.write(Extension::MapStart.new(size))
     end
 
-    # Write the start of a complex object
+    # Write the start of a complex pcore object
     # @param [String] type_ref the name of the type
     # @param [Integer] attr_count the number of attributes in the object
     # @api private
-    def start_object(type_ref, attr_count)
-      @writer.write(Extension::ObjectStart.new(type_ref, attr_count))
+    def start_pcore_object(type_ref, attr_count)
+      @writer.write(Extension::PcoreObjectStart.new(type_ref, attr_count))
+    end
+
+    # Write the start of a complex object
+    # @param [Integer] attr_count the number of attributes in the object
+    # @api private
+    def start_object(attr_count)
+      @writer.write(Extension::ObjectStart.new(attr_count))
+    end
+
+    def push_written(value)
+      @written[value.object_id] = @written.size
+    end
+
+    # Write the start of a sensitive object
+    # @api private
+    def start_sensitive
+      @writer.write(Extension::SensitiveStart::INSTANCE)
     end
 
     # First time write of a tabulated object. This means that the object is written and then remembered. Subsequent writes
@@ -69,17 +86,23 @@ module Serialization
     # @param [Object] value the value to write
     # @api private
     def write_tabulated_first_time(value)
-      @written[value.object_id] = @written.size
       case value
-      when Symbol, Regexp, Semantic::Version, Semantic::VersionRange, Time::Timestamp, Time::Timespan, Types::PSensitiveType::Sensitive, Types::PBinaryType::Binary
+      when Symbol, Regexp, SemanticPuppet::Version, SemanticPuppet::VersionRange, Time::Timestamp, Time::Timespan, Types::PBinaryType::Binary
+        push_written(value)
         @writer.write(value)
       when Array
+        push_written(value)
         start_array(value.size)
         value.each { |elem| write(elem) }
       when Hash
+        push_written(value)
         start_map(value.size)
         value.each_pair { |key, val| write(key); write(val) }
+      when Types::PSensitiveType::Sensitive
+        start_sensitive
+        write(value.unwrap)
       when Types::PTypeReferenceType
+        push_written(value)
         @writer.write(value)
       when Types::PuppetObject
         value._ptype.write(value, self)