puppet 5.0.0 → 6.0.0

data/lib/puppet/pops/serialization/object.rb

@@ -9,8 +9,7 @@ module Serialization
 class ObjectReader
   include InstanceReader
 
-  def read(impl_class, value_count, deserializer)
-    type = impl_class._pcore_type
+  def read(type, impl_class, value_count, deserializer)
     (names, types, required_count) = type.parameter_info(impl_class)
     max = names.size
     unless value_count >= required_count && value_count <= max
@@ -25,7 +24,7 @@ class ObjectReader
         Types::TypeAsserter.assert_instance_of(nil, ptype, arg) { "#{type.name}[#{names[index]}]" } unless arg == :default
       else
         attr = type[names[index]]
-        raise Serialization::SerializationError, _("Missing default value for %{type_name}[%{name}]") % { type_name: type.name, name: names[index] } unless attr.value?
+        raise Serialization::SerializationError, _("Missing default value for %{type_name}[%{name}]") % { type_name: type.name, name: names[index] } unless attr && attr.value?
         args << attr.value
       end
     end
@@ -43,7 +42,7 @@ class ObjectWriter
 
   def write(type, value, serializer)
     impl_class = value.class
-    (names, types, required_count) = type.parameter_info(impl_class)
+    (names, _, required_count) = type.parameter_info(impl_class)
     args = names.map { |name| value.send(name) }
 
     # Pop optional arguments that are default

data/lib/puppet/pops/serialization/serializer.rb

@@ -108,7 +108,8 @@ module Serialization
           value.instance_of?(SemanticPuppet::VersionRange),
           value.instance_of?(Time::Timestamp),
           value.instance_of?(Time::Timespan),
-          value.instance_of?(Types::PBinaryType::Binary)
+          value.instance_of?(Types::PBinaryType::Binary),
+          value.is_a?(URI)
         push_written(value)
         @writer.write(value)
       when value.instance_of?(Array)

data/lib/puppet/pops/serialization/to_data_converter.rb

@@ -11,10 +11,10 @@ module Serialization
     # @param value [Object] the value to convert
     # @param options {Symbol => <Boolean,String>} options hash
     # @option options [Boolean] :rich_data `true` if rich data is enabled
-    # @option options [Boolean] :local_references use local references instead of duplicating complex entries
+    # @option options [Boolean] :local_reference use local references instead of duplicating complex entries
     # @option options [Boolean] :type_by_reference `true` if Object types are converted to references rather than embedded.
     # @option options [Boolean] :symbol_as_string `true` if Symbols should be converted to strings (with type loss)
-    # @option options [String] :path_prefix String to prepend to path in warnings and errors
+    # @option options [String] :message_prefix String to prepend to in warnings and errors
     # @return [Data] the processed result. An object assignable to `Data`.
     #
     # @api public
@@ -71,11 +71,20 @@ module Serialization
     def to_data(value)
       if value.nil? || Types::PScalarDataType::DEFAULT.instance?(value)
         value
-      elsif value.is_a?(Symbol)
-        if value == :default
+      elsif :default == value
+        if @rich_data
           { PCORE_TYPE_KEY => PCORE_TYPE_DEFAULT }
         else
-          @symbol_as_string ? value.to_s : { PCORE_TYPE_KEY => PCORE_TYPE_SYMBOL, PCORE_VALUE_KEY => value.to_s }
+          serialization_issue(Issues::SERIALIZATION_DEFAULT_CONVERTED_TO_STRING, :path => path_to_s)
+          'default'
+        end
+      elsif value.is_a?(Symbol)
+        if @symbol_as_string
+          value.to_s
+        elsif @rich_data
+          { PCORE_TYPE_KEY => PCORE_TYPE_SYMBOL, PCORE_VALUE_KEY => value.to_s }
+        else
+          unknown_to_string_with_warning(value)
         end
       elsif value.instance_of?(Array)
         process(value) do
@@ -87,7 +96,7 @@ module Serialization
         end
       elsif value.instance_of?(Hash)
         process(value) do
-          if value.keys.all? { |key| key.is_a?(String) }
+          if value.keys.all? { |key| key.is_a?(String) && key != PCORE_TYPE_KEY }
             result = {}
             value.each_pair { |key, elem| with(key) { result[key] = to_data(elem) } }
             result
@@ -179,17 +188,21 @@ module Serialization
     end
 
     def unknown_key_to_string_with_warning(value)
-      str = value.to_s
+      str = unknown_to_string(value)
       serialization_issue(Issues::SERIALIZATION_UNKNOWN_KEY_CONVERTED_TO_STRING, :path => path_to_s, :klass => value.class, :value => str)
       str
     end
 
     def unknown_to_string_with_warning(value)
-      str = value.to_s
+      str = unknown_to_string(value)
       serialization_issue(Issues::SERIALIZATION_UNKNOWN_CONVERTED_TO_STRING, :path => path_to_s, :klass => value.class, :value => str)
       str
     end
 
+    def unknown_to_string(value)
+      value.is_a?(Regexp) ? Puppet::Pops::Types::PRegexpType.regexp_to_s_with_delimiters(value) : value.to_s
+    end
+
     def non_string_keyed_hash_to_data(hash)
       if @rich_data
         to_key_extended_hash(hash)
@@ -243,7 +256,7 @@ module Serialization
           end
         else
           process(value) do
-            (names, types, required_count) = pcore_type.parameter_info(value.class)
+            (names, _, required_count) = pcore_type.parameter_info(value.class)
             args = names.map { |name| value.send(name) }
 
             # Pop optional arguments that are default
@@ -277,12 +290,12 @@ module Serialization
     def serialization_issue(issue, options = EMPTY_HASH)
       semantic = @semantic
       if semantic.nil?
-        stacktrace = Puppet::Pops::PuppetStack.stacktrace()
-        if stacktrace.size > 0
-          file, line = stacktrace[0]
-          semantic = Puppet::Pops::SemanticError.new(issue, nil, {:file => file, :line => line})
-        else
+        tos = Puppet::Pops::PuppetStack.top_of_stack
+        if tos.empty?
           semantic = Puppet::Pops::SemanticError.new(issue, nil, EMPTY_HASH)
+        else
+          file, line = stacktrace
+          semantic = Puppet::Pops::SemanticError.new(issue, nil, {:file => file, :line => line})
         end
       end
       optionally_fail(issue,  semantic, options)

data/lib/puppet/pops/serialization/to_stringified_converter.rb

@@ -0,0 +1,226 @@
+module Puppet::Pops
+module Serialization
+
+  # Class that can process an arbitrary object into a value that is assignable to `Data`
+  # and where contents is converted from rich data to one of:
+  # * Numeric (Integer, Float)
+  # * Boolean 
+  # * Undef (nil)
+  # * String
+  # * Array
+  # * Hash
+  # 
+  # The conversion is lossy - the result cannot be deserialized to produce the original data types.
+  # All rich values are transformed to strings..
+  # Hashes with rich keys are transformed to use string representation of such keys.
+  #
+  # @api public
+  class ToStringifiedConverter
+    include Evaluator::Runtime3Support
+
+    # Converts the given _value_ according to the given _options_ and return the result of the conversion
+    #
+    # @param value [Object] the value to convert
+    # @param options {Symbol => <Boolean,String>} options hash
+    # @option options [String] :message_prefix String to prepend to in warnings and errors
+    # @option options [String] :semantic object (AST) to pass to the issue reporter
+    # @return [Data] the processed result. An object assignable to `Data` with rich data stringified.
+    #
+    # @api public
+    def self.convert(value, options = EMPTY_HASH)
+      new(options).convert(value)
+    end
+
+    # Creates a new instance of the processor
+    #
+    # @param options {Symbol => Object} options hash
+    # @option options [String] :message_prefix String to prepend to path in warnings and errors
+    # @option semantic [Object] :semantic object to pass to the issue reporter
+    def initialize(options = EMPTY_HASH)
+      @message_prefix = options[:message_prefix]
+      @semantic = options[:semantic]
+    end
+
+    # Converts the given _value_
+    #
+    # @param value [Object] the value to convert
+    # @return [Data] the processed result. An object assignable to `Data` with rich data stringified.
+    #
+    # @api public
+    def convert(value)
+      @path = []
+      @values = {}
+      to_data(value)
+    end
+
+    private
+
+    def path_to_s
+      s = @message_prefix || ''
+      s << JsonPath.to_json_path(@path)[1..-1]
+      s
+    end
+
+    def to_data(value)
+      if value.is_a?(String)
+        to_string_or_binary(value)
+      elsif value.nil? || Types::PScalarDataType::DEFAULT.instance?(value)
+        value
+      elsif :default == value
+        'default'
+      elsif value.is_a?(Symbol)
+        value.to_s
+      elsif value.instance_of?(Array)
+        process(value) do
+          result = []
+          value.each_with_index do |elem, index|
+            with(index) { result << to_data(elem) }
+          end
+          result
+        end
+      elsif value.instance_of?(Hash)
+        process(value) do
+          if value.keys.all? { |key| key.is_a?(String) && key != PCORE_TYPE_KEY && key != PCORE_VALUE_KEY }
+            result = {}
+            value.each_pair { |key, elem| with(key) { result[key] = to_data(elem) } }
+            result
+          else
+            non_string_keyed_hash_to_data(value)
+          end
+        end
+      else
+        unknown_to_string(value)
+      end
+    end
+
+    # Turns an ASCII-8BIT encoded string into a Binary, returns US_ASCII encoded and transforms all other strings to UTF-8
+    # with replacements for non Unicode characters.
+    # If String cannot be represented as UTF-8
+    def to_string_or_binary(value)
+      encoding = value.encoding
+      if encoding == Encoding::ASCII_8BIT
+        Puppet::Pops::Types::PBinaryType::Binary.from_binary_string(value).to_s
+      else
+        # Transform to UTF-8 (do not assume UTF-8 is correct) with source invalid byte
+        # sequences and UTF-8 undefined characters replaced by the default unicode uFFFD character
+        # (black diamond with question mark).
+        value.encode(Encoding::UTF_8, encoding, :invalid => :replace, :undef => :replace)
+      end
+    end
+
+    # Performs a check for endless recursion before
+    # it yields to the given block. The result of yielding is returned.
+    #
+    # @param value [Object] the value
+    # @yield The block that will produce the data for the value
+    # @return [Data] the result of yielding to the given block, or a hash denoting a reference
+    #
+    # @api private
+    def process(value, &block)
+      with_recursive_guard(value, &block)
+    end
+
+    # Pushes `key` to the end of the path and yields to the given block. The
+    # `key` is popped when the yield returns.
+    # @param key [Object] the key to push on the current path
+    # @yield The block that will produce the returned value
+    # @return [Object] the result of yielding to the given block
+    #
+    # @api private
+    def with(key)
+      @path.push(key)
+      value = yield
+      @path.pop
+      value
+    end
+
+    # @param value [Object] the value to use when checking endless recursion
+    # @yield The block that will produce the data
+    # @return [Data] the result of yielding to the given block
+    def with_recursive_guard(value)
+      id = value.object_id
+      if @recursive_lock
+        if @recursive_lock.include?(id)
+          serialization_issue(Issues::SERIALIZATION_ENDLESS_RECURSION, :type_name => value.class.name)
+        end
+        @recursive_lock[id] = true
+      else
+        @recursive_lock = { id => true }
+      end
+      v = yield
+      @recursive_lock.delete(id)
+      v
+    end
+
+    # A hash key that is non conforming
+    def unknown_key_to_string(value)
+      unknown_to_string(value)
+    end
+
+    def unknown_to_string(value)
+      if value.is_a?(Regexp)
+        return Puppet::Pops::Types::PRegexpType.regexp_to_s_with_delimiters(value)
+
+      elsif value.instance_of?(Types::PSensitiveType::Sensitive)
+        # to_s does not differentiate between instances - if they were used as keys in a hash
+        # the stringified result would override all Sensitive keys with the last such key's value
+        # this adds object_id.
+        #
+        return "#<#{value}:#{value.object_id}>"
+
+      elsif value.is_a?(Puppet::Pops::Types::PObjectType)
+        # regular to_s on an ObjectType gives the entire definition
+        return value.name
+
+      end
+
+      # Do a to_s on anything else
+      result = value.to_s
+
+      # The result may be ascii-8bit encoded without being a binary (low level object.inspect returns ascii-8bit string)
+      # This can be the case if runtime objects have very simple implementation (no to_s or inspect method).
+      # They are most likely not of Binary nature. Therefore the encoding is forced and only if it errors
+      # will the result be taken as binary and encoded as base64 string.
+      if result.encoding == Encoding::ASCII_8BIT
+        begin
+          result.force_encoding(Encoding::UTF_8)
+        rescue
+          # The result cannot be represented in UTF-8, make it a binary Base64 encoded string
+          Puppet::Pops::Types::PBinaryType::Binary.from_binary_string(result).to_s
+        end
+      end
+      result
+    end
+
+    def non_string_keyed_hash_to_data(hash)
+      result = {}
+      hash.each_pair do |key, value|
+        if key.is_a?(Symbol)
+          key = key.to_s
+        elsif !key.is_a?(String)
+          key = unknown_key_to_string(key)
+        end
+        if key == "__ptype" || key =="__pvalue"
+          key = "reserved key: #{key}"
+        end
+        with(key) { result[key] = to_data(value) }
+      end
+      result
+    end
+
+    def serialization_issue(issue, options = EMPTY_HASH)
+      semantic = @semantic
+      if semantic.nil?
+        tos = Puppet::Pops::PuppetStack.top_of_stack
+        if tos.empty?
+          semantic = Puppet::Pops::SemanticError.new(issue, nil, EMPTY_HASH)
+        else
+          file, line = stacktrace
+          semantic = Puppet::Pops::SemanticError.new(issue, nil, {:file => file, :line => line})
+        end
+      end
+      optionally_fail(issue,  semantic, options)
+    end
+  end
+end
+end

data/lib/puppet/pops/serialization.rb

@@ -7,12 +7,12 @@ module Serialization
   class SerializationError < Puppet::Error
   end
 
-  PCORE_TYPE_KEY = '__pcore_type__'.freeze
+  PCORE_TYPE_KEY = '__ptype'.freeze
 
   # Key used when the value can be represented as, and recreated from, a single string that can
   # be passed to a `from_string` method or an array of values that can be passed to the default
   # initializer method.
-  PCORE_VALUE_KEY = '__pcore_value__'.freeze
+  PCORE_VALUE_KEY = '__pvalue'.freeze
 
   # Type key used for hashes that contain keys that are not of type String
   PCORE_TYPE_HASH = 'Hash'.freeze
@@ -34,6 +34,7 @@ end
 require_relative 'serialization/json_path'
 require_relative 'serialization/from_data_converter'
 require_relative 'serialization/to_data_converter'
+require_relative 'serialization/to_stringified_converter'
 require_relative 'serialization/serializer'
 require_relative 'serialization/deserializer'
 require_relative 'serialization/json'

data/lib/puppet/pops/time/timespan.rb

@@ -259,7 +259,7 @@ module Time
           add_unless_zero(result, KEY_NANOSECONDS, n % 1000)
           n /= 1000
           add_unless_zero(result, KEY_MICROSECONDS, n % 1000)
-          add_unless_zero(result, KEY_MILLISECONDS, n /= 1000)
+          add_unless_zero(result, KEY_MILLISECONDS, n / 1000)
         end
         result[KEY_NEGATIVE] = true if negative?
       end

data/lib/puppet/pops/time/timestamp.rb

@@ -1,8 +1,8 @@
 module Puppet::Pops
 module Time
 class Timestamp < TimeData
-  DEFAULT_FORMATS_WO_TZ = ['%FT%T.%N', '%FT%T', '%F']
-  DEFAULT_FORMATS = ['%FT%T.%N %Z', '%FT%T %Z', '%F %Z'] + DEFAULT_FORMATS_WO_TZ
+  DEFAULT_FORMATS_WO_TZ = ['%FT%T.%N', '%FT%T', '%F %T.%N', '%F %T', '%F']
+  DEFAULT_FORMATS = ['%FT%T.%N %Z', '%FT%T %Z', '%F %T.%N %Z', '%F %T %Z', '%F %Z'] + DEFAULT_FORMATS_WO_TZ
 
   CURRENT_TIMEZONE = 'current'.freeze
   KEY_TIMEZONE = 'timezone'.freeze
@@ -71,31 +71,43 @@ class Timestamp < TimeData
     parsed = nil
     if format.is_a?(Array)
       format.each do |fmt|
-        assert_no_tz_extractor(fmt) if has_timezone
-        begin
-          parsed = DateTime.strptime(str, fmt)
-          break
-        rescue ArgumentError
+        parsed = DateTime._strptime(str, fmt)
+        next if parsed.nil?
+        if parsed.include?(:leftover) || (has_timezone && parsed.include?(:zone))
+          parsed = nil
+          next
         end
+        break
+      end
+      if parsed.nil?
+        raise ArgumentError, _(
+          "Unable to parse '%{str}' using any of the formats %{formats}") % { str: str, formats: format.join(', ') }
       end
-      raise ArgumentError, _("Unable to parse '%{str}' using any of the formats %{formats}") % { str: str, formats: format.join(', ') } if parsed.nil?
     else
-      assert_no_tz_extractor(format) if has_timezone
-      begin
-        parsed = DateTime.strptime(str, format)
-      rescue ArgumentError
+      parsed = DateTime._strptime(str, format)
+      if parsed.nil? || parsed.include?(:leftover)
         raise ArgumentError, _("Unable to parse '%{str}' using format '%{format}'") % { str: str, format: format }
       end
+      if has_timezone && parsed.include?(:zone)
+        raise ArgumentError, _(
+          'Using a Timezone designator in format specification is mutually exclusive to providing an explicit timezone argument')
+      end
+    end
+    unless has_timezone
+      timezone = parsed[:zone]
+      has_timezone = !timezone.nil?
     end
-    parsed_time = parsed.to_time
+    fraction = parsed[:sec_fraction]
+
+    # Convert msec rational found in _strptime hash to usec
+    fraction = fraction * 1000000 unless fraction.nil?
+
+    # Create the Time instance and adjust for timezone
+    parsed_time = ::Time.utc(parsed[:year], parsed[:mon], parsed[:mday], parsed[:hour], parsed[:min], parsed[:sec], fraction)
     parsed_time -= utc_offset(timezone) if has_timezone
-    from_time(parsed_time)
-  end
 
-  def self.assert_no_tz_extractor(format)
-    if format =~ /[^%]%[zZ]/
-      raise ArgumentError, _('Using a Timezone designator in format specification is mutually exclusive to providing an explicit timezone argument')
-    end
+    # Convert to Timestamp
+    from_time(parsed_time)
   end
 
   undef_method :-@, :+@, :div, :fdiv, :abs, :abs2, :magnitude # does not make sense on a Timestamp

data/lib/puppet/pops/types/annotatable.rb

@@ -7,9 +7,9 @@ KEY_ANNOTATIONS = 'annotations'.freeze
 #
 # @api public
 module Annotatable
-  TYPE_ANNOTATIONS = PHashType.new(PType.new(PTypeReferenceType.new('Annotation')), PHashType::DEFAULT)
+  TYPE_ANNOTATIONS = PHashType.new(PTypeType.new(PTypeReferenceType.new('Annotation')), PHashType::DEFAULT)
 
-  # @return [{PType => PStructType}] the map of annotations
+  # @return [{PTypeType => PStructType}] the map of annotations
   # @api public
   def annotations
     @annotations.nil? ? EMPTY_HASH : @annotations

data/lib/puppet/pops/types/annotation.rb

@@ -21,29 +21,29 @@ module Types
     # If no annotation was found and no block is given, this method returns `nil`
     #
     # @param o [Object] object to annotate
-    # @param block [Proc] optional, evaluated when a new annotation must be created. Should return the i12n hash
+    # @param block [Proc] optional, evaluated when a new annotation must be created. Should return the init hash
     # @return [Annotation<self>] an annotation of the same class as the receiver of the call
     #
     def self.annotate(o)
       adapter = get(o)
       if adapter.nil?
         if o.is_a?(Annotatable)
-          i12n = o.annotations[_pcore_type]
-          i12n = yield if i12n.nil? && block_given?
+          init_hash = o.annotations[_pcore_type]
+          init_hash = yield if init_hash.nil? && block_given?
         else
-          i12n = yield if block_given?
+          init_hash = yield if block_given?
         end
-        adapter = associate_adapter(_pcore_type.from_hash(i12n), o) unless i12n.nil?
+        adapter = associate_adapter(_pcore_type.from_hash(init_hash), o) unless init_hash.nil?
       end
       adapter
     end
 
     # Forces the creation or removal of an annotation of this type.
-    # If `i21n` is a hash, a new annotation is created and returned
-    # If `i12n` is `nil`, then the annotation is cleared and the previous annotation is returned.
+    # If `init_hash` is a hash, a new annotation is created and returned
+    # If `init_hash` is `nil`, then the annotation is cleared and the previous annotation is returned.
     #
     # @param o [Object] object to annotate
-    # @param i12n [Hash{String,Object},nil] the initializer for the annotation or `nil` to clear the annotation
+    # @param init_hash [Hash{String,Object},nil] the initializer for the annotation or `nil` to clear the annotation
     # @return [Annotation<self>] an annotation of the same class as the receiver of the call
     #
     def self.annotate_new(o, init_hash)

data/lib/puppet/pops/types/class_loader.rb

@@ -24,7 +24,7 @@ class ClassLoader
     when Array
       provide_from_name_path(name.join('::'), name)
 
-    when PAnyType, PType
+    when PAnyType, PTypeType
       provide_from_type(name)
 
     else
@@ -32,8 +32,6 @@ class ClassLoader
     end
   end
 
-  private
-
   def self.provide_from_type(type)
     case type
     when PRuntimeType
@@ -44,8 +42,8 @@ class ClassLoader
       # There is no other thing to load except this Enum meta type
       RGen::MetamodelBuilder::MMBase::Boolean
 
-    when PType
-      # TODO: PType should has a type argument (a PAnyType) so the Class' class could be returned
+    when PTypeType
+      # TODO: PTypeType should have a type argument (a PAnyType) so the Class' class could be returned
       #       (but this only matters in special circumstances when meta programming has been used).
       Class
 
@@ -66,12 +64,13 @@ class ClassLoader
     when PPatternType  ; String
     when PEnumType     ; String
     when PFloatType    ; Float
-    when PUndefType      ; NilClass
+    when PUndefType    ; NilClass
     when PCallableType ; Proc
     else
       nil
     end
   end
+  private_class_method :provide_from_type
 
   def self.provide_from_string(name)
     name_path = name.split(TypeFormatter::NAME_SEGMENT_SEPARATOR)
@@ -97,6 +96,7 @@ class ClassLoader
     return nil unless result.is_a?(Module)
     result
   end
+  private_class_method :provide_from_string
 
   def self.find_class(name_path)
     name_path.reduce(Object) do |ns, name|
@@ -107,6 +107,7 @@ class ClassLoader
       end
     end
   end
+  private_class_method :find_class
 
   def self.paths_for_name(fq_named_parts)
     # search two entries, one where all parts are decamelized, and one with names just downcased
@@ -115,6 +116,7 @@ class ClassLoader
     #
     [fq_named_parts.map {|part| de_camel(part)}.join('/'), fq_named_parts.join('/').downcase ]
   end
+  private_class_method :paths_for_name
 
   def self.de_camel(fq_name)
     fq_name.to_s.gsub(/::/, '/').
@@ -123,6 +125,7 @@ class ClassLoader
     tr("-", "_").
     downcase
   end
+  private_class_method :de_camel
 
 end
 end