puppet 4.0.0 → 4.1.0

data/lib/puppet/pops/parser/slurp_support.rb

@@ -7,10 +7,13 @@
 # TODO: More detailed performance analysis of excessive character escaping and interpolation.
 #
 module Puppet::Pops::Parser::SlurpSupport
+  include Puppet::Pops::Parser::LexerSupport
 
   SLURP_SQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*[']/
   SLURP_DQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*(["]|[$]\{?)/
   SLURP_UQ_PATTERN  = /(?:[^\\]|^|[^\\])(?:[\\]{2})*([$]\{?|\z)/
+  # unquoted, no escapes
+  SLURP_UQNE_PATTERN  = /(\$\{?|\z)/m
   SLURP_ALL_PATTERN = /.*(\z)/m
   SQ_ESCAPES = %w{ \\ ' }
   DQ_ESCAPES = %w{ \\  $ ' " r n t s u}+["\r\n", "\n"]
@@ -19,7 +22,8 @@ module Puppet::Pops::Parser::SlurpSupport
   def slurp_sqstring
     # skip the leading '
     @scanner.pos += 1
-    str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes) || lex_error("Unclosed quote after \"'\" followed by '#{followed_by}'")
+    str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes)
+    lex_error(Puppet::Pops::Issues::UNCLOSED_QUOTE, :after => "\"'\"", :followed_by => followed_by) unless str
     str[0..-2] # strip closing "'" from result
   end
 
@@ -28,7 +32,7 @@ module Puppet::Pops::Parser::SlurpSupport
     last = scn.matched
     str = slurp(scn, SLURP_DQ_PATTERN, DQ_ESCAPES, false)
     unless str
-      lex_error("Unclosed quote after #{format_quote(last)} followed by '#{followed_by}'")
+      lex_error(Puppet::Pops::Issues::UNCLOSED_QUOTE, :after => format_quote(last), :followed_by => followed_by)
     end
 
     # Terminator may be a single char '"', '$', or two characters '${' group match 1 (scn[1]) from the last slurp holds this
@@ -41,6 +45,7 @@ module Puppet::Pops::Parser::SlurpSupport
      scn = @scanner
      last = scn.matched
      ignore = true
+
      str = slurp(scn, @lexing_context[:uq_slurp_pattern], @lexing_context[:escapes], :ignore_invalid_escapes)
 
      # Terminator may be a single char '$', two characters '${', or empty string '' at the end of intput.
@@ -65,11 +70,12 @@ module Puppet::Pops::Parser::SlurpSupport
     # Process unicode escapes first as they require getting 4 hex digits
     # If later a \u is found it is warned not to be a unicode escape
     if escapes.include?('u')
-      str.gsub!(/\\u([\da-fA-F]{4})/m) {
-        [$1.hex].pack("U")
+      str.gsub!(/\\u(?:([\da-fA-F]{4})|\{([\da-fA-F]{1,6})\})/m) {
+        [($1 || $2).hex].pack("U")
       }
     end
 
+    begin
     str.gsub!(/\\([^\r\n]|(?:\r?\n))/m) {
       ch = $1
       if escapes.include? ch
@@ -79,17 +85,27 @@ module Puppet::Pops::Parser::SlurpSupport
         when 't'   ; "\t"
         when 's'   ; " "
         when 'u'
-          Puppet.warning(positioned_message("Unicode escape '\\u' was not followed by 4 hex digits"))
+          lex_warning(Puppet::Pops::Issues::ILLEGAL_UNICODE_ESCAPE)
           "\\u"
         when "\n"  ; ''
         when "\r\n"; ''
         else      ch
         end
       else
-        Puppet.warning(positioned_message("Unrecognized escape sequence '\\#{ch}'")) unless ignore_invalid_escapes
+        lex_warning(Puppet::Pops::Issues::UNRECOGNIZED_ESCAPE, :ch => ch) unless ignore_invalid_escapes
         "\\#{ch}"
       end
     }
+    rescue ArgumentError => e
+      # A invalid byte sequence may be the result of faulty input as well, but that could not possibly
+      # have reached this far... Unfortunately there is no more specific error and a match on message is
+      # required to differentiate from other internal problems.
+      if e.message =~ /invalid byte sequence/
+        lex_error(Puppet::Pops::Issues::ILLEGAL_UNICODE_ESCAPE)
+      else
+        raise e
+      end
+    end
     str
   end
 end

data/lib/puppet/pops/types/type_calculator.rb

@@ -268,14 +268,21 @@ class Puppet::Pops::Types::TypeCalculator
     if t2.is_a?(Class)
       t2 = type(t2)
     end
+    t2_class = t2.class
+
     # Unit can be assigned to anything
-    return true if t2.class == Types::PUnitType
+    return true if t2_class == Types::PUnitType
 
-    if t2.class == Types::PVariantType
+    if t2_class == Types::PVariantType
       # Assignable if all contained types are assignable
       t2.types.all? { |vt| @@assignable_visitor.visit_this_1(self, t, vt) }
     else
-      @@assignable_visitor.visit_this_1(self, t, t2)
+      # Turn NotUndef[T] into T when T is not assignable from Undef
+      if t2_class == Types::PNotUndefType && !(t2.type.nil? || assignable?(t2.type, @nil_t))
+        assignable?(t, t2.type)
+      else
+        @@assignable_visitor.visit_this_1(self, t, t2)
+      end
     end
  end
 
@@ -353,10 +360,23 @@ class Puppet::Pops::Types::TypeCalculator
     # do nothing, there is nothing to change for most types
   end
 
+  # @return [Boolean] true if the given argument is contained in a struct element key
+  def is_struct_element_key?(o)
+    c = o.eContainer
+    if c.is_a?(Types::POptionalType)
+      o = c
+      c = c.eContainer
+    end
+    c.is_a?(Types::PStructElement) && c.key_type.equal?(o)
+  end
+  private :is_struct_element_key?
+
   def generalize_PStringType(o)
-    o.values = []
-    o.size_type = nil
-    []
+    # Skip generalization if the string is contained in a PStructElement key.
+    unless is_struct_element_key?(o)
+      o.values = []
+      o.size_type = nil
+    end
   end
 
   def generalize_PCollectionType(o)
@@ -433,8 +453,13 @@ class Puppet::Pops::Types::TypeCalculator
   def instance_of_PStringType(t, o)
     return false unless o.is_a?(String)
     # true if size compliant
-    size_t = t.size_type || @collection_default_size_t
-    instance_of_PIntegerType(size_t, o.size)
+    size_t = t.size_type
+    if size_t.nil? || instance_of_PIntegerType(size_t, o.size)
+      values = t.values
+      values.empty? || values.include?(o)
+    else
+      false
+    end
   end
 
   def instance_of_PTupleType(t, o)
@@ -451,9 +476,18 @@ class Puppet::Pops::Types::TypeCalculator
 
   def instance_of_PStructType(t, o)
     return false unless o.is_a?(Hash)
-    h = t.hashed_elements
-    # all keys must be present and have a value (even if nil/undef)
-    (o.keys - h.keys).empty? && h.all? { |k,v| instance_of(v, o[k]) }
+    matched = 0
+    t.elements.all? do |e|
+      key = e.name
+      v = o[key]
+      if v.nil? && !o.include?(key)
+        # Entry is missing. Only OK when key is optional
+        assignable?(e.key_type, @nil_t)
+      else
+        matched += 1
+        instance_of(e.value_type, v)
+      end
+    end && matched == o.size
   end
 
   def instance_of_PHashType(t, o)
@@ -470,6 +504,10 @@ class Puppet::Pops::Types::TypeCalculator
     instance_of(@data_variant_t, o)
   end
 
+  def instance_of_PNotUndefType(t, o)
+    !(o.nil? || o == :undef) && (t.type.nil? || instance_of(t.type, o))
+  end
+
   def instance_of_PUndefType(t, o)
     o.nil? || o == :undef
   end
@@ -597,7 +635,8 @@ class Puppet::Pops::Types::TypeCalculator
 
     if t1.is_a?(Types::PStringType) && t2.is_a?(Types::PStringType)
       t = Types::PStringType.new()
-      t.values = t1.values | t2.values
+      t.values = t1.values | t2.values unless t1.values.empty? || t2.values.empty?
+      t.size_type = common_type(t1.size_type, t2.size_type) unless t1.size_type.nil? || t2.size_type.nil?
       return t
     end
 
@@ -919,27 +958,23 @@ class Puppet::Pops::Types::TypeCalculator
       type.key_type = Types::PUndefType.new
       type.element_type = Types::PUndefType.new
       type.size_type = size_as_type(o)
-    else
-      if o.keys.find {|k| !instance_of_PStringType(@non_empty_string_t, k) }
-        type = Types::PHashType.new
-        ktype = Types::PVariantType.new
-        ktype.types = o.keys.map {|k| infer_set(k) }
-        etype = Types::PVariantType.new
-        etype.types = o.values.map {|e| infer_set(e) }
-        type.key_type = unwrap_single_variant(ktype)
-        type.element_type = unwrap_single_variant(etype)
-        type.size_type = size_as_type(o)
-      else
-        elements = []
-        o.each_pair do |k,v|
-          element = Types::PStructElement.new
-          element.name = k
-          element.type = infer_set(v)
-          elements << element
-        end
-        type = Types::PStructType.new
-        type.elements = elements
+    elsif o.keys.all? {|k| instance_of_PStringType(@non_empty_string_t, k) }
+      type = Types::PStructType.new
+      type.elements = o.map do |k,v|
+        element = Types::PStructElement.new
+        element.key_type = infer_String(k)
+        element.value_type = infer_set(v)
+        element
       end
+    else
+      type = Types::PHashType.new
+      ktype = Types::PVariantType.new
+      ktype.types = o.keys.map {|k| infer_set(k) }
+      etype = Types::PVariantType.new
+      etype.types = o.values.map {|e| infer_set(e) }
+      type.key_type = unwrap_single_variant(ktype)
+      type.element_type = unwrap_single_variant(etype)
+      type.size_type = size_as_type(o)
     end
     type
   end
@@ -963,6 +998,11 @@ class Puppet::Pops::Types::TypeCalculator
     t2.is_a?(Types::PAnyType)
   end
 
+  # @api private
+  def assignable_PNotUndefType(t, t2)
+    !assignable?(t2, @nil_t) && (t.type.nil? || assignable?(t.type, t2))
+  end
+
   # @api private
   def assignable_PUndefType(t, t2)
     # Only undef/nil is assignable to nil type
@@ -1103,6 +1143,17 @@ class Puppet::Pops::Types::TypeCalculator
     end
   end
 
+  # @api private
+  def self.is_kind_of_optional?(t, optional = true)
+    case t
+    when Types::POptionalType
+      true
+    when Types::PVariantType
+      t.types.all? {|t2| is_kind_of_optional?(t2, optional) }
+    else
+      false
+    end
+  end
 
   def callable_PArrayType(args_array, callable_t)
     return false unless assignable?(callable_t.param_types, args_array)
@@ -1186,22 +1237,34 @@ class Puppet::Pops::Types::TypeCalculator
   #
   def assignable_PStructType(t, t2)
     if t2.is_a?(Types::PStructType)
-      h = t.hashed_elements
       h2 = t2.hashed_elements
-      (h2.keys - h.keys).empty? && h.all? {|k, v| v2 = h2[k]; assignable?(v, v2.nil? ? @nil_t : v2) }
+      matched = 0
+      t.elements.all? do |e1|
+        e2 = h2[e1.name]
+        if e2.nil?
+          assignable?(e1.key_type, @nil_t)
+        else
+          matched += 1
+          assignable?(e1.key_type, e2.key_type) && assignable?(e1.value_type, e2.value_type)
+        end
+      end && matched == h2.size
     elsif t2.is_a?(Types::PHashType)
-      size_t2 = t2.size_type || @collection_default_size_t
-      size_t = Types::PIntegerType.new
-      elements = t.elements
-      size_t.from = elements.count {|e| !assignable?(e.type, @nil_t) }
-      size_t.to = elements.size
-      # compatible size
-      # hash key type must be string of min 1 size
-      # hash value t must be assignable to each key
-      element_type = t2.element_type
-      assignable_PIntegerType(size_t, size_t2) &&
-        (size_t2.to == 0 || assignable?(@non_empty_string_t, t2.key_type)) &&
-          elements.all? {|e| assignable?(e.type, element_type) }
+      required = 0
+      required_elements_assignable = t.elements.all? do |e|
+        if assignable?(e.key_type, @nil_t)
+          true
+        else
+          required += 1
+          assignable?(e.value_type, t2.element_type)
+        end
+      end
+      if required_elements_assignable
+        size_t2 = t2.size_type || @collection_default_size_t
+        size_t = Types::PIntegerType.new
+        size_t.from = required
+        size_t.to = t.elements.size
+        assignable_PIntegerType(size_t, size_t2)
+      end
     else
       false
     end
@@ -1210,8 +1273,9 @@ class Puppet::Pops::Types::TypeCalculator
   # @api private
   def assignable_POptionalType(t, t2)
     return true if t2.is_a?(Types::PUndefType)
+    return true if t.optional_type.nil?
     if t2.is_a?(Types::POptionalType)
-      assignable?(t.optional_type, t2.optional_type)
+      assignable?(t.optional_type, t2.optional_type || @t)
     else
       assignable?(t.optional_type, t2)
     end
@@ -1381,11 +1445,11 @@ class Puppet::Pops::Types::TypeCalculator
   # @api private
   def assignable_PArrayType(t, t2)
     if t2.is_a?(Types::PArrayType)
-      return false unless assignable?(t.element_type, t2.element_type)
+      return false unless t.element_type.nil? || assignable?(t.element_type, t2.element_type || @t)
       assignable_PCollectionType(t, t2)
 
     elsif t2.is_a?(Types::PTupleType)
-      return false unless t2.types.all? {|t2_element| assignable?(t.element_type, t2_element) }
+      return false unless t.element_type.nil? || t2.types.all? {|t2_element| assignable?(t.element_type, t2_element) }
       t2_regular = t2.types[0..-2]
       t2_ranged = t2.types[-1]
       t2_from, t2_to = size_range(t2.size_type)
@@ -1423,7 +1487,8 @@ class Puppet::Pops::Types::TypeCalculator
     case t2
     when Types::PHashType
       return true if (t.size_type.nil? || t.size_type.from == 0) && t2.is_the_empty_hash?
-      return false unless assignable?(t.key_type, t2.key_type) && assignable?(t.element_type, t2.element_type)
+      return false unless t.key_type.nil? || assignable?(t.key_type, t2.key_type || @t)
+      return false unless t.element_type.nil? || assignable?(t.element_type, t2.element_type || @t)
       assignable_PCollectionType(t, t2)
     when Types::PStructType
       # hash must accept String as key type
@@ -1435,7 +1500,7 @@ class Puppet::Pops::Types::TypeCalculator
       key_type = t.key_type
       element_type = t.element_type
       ( struct_size >= min && struct_size <= max &&
-        t2.elements.all? {|e| instance_of(key_type, e.name) && assignable?(element_type, e.type) })
+        t2.elements.all? {|e| (key_type.nil? || instance_of(key_type, e.name)) && (element_type.nil? || assignable?(element_type, e.value_type)) })
     else
       false
     end
@@ -1467,7 +1532,15 @@ class Puppet::Pops::Types::TypeCalculator
   # Data is assignable by other Data and by Array[Data] and Hash[Scalar, Data]
   # @api private
   def assignable_PDataType(t, t2)
-    t2.is_a?(Types::PDataType) || assignable?(@data_variant_t, t2)
+    # We cannot put the NotUndefType[Data] in the @data_variant_t since that causes an endless recursion
+    case t2
+    when Types::PDataType
+      true
+    when Types::PNotUndefType
+      assignable?(t, t2.type || @t)
+    else
+      assignable?(@data_variant_t, t2)
+    end
   end
 
   # Assignable if t2's has the same runtime and the runtime name resolves to
@@ -1638,7 +1711,16 @@ class Puppet::Pops::Types::TypeCalculator
   end
 
   def string_PStructElement(t)
-    "'#{t.name}'=>#{string(t.type)}"
+    k = t.key_type
+    value_optional = assignable?(t.value_type, @nil_t)
+    key_string =
+      if k.is_a?(Types::POptionalType)
+        # Output as literal String
+        value_optional ? "'#{t.name}'" : string(k)
+      else
+        value_optional ? "NotUndef['#{t.name}']" : "'#{t.name}'"
+      end
+    "#{key_string}=>#{string(t.value_type)}"
   end
 
   # @api private
@@ -1704,11 +1786,30 @@ class Puppet::Pops::Types::TypeCalculator
     end
   end
 
+  # @api private
+  def string_PNotUndefType(t)
+    contained_type = t.type
+    if contained_type.nil? || contained_type.class == Puppet::Pops::Types::PAnyType
+      'NotUndef'
+    else
+      if contained_type.is_a?(Puppet::Pops::Types::PStringType) && contained_type.values.size == 1
+        "NotUndef['#{contained_type.values[0]}']"
+      else
+        "NotUndef[#{string(contained_type)}]"
+      end
+    end
+  end
+
   def string_POptionalType(t)
-    if t.optional_type.nil?
+    optional_type = t.optional_type
+    if optional_type.nil?
       "Optional"
     else
-      "Optional[#{string(t.optional_type)}]"
+      if optional_type.is_a?(Puppet::Pops::Types::PStringType) && optional_type.values.size == 1
+        "Optional['#{optional_type.values[0]}']"
+      else
+        "Optional[#{string(optional_type)}]"
+      end
     end
   end
 

data/lib/puppet/pops/types/type_factory.rb

@@ -2,9 +2,9 @@
 # @api public
 #
 module Puppet::Pops::Types::TypeFactory
-  @type_calculator = Puppet::Pops::Types::TypeCalculator.new()
-
   Types = Puppet::Pops::Types
+  @type_calculator = Types::TypeCalculator.singleton
+  @undef_t = Types::PUndefType.new
 
   # Produces the Integer type
   # @api public
@@ -66,9 +66,17 @@ module Puppet::Pops::Types::TypeFactory
   end
 
   # Produces the Optional type, i.e. a short hand for Variant[T, Undef]
+  # If the given 'optional_type' argument is a String, then it will be
+  # converted into a String type that represents that string.
+  #
+  # @param optional_type [String,PAnyType,nil] the optional type
+  # @return [POptionalType] the created type
+  #
+  # @api public
+  #
   def self.optional(optional_type = nil)
     t = Types::POptionalType.new
-    t.optional_type = type_of(optional_type)
+    t.optional_type = optional_type.is_a?(String) ? string(optional_type) : type_of(optional_type)
     t
   end
 
@@ -91,19 +99,48 @@ module Puppet::Pops::Types::TypeFactory
   end
 
   # Produces the Struct type, either a non parameterized instance representing
-  # all structs (i.e. all hashes) or a hash with a given set of keys of String
-  # type (names), bound to a value of a given type. Type may be a Ruby Class, a
-  # Puppet Type, or an instance from which the type is inferred.
+  # all structs (i.e. all hashes) or a hash with entries where the key is
+  # either a literal String, an Enum with one entry, or a String representing exactly one value.
+  # The key type may also be wrapped in a NotUndef or an Optional.
   #
-  def self.struct(name_type_hash = {})
+  # The value can be a ruby class, a String (interpreted as the name of a ruby class) or
+  # a Type.
+  #
+  # @param hash [Hash<Object, Object>] key => value hash
+  # @return [PStructType] the created Struct type
+  #
+  def self.struct(hash = {})
+    tc = @type_calculator
     t = Types::PStructType.new
-    t.elements = name_type_hash.map do |name, type|
-      elem = Types::PStructElement.new
-      if name.is_a?(String) && name.empty?
-        raise ArgumentError, "An empty String can not be used where a String[1, default] is expected"
+    t.elements = hash.map do |key_type, value_type|
+      value_type = type_of(value_type)
+      raise ArgumentError, 'Struct element value_type must be a Type' unless value_type.is_a?(Types::PAnyType)
+
+      # TODO: Should have stricter name rule
+      if key_type.is_a?(String)
+        raise ArgumentError, 'Struct element key cannot be an empty String' if key_type.empty?
+        key_type = string(key_type)
+        # Must make key optional if the value can be Undef
+        key_type = optional(key_type) if tc.assignable?(value_type, @undef_t)
+      else
+        # assert that the key type is one of String[1], NotUndef[String[1]] and Optional[String[1]]
+        case key_type
+        when Types::PNotUndefType
+          # We can loose the NotUndef wrapper here since String[1] isn't optional anyway
+          key_type = key_type.type
+          s = key_type
+        when Types::POptionalType
+          s = key_type.optional_type
+        else
+          s = key_type
+        end
+        unless (s.is_a?(Puppet::Pops::Types::PStringType) || s.is_a?(Puppet::Pops::Types::PEnumType)) && s.values.size == 1 && !s.values[0].empty?
+          raise ArgumentError, 'Unable to extract a non-empty literal string from Struct member key type' if key_type.empty?
+        end
       end
-      elem.name = name
-      elem.type = type_of(type)
+      elem = Types::PStructElement.new
+      elem.key_type = key_type
+      elem.value_type = value_type
       elem
     end
     t
@@ -351,6 +388,22 @@ module Puppet::Pops::Types::TypeFactory
     type
   end
 
+  # Produces a type for NotUndef[T]
+  # The given 'inst_type' can be a string in which case it will be converted into
+  # the type String[inst_type].
+  #
+  # @param inst_type [Type,String] the type to qualify
+  # @return [Puppet::Pops::Types::PNotUndefType] the NotUndef type
+  #
+  # @api public
+  #
+  def self.not_undef(inst_type = nil)
+    type = Types::PNotUndefType.new()
+    inst_type = string(inst_type) if inst_type.is_a?(String)
+    type.type = inst_type
+    type
+  end
+
   # Produces a type for Type[T]
   # @api public
   #