puppet 4.4.0 → 4.5.0

data/lib/puppet/pops/lookup/explainer.rb

@@ -367,13 +367,46 @@ module Puppet::Pops::Lookup
     end
   end
 
+  class ExplainSubLookup < ExplainTreeNode
+    def initialize(parent, sub_key)
+      super(parent)
+      @sub_key = sub_key
+    end
+
+    def dump_on(io, indent, first_indent)
+      io << indent << 'Sub key: "' << @sub_key.join('.') << "\"\n"
+      indent = increase_indent(indent)
+      branches.each {|b| b.dump_on(io, indent, indent)}
+      dump_outcome(io, indent)
+    end
+
+    def type
+      :sub_key
+    end
+  end
+
+  class ExplainKeySegment < ExplainTreeNode
+    def initialize(parent, segment)
+      super(parent)
+      @segment = segment
+    end
+
+    def dump_on(io, indent, first_indent)
+      dump_outcome(io, indent)
+    end
+
+    def type
+      :segment
+    end
+  end
+
   class ExplainScope < ExplainTreeNode
     def initialize(parent)
       super(parent)
     end
 
     def dump_on(io, indent, first_indent)
-      io << indent << 'Global Scope' << "\"\n"
+      io << indent << 'Global Scope' << "\n"
       indent = increase_indent(indent)
       dump_outcome(io, indent)
     end
@@ -406,6 +439,10 @@ module Puppet::Pops::Lookup
           ExplainMerge.new(@current, qualifier)
         when :scope
           ExplainScope.new(@current)
+        when :sub_lookup
+          ExplainSubLookup.new(@current, qualifier)
+        when :segment
+          ExplainKeySegment.new(@current, qualifier)
         when :meta, :data
           ExplainTop.new(@current, qualifier_type, qualifier)
         when :invalid_key

data/lib/puppet/pops/lookup/interpolation.rb

@@ -0,0 +1,115 @@
+require_relative 'sub_lookup'
+
+# Add support for Hiera-like interpolation expressions. The expressions may contain keys that uses dot-notation
+# to further navigate into hashes and arrays
+#
+module Puppet::Pops
+module Lookup
+module Interpolation
+  include SubLookup
+
+  def interpolate(subject, lookup_invocation, allow_methods)
+    case subject
+    when String
+      subject.index('%{').nil? ? subject : interpolate_string(subject, lookup_invocation, allow_methods)
+    when Array
+      subject.map { |element| interpolate(element, lookup_invocation, allow_methods) }
+    when Hash
+      Hash[subject.map { |k, v| [k, interpolate(v, lookup_invocation, allow_methods)] }]
+    else
+      subject
+    end
+  end
+
+  private
+
+  EMPTY_INTERPOLATIONS = {
+    '' => true,
+    '::' => true,
+    '""' => true,
+    "''" => true,
+    '"::"' => true,
+    "'::'" => true
+  }.freeze
+
+  # Matches a key that is quoted using a matching pair of either single or double quotes.
+  QUOTED_KEY = /^(?:"([^"]+)"|'([^']+)')$/
+
+  def interpolate_string(subject, lookup_invocation, allow_methods)
+    lookup_invocation.with(:interpolate, subject) do
+      subject.gsub(/%\{([^\}]*)\}/) do |match|
+        expr = $1
+        # Leading and trailing spaces inside an interpolation expression are insignificant
+        expr.strip!
+        value = nil
+        unless EMPTY_INTERPOLATIONS[expr]
+          method_key, key = get_method_and_data(expr, allow_methods)
+          is_alias = method_key == 'alias'
+
+          # Alias is only permitted if the entire string is equal to the interpolate expression
+          raise Puppet::DataBinding::LookupError, "'alias' interpolation is only permitted if the expression is equal to the entire string" if is_alias && subject != match
+
+          segments = split_key(key) { |problem| Puppet::DataBinding::LookupError.new("#{problem} in string: #{subject}") }
+          root_key = segments.shift
+          value = interpolate_method(method_key).call(root_key, lookup_invocation)
+          value = sub_lookup(key, lookup_invocation, segments, value) unless segments.empty?
+          value = lookup_invocation.check(key) { interpolate(value, lookup_invocation, allow_methods) }
+
+          # break gsub and return value immediately if this was an alias substitution. The value might be something other than a String
+          return value if is_alias
+        end
+        value || ''
+      end
+    end
+  end
+
+  def interpolate_method(method_key)
+    @@interpolate_methods ||= begin
+      global_lookup = lambda { |key, lookup_invocation| Lookup.lookup(key, nil, '', true, nil, lookup_invocation) }
+      scope_lookup = lambda do |key, lookup_invocation|
+        lookup_invocation.with(:scope, nil) do
+          ovr = lookup_invocation.override_values
+          if ovr.include?(key)
+            lookup_invocation.report_found_in_overrides(key, ovr[key])
+          else
+            scope = lookup_invocation.scope
+            if scope.include?(key)
+              lookup_invocation.report_found(key, scope[key])
+            else
+              defaults = lookup_invocation.default_values
+              if defaults.include?(key)
+                lookup_invocation.report_found_in_defaults(key, defaults[key])
+              else
+                nil
+              end
+            end
+          end
+        end
+      end
+
+      {
+        'lookup' => global_lookup,
+        'hiera' => global_lookup, # this is just an alias for 'lookup'
+        'alias' => global_lookup, # same as 'lookup' but expression must be entire string and result is not subject to string substitution
+        'scope' => scope_lookup,
+        'literal' => lambda { |key, _| key }
+      }.freeze
+    end
+    interpolate_method = @@interpolate_methods[method_key]
+    raise Puppet::DataBinding::LookupError, "Unknown interpolation method '#{method_key}'" unless interpolate_method
+    interpolate_method
+  end
+
+  def get_method_and_data(data, allow_methods)
+    if match = data.match(/^(\w+)\((?:["]([^"]+)["]|[']([^']+)['])\)$/)
+      raise Puppet::DataBinding::LookupError, 'Interpolation using method syntax is not allowed in this context' unless allow_methods
+      key = match[1]
+      data = match[2] || match[3] # double or single qouted
+    else
+      key = 'scope'
+    end
+    [key, data]
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/sub_lookup.rb

@@ -0,0 +1,86 @@
+module Puppet::Pops
+module Lookup
+module SubLookup
+  # Split key into segments. A segment may be a quoted string (both single and double quotes can
+  # be used) and the segment separator is the '.' character. Whitespace will be trimmed off on
+  # both sides of each segment. Whitespace within quotes are not trimmed.
+  #
+  # If the key cannot be parsed, this method will yield a string describing the problem to a one
+  # parameter block. The block must return an exception instance.
+  #
+  # @param key [String] the string to split
+  # @return Array<String> the array of segments
+  # @yieldparam problem [String] the problem, i.e. 'Syntax error'
+  # @yieldreturn [Exception] the exception to raise
+  #
+  # @api public
+  def split_key(key)
+    segments = key.split(/(\s*"[^"]+"\s*|\s*'[^']+'\s*|[^'".]+)/)
+    if segments.empty?
+      # Only happens if the original key was an empty string
+      ''
+    elsif segments.shift == ''
+      count = segments.size
+      raise yield('Syntax error') unless count > 0
+
+      segments.keep_if { |seg| seg != '.' }
+      raise yield('Syntax error') unless segments.size * 2 == count + 1
+      segments.map! do |segment|
+        segment.strip!
+        segment.start_with?('"') || segment.start_with?("'") ? segment[1..-2] : segment
+      end
+    else
+      raise yield('Syntax error')
+    end
+  end
+
+  # Perform a sub-lookup using the given _segments_ to access the given _value_. Each segment must be a string. A string
+  # consisting entirely of digits will be treated as an indexed lookup which means that the value that it is applied to
+  # must be an array. Other types of segments will expect that the given value is something other than a String that
+  # implements the '#[]' method.
+  #
+  # @param key [String] the original key (only used for error messages)
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param segments [Array<String>] the segments to use for lookup
+  # @param value [Object] the value to access using the segments
+  # @return [Object] the value obtained when accessing the value
+  #
+  # @api public
+  def sub_lookup(key, lookup_invocation, segments, value)
+    lookup_invocation.with(:sub_lookup, segments) do
+      segments.each do |segment|
+        lookup_invocation.with(:segment, segment) do
+          if value.nil?
+            lookup_invocation.report_not_found(segment)
+            throw :no_such_key
+          end
+          if segment =~ /^[0-9]+$/
+            segment = segment.to_i
+            unless value.instance_of?(Array)
+              raise Puppet::DataBinding::LookupError,
+                "Data Provider type mismatch: Got #{value.class.name} when Array was expected to access value using '#{segment}' from key '#{key}'"
+            end
+            unless segment < value.size
+              lookup_invocation.report_not_found(segment)
+              throw :no_such_key
+            end
+          else
+            unless value.respond_to?(:'[]') && !(value.instance_of?(Array) || value.instance_of?(String))
+              raise Puppet::DataBinding::LookupError,
+                "Data Provider type mismatch: Got #{value.class.name} when a hash-like object was expected to access value using '#{segment}' from key '#{key}'"
+            end
+            unless value.include?(segment)
+              lookup_invocation.report_not_found(segment)
+              throw :no_such_key
+            end
+          end
+          value = value[segment]
+          lookup_invocation.report_found(segment, value)
+        end
+      end
+      value
+    end
+  end
+end
+end
+end

data/lib/puppet/pops/merge_strategy.rb

@@ -279,7 +279,6 @@ module Puppet::Pops
   #   hash.deep_merge!({:x => [1,2]}, {:knockout_prefix => '--'})
   #   - 'knockout_prefix' Set to string value to signify prefix which deletes elements from existing element. Defaults is _undef_
   #   - 'sort_merged_arrays' Set to _true_ to sort all arrays that are merged together. Default is _false_
-  #   - 'unpack_arrays' Set to string value used as a deliminator to join all array values and then split them again. Default is _undef_
   #   - 'merge_hash_arrays' Set to _true_ to merge hashes within arrays. Default is _false_
   #
   # Selected Options Details:
@@ -295,14 +294,6 @@ module Puppet::Pops
   #    dest   = {:x => [1,2,3]}
   #    dest.ko_deep_merge!(source)
   #    Results: {:x => ""}
-  # :unpack_arrays => The purpose of this is to permit compound elements to be passed
-  #   in as strings and to be converted into discrete array elements
-  #   irsource = {:x => ['1,2,3', '4']}
-  #   dest   = {:x => ['5','6','7,8']}
-  #   dest.deep_merge!(source, {:unpack_arrays => ','})
-  #   Results: {:x => ['1','2','3','4','5','6','7','8']}
-  #   Why: If receiving data from an HTML form, this makes it easy for a checkbox
-  #    to pass multiple values from within a single HTML element
   #
   # :merge_hash_arrays => merge hashes within arrays
   #   source = {:x => [{:y => 1}]}
@@ -335,7 +326,6 @@ module Puppet::Pops
           'merge_debug=>Optional[Boolean],'\
           'merge_hash_arrays=>Optional[Boolean],'\
           'sort_merge_arrays=>Optional[Boolean],'\
-          'unpack_arrays=>Optional[String]'\
           '}]')
     end
 

data/lib/puppet/pops/model/ast_transformer.rb

@@ -47,8 +47,11 @@ class Puppet::Pops::Model::AstTransformer
       if source_pos
         pos[:line] = source_pos.line
         pos[:pos]  = source_pos.pos
+        pos[:file] = source_pos.locator.file
+      end
+      if nil_or_empty?(pos[:file]) && !nil_or_empty?(@source_file)
+        pos[:file] = @source_file
       end
-      pos[:file] = @source_file if @source_file
       hash = hash.merge(pos)
     end
     hash
@@ -122,4 +125,8 @@ class Puppet::Pops::Model::AstTransformer
   def is_nop?(o)
     o.nil? || o.is_a?(Model::Nop)
   end
+
+  def nil_or_empty?(x)
+    x.nil? || x == ''
+  end
 end

data/lib/puppet/pops/model/factory.rb

@@ -157,8 +157,8 @@ class Factory
   end
 
   def build_KeyedEntry(o, k, v)
-    o.key = to_ops(k)
-    o.value = to_ops(v)
+    o.key = to_collection_entry(to_ops(k))
+    o.value = to_collection_entry(to_ops(v))
     o
   end
 
@@ -167,8 +167,21 @@ class Factory
     o
   end
 
+  def to_collection_entry(o)
+    if o.is_a?(Model::ReservedWord)
+      case o.word
+      when 'application', 'site', 'produces', 'consumes'
+        build(o.word)
+      else
+        o
+      end
+    else
+      o
+    end
+  end
+
   def build_LiteralList(o, *values)
-    o.values = values.map {|v| build(v) }
+    o.values = values.map {|v| to_collection_entry(build(v)) }
     o
   end
 
@@ -266,7 +279,7 @@ class Factory
   end
 
   def build_QualifiedReference(o, name)
-    o.value = name.to_s.downcase
+    o.cased_value = name.to_s
     o
   end
 
@@ -335,8 +348,14 @@ class Factory
   end
 
   def build_TypeAlias(o, name, type_expr)
-    o.type_expr = build(type_expr)
-    o.name = name
+    o.type_expr = to_ops(type_expr)
+    o.name = to_ops(name).cased_value
+    o
+  end
+
+  def build_TypeMapping(o, lhs, rhs)
+    o.type_expr = to_ops(lhs)
+    o.mapping_expr = to_ops(rhs)
     o
   end
 
@@ -806,8 +825,12 @@ class Factory
     new(LambdaExpression, parameters, body)
   end
 
-  def self.TYPE_ALIAS(name, type_expr)
-    new(TypeAlias, name, type_expr)
+  def self.TYPE_ASSIGNMENT(lhs, rhs)
+    if lhs.current.is_a?(AccessExpression)
+      new(TypeMapping, lhs, rhs)
+    else
+      new(TypeAlias, lhs, rhs)
+    end
   end
 
   def self.TYPE_DEFINITION(name, parent, body)

data/lib/puppet/pops/model/model.rb

@@ -107,6 +107,14 @@ module Puppet::Pops
       end
     end
 
+    class QualifiedReference
+      module ClassModule
+        def value
+          @value ||= cased_value.downcase
+        end
+      end
+    end
+
     class Program < PopsObject
       module ClassModule
         def locator

data/lib/puppet/pops/model/model_label_provider.rb

@@ -96,6 +96,7 @@ class ModelLabelProvider
   def label_CatalogCollector o            ; "Catalog-Collector"                 end
   def label_ExportedCollector o           ; "Exported-Collector"                end
   def label_TypeAlias o                   ; "Type Alias"                        end
+  def label_TypeMapping o                 ; "Type Mapping"                      end
   def label_TypeDefinition o              ; "Type Definition"                   end
 
   def label_PResourceType o

data/lib/puppet/pops/model/model_meta.rb

@@ -305,6 +305,12 @@ module Puppet::Pops::Model
     contains_one_uni 'type_expr', Expression
   end
 
+  # A type mapping assignment
+  class TypeMapping < Definition
+    contains_one_uni 'type_expr', Expression
+    contains_one_uni 'mapping_expr', Expression
+  end
+
   # A type definition
   class TypeDefinition < QRefDefinition
     has_attr 'parent', String
@@ -502,7 +508,7 @@ module Puppet::Pops::Model
   # A DSL CLASSREF (one or multiple parts separated by '::' where (at least) the first part starts with an upper case letter).
   #
   class QualifiedReference < LiteralValue
-    has_attr 'value', String, :lowerBound => 1
+    has_attr 'cased_value', String, :lowerBound => 1
   end
 
   # A Variable expression looks up value of expr (some kind of name) in scope.

data/lib/puppet/pops/model/model_tree_dumper.rb

@@ -419,6 +419,10 @@ class Puppet::Pops::Model::ModelTreeDumper < Puppet::Pops::Model::TreeDumper
     ['type-alias', o.name, do_dump(o.type_expr)]
   end
 
+  def dump_TypeMapping(o)
+    ['type-mapping', do_dump(o.type_expr), do_dump(o.mapping_expr)]
+  end
+
   def dump_TypeDefinition(o)
     ['type-definition', o.name, o.parent, do_dump(o.body)]
   end

data/lib/puppet/pops/parser/egrammar.ra

@@ -684,8 +684,8 @@ parameter_type
 
 #--TYPE ALIAS
 type_alias
-  : TYPE CLASSREF EQUALS expression {
-      result = add_definition(Factory.TYPE_ALIAS(val[1][:value], val[3]))
+  : TYPE parameter_type EQUALS expression {
+      result = add_definition(Factory.TYPE_ASSIGNMENT(val[1], val[3]))
       loc(result, val[0], val[3])
     }
 
@@ -720,10 +720,10 @@ reserved_word
 #
 
 array
-  : LISTSTART assignments endcomma  RBRACK { result = Factory.LIST(val[1]); loc result, val[0], val[3] }
-  | LISTSTART                       RBRACK { result = Factory.literal([]) ; loc result, val[0], val[1] }
-  | LBRACK    assignments endcomma  RBRACK { result = Factory.LIST(val[1]); loc result, val[0], val[3] }
-  | LBRACK                          RBRACK { result = Factory.literal([]) ; loc result, val[0], val[1] }
+  : LISTSTART collection_entries endcomma  RBRACK { result = Factory.LIST(val[1]); loc result, val[0], val[3] }
+  | LISTSTART                              RBRACK { result = Factory.literal([]) ; loc result, val[0], val[1] }
+  | LBRACK    collection_entries endcomma  RBRACK { result = Factory.LIST(val[1]); loc result, val[0], val[3] }
+  | LBRACK                                 RBRACK { result = Factory.literal([]) ; loc result, val[0], val[1] }
 
 hash
   : LBRACE hashpairs RBRACE       { result = Factory.HASH(val[1]); loc result, val[0], val[2] }
@@ -735,7 +735,24 @@ hash
     | hashpairs COMMA hashpair { result = val[0].push val[2] }
 
   hashpair
-    : assignment FARROW assignment { result = Factory.KEY_ENTRY(val[0], val[2]); loc result, val[1] }
+    : collection_entry FARROW collection_entry { result = Factory.KEY_ENTRY(val[0], val[2]); loc result, val[1] }
+
+collection_entry:
+  : assignment
+  | collection_entry_keyword { result = Factory.literal(val[0][:value]) ; loc result, val[0] }
+
+collection_entries
+  : collection_entry                           { result = [val[0]] }
+  | collection_entries COMMA collection_entry  { result = val[0].push(val[2]) }
+
+# Keywords valid as a collection value
+collection_entry_keyword
+  : TYPE
+  | FUNCTION
+  | APPLICATION
+  | CONSUMES
+  | PRODUCES
+  | SITE
 
 quotedtext
   : string