puppet 4.8.2-x64-mingw32 → 4.9.0-x64-mingw32

data/lib/puppet/pops/lookup/lookup_key.rb

@@ -0,0 +1,99 @@
+require_relative 'sub_lookup'
+
+module Puppet::Pops
+module Lookup
+# @api private
+class LookupKey
+  include SubLookup
+
+  attr_reader :module_name, :root_key, :segments
+
+  def initialize(key)
+    segments = split_key(key) { |problem| Puppet::DataBinding::LookupError.new("#{problem} in key: '#{key}'") }
+    root_key = segments.shift.freeze
+    qual_index = root_key.index(DOUBLE_COLON)
+
+    @key = key
+    @module_name = qual_index.nil? ? nil : root_key[0..qual_index-1].freeze
+    @root_key = root_key
+    @segments = segments.empty? ? nil : segments.map { |segment| segment =~ /^[0-9]+$/ ? segment.to_i : segment }.freeze
+  end
+
+  def dig(lookup_invocation, value)
+    @segments.nil? ? value : sub_lookup(@key, lookup_invocation, @segments, value)
+  end
+
+  # Prunes a found root value with respect to subkeys in this key. The given _value_ is returned untouched
+  # if this key has no subkeys. Otherwise an attempt is made to create a Hash or Array that contains only the
+  # path to the appointed value and that value.
+  #
+  # If subkeys exists and no value is found, then this method will return `nil`, an empty `Array` or an empty `Hash`
+  # to enable further merges to be applied. The returned type depends on the given _value_.
+  #
+  # @param value [Object] the value to prune
+  # @return the possibly pruned value
+  def prune(value)
+    if @segments.nil?
+      value
+    else
+      pruned = @segments.reduce(value) do |memo, segment|
+        memo.is_a?(Hash) || memo.is_a?(Array) && segment.is_a?(Integer) ? memo[segment] : nil
+      end
+      if pruned.nil?
+        case value
+        when Hash
+          EMPTY_HASH
+        when Array
+          EMPTY_ARRAY
+        else
+          nil
+        end
+      else
+        undig(pruned)
+      end
+    end
+  end
+
+  # Create a structure that can be dug into using the subkeys of this key in order to find the
+  # given _value_. If this key has no subkeys, the _value_ is returned.
+  #
+  # @param value [Object] the value to wrap in a structure in case this value has subkeys
+  # @return [Object] the possibly wrapped value
+  def undig(value)
+    @segments.nil? ? value : segments.reverse.reduce(value) do |memo, segment|
+      if segment.is_a?(Integer)
+        x = []
+        x[segment] = memo
+      else
+        x = { segment => memo }
+      end
+      x
+    end
+  end
+
+  def to_a
+    unless instance_variable_defined?(:@all_segments)
+      a = [@root_key]
+      a += @segments unless @segments.nil?
+      @all_segments = a.freeze
+    end
+    @all_segments
+  end
+
+  def eql?(v)
+    v.is_a?(LookupKey) && @key == v.to_s
+  end
+  alias == eql?
+
+  def hash
+    @key.hash
+  end
+
+  def to_s
+    @key
+  end
+
+  LOOKUP_OPTIONS = LookupKey.new('lookup_options')
+end
+end
+end

data/lib/puppet/pops/lookup/lookup_key_function_provider.rb

@@ -0,0 +1,119 @@
+require_relative 'function_provider'
+
+module Puppet::Pops
+module Lookup
+# @api private
+class LookupKeyFunctionProvider < FunctionProvider
+  TAG = 'lookup_key'.freeze
+
+  # Performs a lookup with the assumption that a recursive check has been made.
+  #
+  # @param key [LookupKey] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String => Object},nil] Merge strategy, merge strategy name, strategy and options hash, or nil (implies "first found")
+  # @return [Object] the found object
+  # @throw :no_such_key when the object is not found
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    lookup_invocation.with(:data_provider, self) do
+      MergeStrategy.strategy(merge).lookup(locations, lookup_invocation) do |location|
+        if location.nil?
+          value = lookup_key(key.root_key, lookup_invocation, nil, merge)
+          lookup_invocation.report_found(key.root_key, validate_data_value(self, value))
+        else
+          lookup_invocation.with(:location, location) do
+            value = lookup_key(key.root_key, lookup_invocation, location.location, merge)
+            lookup_invocation.report_found(key.root_key, validate_data_value(self, value))
+          end
+        end
+      end
+    end
+  end
+
+  def label
+    'Lookup Key'
+  end
+
+  private
+
+  def lookup_key(key, lookup_invocation, location, merge)
+    unless location.nil? || location.exist?
+      lookup_invocation.report_location_not_found
+      throw :no_such_key
+    end
+    ctx = function_context(lookup_invocation, location)
+    ctx.data_hash ||= {}
+    catch(:no_such_key) do
+      hash = ctx.data_hash
+      hash[key] = ctx.function.call(lookup_invocation.scope, key, options(location), Context.new(ctx, lookup_invocation)) unless hash.include?(key)
+      return hash[key]
+    end
+    lookup_invocation.report_not_found(key)
+    throw :no_such_key
+  end
+end
+
+class V3BackendFunctionProvider < LookupKeyFunctionProvider
+  TAG = 'hiera3_backend'.freeze
+
+  def lookup_key(key, lookup_invocation, location, merge)
+    @backend ||= instantiate_backend(lookup_invocation)
+    @backend.lookup(key, lookup_invocation.scope, lookup_invocation.hiera_v3_location_overrides, convert_merge(merge), context = {:recurse_guard => nil})
+  end
+
+  private
+
+  def instantiate_backend(lookup_invocation)
+    backend_name = options[HieraConfig::KEY_BACKEND]
+    begin
+      require 'hiera/backend'
+      require "hiera/backend/#{backend_name.downcase}_backend"
+      backend = Hiera::Backend.const_get("#{backend_name.capitalize}_backend").new
+      return backend.method(:lookup).arity == 4 ? Hiera::Backend::Backend1xWrapper.new(backend) : backend
+    rescue LoadError => e
+      lookup_invocation.report_text { "Unable to load backend '#{backend_name}': #{e.message}" }
+      throw :no_such_key
+    rescue NameError => e
+      lookup_invocation.report_text { "Unable to instantiate backend '#{backend_name}': #{e.message}" }
+      throw :no_such_key
+    end
+  end
+
+  # Converts a lookup 'merge' parameter argument into a Hiera 'resolution_type' argument.
+  #
+  # @param merge [String,Hash,nil] The lookup 'merge' argument
+  # @return [Symbol,Hash,nil] The Hiera 'resolution_type'
+  def convert_merge(merge)
+    case merge
+    when nil
+    when 'first', 'default'
+      # Nil is OK. Defaults to Hiera :priority
+      nil
+    when Puppet::Pops::MergeStrategy
+      convert_merge(merge.configuration)
+    when 'unique'
+      # Equivalent to Hiera :array
+      :array
+    when 'hash'
+      # Equivalent to Hiera :hash with default :native merge behavior. A Hash must be passed here
+      # to override possible Hiera deep merge config settings.
+      { :behavior => :native }
+    when 'deep'
+      # Equivalent to Hiera :hash with :deeper merge behavior.
+      { :behavior => :deeper }
+    when Hash
+      strategy = merge['strategy']
+      if strategy == 'deep'
+        result = { :behavior => :deeper }
+        # Remaining entries must have symbolic keys
+        merge.each_pair { |k,v| result[k.to_sym] = v unless k == 'strategy' }
+        result
+      else
+        convert_merge(strategy)
+      end
+    else
+      raise Puppet::DataBinding::LookupError, "Unrecognized value for request 'merge' parameter: '#{merge}'"
+    end
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/module_data_provider.rb

@@ -0,0 +1,58 @@
+require_relative 'configured_data_provider'
+
+module Puppet::Pops
+module Lookup
+# @api private
+class ModuleDataProvider < ConfiguredDataProvider
+
+  attr_reader :module_name
+
+  def initialize(module_name, config = nil)
+    super(config)
+    @module_name = module_name
+  end
+
+  def place
+    'Module'
+  end
+
+  # Asserts that all keys in the given _data_hash_ are prefixed with the configured _module_name_. Removes entries
+  # that does not follow the convention and logs a warning.
+  #
+  # @param data_hash [Hash] The data hash
+  # @return [Hash] The possibly pruned hash
+  def validate_data_hash(data_provider, data_hash)
+    super
+    module_prefix = "#{module_name}::"
+    data_hash.each_key.reduce(data_hash) do |memo, k|
+      next memo if k == LOOKUP_OPTIONS || k.start_with?(module_prefix)
+      msg = 'must use keys qualified with the name of the module'
+      memo = memo.clone if memo.equal?(data_hash)
+      memo.delete(k)
+      Puppet.warning("Module '#{module_name}': #{data_provider.name} #{msg}")
+      memo
+    end
+  end
+
+  protected
+
+  def assert_config_version(config)
+    raise Puppet::DataBinding::LookupError, "#{config.name} cannot be used in a module" unless config.version > 3
+    config
+  end
+
+  # Return the root of the module with the name equal to the configured module name
+  #
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @return [Pathname] Path to root of the module
+  # @raise [Puppet::DataBinding::LookupError] if the module can not be found
+  #
+  def provider_root(lookup_invocation)
+    env = lookup_invocation.scope.environment
+    mod = env.module(module_name)
+    raise Puppet::DataBinding::LookupError, "Environment '#{env.name}', cannot find module '#{module_name}'" unless mod
+    Pathname.new(mod.path)
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/sub_lookup.rb

@@ -1,6 +1,8 @@
 module Puppet::Pops
 module Lookup
 module SubLookup
+  SPECIAL = /['"\.]/
+
   # Split key into segments. A segment may be a quoted string (both single and double quotes can
   # be used) and the segment separator is the '.' character. Whitespace will be trimmed off on
   # both sides of each segment. Whitespace within quotes are not trimmed.
@@ -15,10 +17,11 @@ module SubLookup
   #
   # @api public
   def split_key(key)
+    return [key] if key.match(SPECIAL).nil?
     segments = key.split(/(\s*"[^"]+"\s*|\s*'[^']+'\s*|[^'".]+)/)
     if segments.empty?
       # Only happens if the original key was an empty string
-      ''
+      raise yield('Syntax error')
     elsif segments.shift == ''
       count = segments.size
       raise yield('Syntax error') unless count > 0
@@ -40,13 +43,14 @@ module SubLookup
   # implements the '#[]' method.
   #
   # @param key [String] the original key (only used for error messages)
-  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param context [Context] The current lookup context
   # @param segments [Array<String>] the segments to use for lookup
   # @param value [Object] the value to access using the segments
   # @return [Object] the value obtained when accessing the value
   #
   # @api public
-  def sub_lookup(key, lookup_invocation, segments, value)
+  def sub_lookup(key, context, segments, value)
+    lookup_invocation = context.is_a?(Invocation) ? context : context.invocation
     lookup_invocation.with(:sub_lookup, segments) do
       segments.each do |segment|
         lookup_invocation.with(:segment, segment) do
@@ -54,7 +58,7 @@ module SubLookup
             lookup_invocation.report_not_found(segment)
             throw :no_such_key
           end
-          if segment =~ /^[0-9]+$/
+          if segment.is_a?(Integer) || segment =~ /^[0-9]+$/
             segment = segment.to_i
             unless value.instance_of?(Array)
               raise Puppet::DataBinding::LookupError,

data/lib/puppet/pops/merge_strategy.rb

@@ -6,12 +6,6 @@ module Puppet::Pops
   class MergeStrategy
     NOT_FOUND = Object.new.freeze
 
-    # The type used for validation of the _merge_ argument
-    def self.merge_t
-      @@merge_t ||=  Types::TypeParser.singleton.parse("Variant[String[1],Runtime[ruby,'Symbol'],Hash[Variant[String[1],Runtime[ruby,'Symbol']],Scalar,1]]")
-    end
-    private_class_method :merge_t
-
     def self.strategies
       @@strategies ||= {}
     end
@@ -24,22 +18,23 @@ module Puppet::Pops
     # @return [MergeStrategy] The matching merge strategy
     #
     def self.strategy(merge)
+      return DefaultMergeStrategy::INSTANCE unless merge
       return merge if merge.is_a?(MergeStrategy)
-      merge = :first if merge.nil?
-      Types::TypeAsserter.assert_instance_of("MergeStrategy 'merge' parameter", merge_t, merge)
+
       if merge.is_a?(Hash)
         merge_strategy = merge['strategy']
         if merge_strategy.nil?
           raise ArgumentError, "The hash given as 'merge' must contain the name of a strategy in string form for the key 'strategy'"
         end
-        merge_options  = merge
+        merge_options  = merge.size == 1 ? EMPTY_HASH : merge
       else
         merge_strategy = merge
-        merge_options  = {}
+        merge_options = EMPTY_HASH
       end
-      strategy = strategies[merge_strategy.to_sym]
-      raise ArgumentError, "Unknown merge strategy: '#{merge_strategy}'" if strategy.nil?
-      strategy.new(merge_options)
+      merge_strategy = merge_strategy.to_sym if merge_strategy.is_a?(String)
+      strategy_class = strategies[merge_strategy]
+      raise ArgumentError, "Unknown merge strategy: '#{merge_strategy}'" if strategy_class.nil?
+      merge_options == EMPTY_HASH ? strategy_class::INSTANCE : strategy_class.new(merge_options)
     end
 
     # Returns the list of merge strategy keys known to this class
@@ -47,7 +42,7 @@ module Puppet::Pops
     # @return [Array<Symbol>] List of strategy keys
     #
     def self.strategy_keys
-      strategies.keys
+      strategies.keys - [:default, :unconstrained_deep, :reverse_deep]
     end
 
     # Adds a new merge strategy to the map of strategies known to this class
@@ -79,7 +74,7 @@ module Puppet::Pops
     # Create a new instance of this strategy configured with the given _options_
     # @param merge_options [Hash<String,Object>] Merge options
     def initialize(options)
-      assert_type('The merge options', options_t, options)
+      assert_type('The merge options', self.class.options_t, options) unless options.empty?
       @options = options
     end
 
@@ -96,6 +91,12 @@ module Puppet::Pops
         assert_type('The second element of the merge', value_t, e2))
     end
 
+    # TODO: API 5.0 Remove this method
+    # @deprecated
+    def merge_lookup(lookup_variants)
+      lookup(lookup_variants, Lookup::Invocation.current)
+    end
+
     # Merges the result of yielding the given _lookup_variants_ to a given block.
     #
     # @param lookup_variants [Array] The variants to pass as second argument to the given block
@@ -105,22 +106,40 @@ module Puppet::Pops
     # @yieldreturn [Object] the value to merge with other values
     # @throws :no_such_key if the lookup was unsuccessful
     #
-    def merge_lookup(lookup_variants)
-      result = lookup_variants.reduce(NOT_FOUND) do |memo, lookup_variant|
-        not_found = true
-        value = catch(:no_such_key) do
-          v = yield(lookup_variant)
-          not_found = false
-          v
-        end
-        if not_found
-          memo
-        else
-          memo.equal?(NOT_FOUND) ? convert_value(value) : merge(memo, value)
+    # Merges the result of yielding the given _lookup_variants_ to a given block.
+    #
+    # @param lookup_variants [Array] The variants to pass as second argument to the given block
+    # @return [Object] the merged value.
+    # @yield [} ]
+    # @yieldparam variant [Object] each variant given in the _lookup_variants_ array.
+    # @yieldreturn [Object] the value to merge with other values
+    # @throws :no_such_key if the lookup was unsuccessful
+    #
+    def lookup(lookup_variants, lookup_invocation)
+      case lookup_variants.size
+      when 0
+        throw :no_such_key
+      when 1
+        yield(lookup_variants[0])
+      else
+        lookup_invocation.with(:merge, self) do
+          result = lookup_variants.reduce(NOT_FOUND) do |memo, lookup_variant|
+            not_found = true
+            value = catch(:no_such_key) do
+              v = yield(lookup_variant)
+              not_found = false
+              v
+            end
+            if not_found
+              memo
+            else
+              memo.equal?(NOT_FOUND) ? convert_value(value) : merge(memo, value)
+            end
+          end
+          throw :no_such_key if result == NOT_FOUND
+          lookup_invocation.report_result(result)
         end
       end
-      throw :no_such_key if result == NOT_FOUND
-      result
     end
 
     # Converts a single value to the type expected when merging two elements
@@ -148,8 +167,8 @@ module Puppet::Pops
     #
     # @return [Types::PStructType] the puppet type
     #
-    def options_t
-      @options_t ||=Types::TypeParser.singleton.parse("Struct[{strategy=>Optional[Pattern[#{self.class.key}]]}]")
+    def self.options_t
+      @options_t ||=Types::TypeParser.singleton.parse("Struct[{strategy=>Optional[Pattern[/#{key}/]]}]")
     end
 
     # Returns the type used to validate the options hash
@@ -172,6 +191,8 @@ module Puppet::Pops
   # Simple strategy that returns the first value found. It never merges any values.
   #
   class FirstFoundStrategy < MergeStrategy
+    INSTANCE = self.new(EMPTY_HASH)
+
     def self.key
       :first
     end
@@ -182,10 +203,10 @@ module Puppet::Pops
     # @return [Object] the merged value
     # @throws :no_such_key unless the lookup was successful
     #
-    def merge_lookup(lookup_variants)
-      lookup_variants.each do |lookup_variant|
-        catch(:no_such_key) { return yield(lookup_variant) }
-      end
+    def lookup(lookup_variants, _)
+      # First found does not continue when a root key was found and a subkey wasn't since that would
+      # simulate a hash merge
+      lookup_variants.each { |lookup_variant| catch(:no_such_key) { return yield(lookup_variant) } }
       throw :no_such_key
     end
 
@@ -198,10 +219,23 @@ module Puppet::Pops
     MergeStrategy.add_strategy(self)
   end
 
+  # Same as {FirstFoundStrategy} but used when no strategy has been explicitly given
+  class DefaultMergeStrategy < FirstFoundStrategy
+    INSTANCE = self.new(EMPTY_HASH)
+
+    def self.key
+      :default
+    end
+
+    MergeStrategy.add_strategy(self)
+  end
+
   # Produces a new hash by merging hash e1 with hash e2 in such a way that the values of duplicate keys
   # will be those of e1
   #
   class HashMergeStrategy < MergeStrategy
+    INSTANCE = self.new(EMPTY_HASH)
+
     def self.key
       :hash
     end
@@ -230,6 +264,8 @@ module Puppet::Pops
   # first contributor of elements and e2 the second.
   #
   class UniqueMergeStrategy < MergeStrategy
+    INSTANCE = self.new(EMPTY_HASH)
+
     def self.key
       :unique
     end
@@ -302,6 +338,8 @@ module Puppet::Pops
   #   Results: {:x => [{:y => 1, :z => 2}]}
   #
   class DeepMergeStrategy < MergeStrategy
+    INSTANCE = self.new(EMPTY_HASH)
+
     def self.key
       :deep
     end
@@ -309,8 +347,20 @@ module Puppet::Pops
     def checked_merge(e1, e2)
       dm_options = { :preserve_unmergeables => false }
       options.each_pair { |k,v| dm_options[k.to_sym] = v unless k == 'strategy' }
-      # e2 (the destination) is dup'ed to avoid that the passed in object mutates
-      DeepMerge.deep_merge!(e1, e2.dup, dm_options)
+      # e2 (the destination) is deep cloned to avoid that the passed in object mutates
+      DeepMerge.deep_merge!(e1, deep_clone(e2), dm_options)
+    end
+
+    def deep_clone(value)
+      if value.is_a?(Hash)
+        result = value.clone
+        value.each{ |k, v| result[k] = deep_clone(v) }
+        result
+      elsif value.is_a?(Array)
+        value.map{ |v| deep_clone(v) }
+      else
+        value
+      end
     end
 
     protected
@@ -319,9 +369,9 @@ module Puppet::Pops
     # the setting of that option to false
     #
     # @return [Types::PAnyType] the puppet type used when validating the options hash
-    def options_t
+    def self.options_t
       @options_t ||= Types::TypeParser.singleton.parse('Struct[{'\
-          "strategy=>Optional[Pattern[#{self.class.key}]],"\
+          "strategy=>Optional[Pattern[#{key}]],"\
           'knockout_prefix=>Optional[String],'\
           'merge_debug=>Optional[Boolean],'\
           'merge_hash_arrays=>Optional[Boolean],'\
@@ -335,4 +385,35 @@ module Puppet::Pops
 
     MergeStrategy.add_strategy(self)
   end
+
+  # Same as {DeepMergeStrategy} but without constraint on valid merge options
+  # (needed for backward compatibility with Hiera v3)
+  class UnconstrainedDeepMergeStrategy < DeepMergeStrategy
+    def self.key
+      :unconstrained_deep
+    end
+
+    # @return [Types::PAnyType] the puppet type used when validating the options hash
+    def self.options_t
+      @options_t ||= Types::TypeParser.singleton.parse('Hash[String[1],Any]')
+    end
+
+    MergeStrategy.add_strategy(self)
+  end
+
+  # Same as {UnconstrainedDeepMergeStrategy} but with reverse priority of merged elements.
+  # (needed for backward compatibility with Hiera v3)
+  class ReverseDeepMergeStrategy < UnconstrainedDeepMergeStrategy
+    INSTANCE = self.new(EMPTY_HASH)
+
+    def self.key
+      :reverse_deep
+    end
+
+    def checked_merge(e1, e2)
+      super(e2, e1)
+    end
+
+    MergeStrategy.add_strategy(self)
+  end
 end