puppet 4.9.4 → 4.10.0

data/lib/puppet/pops/lookup/data_provider.rb

@@ -3,36 +3,32 @@ module Lookup
 # @api private
 module DataProvider
   def self.key_type
-    ensure_types_initialized
     @key_type
   end
 
   def self.value_type
-    ensure_types_initialized
     @value_type
   end
 
-  def self.ensure_types_initialized
-    if @key_type.nil?
-      (@key_type, @value_type) = Pcore::register_aliases(
-        # The Pcore type for all keys and subkeys in a data hash.
-        'Puppet::LookupKey' => 'Variant[String,Numeric]',
+  def self.register_types(loader)
+    (@key_type, @value_type) = Pcore::register_aliases({
+      # The Pcore type for all keys and subkeys in a data hash.
+      'Puppet::LookupKey' => 'Variant[String,Numeric]',
 
-        # The Pcore type for all values and sub-values in a data hash. The
-        # type is self-recursive to enforce the same constraint on values contained
-        # in arrays and hashes
-        'Puppet::LookupValue' => <<-PUPPET
-          Variant[
-            Scalar,
-            Undef,
-            Sensitive,
-            Type,
-            Hash[Puppet::LookupKey, Puppet::LookupValue],
-            Array[Puppet::LookupValue]
-          ]
+      # The Pcore type for all values and sub-values in a data hash. The
+      # type is self-recursive to enforce the same constraint on values contained
+      # in arrays and hashes
+      'Puppet::LookupValue' => <<-PUPPET
+        Variant[
+          Scalar,
+          Undef,
+          Sensitive,
+          Type,
+          Hash[Puppet::LookupKey, Puppet::LookupValue],
+          Array[Puppet::LookupValue]
+        ]
         PUPPET
-      )
-    end
+    }, Pcore::RUNTIME_NAME_AUTHORITY, loader)
   end
 
   # Performs a lookup with an endless recursion check.
@@ -45,6 +41,17 @@ module DataProvider
     lookup_invocation.check(key.to_s) { unchecked_key_lookup(key, lookup_invocation, merge) }
   end
 
+  # Performs a lookup using a module default hierarchy with an endless recursion check. All providers except
+  # the `ModuleDataProvider` will throw `:no_such_key` if this method is called.
+  #
+  # @param key [LookupKey] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String=>Object},nil] Merge strategy or hash with strategy and options
+  #
+  def key_lookup_in_default(key, lookup_invocation, merge)
+    throw :no_such_key
+  end
+
   def lookup(key, lookup_invocation, merge)
     lookup_invocation.check(key.to_s) { unchecked_key_lookup(key, lookup_invocation, merge) }
   end
@@ -70,23 +77,30 @@ module DataProvider
     raise NotImplementedError, "Subclass of #{DataProvider.name} must implement 'name' method"
   end
 
-  # Asserts that _data_hash_ is a hash.
+  # @returns `true` if the value provided by this instance can always be trusted, `false` otherwise
+  def value_is_validated?
+    false
+  end
+
+  # Asserts that _data_hash_ is a hash. Will yield to obtain origin of value in case an error is produced
   #
-  # @param data_provider [DataProvider] The data provider that produced the hash
   # @param data_hash [Hash{String=>Object}] The data hash
   # @return [Hash{String=>Object}] The data hash
-  def validate_data_hash(data_provider, data_hash)
-    Types::TypeAsserter.assert_instance_of(nil, Types::PHashType::DEFAULT, data_hash) { "Value returned from #{data_provider.name}" }
+  def validate_data_hash(data_hash, &block)
+    Types::TypeAsserter.assert_instance_of(nil, Types::PHashType::DEFAULT, data_hash, &block)
   end
 
-  # Asserts that _data_value_ is of valid type.
+  # Asserts that _data_value_ is of valid type. Will yield to obtain origin of value in case an error is produced
   #
   # @param data_provider [DataProvider] The data provider that produced the hash
-  # @param data_value [Object] The data value
   # @return [Object] The data value
-  def validate_data_value(data_provider, value)
+  def validate_data_value(value, &block)
     # The DataProvider.value_type is self recursive so further recursive check of collections is needed here
-    Types::TypeAsserter.assert_instance_of(nil, DataProvider.value_type, value) { "Value returned from #{data_provider.name}" }
+    unless value_is_validated? || DataProvider.value_type.instance?(value)
+      actual_type = Types::TypeCalculator.singleton.infer(value)
+      raise Types::TypeAssertionError.new("#{yield} has wrong type, expects Puppet::LookupValue, got #{actual_type}", DataProvider.value_type, actual_type)
+    end
+    value
   end
 end
 end

data/lib/puppet/pops/lookup/environment_data_provider.rb

@@ -15,7 +15,7 @@ class EnvironmentDataProvider < ConfiguredDataProvider
       config
     else
       if Puppet[:strict] == :error
-        raise Puppet::DataBinding::LookupError, "#{config.name} cannot be used in an environment"
+        config.fail(Issues::HIERA_VERSION_3_NOT_GLOBAL, :where => 'environment')
       else
         Puppet.warn_once(:hiera_v3_at_env_root, config.config_path, 'hiera.yaml version 3 found at the environment root was ignored')
       end

data/lib/puppet/pops/lookup/explainer.rb

@@ -444,6 +444,7 @@ module Lookup
     def dump_on(io, indent, first_indent)
       io << indent << @name << "\n"
       indent = increase_indent(indent)
+      branches.each {|b| b.dump_on(io, indent, indent)}
       dump_outcome(io, indent)
     end
 

data/lib/puppet/pops/lookup/function_provider.rb

@@ -10,6 +10,15 @@ class FunctionProvider
 
   attr_reader :parent_data_provider, :function_name, :locations
 
+  # Returns the type that all the return type of all functions must be assignable to.
+  # For `lookup_key` and `data_dig`, that will be the `Puppet::LookupValue` type. For
+  # `data_hash` it will be a Hash[Puppet::LookupKey,Puppet::LookupValue]`
+  #
+  # @return [Type] the trusted return type
+  def self.trusted_return_type
+    DataProvider.value_type
+  end
+
   def initialize(name, parent_data_provider, function_name, options, locations)
     @name = name
     @parent_data_provider = parent_data_provider
@@ -25,8 +34,7 @@ class FunctionProvider
   end
 
   def create_function_context(lookup_invocation)
-    scope = lookup_invocation.scope
-    FunctionContext.new(EnvironmentContext.adapt(scope.compiler.environment), module_name, function(scope))
+    FunctionContext.new(EnvironmentContext.adapt(lookup_invocation.scope.compiler.environment), module_name, function(lookup_invocation))
   end
 
   def module_name
@@ -37,6 +45,10 @@ class FunctionProvider
     "Hierarchy entry \"#{@name}\""
   end
 
+  def full_name
+    "#{self.class::TAG} function '#{@function_name}'"
+  end
+
   def to_s
     name
   end
@@ -46,9 +58,8 @@ class FunctionProvider
   # @param [Pathname,URI] location The location to add to the options
   # @return [Hash{String => Object}] The options hash
   def options(location = nil)
+    location = location.location unless location.nil?
     case location
-    when nil
-      @options
     when Pathname
       @options.merge(HieraConfig::KEY_PATH => location.to_s)
     when URI
@@ -58,14 +69,18 @@ class FunctionProvider
     end
   end
 
+  def value_is_validated?
+    @value_is_validated
+  end
+
   private
 
-  def function(scope)
-    @function ||= load_function(scope)
+  def function(lookup_invocation)
+    @function ||= load_function(lookup_invocation)
   end
 
-  def load_function(scope)
-    loaders = scope.compiler.loaders
+  def load_function(lookup_invocation)
+    loaders = lookup_invocation.scope.compiler.loaders
     typed_name = Loader::TypedName.new(:function, @function_name)
     loader = if typed_name.qualified?
       qualifier = typed_name.name_parts[0]
@@ -75,10 +90,20 @@ class FunctionProvider
     end
     te = loader.load_typed(typed_name)
     if te.nil? || te.value.nil?
-      raise Puppet::DataBinding::LookupError,
-        "#{@options[HieraConfig::KEY_NAME]}: Unable to find '#{self.class::TAG}' function named '#{function_name}'"
+      @parent_data_provider.config(lookup_invocation).fail(Issues::HIERA_DATA_PROVIDER_FUNCTION_NOT_FOUND,
+        :function_type => self.class::TAG, :function_name => @function_name)
+    end
+    func = te.value
+    @value_is_validated = func.class.dispatcher.dispatchers.all? do |dispatcher|
+      rt = dispatcher.type.return_type
+      if rt.nil?
+        false
+      else
+        Types::TypeAsserter.assert_assignable(nil, self.class.trusted_return_type, rt) { "Return type of '#{self.class::TAG}' function named '#{function_name}'" }
+        true
+      end
     end
-    te.value
+    func
   end
 end
 end

data/lib/puppet/pops/lookup/global_data_provider.rb

@@ -15,11 +15,16 @@ class GlobalDataProvider < ConfiguredDataProvider
       # Hiera version 3 needs access to special scope variables
       scope = lookup_invocation.scope
       unless scope.is_a?(Hiera::Scope)
-        lookup_invocation = Invocation.new(
+        hiera_invocation = Invocation.new(
           Hiera::Scope.new(scope),
           lookup_invocation.override_values,
           lookup_invocation.default_values,
           lookup_invocation.explainer)
+
+        # Confine to global scope unless an environment data provider has been defined (same as for hiera_xxx functions)
+        adapter = lookup_invocation.lookup_adapter
+        hiera_invocation.set_global_only unless adapter.global_only? || adapter.has_environment_data_provider?(lookup_invocation)
+        lookup_invocation = hiera_invocation
       end
 
       merge = MergeStrategy.strategy(merge)
@@ -32,9 +37,17 @@ class GlobalDataProvider < ConfiguredDataProvider
       end
 
       value = super(key, lookup_invocation, merge)
-      if lookup_invocation.hiera_xxx_call? && (merge.is_a?(HashMergeStrategy) || merge.is_a?(DeepMergeStrategy))
-        # hiera_hash calls should error when found values are not hashes
-        Types::TypeAsserter.assert_instance_of('value', Types::PHashType::DEFAULT, value)
+      if lookup_invocation.hiera_xxx_call?
+        if merge.is_a?(HashMergeStrategy) || merge.is_a?(DeepMergeStrategy)
+          # hiera_hash calls should error when found values are not hashes
+          Types::TypeAsserter.assert_instance_of('value', Types::PHashType::DEFAULT, value)
+        end
+        if !key.segments.nil? && (merge.is_a?(HashMergeStrategy) || merge.is_a?(UniqueMergeStrategy))
+          strategy = merge.is_a?(HashMergeStrategy) ? 'hash' : 'array'
+
+          # Fail with old familiar message from Hiera 3
+          raise Puppet::DataBinding::LookupError, "Resolution type :#{strategy} is illegal when accessing values using dotted keys. Offending key was '#{key}'"
+        end
       end
       value
     else
@@ -45,7 +58,7 @@ class GlobalDataProvider < ConfiguredDataProvider
   protected
 
   def assert_config_version(config)
-    raise Puppet::DataBinding::LookupError, "#{config.name} cannot be used in the global layer" if config.version == 4
+    config.fail(Issues::HIERA_UNSUPPORTED_VERSION_IN_GLOBAL) if config.version == 4
     config
   end
 

data/lib/puppet/pops/lookup/hiera_config.rb

@@ -22,6 +22,16 @@ class ScopeLookupCollectingInvocation < Invocation
     @scope_interpolations.uniq! { |si| si[0] }
     @scope_interpolations
   end
+
+  # Yield invocation that remembers all but the given name
+  def with_local_memory_eluding(name)
+    save_si = @scope_interpolations
+    @scope_interpolations = []
+    result = yield
+    save_si.concat(@scope_interpolations.reject { |entry| entry[1] == name })
+    @scope_interpolations = save_si
+    result
+  end
 end
 
 # @api private
@@ -34,11 +44,13 @@ class HieraConfig
   KEY_NAME = 'name'.freeze
   KEY_VERSION = 'version'.freeze
   KEY_DATADIR = 'datadir'.freeze
+  KEY_DEFAULT_HIERARCHY = 'default_hierarchy'.freeze
   KEY_HIERARCHY = 'hierarchy'.freeze
   KEY_LOGGER = 'logger'.freeze
   KEY_OPTIONS = 'options'.freeze
   KEY_PATH = 'path'.freeze
   KEY_PATHS = 'paths'.freeze
+  KEY_MAPPED_PATHS = 'mapped_paths'.freeze
   KEY_GLOB = 'glob'.freeze
   KEY_GLOBS = 'globs'.freeze
   KEY_URI = 'uri'.freeze
@@ -56,7 +68,7 @@ class HieraConfig
 
   FUNCTION_KEYS = [KEY_DATA_HASH, KEY_LOOKUP_KEY, KEY_DATA_DIG, KEY_V3_BACKEND]
   ALL_FUNCTION_KEYS = FUNCTION_KEYS + [KEY_V4_DATA_HASH]
-  LOCATION_KEYS = [KEY_PATH, KEY_PATHS, KEY_GLOB, KEY_GLOBS, KEY_URI, KEY_URIS]
+  LOCATION_KEYS = [KEY_PATH, KEY_PATHS, KEY_GLOB, KEY_GLOBS, KEY_URI, KEY_URIS, KEY_MAPPED_PATHS]
   FUNCTION_PROVIDERS = {
     KEY_DATA_HASH => DataHashFunctionProvider,
     KEY_DATA_DIG => DataDigFunctionProvider,
@@ -67,7 +79,7 @@ class HieraConfig
     KEY_V4_DATA_HASH => V4DataHashFunctionProvider
   }
 
-  def self.v4_function_config(config_root, function_name)
+  def self.v4_function_config(config_root, function_name, owner)
     unless Puppet[:strict] == :off
       Puppet.warn_once(:deprecation, 'legacy_provider_function',
         "Using of legacy data provider function '#{function_name}'. Please convert to a 'data_hash' function")
@@ -81,7 +93,8 @@ class HieraConfig
             KEY_V4_DATA_HASH => function_name
           }
         ]
-      }.freeze
+      }.freeze,
+      owner
     )
   end
 
@@ -106,8 +119,9 @@ class HieraConfig
   #
   # @param lookup_invocation [Invocation] Invocation data containing scope, overrides, and defaults
   # @param config_path [Pathname] Absolute path to the configuration file
+  # @param owner [ConfiguredDataProvider] The data provider that will own the created configuration
   # @return [LookupConfiguration] the configuration
-  def self.create(lookup_invocation, config_path)
+  def self.create(lookup_invocation, config_path, owner)
     if config_path.is_a?(Hash)
       config_path = nil
       loaded_config = config_path
@@ -137,13 +151,15 @@ class HieraConfig
     version = version.nil? ? 3 : version.to_i
     case version
     when 5
-      HieraConfigV5.new(config_root, config_path, loaded_config)
+      HieraConfigV5.new(config_root, config_path, loaded_config, owner)
     when 4
-      HieraConfigV4.new(config_root, config_path, loaded_config)
+      HieraConfigV4.new(config_root, config_path, loaded_config, owner)
     when 3
-      HieraConfigV3.new(config_root, config_path, loaded_config)
+      HieraConfigV3.new(config_root, config_path, loaded_config, owner)
     else
-      raise Puppet::DataBinding::LookupError, "#{@config_path}: This runtime does not support #{CONFIG_FILE_NAME} version '#{version}'"
+      issue = Issues::HIERA_UNSUPPORTED_VERSION
+      raise Puppet::DataBinding::LookupError.new(
+        issue.format(:version => version),  config_path, nil, nil, nil, issue.issue_code)
     end
   end
 
@@ -154,29 +170,76 @@ class HieraConfig
   #
   # @param config_path [Pathname] Absolute path to the configuration
   # @param loaded_config [Hash] the loaded configuration
-  def initialize(config_root, config_path, loaded_config)
+  def initialize(config_root, config_path, loaded_config, owner)
     @config_root = config_root
     @config_path = config_path
     @loaded_config = loaded_config
-    @config = validate_config(self.class.symkeys_to_string(@loaded_config))
+    @config = validate_config(self.class.symkeys_to_string(@loaded_config), owner)
     @data_providers = nil
   end
 
+  def fail(issue, args = EMPTY_HASH, line = nil)
+    raise Puppet::DataBinding::LookupError.new(
+      issue.format(args.merge(:label => self)),  @config_path, line, nil, nil, issue.issue_code)
+  end
+
+  def has_default_hierarchy?
+    false
+  end
+
   # Returns the data providers for this config
   #
   # @param lookup_invocation [Invocation] Invocation data containing scope, overrides, and defaults
   # @param parent_data_provider [DataProvider] The data provider that loaded this configuration
   # @return [Array<DataProvider>] the data providers
-  def configured_data_providers(lookup_invocation, parent_data_provider)
+  def configured_data_providers(lookup_invocation, parent_data_provider, use_default_hierarchy = false)
     unless @data_providers && scope_interpolations_stable?(lookup_invocation)
       if @data_providers
         lookup_invocation.report_text { 'Hiera configuration recreated due to change of scope variables used in interpolation expressions' }
       end
       slc_invocation = ScopeLookupCollectingInvocation.new(lookup_invocation.scope)
-      @data_providers = create_configured_data_providers(slc_invocation, parent_data_provider)
+      begin
+        @data_providers = create_configured_data_providers(slc_invocation, parent_data_provider, false)
+        if has_default_hierarchy?
+          @default_data_providers = create_configured_data_providers(slc_invocation, parent_data_provider, true)
+        end
+      rescue StandardError => e
+        # Raise a LookupError with a RUNTIME_ERROR issue to prevent this being translated to an evaluation error triggered in the pp file
+        # where the lookup started
+        if e.message =~ /^Undefined variable '([^']+)'/
+          var = $1
+          fail(Issues::HIERA_UNDEFINED_VARIABLE, { :name => var }, find_line_matching(/%\{['"]?#{var}['"]?}/))
+        end
+        raise e
+      end
       @scope_interpolations = slc_invocation.scope_interpolations
     end
-    @data_providers
+    use_default_hierarchy ? @default_data_providers : @data_providers
+  end
+
+  # Find first line in configuration that matches regexp after given line. Comments are stripped
+  def find_line_matching(regexp, start_line = 1)
+    line_number = 0
+    File.foreach(@config_path) do |line|
+      line_number += 1
+      next if line_number < start_line
+      quote = nil
+      stripped = ''
+      line.each_codepoint do |cp|
+        if cp == 0x22 || cp == 0x27 # double or single quote
+          if quote == cp
+            quote = nil
+          elsif quote.nil?
+            quote = cp
+          end
+        elsif cp == 0x23 # unquoted hash mark
+          break
+        end
+        stripped << cp
+      end
+      return line_number if stripped =~ regexp
+    end
+    nil
   end
 
   def scope_interpolations_stable?(lookup_invocation)
@@ -188,7 +251,7 @@ class HieraConfig
         @scope_interpolations.all? do |key, root_key, segments, old_value|
           value = scope[root_key]
           unless value.nil? || segments.empty?
-            found = '';
+            found = nil;
             catch(:no_such_key) { found = sub_lookup(key, lookup_invocation, segments, value) }
             value = found;
           end
@@ -199,11 +262,11 @@ class HieraConfig
   end
 
   # @api private
-  def create_configured_data_providers(lookup_invocation, parent_data_provider)
+  def create_configured_data_providers(lookup_invocation, parent_data_provider, use_default_hierarchy)
     self.class.not_implemented(self, 'create_configured_data_providers')
   end
 
-  def validate_config(config)
+  def validate_config(config, owner)
     self.class.not_implemented(self, 'validate_config')
   end
 
@@ -298,7 +361,7 @@ class HieraConfigV3 < HieraConfig
     }
   end
 
-  def create_configured_data_providers(lookup_invocation, parent_data_provider)
+  def create_configured_data_providers(lookup_invocation, parent_data_provider, _)
     scope = lookup_invocation.scope
     unless scope.is_a?(Hiera::Scope)
       lookup_invocation = Invocation.new(
@@ -312,7 +375,15 @@ class HieraConfigV3 < HieraConfig
     data_providers = {}
 
     [@config[KEY_BACKENDS]].flatten.each do |backend|
-      raise Puppet::DataBinding::LookupError, "#{@config_path}: Backend '#{backend}' defined more than once" if data_providers.include?(backend)
+      if data_providers.include?(backend)
+        first_line = find_line_matching(/[^\w]#{backend}(?:[^\w]|$)/)
+        line = find_line_matching(/[^\w]#{backend}(?:[^\w]|$)/, first_line + 1) if first_line
+        unless line
+          line = first_line
+          first_line = nil
+        end
+        fail(Issues::HIERA_BACKEND_MULTIPLY_DEFINED, { :name => backend, :first_line => first_line }, line)
+      end
       original_paths = [@config[KEY_HIERARCHY]].flatten
       backend_config = @config[backend]
       if backend_config.nil?
@@ -348,7 +419,7 @@ class HieraConfigV3 < HieraConfig
     KEY_MERGE_BEHAVIOR => 'native'
   }
 
-  def validate_config(config)
+  def validate_config(config, owner)
     unless Puppet[:strict] == :off
       Puppet.warn_once(:deprecation, 'hiera.yaml',
         "#{@config_path}: Use of 'hiera.yaml' version 3 is deprecated. It should be converted to version 5", config_path.to_s)
@@ -423,7 +494,9 @@ class HieraConfigV4 < HieraConfig
   def factory_create_data_provider(lookup_invocation, name, parent_data_provider, provider_name, datadir, original_paths)
     service_type = Registry.hash_of_path_based_data_provider_factories
     provider_factory = Puppet.lookup(:injector).lookup(nil, service_type, PATH_BASED_DATA_PROVIDER_FACTORIES_KEY)[provider_name]
-    raise Puppet::DataBinding::LookupError, "#{@config_path}: No data provider is registered for backend '#{provider_name}' " unless provider_factory
+    unless provider_factory
+      fail(Issues::HIERA_NO_PROVIDER_FOR_BACKEND, { :name => provider_name }, find_line_matching(/[^\w]#{provider_name}(?:[^\w]|$)/))
+    end
 
     paths = original_paths.map { |path| interpolate(path, lookup_invocation, false) }
     paths = provider_factory.resolve_paths(datadir, original_paths, paths, lookup_invocation)
@@ -437,13 +510,21 @@ class HieraConfigV4 < HieraConfig
     end
   end
 
-  def create_configured_data_providers(lookup_invocation, parent_data_provider)
+  def create_configured_data_providers(lookup_invocation, parent_data_provider, _)
     default_datadir = @config[KEY_DATADIR]
     data_providers = {}
 
     @config[KEY_HIERARCHY].each do |he|
       name = he[KEY_NAME]
-      raise Puppet::DataBinding::LookupError, "#{@config_path}: Name '#{name}' defined more than once" if data_providers.include?(name)
+      if data_providers.include?(name)
+        first_line = find_line_matching(/\s+name:\s+['"]?#{name}(?:[^\w]|$)/)
+        line = find_line_matching(/\s+name:\s+['"]?#{name}(?:[^\w]|$)/, first_line + 1) if first_line
+        unless line
+          line = first_line
+          first_line = nil
+        end
+        fail(Issues::HIERA_HIERARCHY_NAME_MULTIPLY_DEFINED, { :name => name, :first_line => first_line }, line)
+      end
       original_paths = he[KEY_PATHS] || [he[KEY_PATH] || name]
       datadir = @config_root + (he[KEY_DATADIR] || default_datadir)
       provider_name = he[KEY_BACKEND]
@@ -461,7 +542,7 @@ class HieraConfigV4 < HieraConfig
     data_providers.values
   end
 
-  def validate_config(config)
+  def validate_config(config, owner)
     unless Puppet[:strict] == :off
       Puppet.warn_once(:deprecation, 'hiera.yaml',
         "#{@config_path}: Use of 'hiera.yaml' version 4 is deprecated. It should be converted to version 5", config_path.to_s)
@@ -489,6 +570,25 @@ class HieraConfigV5 < HieraConfig
     # The option name must start with a letter and end with a letter or digit. May contain underscore and dash.
     option_name_t = tf.pattern(/\A[A-Za-z](:?[0-9A-Za-z_-]*[0-9A-Za-z])?\z/)
 
+    hierarchy_t = tf.array_of(tf.struct(
+      {
+        KEY_NAME => nes_t,
+        tf.optional(KEY_OPTIONS) => tf.hash_kv(option_name_t, tf.data),
+        tf.optional(KEY_DATA_HASH) => nes_t,
+        tf.optional(KEY_LOOKUP_KEY) => nes_t,
+        tf.optional(KEY_V3_BACKEND) => nes_t,
+        tf.optional(KEY_V4_DATA_HASH) => nes_t,
+        tf.optional(KEY_DATA_DIG) => nes_t,
+        tf.optional(KEY_PATH) => nes_t,
+        tf.optional(KEY_PATHS) => tf.array_of(nes_t, tf.range(1, :default)),
+        tf.optional(KEY_GLOB) => nes_t,
+        tf.optional(KEY_GLOBS) => tf.array_of(nes_t, tf.range(1, :default)),
+        tf.optional(KEY_URI) => uri_t,
+        tf.optional(KEY_URIS) => tf.array_of(uri_t, tf.range(1, :default)),
+        tf.optional(KEY_MAPPED_PATHS) => tf.array_of(nes_t, tf.range(3, 3)),
+        tf.optional(KEY_DATADIR) => nes_t
+      }))
+
     @@CONFIG_TYPE = tf.struct({
       KEY_VERSION => tf.range(5, 5),
       tf.optional(KEY_DEFAULTS) => tf.struct(
@@ -496,38 +596,41 @@ class HieraConfigV5 < HieraConfig
           tf.optional(KEY_DATA_HASH) => nes_t,
           tf.optional(KEY_LOOKUP_KEY) => nes_t,
           tf.optional(KEY_DATA_DIG) => nes_t,
-          tf.optional(KEY_DATADIR) => nes_t
-        }),
-      tf.optional(KEY_HIERARCHY) => tf.array_of(tf.struct(
-        {
-          KEY_NAME => nes_t,
+          tf.optional(KEY_DATADIR) => nes_t,
           tf.optional(KEY_OPTIONS) => tf.hash_kv(option_name_t, tf.data),
-          tf.optional(KEY_DATA_HASH) => nes_t,
-          tf.optional(KEY_LOOKUP_KEY) => nes_t,
-          tf.optional(KEY_V3_BACKEND) => nes_t,
-          tf.optional(KEY_V4_DATA_HASH) => nes_t,
-          tf.optional(KEY_DATA_DIG) => nes_t,
-          tf.optional(KEY_PATH) => nes_t,
-          tf.optional(KEY_PATHS) => tf.array_of(nes_t, tf.range(1, :default)),
-          tf.optional(KEY_GLOB) => nes_t,
-          tf.optional(KEY_GLOBS) => tf.array_of(nes_t, tf.range(1, :default)),
-          tf.optional(KEY_URI) => uri_t,
-          tf.optional(KEY_URIS) => tf.array_of(uri_t, tf.range(1, :default)),
-          tf.optional(KEY_DATADIR) => nes_t
-        }))
+        }),
+      tf.optional(KEY_HIERARCHY) => hierarchy_t,
+      tf.optional(KEY_DEFAULT_HIERARCHY) => hierarchy_t
     })
   end
 
-  def create_configured_data_providers(lookup_invocation, parent_data_provider)
+  def create_configured_data_providers(lookup_invocation, parent_data_provider, use_default_hierarchy)
     defaults = @config[KEY_DEFAULTS] || EMPTY_HASH
     datadir = defaults[KEY_DATADIR] || 'data'
 
     # Hashes enumerate their values in the order that the corresponding keys were inserted so it's safe to use
     # a hash for the data_providers.
     data_providers = {}
-    @config[KEY_HIERARCHY].each do |he|
+
+    if @config.include?(KEY_DEFAULT_HIERARCHY)
+      unless parent_data_provider.is_a?(ModuleDataProvider)
+        fail(Issues::HIERA_DEFAULT_HIERARCHY_NOT_IN_MODULE, EMPTY_HASH, find_line_matching(/\s+default_hierarchy:/))
+      end
+    elsif use_default_hierarchy
+      return data_providers
+    end
+
+    @config[use_default_hierarchy ? KEY_DEFAULT_HIERARCHY : KEY_HIERARCHY].each do |he|
       name = he[KEY_NAME]
-      raise Puppet::DataBinding::LookupError, "#{@config_path}: Name '#{name}' defined more than once" if data_providers.include?(name)
+      if data_providers.include?(name)
+        first_line = find_line_matching(/\s+name:\s+['"]?#{name}(?:[^\w]|$)/)
+        line = find_line_matching(/\s+name:\s+['"]?#{name}(?:[^\w]|$)/, first_line + 1) if first_line
+        unless line
+          line = first_line
+          first_line = nil
+        end
+        fail(Issues::HIERA_HIERARCHY_NAME_MULTIPLY_DEFINED, { :name => name, :first_line => first_line }, line)
+      end
       function_kind = ALL_FUNCTION_KEYS.find { |key| he.include?(key) }
       if function_kind.nil?
         function_kind = FUNCTION_KEYS.find { |key| defaults.include?(key) }
@@ -552,22 +655,15 @@ class HieraConfigV5 < HieraConfig
         expand_uris(he[location_key], lookup_invocation)
       when KEY_URI
         expand_uris([he[location_key]], lookup_invocation)
+      when KEY_MAPPED_PATHS
+        expand_mapped_paths(entry_datadir, he[location_key], lookup_invocation)
       else
         nil
       end
       next if @config_path.nil? && !locations.nil? && locations.empty? # Default config and no existing paths found
-      options = he[KEY_OPTIONS]
+      options = he[KEY_OPTIONS] || defaults[KEY_OPTIONS]
       options = options.nil? ? EMPTY_HASH : interpolate(options, lookup_invocation, false)
       if(function_kind == KEY_V3_BACKEND)
-        unless parent_data_provider.is_a?(GlobalDataProvider)
-          # hiera3_backend is not allowed in environments and modules
-          raise Puppet::DataBinding::LookupError, "#{@config_path}: '#{KEY_V3_BACKEND}' is only allowed in the global layer"
-        end
-
-        if function_name == 'json' || function_name == 'yaml' || function_name == 'hocon' &&  Puppet.features.hocon?
-          # Disallow use of backends that have corresponding "data_hash" functions in version 5
-          raise Puppet::DataBinding::LookupError, "#{@config_path}: Use \"#{KEY_DATA_HASH}: #{function_name}_data\" instead of \"#{KEY_V3_BACKEND}: #{function_name}\""
-        end
         v3options = { :datadir => entry_datadir.to_s }
         options.each_pair { |k, v| v3options[k.to_sym] = v }
         data_providers[name] = create_hiera3_backend_provider(name, function_name, parent_data_provider, entry_datadir, locations, {
@@ -587,6 +683,10 @@ class HieraConfigV5 < HieraConfig
     data_providers.values
   end
 
+  def has_default_hierarchy?
+    @config.include?(KEY_DEFAULT_HIERARCHY)
+  end
+
   RESERVED_OPTION_KEYS = ['path', 'uri'].freeze
 
   DEFAULT_CONFIG_HASH = {
@@ -603,46 +703,59 @@ class HieraConfigV5 < HieraConfig
     ]
   }.freeze
 
-  def validate_config(config)
+  def validate_config(config, owner)
     config[KEY_DEFAULTS] ||= DEFAULT_CONFIG_HASH[KEY_DEFAULTS]
     config[KEY_HIERARCHY] ||= DEFAULT_CONFIG_HASH[KEY_HIERARCHY]
 
     Types::TypeAsserter.assert_instance_of(["The Lookup Configuration at '%s'", @config_path], self.class.config_type, config)
     defaults = config[KEY_DEFAULTS]
     validate_defaults(defaults) unless defaults.nil?
-    config[KEY_HIERARCHY].each do |he|
-      name = he[KEY_NAME]
-      case ALL_FUNCTION_KEYS.count { |key| he.include?(key) }
-      when 0
-        if defaults.nil? || FUNCTION_KEYS.count { |key| defaults.include?(key) } == 0
-          raise Puppet::DataBinding::LookupError,
-            "#{@config_path}: One of #{combine_strings(FUNCTION_KEYS)} must defined in hierarchy '#{name}'"
-        end
-      when 1
-        # OK
-      when 0
-        raise Puppet::DataBinding::LookupError,
-          "#{@config_path}: One of #{combine_strings(FUNCTION_KEYS)} must defined in hierarchy '#{name}'"
-      else
-        raise Puppet::DataBinding::LookupError,
-          "#{@config_path}: Only one of #{combine_strings(FUNCTION_KEYS)} can be defined in hierarchy '#{name}'"
+    config[KEY_HIERARCHY].each { |he| validate_hierarchy(he, defaults, owner) }
+
+    if config.include?(KEY_DEFAULT_HIERARCHY)
+      unless owner.is_a?(ModuleDataProvider)
+        fail(Issues::HIERA_DEFAULT_HIERARCHY_NOT_IN_MODULE, EMPTY_HASH, find_line_matching(/(?:^|\s+)#{KEY_DEFAULT_HIERARCHY}:/))
+      end
+      config[KEY_DEFAULT_HIERARCHY].each { |he| validate_hierarchy(he, defaults, owner) }
+    end
+    config
+  end
+
+  def validate_hierarchy(he, defaults, owner)
+    name = he[KEY_NAME]
+    case ALL_FUNCTION_KEYS.count { |key| he.include?(key) }
+    when 0
+      if defaults.nil? || FUNCTION_KEYS.count { |key| defaults.include?(key) } == 0
+        fail(Issues::HIERA_MISSING_DATA_PROVIDER_FUNCTION, :name => name)
       end
+    when 1
+      # OK
+    else
+      fail(Issues::HIERA_MULTIPLE_DATA_PROVIDER_FUNCTIONS, :name => name)
+    end
 
-      if LOCATION_KEYS.count { |key| he.include?(key) } > 1
-        raise Puppet::DataBinding::LookupError,
-          "#{@config_path}: Only one of #{combine_strings(LOCATION_KEYS)} can be defined in hierarchy '#{name}'"
+    v3_backend = he[KEY_V3_BACKEND]
+    unless v3_backend.nil?
+      unless owner.is_a?(GlobalDataProvider)
+        fail(Issues::HIERA_V3_BACKEND_NOT_GLOBAL, EMPTY_HASH, find_line_matching(/\s+#{KEY_V3_BACKEND}:/))
       end
+      if v3_backend == 'json' || v3_backend == 'yaml' || v3_backend == 'hocon' &&  Puppet.features.hocon?
+        # Disallow use of backends that have corresponding "data_hash" functions in version 5
+        fail(Issues::HIERA_V3_BACKEND_REPLACED_BY_DATA_HASH, { :function_name => v3_backend },
+          find_line_matching(/\s+#{KEY_V3_BACKEND}:\s*['"]?#{v3_backend}(?:[^\w]|$)/))
+      end
+    end
 
-      options = he[KEY_OPTIONS]
-      unless options.nil?
-        RESERVED_OPTION_KEYS.each do |key|
-          if options.include?(key)
-            raise Puppet::DataBinding::LookupError, "#{@config_path}: Option key '#{key}' used in hierarchy '#{name}' is reserved by Puppet"
-          end
-        end
+    if LOCATION_KEYS.count { |key| he.include?(key) } > 1
+      fail(Issues::HIERA_MULTIPLE_LOCATION_SPECS, :name => name)
+    end
+
+    options = he[KEY_OPTIONS]
+    unless options.nil?
+      RESERVED_OPTION_KEYS.each do |key|
+        fail(Issues::HIERA_OPTION_RESERVED_BY_PUPPET, :key => key, :name => name) if options.include?(key)
       end
     end
-    config
   end
 
   def validate_defaults(defaults)
@@ -650,8 +763,14 @@ class HieraConfigV5 < HieraConfig
     when 0, 1
       # OK
     else
-      raise Puppet::DataBinding::LookupError,
-        "#{@config_path}: Only one of #{combine_strings(FUNCTION_KEYS)} can be defined in defaults"
+      fail(Issues::HIERA_MULTIPLE_DATA_PROVIDER_FUNCTIONS_IN_DEFAULT)
+    end
+
+    options = defaults[KEY_OPTIONS]
+    unless options.nil?
+      RESERVED_OPTION_KEYS.each do |key|
+        fail(Issues::HIERA_DEFAULT_OPTION_RESERVED_BY_PUPPET, :key => key) if options.include?(key)
+      end
     end
   end