puppet 4.9.4 → 4.10.0

data/lib/puppet/pops/evaluator/callable_signature.rb

@@ -97,4 +97,11 @@ class Puppet::Pops::Evaluator::CallableSignature
   def infinity?(x)
     x == Float::INFINITY
   end
+
+  # @return [Boolean] true if this signature represents an argument mismatch, false otherwise
+  #
+  # @api private
+  def argument_mismatch_handler?
+    false
+  end
 end

data/lib/puppet/pops/functions/dispatch.rb

@@ -21,15 +21,24 @@ class Dispatch < Evaluator::CallableSignature
   # @api public
   attr_reader :block_name
 
+  # @param type [Puppet::Pops::Types::PArrayType, Puppet::Pops::Types::PTupleType] - type describing signature
+  # @param method_name [String] the name of the method that will be called when type matches given arguments
+  # @param param_names [Array<String>] names matching the number of parameters specified by type (or empty array)
+  # @param block_name [String,nil] name of block parameter, no nil
+  # @param injections [Array<Array>] injection data for weaved parameters
+  # @param weaving [Array<Integer,Array>] weaving knits
+  # @param last_captures [Boolean] true if last parameter is captures rest
+  # @param argument_mismatch_handler [Boolean] true if this is a dispatch for an argument mismatch
   # @api private
-  def initialize(type, method_name, param_names, block_name, injections, weaving, last_captures)
+  def initialize(type, method_name, param_names, last_captures = false, block_name = nil, injections = EMPTY_ARRAY, weaving = EMPTY_ARRAY, argument_mismatch_handler = false)
     @type = type
     @method_name = method_name
-    @param_names = param_names || []
+    @param_names = param_names
+    @last_captures = last_captures
     @block_name = block_name
-    @injections = injections || []
+    @injections = injections
     @weaving = weaving
-    @last_captures = last_captures
+    @argument_mismatch_handler = argument_mismatch_handler
   end
 
   # @api private
@@ -39,7 +48,11 @@ class Dispatch < Evaluator::CallableSignature
 
   # @api private
   def last_captures_rest?
-    !! @last_captures
+    @last_captures
+  end
+
+  def argument_mismatch_handler?
+    @argument_mismatch_handler
   end
 
   # @api private

data/lib/puppet/pops/functions/dispatcher.rb

@@ -31,7 +31,12 @@ class Puppet::Pops::Functions::Dispatcher
     found = @dispatchers.find { |d| d.type.callable_with?(args, block) }
     unless found
       args_type = Puppet::Pops::Types::TypeCalculator.singleton.infer_set(block_given? ? args + [block] : args)
-      raise ArgumentError, Puppet::Pops::Types::TypeMismatchDescriber.describe_signatures(instance.class.name, @dispatchers, args_type)
+      raise ArgumentError, Puppet::Pops::Types::TypeMismatchDescriber.describe_signatures(instance.class.name, signatures, args_type)
+    end
+
+    if found.argument_mismatch_handler?
+      msg = found.invoke(instance, calling_scope, args)
+      raise ArgumentError, "'#{instance.class.name}' #{msg}"
     end
 
     catch(:next) do
@@ -39,17 +44,6 @@ class Puppet::Pops::Functions::Dispatcher
     end
   end
 
-  # Adds a regular dispatch for one method name
-  #
-  # @param type [Puppet::Pops::Types::PArrayType, Puppet::Pops::Types::PTupleType] - type describing signature
-  # @param method_name [String] - the name of the method that will be called when type matches given arguments
-  # @param names [Array<String>] - array with names matching the number of parameters specified by type (or empty array)
-  #
-  # @api private
-  def add_dispatch(type, method_name, param_names, block_name, injections, weaving, last_captures)
-    add(Puppet::Pops::Functions::Dispatch.new(type, method_name, param_names, block_name, injections, weaving, last_captures))
-  end
-
   # Adds a dispatch directly to the set of dispatchers.
   # @api private
   def add(a_dispatch)
@@ -72,6 +66,6 @@ class Puppet::Pops::Functions::Dispatcher
 
   # @api private
   def signatures
-    @dispatchers
+    @dispatchers.reject { |dispatcher| dispatcher.argument_mismatch_handler? }
   end
 end

data/lib/puppet/pops/issue_reporter.rb

@@ -69,7 +69,7 @@ class IssueReporter
         break if emitted >= max_errors
       end
       warnings_message = (emit_warnings && warnings.size > 0) ? ", and #{warnings.size} warnings" : ""
-      giving_up_message = "Found #{errors.size} errors#{warnings_message}. Giving up"
+      giving_up_message = "Language validation logged #{errors.size} errors#{warnings_message}. Giving up"
       exception = emit_exception.new(giving_up_message)
       exception.file = errors[0].file
       raise exception

data/lib/puppet/pops/issues.rb

@@ -738,5 +738,89 @@ module Issues
   ILLEGAL_BOM = hard_issue :ILLEGAL_BOM, :format_name, :bytes do
     "Illegal #{format_name} Byte Order mark at beginning of input: #{bytes} - remove these from the puppet source"
   end
+
+  HIERA_UNSUPPORTED_VERSION = hard_issue :HIERA_UNSUPPORTED_VERSION, :version do
+    "This runtime does not support hiera.yaml version #{version}"
+  end
+
+  HIERA_VERSION_3_NOT_GLOBAL = hard_issue :HIERA_VERSION_3_NOT_GLOBAL, :where do
+    "hiera.yaml version 3 cannot be used in #{label.a_an(where)}"
+  end
+
+  HIERA_UNSUPPORTED_VERSION_IN_GLOBAL = hard_issue :HIERA_UNSUPPORTED_VERSION_IN_GLOBAL do
+    'hiera.yaml version 4 cannot be used in the global layer'
+  end
+
+  HIERA_UNDEFINED_VARIABLE = hard_issue :HIERA_UNDEFINED_VARIABLE, :name do
+    "Undefined variable '#{name}'"
+  end
+
+  HIERA_BACKEND_MULTIPLY_DEFINED = hard_issue :HIERA_BACKEND_MULTIPLY_DEFINED, :name, :first_line do
+    msg = "Backend '#{name}' is defined more than once"
+    fl = first_line
+    fl ? "#{msg}. First defined at line #{fl}" : msg
+  end
+
+  HIERA_NO_PROVIDER_FOR_BACKEND = hard_issue :HIERA_NO_PROVIDER_FOR_BACKEND, :name do
+    "No data provider is registered for backend '#{name}'"
+  end
+
+  HIERA_HIERARCHY_NAME_MULTIPLY_DEFINED = hard_issue :HIERA_HIERARCHY_NAME_MULTIPLY_DEFINED, :name, :first_line do
+    msg = "Hierarchy name '#{name}' defined more than once"
+    fl = first_line
+    fl ? "#{msg}. First defined at line #{fl}" : msg
+  end
+
+  HIERA_V3_BACKEND_NOT_GLOBAL = hard_issue :HIERA_V3_BACKEND_NOT_GLOBAL do
+    "'hiera3_backend' is only allowed in the global layer"
+  end
+
+  HIERA_DEFAULT_HIERARCHY_NOT_IN_MODULE = hard_issue :HIERA_DEFAULT_HIERARCHY_NOT_IN_MODULE do
+    "'default_hierarchy' is only allowed in the module layer"
+  end
+
+  HIERA_V3_BACKEND_REPLACED_BY_DATA_HASH = hard_issue :HIERA_V3_BACKEND_REPLACED_BY_DATA_HASH, :function_name do
+    "Use \"data_hash: #{function_name}_data\" instead of \"hiera3_backend: #{function_name}\""
+  end
+
+  HIERA_MISSING_DATA_PROVIDER_FUNCTION = hard_issue :HIERA_MISSING_DATA_PROVIDER_FUNCTION, :name do
+    "One of #{label.combine_strings(Lookup::HieraConfig::FUNCTION_KEYS)} must be defined in hierarchy '#{name}'"
+  end
+
+  HIERA_MULTIPLE_DATA_PROVIDER_FUNCTIONS = hard_issue :HIERA_MULTIPLE_DATA_PROVIDER_FUNCTIONS, :name do
+    "Only one of #{label.combine_strings(Lookup::HieraConfig::FUNCTION_KEYS)} can be defined in hierarchy '#{name}'"
+  end
+
+  HIERA_MULTIPLE_DATA_PROVIDER_FUNCTIONS_IN_DEFAULT = hard_issue :HIERA_MULTIPLE_DATA_PROVIDER_FUNCTIONS_IN_DEFAULT do
+    "Only one of #{label.combine_strings(Lookup::HieraConfig::FUNCTION_KEYS)} can be defined in defaults"
+  end
+
+  HIERA_MULTIPLE_LOCATION_SPECS = hard_issue :HIERA_MULTIPLE_LOCATION_SPECS, :name do
+    "Only one of #{label.combine_strings(Lookup::HieraConfig::LOCATION_KEYS)} can be defined in hierarchy '#{name}'"
+  end
+
+  HIERA_OPTION_RESERVED_BY_PUPPET = hard_issue :HIERA_OPTION_RESERVED_BY_PUPPET, :key, :name do
+    "Option key '#{key}' used in hierarchy '#{name}' is reserved by Puppet"
+  end
+
+  HIERA_DEFAULT_OPTION_RESERVED_BY_PUPPET = hard_issue :HIERA_DEFAULT_OPTION_RESERVED_BY_PUPPET, :key do
+    "Option key '#{key}' used in defaults is reserved by Puppet"
+  end
+
+  HIERA_DATA_PROVIDER_FUNCTION_NOT_FOUND = hard_issue :HIERA_DATA_PROVIDER_FUNCTION_NOT_FOUND, :function_type, :function_name do
+    "Unable to find '#{function_type}' function named '#{function_name}'"
+  end
+
+  HIERA_INTERPOLATION_ALIAS_NOT_ENTIRE_STRING = hard_issue :HIERA_INTERPOLATION_ALIAS_NOT_ENTIRE_STRING do
+    "'alias' interpolation is only permitted if the expression is equal to the entire string"
+  end
+
+  HIERA_INTERPOLATION_UNKNOWN_INTERPOLATION_METHOD = hard_issue :HIERA_INTERPOLATION_UNKNOWN_INTERPOLATION_METHOD, :name do
+    "Unknown interpolation method '#{name}'"
+  end
+
+  HIERA_INTERPOLATION_METHOD_SYNTAX_NOT_ALLOWED = hard_issue :HIERA_INTERPOLATION_METHOD_SYNTAX_NOT_ALLOWED do
+    'Interpolation using method syntax is not allowed in this context'
+  end
 end
 end

data/lib/puppet/pops/loader/base_loader.rb

@@ -136,13 +136,21 @@ class BaseLoader < Loader
   # 4. give up
   #
   def internal_load(typed_name)
-    # avoid calling get_entry, by looking it up
+    # avoid calling get_entry by looking it up
     te = @named_values[typed_name]
-    te = parent.load_typed(typed_name) if te.nil? || te.value.nil?
-    te = find(typed_name) if te.nil? || te.value.nil?
-    te
-  end
+    return te unless te.nil? || te.value.nil?
+
+    te = parent.load_typed(typed_name)
+    return te unless te.nil? || te.value.nil?
 
+    # Under some circumstances, the call to the parent loader will have resulted in files being
+    # parsed that in turn contained references to the requested entity and hence, caused a
+    # recursive call into this loader. This means that the entry might be present now, so a new
+    # check must be made.
+    te = @named_values[typed_name]
+    te.nil? || te.value.nil? ? find(typed_name) : te
+  end
 end
 end
 end
+

data/lib/puppet/pops/lookup/configured_data_provider.rb

@@ -13,7 +13,13 @@ class ConfiguredDataProvider
   end
 
   def config(lookup_invocation)
-    @config ||= assert_config_version(HieraConfig.create(lookup_invocation, configuration_path(lookup_invocation)))
+    @config ||= assert_config_version(HieraConfig.create(lookup_invocation, configuration_path(lookup_invocation), self))
+  end
+
+  # Needed to assign generated version 4 config
+  # @deprecated
+  def config=(config)
+    @config = config
   end
 
   # @return [Pathname] the path to the configuration
@@ -46,7 +52,7 @@ class ConfiguredDataProvider
         lookup_invocation.report_not_found(key)
         throw :no_such_key
       end
-      merge_strategy.lookup(data_providers(lookup_invocation), lookup_invocation) do |data_provider|
+      merge_strategy.lookup(dps, lookup_invocation) do |data_provider|
         data_provider.unchecked_key_lookup(key, lookup_invocation, merge_strategy)
       end
     end

data/lib/puppet/pops/lookup/data_dig_function_provider.rb

@@ -14,42 +14,132 @@ class DataDigFunctionProvider < FunctionProvider
   # @return [Object] the found object
   # @throw :no_such_key when the object is not found
   def unchecked_key_lookup(key, lookup_invocation, merge)
-    segments = key.to_a
     lookup_invocation.with(:data_provider, self) do
       MergeStrategy.strategy(merge).lookup(locations, lookup_invocation) do |location|
-        if location.nil?
-          value = data_dig(key, segments, lookup_invocation, nil)
-          lookup_invocation.report_found(key, validate_data_value(self, value))
-          key.undig(value)
-        else
-          lookup_invocation.with(:location, location) do
-            if location.exist?
-              value = data_dig(key, segments, lookup_invocation, location.location)
-              lookup_invocation.report_found(key, validate_data_value(self, value))
-              key.undig(value)
-            else
-              lookup_invocation.report_path_not_found
-              throw :no_such_key
-            end
-          end
-        end
+        invoke_with_location(lookup_invocation, location, key, merge)
       end
     end
   end
 
+  def invoke_with_location(lookup_invocation, location, key, merge)
+    if location.nil?
+      key.undig(lookup_invocation.report_found(key, validated_data_dig(key, lookup_invocation, nil, merge)))
+    else
+      lookup_invocation.with(:location, location) do
+        key.undig(lookup_invocation.report_found(key, validated_data_dig(key, lookup_invocation, location, merge)))
+      end
+    end
+  end
+
+  def label
+    'Data Dig'
+  end
+
+  def validated_data_dig(key, lookup_invocation, location, merge)
+    validate_data_value(data_dig(key, lookup_invocation, location, merge)) do
+      msg = "Value for key '#{key}', returned from #{full_name}"
+      location.nil? ? msg : "#{msg}, when using location '#{location}',"
+    end
+  end
+
   private
 
-  def data_dig(key, segments, lookup_invocation, location)
+  def data_dig(key, lookup_invocation, location, merge)
+    unless location.nil? || location.exist?
+      lookup_invocation.report_location_not_found
+      throw :no_such_key
+    end
     ctx = function_context(lookup_invocation, location)
     ctx.data_hash ||= {}
     catch(:no_such_key) do
       hash = ctx.data_hash
-      hash[key] = ctx.function.call(lookup_invocation.scope, segments, options(location), Context.new(ctx, lookup_invocation)) unless hash.include?(key)
+      hash[key] = ctx.function.call(lookup_invocation.scope, key.to_a, options(location), Context.new(ctx, lookup_invocation)) unless hash.include?(key)
       return hash[key]
     end
     lookup_invocation.report_not_found(key)
     throw :no_such_key
   end
 end
+
+# @api private
+class V3BackendFunctionProvider < DataDigFunctionProvider
+  TAG = 'hiera3_backend'.freeze
+
+  def data_dig(key, lookup_invocation, location, merge)
+    @backend ||= instantiate_backend(lookup_invocation)
+
+    # A merge_behavior retrieved from hiera.yaml must not be converted here. Instead, passing the symbol :hash
+    # tells the V3 backend to pick it up from the config.
+    resolution_type = lookup_invocation.hiera_v3_merge_behavior? ? :hash : convert_merge(merge)
+    @backend.lookup(key.to_s, lookup_invocation.scope, lookup_invocation.hiera_v3_location_overrides, resolution_type, context = {:recurse_guard => nil})
+  end
+
+  def full_name
+    "hiera version 3 backend '#{options[HieraConfig::KEY_BACKEND]}'"
+  end
+
+  def value_is_validated?
+    false
+  end
+
+  private
+
+  def instantiate_backend(lookup_invocation)
+    backend_name = options[HieraConfig::KEY_BACKEND]
+    begin
+      require 'hiera/backend'
+      require "hiera/backend/#{backend_name.downcase}_backend"
+      backend = Hiera::Backend.const_get("#{backend_name.capitalize}_backend").new
+      return backend.method(:lookup).arity == 4 ? Hiera::Backend::Backend1xWrapper.new(backend) : backend
+    rescue LoadError => e
+      lookup_invocation.report_text { "Unable to load backend '#{backend_name}': #{e.message}" }
+      throw :no_such_key
+    rescue NameError => e
+      lookup_invocation.report_text { "Unable to instantiate backend '#{backend_name}': #{e.message}" }
+      throw :no_such_key
+    end
+  end
+
+  # Converts a lookup 'merge' parameter argument into a Hiera 'resolution_type' argument.
+  #
+  # @param merge [String,Hash,nil] The lookup 'merge' argument
+  # @return [Symbol,Hash,nil] The Hiera 'resolution_type'
+  def convert_merge(merge)
+    case merge
+    when nil
+    when 'first', 'default'
+      # Nil is OK. Defaults to Hiera :priority
+      nil
+    when Puppet::Pops::MergeStrategy
+      convert_merge(merge.configuration)
+    when 'unique'
+      # Equivalent to Hiera :array
+      :array
+    when 'hash'
+      # Equivalent to Hiera :hash with default :native merge behavior. A Hash must be passed here
+      # to override possible Hiera deep merge config settings.
+      { :behavior => :native }
+    when 'deep', 'unconstrained_deep'
+      # Equivalent to Hiera :hash with :deeper merge behavior.
+      { :behavior => :deeper }
+    when 'reverse_deep'
+      # Equivalent to Hiera :hash with :deep merge behavior.
+      { :behavior => :deep }
+    when Hash
+      strategy = merge['strategy']
+      case strategy
+      when 'deep', 'unconstrained_deep', 'reverse_deep'
+        result = { :behavior => strategy == 'reverse_deep' ? :deep : :deeper }
+        # Remaining entries must have symbolic keys
+        merge.each_pair { |k,v| result[k.to_sym] = v unless k == 'strategy' }
+        result
+      else
+        convert_merge(strategy)
+      end
+    else
+      raise Puppet::DataBinding::LookupError, "Unrecognized value for request 'merge' parameter: '#{merge}'"
+    end
+  end
+end
 end
 end

data/lib/puppet/pops/lookup/data_hash_function_provider.rb

@@ -10,6 +10,10 @@ class DataHashFunctionProvider < FunctionProvider
 
   TAG = 'data_hash'.freeze
 
+  def self.trusted_return_type
+    @trusted_return_type ||= Types::PHashType.new(DataProvider.key_type, DataProvider.value_type)
+  end
+
   # Performs a lookup with the assumption that a recursive check has been made.
   #
   # @param key [LookupKey] The key to lookup
@@ -34,7 +38,7 @@ class DataHashFunctionProvider < FunctionProvider
     else
       lookup_invocation.with(:location, location) do
         if location.exist?
-          lookup_key(lookup_invocation, location.location, root_key)
+          lookup_key(lookup_invocation, location, root_key)
         else
           lookup_invocation.report_location_not_found
           throw :no_such_key
@@ -54,12 +58,19 @@ class DataHashFunctionProvider < FunctionProvider
       lookup_invocation.report_not_found(root_key)
       throw :no_such_key
     end
-    interpolate(parent_data_provider.validate_data_value(self, value), lookup_invocation, true)
+    value = validate_data_value(value) do
+      msg = "Value for key '#{root_key}', in hash returned from #{full_name}"
+      location.nil? ? msg : "#{msg}, when using location '#{location}',"
+    end
+    interpolate(value, lookup_invocation, true)
   end
 
   def data_hash(lookup_invocation, location)
     ctx = function_context(lookup_invocation, location)
-    ctx.data_hash ||= parent_data_provider.validate_data_hash(self, call_data_hash_function(ctx, lookup_invocation, location))
+    ctx.data_hash ||= parent_data_provider.validate_data_hash(call_data_hash_function(ctx, lookup_invocation, location)) do
+      msg = "Value returned from #{full_name}"
+      location.nil? ? msg : "#{msg}, when using location '#{location}',"
+    end
   end
 
   def call_data_hash_function(ctx, lookup_invocation, location)
@@ -100,7 +111,11 @@ class V4DataHashFunctionProvider < DataHashFunctionProvider
   TAG = 'v4_data_hash'.freeze
 
   def name
-    "deprecated API function \"#{function_name}\""
+    "Deprecated API function \"#{function_name}\""
+  end
+
+  def full_name
+    "deprecated API function '#{function_name}'"
   end
 
   def call_data_hash_function(ctx, lookup_invocation, location)