puppet 4.9.3 → 4.9.4

data/lib/puppet/functions/eyaml_lookup_key.rb

@@ -28,21 +28,29 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
           context.explain { "Setting Eyaml option '#{k}' to '#{v}'" }
         end
       end
-      raw_data = load_data_hash(options)
+      raw_data = load_data_hash(options, context)
       context.cache(nil, raw_data)
     end
     context.not_found unless raw_data.include?(key)
     context.cache(key, decrypt_value(raw_data[key], context))
   end
 
-  def load_data_hash(options)
-    begin
-      data = YAML.load_file(options['path'])
-      Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data.is_a?(Hash) ? data : {})
-    rescue YAML::SyntaxError => ex
-      # Psych errors includes the absolute path to the file, so no need to add that
-      # to the message
-      raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
+  def load_data_hash(options, context)
+    path = options['path']
+    context.cached_file_data(path) do |content|
+      begin
+        data = YAML.load(content, path)
+        if data.is_a?(Hash)
+          Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data)
+        else
+          Puppet.warning("#{path}: file does not contain a valid yaml hash")
+          {}
+        end
+      rescue YAML::SyntaxError => ex
+        # Psych errors includes the absolute path to the file, so no need to add that
+        # to the message
+        raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
+      end
     end
   end
 
@@ -52,7 +60,7 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
       decrypt(value, context)
     when Hash
       result = {}
-      value.each_pair { |k, v| result[k] = decrypt_value(v, context) }
+      value.each_pair { |k, v| result[context.interpolate(k)] = decrypt_value(v, context) }
       result
     when Array
       value.map { |v| decrypt_value(v, context) }

data/lib/puppet/functions/hocon_data.rb

@@ -15,10 +15,12 @@ Puppet::Functions.create_function(:hocon_data) do
 
   def hocon_data(options, context)
     path = options['path']
-    begin
-      Hocon.parse(Puppet::FileSystem.read(path, :encoding => 'utf-8'))
-    rescue Hocon::ConfigError => ex
-      raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+    context.cached_file_data(path) do |content|
+      begin
+        Hocon.parse(content)
+      rescue Hocon::ConfigError => ex
+        raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+      end
     end
   end
 end

data/lib/puppet/functions/json_data.rb

@@ -8,11 +8,13 @@ Puppet::Functions.create_function(:json_data) do
 
   def json_data(options, context)
     path = options['path']
-    begin
-      JSON.parse(Puppet::FileSystem.read(path, :encoding => 'utf-8'))
-    rescue JSON::ParserError => ex
-      # Filename not included in message, so we add it here.
-      raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+    context.cached_file_data(path) do |content|
+      begin
+        JSON.parse(content)
+      rescue JSON::ParserError => ex
+        # Filename not included in message, so we add it here.
+        raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+      end
     end
   end
 end

data/lib/puppet/functions/yaml_data.rb

@@ -9,18 +9,21 @@ Puppet::Functions.create_function(:yaml_data) do
   end
 
   def yaml_data(options, context)
-    begin
-      path = options['path']
-      data = YAML.load_file(path)
-      unless data.is_a?(Hash)
-        Puppet.warning("#{path}: file does not contain a valid yaml hash")
-        data = {}
+    path = options['path']
+    context.cached_file_data(path) do |content|
+      begin
+        data = YAML.load(content, path)
+        if data.is_a?(Hash)
+          Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data)
+        else
+          Puppet.warning("#{path}: file does not contain a valid yaml hash")
+          {}
+        end
+      rescue YAML::SyntaxError => ex
+        # Psych errors includes the absolute path to the file, so no need to add that
+        # to the message
+        raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
       end
-      Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data.nil? ? {} : data)
-    rescue YAML::SyntaxError => ex
-      # Psych errors includes the absolute path to the file, so no need to add that
-      # to the message
-      raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
     end
   end
 end

data/lib/puppet/pops/functions/dispatcher.rb

@@ -28,15 +28,14 @@ class Puppet::Pops::Functions::Dispatcher
   #
   # @api private
   def dispatch(instance, calling_scope, args, &block)
-    tc = Puppet::Pops::Types::TypeCalculator.singleton
-    actual = tc.infer_set(block_given? ? args + [block] : args)
-    found = @dispatchers.find { |d| tc.callable?(d.type, actual) }
-    if found
-      catch(:next) do
-        found.invoke(instance, calling_scope, args, &block)
-      end
-    else
-      raise ArgumentError, Puppet::Pops::Types::TypeMismatchDescriber.describe_signatures(instance.class.name, @dispatchers, actual)
+    found = @dispatchers.find { |d| d.type.callable_with?(args, block) }
+    unless found
+      args_type = Puppet::Pops::Types::TypeCalculator.singleton.infer_set(block_given? ? args + [block] : args)
+      raise ArgumentError, Puppet::Pops::Types::TypeMismatchDescriber.describe_signatures(instance.class.name, @dispatchers, args_type)
+    end
+
+    catch(:next) do
+      found.invoke(instance, calling_scope, args, &block)
     end
   end
 

data/lib/puppet/pops/lookup.rb

@@ -73,7 +73,9 @@ module Lookup
 
   # @api private
   def self.search_and_merge(name, lookup_invocation, merge)
-    lookup_invocation.lookup_adapter.lookup(name, lookup_invocation, merge)
+    answer = lookup_invocation.lookup_adapter.lookup(name, lookup_invocation, merge)
+    lookup_invocation.emit_debug_info("Automatic Parameter Lookup of '#{name}") if Puppet[:debug]
+    answer
   end
 
   def self.assert_type(subject, type, value)

data/lib/puppet/pops/lookup/configured_data_provider.rb

@@ -13,7 +13,7 @@ class ConfiguredDataProvider
   end
 
   def config(lookup_invocation)
-    @config ||= assert_config_version(HieraConfig.create(configuration_path(lookup_invocation)))
+    @config ||= assert_config_version(HieraConfig.create(lookup_invocation, configuration_path(lookup_invocation)))
   end
 
   # @return [Pathname] the path to the configuration

data/lib/puppet/pops/lookup/context.rb

@@ -2,19 +2,73 @@ require_relative 'interpolation'
 
 module Puppet::Pops
 module Lookup
+# The EnvironmentContext is adapted to the current environment
+#
+class EnvironmentContext < Adaptable::Adapter
+  class FileData
+    attr_reader :data
+
+    def initialize(path, inode, mtime, size, data)
+      @path = path
+      @inode = inode
+      @mtime = mtime
+      @size = size
+      @data = data
+    end
+
+    def valid?(stat)
+      stat.ino == @inode && stat.mtime == @mtime && stat.size == @size
+    end
+  end
+
+  attr_reader :environment_name
+
+  def self.create_adapter(environment)
+    new(environment)
+  end
+
+  def initialize(environment)
+    @environment_name = environment.name
+    @file_data_cache = {}
+  end
+
+  # Loads the contents of the file given by _path_. The content is then yielded to the provided block in
+  # case a block is given, and the returned value from that block is cached and returned by this method.
+  # If no block is given, the content is stored instead.
+  #
+  # The cache is retained as long as the inode, mtime, and size of the file remains unchanged.
+  #
+  # @param path [String] path to the file to be read
+  # @yieldparam content [String] the content that was read from the file
+  # @yieldreturn [Object] some result based on the content
+  # @return [Object] the content, or if a block was given, the return value of the block
+  #
+  def cached_file_data(path)
+    file_data = @file_data_cache[path]
+    stat = Puppet::FileSystem.stat(path)
+    unless file_data && file_data.valid?(stat)
+      Puppet.debug("File at '#{path}' was changed, reloading") if file_data
+      content = Puppet::FileSystem.read(path, :encoding => 'utf-8')
+      file_data = FileData.new(path, stat.ino, stat.mtime, stat.size, block_given? ? yield(content) : content)
+      @file_data_cache[path] = file_data
+    end
+    file_data.data
+  end
+end
+
 # A FunctionContext is created for each unique hierarchy entry and adapted to the Compiler (and hence shares
 # the compiler's life-cycle).
 # @api private
 class FunctionContext
   include Interpolation
 
-  attr_reader :environment_name, :module_name, :function
+  attr_reader :module_name, :function
   attr_accessor :data_hash
 
-  def initialize(environment_name, module_name, function)
+  def initialize(environment_context, module_name, function)
     @data_hash = nil
     @cache = {}
-    @environment_name = environment_name
+    @environment_context = environment_context
     @module_name = module_name
     @function = function
   end
@@ -51,6 +105,14 @@ class FunctionContext
       Types::Iterable.on(@cache)
     end
   end
+
+  def cached_file_data(path, &block)
+    @environment_context.cached_file_data(path, &block)
+  end
+
+  def environment_name
+    @environment_context.environment_name
+  end
 end
 
 # The Context is created once for each call to a function. It provides a combination of the {Invocation} object needed
@@ -73,10 +135,12 @@ class Context
     key_type = tf.optional(tf.scalar)
     @type = Pcore::create_object_type(loader, ir, self, 'Puppet::LookupContext', 'Any',
       {
-        'environment_name' => Types::PStringType::NON_EMPTY,
+        'environment_name' => {
+          Types::KEY_TYPE => Types::PStringType::NON_EMPTY,
+          Types::KEY_KIND => Types::PObjectType::ATTRIBUTE_KIND_DERIVED
+        },
         'module_name' => {
-          Types::KEY_TYPE => tf.optional(Types::PStringType::NON_EMPTY),
-          Types::KEY_VALUE => nil
+          Types::KEY_TYPE => tf.variant(Types::PStringType::NON_EMPTY, Types::PUndefType::DEFAULT)
         }
       },
       {
@@ -90,19 +154,19 @@ class Context
         'cached_entries' => tf.variant(
           tf.callable([0, 0, tf.callable(1,1)], tf.undef),
           tf.callable([0, 0, tf.callable(2,2)], tf.undef),
-          tf.callable([0, 0], tf.iterable(tf.tuple([key_type, tf.any])))
-        )
+          tf.callable([0, 0], tf.iterable(tf.tuple([key_type, tf.any])))),
+        'cached_file_data' => tf.callable(tf.string, tf.optional(tf.callable([1, 1])))
       }
     ).resolve(Types::TypeParser.singleton, loader)
   end
 
   # Mainly for test purposes. Makes it possible to create a {Context} in Puppet code provided that a current {Invocation} exists.
-  def self.from_asserted_args(environment_name, module_name)
-    new(FunctionContext.new(environment_name, module_name, nil), Invocation.current)
+  def self.from_asserted_args(module_name)
+    new(FunctionContext.new(EnvironmentContext.adapt(Puppet.lookup(:environments).get(Puppet[:environment])), module_name, nil), Invocation.current)
   end
 
   # Public methods delegated to the {FunctionContext}
-  def_delegators :@function_context, :cache, :cache_all, :cache_has_key, :cached_value, :cached_entries, :environment_name, :module_name
+  def_delegators :@function_context, :cache, :cache_all, :cache_has_key, :cached_value, :cached_entries, :environment_name, :module_name, :cached_file_data
 
   def initialize(function_context, lookup_invocation)
     @lookup_invocation = lookup_invocation

data/lib/puppet/pops/lookup/data_hash_function_provider.rb

@@ -54,7 +54,7 @@ class DataHashFunctionProvider < FunctionProvider
       lookup_invocation.report_not_found(root_key)
       throw :no_such_key
     end
-    interpolate(value, lookup_invocation, true)
+    interpolate(parent_data_provider.validate_data_value(self, value), lookup_invocation, true)
   end
 
   def data_hash(lookup_invocation, location)

data/lib/puppet/pops/lookup/data_provider.rb

@@ -70,32 +70,23 @@ module DataProvider
     raise NotImplementedError, "Subclass of #{DataProvider.name} must implement 'name' method"
   end
 
-  # Asserts that _data_hash_ is a valid hash.
+  # Asserts that _data_hash_ is a hash.
   #
   # @param data_provider [DataProvider] The data provider that produced the hash
   # @param data_hash [Hash{String=>Object}] The data hash
   # @return [Hash{String=>Object}] The data hash
   def validate_data_hash(data_provider, data_hash)
     Types::TypeAsserter.assert_instance_of(nil, Types::PHashType::DEFAULT, data_hash) { "Value returned from #{data_provider.name}" }
-    data_hash.each_pair { |k, v| validate_data_entry(data_provider, k, v) }
-    data_hash
   end
 
-  def validate_data_value(data_provider, value, where = '')
-    Types::TypeAsserter.assert_instance_of(nil, DataProvider.value_type, value) { "Value #{where}returned from #{data_provider.name}" }
-    case value
-    when Hash
-      value.each_pair { |k, v| validate_data_entry(data_provider, k, v) }
-    when Array
-      value.each {|v| validate_data_value(data_provider, v, 'in array ') }
-    end
-    value
-  end
-
-  def validate_data_entry(data_provider, key, value)
-    Types::TypeAsserter.assert_instance_of(nil, DataProvider.key_type, key) { "Key in hash returned from #{data_provider.name}" }
-    validate_data_value(data_provider, value, 'in hash ')
-    nil
+  # Asserts that _data_value_ is of valid type.
+  #
+  # @param data_provider [DataProvider] The data provider that produced the hash
+  # @param data_value [Object] The data value
+  # @return [Object] The data value
+  def validate_data_value(data_provider, value)
+    # The DataProvider.value_type is self recursive so further recursive check of collections is needed here
+    Types::TypeAsserter.assert_instance_of(nil, DataProvider.value_type, value) { "Value returned from #{data_provider.name}" }
   end
 end
 end

data/lib/puppet/pops/lookup/function_provider.rb

@@ -21,9 +21,12 @@ class FunctionProvider
 
   # @return [FunctionContext] the function context associated with this provider
   def function_context(lookup_invocation, location)
+    @contexts[location] ||= create_function_context(lookup_invocation)
+  end
+
+  def create_function_context(lookup_invocation)
     scope = lookup_invocation.scope
-    compiler = scope.compiler
-    @contexts[location] ||= FunctionContext.new(compiler.environment.name, module_name, function(scope))
+    FunctionContext.new(EnvironmentContext.adapt(scope.compiler.environment), module_name, function(scope))
   end
 
   def module_name

data/lib/puppet/pops/lookup/global_data_provider.rb

@@ -22,19 +22,24 @@ class GlobalDataProvider < ConfiguredDataProvider
           lookup_invocation.explainer)
       end
 
+      merge = MergeStrategy.strategy(merge)
       unless config.merge_strategy.is_a?(DefaultMergeStrategy)
-        if lookup_invocation.hiera_xxx_call?
-          # Merge strategy of the hiera_xxx call should only be applied when no merge strategy is defined in the hiera config
-          merge = config.merge_strategy
-          lookup_invocation.set_hiera_v3_merge_behavior
-        elsif merge.is_a?(DefaultMergeStrategy)
-          # For all other calls, the strategy of the call overrides the strategy defined in the hiera config
+        if lookup_invocation.hiera_xxx_call? && merge.is_a?(HashMergeStrategy)
+          # Merge strategy defined in the hiera config only applies when the call stems from a hiera_hash call.
           merge = config.merge_strategy
           lookup_invocation.set_hiera_v3_merge_behavior
         end
       end
+
+      value = super(key, lookup_invocation, merge)
+      if lookup_invocation.hiera_xxx_call? && (merge.is_a?(HashMergeStrategy) || merge.is_a?(DeepMergeStrategy))
+        # hiera_hash calls should error when found values are not hashes
+        Types::TypeAsserter.assert_instance_of('value', Types::PHashType::DEFAULT, value)
+      end
+      value
+    else
+      super
     end
-    super(key, lookup_invocation, merge)
   end
 
   protected

data/lib/puppet/pops/lookup/hiera_config.rb

@@ -8,15 +8,19 @@ module Lookup
 
 # @api private
 class ScopeLookupCollectingInvocation < Invocation
-  attr_reader :scope_interpolations
-
   def initialize(scope)
     super(scope)
     @scope_interpolations = []
   end
 
-  def remember_scope_lookup(*lookup_result)
-    @scope_interpolations << lookup_result
+  def remember_scope_lookup(key, root_key, segments, value)
+    @scope_interpolations << [key, root_key, segments, value] unless !value.nil? && key.start_with?('::')
+  end
+
+  def scope_interpolations
+    # Save extra checks by keeping the array unique with respect to the key (first entry)
+    @scope_interpolations.uniq! { |si| si[0] }
+    @scope_interpolations
   end
 end
 
@@ -48,6 +52,7 @@ class HieraConfig
   KEY_V3_BACKEND = V3BackendFunctionProvider::TAG
   KEY_V4_DATA_HASH = V4DataHashFunctionProvider::TAG
   KEY_BACKEND = 'backend'.freeze
+  KEY_EXTENSION = 'extension'.freeze
 
   FUNCTION_KEYS = [KEY_DATA_HASH, KEY_LOOKUP_KEY, KEY_DATA_DIG, KEY_V3_BACKEND]
   ALL_FUNCTION_KEYS = FUNCTION_KEYS + [KEY_V4_DATA_HASH]
@@ -99,22 +104,28 @@ class HieraConfig
   # Creates a new HieraConfig from the given _config_root_. This is where the 'hiera.yaml' is expected to be found
   # and is also the base location used when resolving relative paths.
   #
+  # @param lookup_invocation [Invocation] Invocation data containing scope, overrides, and defaults
   # @param config_path [Pathname] Absolute path to the configuration file
   # @return [LookupConfiguration] the configuration
-  def self.create(config_path)
+  def self.create(lookup_invocation, config_path)
     if config_path.is_a?(Hash)
       config_path = nil
       loaded_config = config_path
     else
       config_root = config_path.parent
       if config_path.exist?
-        loaded_config = YAML.load_file(config_path)
-
-        # For backward compatibility, we must treat an empty file, or a yaml that doesn't
-        # produce a Hash as Hiera version 3 default.
-        unless loaded_config.is_a?(Hash)
-          Puppet.warning("#{config_path}: File exists but does not contain a valid YAML hash. Falling back to Hiera version 3 default config")
-          loaded_config = HieraConfigV3::DEFAULT_CONFIG_HASH
+        env_context = EnvironmentContext.adapt(lookup_invocation.scope.compiler.environment)
+        loaded_config = env_context.cached_file_data(config_path) do |content|
+          parsed = YAML.load(content, config_path)
+
+          # For backward compatibility, we must treat an empty file, or a yaml that doesn't
+          # produce a Hash as Hiera version 3 default.
+          if parsed.is_a?(Hash)
+            parsed
+          else
+            Puppet.warning("#{config_path}: File exists but does not contain a valid YAML hash. Falling back to Hiera version 3 default config")
+            HieraConfigV3::DEFAULT_CONFIG_HASH
+          end
         end
       else
         config_path = nil
@@ -169,15 +180,21 @@ class HieraConfig
   end
 
   def scope_interpolations_stable?(lookup_invocation)
-    scope = lookup_invocation.scope
-    @scope_interpolations.all? do |key, root_key, segments, old_value|
-      value = scope[root_key]
-      unless value.nil? || segments.empty?
-        found = '';
-        catch(:no_such_key) { found = sub_lookup(key, lookup_invocation, segments, value) }
-        value = found;
+    if @scope_interpolations.empty?
+      true
+    else
+      scope = lookup_invocation.scope
+      lookup_invocation.without_explain do
+        @scope_interpolations.all? do |key, root_key, segments, old_value|
+          value = scope[root_key]
+          unless value.nil? || segments.empty?
+            found = '';
+            catch(:no_such_key) { found = sub_lookup(key, lookup_invocation, segments, value) }
+            value = found;
+          end
+          old_value.eql?(value)
+        end
       end
-      old_value.eql?(value)
     end
   end
 
@@ -297,9 +314,19 @@ class HieraConfigV3 < HieraConfig
     [@config[KEY_BACKENDS]].flatten.each do |backend|
       raise Puppet::DataBinding::LookupError, "#{@config_path}: Backend '#{backend}' defined more than once" if data_providers.include?(backend)
       original_paths = [@config[KEY_HIERARCHY]].flatten
-      backend_config = @config[backend] || EMPTY_HASH
-      datadir = Pathname(interpolate(backend_config[KEY_DATADIR] || default_datadir, lookup_invocation, false))
-      ext = backend == 'hocon' ? '.conf' : ".#{backend}"
+      backend_config = @config[backend]
+      if backend_config.nil?
+        backend_config = EMPTY_HASH
+      else
+        backend_config = interpolate(backend_config, lookup_invocation, false)
+      end
+      datadir = Pathname(backend_config[KEY_DATADIR] || interpolate(default_datadir, lookup_invocation, false))
+      ext = backend_config[KEY_EXTENSION]
+      if ext.nil?
+        ext = backend == 'hocon' ? '.conf' : ".#{backend}"
+      else
+        ext = ".#{ext}"
+      end
       paths = resolve_paths(datadir, original_paths, lookup_invocation, @config_path.nil?, ext)
       data_providers[backend] = case
       when backend == 'json', backend == 'yaml'
@@ -510,6 +537,7 @@ class HieraConfigV5 < HieraConfig
       end
 
       entry_datadir = @config_root + (he[KEY_DATADIR] || datadir)
+      entry_datadir = Pathname(interpolate(entry_datadir.to_s, lookup_invocation, false))
       location_key = LOCATION_KEYS.find { |key| he.include?(key) }
       locations = case location_key
       when KEY_PATHS