puppet 4.8.2-universal-darwin → 4.9.0-universal-darwin

data/lib/puppet/pops/loaders.rb

@@ -17,16 +17,16 @@ class Loaders
   attr_reader :private_environment_loader
   attr_reader :implementation_registry
 
-  def self.new(environment)
+  def self.new(environment, for_agent = false)
     obj = environment.loaders
     if obj.nil?
       obj = self.allocate
-      obj.send(:initialize, environment)
+      obj.send(:initialize, environment, for_agent)
     end
     obj
   end
 
-  def initialize(environment)
+  def initialize(environment, for_agent)
     # Protect against environment havoc
     raise ArgumentError.new("Attempt to redefine already initialized loaders for environment") unless environment.loaders.nil?
     environment.loaders = self
@@ -47,12 +47,16 @@ class Loaders
     #    concept of environment the same way as when running as a master (except when doing apply).
     #    The creation mechanisms should probably differ between the two.
     #
-    @private_environment_loader = create_environment_loader(environment)
+    @private_environment_loader = if for_agent
+      add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@puppet_system_loader, 'agent environment'))
+    else
+      create_environment_loader(environment)
+    end
 
     # 3. The implementation registry maintains mappings between Puppet types and Runtime types for
     #    the current environment
     @implementation_registry = Types::ImplementationRegistry.new(@private_environment_loader)
-    Pcore.init(@puppet_system_loader, @implementation_registry)
+    Pcore.init(@puppet_system_loader, @implementation_registry, for_agent)
 
     # 4. module loaders are set up from the create_environment_loader, they register themselves
   end
@@ -106,15 +110,30 @@ class Loaders
   # @api private
   def self.register_runtime3_type(name, origin)
     loaders = Puppet.lookup(:loaders) { nil }
-    unless loaders.nil?
-      name = name.to_s
-      caps_name = Types::TypeFormatter.singleton.capitalize_segments(name)
-      typed_name = Loader::TypedName.new(:type, name.downcase)
-      loaders.runtime3_type_loader.set_entry(typed_name, Types::PResourceType.new(caps_name), origin)
-    end
+    return nil if loaders.nil?
+
+    rt3_loader = loaders.runtime3_type_loader
+    return nil if rt3_loader.nil?
+
+    name = name.to_s
+    caps_name = Types::TypeFormatter.singleton.capitalize_segments(name)
+    typed_name = Loader::TypedName.new(:type, name.downcase)
+    rt3_loader.set_entry(typed_name, Types::PResourceType.new(caps_name), origin)
     nil
   end
 
+  # Finds a loader to use when deserializing a catalog and then subsequenlty use user
+  # defined types found in that catalog.
+  #
+  def self.catalog_loader
+    loaders = Puppet.lookup(:loaders) { nil }
+    if loaders.nil?
+      loaders = Loaders.new(Puppet.lookup(:current_environment), true)
+      Puppet.push_context(:loaders => loaders)
+    end
+    loaders.find_loader(nil)
+  end
+
   # Finds the `Loaders` instance by looking up the :loaders in the global Puppet context
   #
   # @return [Loaders] the loaders instance
@@ -241,7 +260,7 @@ class Loaders
       loader = add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@runtime3_type_loader, loader_name))
     else
       # View the environment as a module to allow loading from it - this module is always called 'environment'
-      loader = Loader::ModuleLoaders.module_loader_from(@runtime3_type_loader, self, 'environment', env_path)
+      loader = Loader::ModuleLoaders.environment_loader_from(@runtime3_type_loader, self, env_path)
     end
 
     # An environment has a module path even if it has a null loader

data/lib/puppet/pops/lookup/configured_data_provider.rb

@@ -0,0 +1,87 @@
+require_relative 'hiera_config'
+require_relative 'data_provider'
+
+module Puppet::Pops
+module Lookup
+# @api private
+class ConfiguredDataProvider
+  include DataProvider
+
+  # @param config [HieraConfig,nil] the configuration
+  def initialize(config = nil)
+    @config = config.nil? ? nil : assert_config_version(config)
+  end
+
+  def config(lookup_invocation)
+    @config ||= assert_config_version(HieraConfig.create(configuration_path(lookup_invocation)))
+  end
+
+  # @return [Pathname] the path to the configuration
+  def config_path
+    @config.nil? ? nil : @config.config_path
+  end
+
+  # @return [String] the name of this provider
+  def name
+    n = "#{place} "
+    n << '"' << module_name << '" ' unless module_name.nil?
+    n << 'Data Provider'
+    n << " (#{@config.name})" unless @config.nil?
+    n
+  end
+
+  # Performs a lookup by searching all configured locations for the given _key_. A merge will be performed if
+  # the value is found in more than one location.
+  #
+  # @param key [String] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String => Object},nil] Merge strategy, merge strategy name, strategy and options hash, or nil (implies "first found")
+  # @return [Object] the found object
+  # @throw :no_such_key when the object is not found
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    lookup_invocation.with(:data_provider, self) do
+      merge_strategy = MergeStrategy.strategy(merge)
+      dps = data_providers(lookup_invocation)
+      if dps.empty?
+        lookup_invocation.report_not_found(key)
+        throw :no_such_key
+      end
+      merge_strategy.lookup(data_providers(lookup_invocation), lookup_invocation) do |data_provider|
+        data_provider.unchecked_key_lookup(key, lookup_invocation, merge_strategy)
+      end
+    end
+  end
+
+  protected
+
+  # Assert that the given config version is accepted by this data provider.
+  #
+  # @param config [HieraConfig] the configuration to check
+  # @return [HieraConfig] the argument
+  # @raise [Puppet::DataBinding::LookupError] if the configuration version is unacceptable
+  def assert_config_version(config)
+    config
+  end
+
+  # Return the root of the configured entity
+  #
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @return [Pathname] Path to root of the module
+  # @raise [Puppet::DataBinding::LookupError] if the given module is can not be found
+  #
+  def provider_root(lookup_invocation)
+    raise NotImplementedError, "#{self.class.name} must implement method '#provider_root'"
+  end
+
+  def configuration_path(lookup_invocation)
+    provider_root(lookup_invocation) + HieraConfig::CONFIG_FILE_NAME
+  end
+
+  private
+
+  def data_providers(lookup_invocation)
+    config(lookup_invocation).configured_data_providers(lookup_invocation, self)
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/context.rb

@@ -1,86 +1,136 @@
+require_relative 'interpolation'
+
 module Puppet::Pops
-  module Lookup
-    class Context
-      include Types::PuppetObject
+module Lookup
+# A FunctionContext is created for each unique hierarchy entry and adapted to the Compiler (and hence shares
+# the compiler's life-cycle).
+# @api private
+class FunctionContext
+  include Interpolation
 
-      def self._ptype
-        @type
-      end
+  attr_reader :environment_name, :module_name, :function
+  attr_accessor :data_hash
 
-      def self.register_ptype(loader, ir)
-        tf = Types::TypeFactory
-        @type = Pcore::create_object_type(loader, ir, self, 'Puppet::LookupContext', 'Any',
-          {
-            'environment_name' => Types::PStringType::NON_EMPTY,
-            'module_name' => {
-              Types::KEY_TYPE => tf.optional(Types::PStringType::NON_EMPTY),
-              Types::KEY_VALUE => nil
-            }
-          },
-          {
-            'not_found' => tf.callable([0, 0], tf.undef),
-            'explain' => tf.callable([0, 0, tf.callable(0,0)], tf.undef),
-            'cache' => tf.callable([tf.scalar, tf.any], tf.undef),
-            'cache_all' => tf.callable([tf.hash_kv(tf.scalar, tf.any)], tf.undef),
-            'cached_value' => tf.callable([tf.scalar], tf.any),
-            'cached_entries' => tf.variant(
-              tf.callable([0, 0, tf.callable(1,1)], tf.undef),
-              tf.callable([0, 0, tf.callable(2,2)], tf.undef),
-              tf.callable([0, 0], tf.iterable(tf.tuple([tf.scalar, tf.any])))
-            )
-          }
-        ).resolve(Types::TypeParser.singleton, loader)
-      end
-
-      attr_reader :environment_name
-      attr_reader :module_name
+  def initialize(environment_name, module_name, function)
+    @data_hash = nil
+    @cache = {}
+    @environment_name = environment_name
+    @module_name = module_name
+    @function = function
+  end
 
-      def initialize(environment_name, module_name, lookup_invocation = Invocation.current)
-        @lookup_invocation = lookup_invocation
-        @environment_name = environment_name
-        @module_name = module_name
-        @cache = {}
-      end
+  def cache(key, value)
+    @cache[key] = value
+  end
 
-      def cache(key, value)
-        @cache[key] = value
-        nil
-      end
+  def cache_all(hash)
+    @cache.merge!(hash)
+    nil
+  end
 
-      def cache_all(hash)
-        @cache.merge!(hash)
-        nil
-      end
+  def cache_has_key(key)
+    @cache.include?(key)
+  end
 
-      def cached_value(key)
-        @cache[key]
-      end
+  def cached_value(key)
+    @cache[key]
+  end
 
-      def cached_entries(&block)
-        @cache
-        if block_given?
-          enumerator = @cache.each_pair
-          @cache.size.times do
-            if block.arity == 2
-              yield(*enumerator.next)
-            else
-              yield(enumerator.next)
-            end
-          end
-          nil
+  def cached_entries(&block)
+    if block_given?
+      enumerator = @cache.each_pair
+      @cache.size.times do
+        if block.arity == 2
+          yield(*enumerator.next)
         else
-          Types::Iterable.on(@cache)
+          yield(enumerator.next)
         end
       end
+      nil
+    else
+      Types::Iterable.on(@cache)
+    end
+  end
+end
 
-      def explain(&block)
-        @lookup_invocation.report_text(&block) unless @lookup_invocation.nil?
-        nil
-      end
+# The Context is created once for each call to a function. It provides a combination of the {Invocation} object needed
+# to provide explanation support and the {FunctionContext} object needed to provide the private cache.
+# The {Context} is part of the public API. It will be passed to a _data_hash_, _data_dig_, or _lookup_key_ function and its
+# attributes and methods can be used in a Puppet function as well as in a Ruby function.
+# The {Context} is maps to the Pcore type 'Puppet::LookupContext'
+#
+# @api public
+class Context
+  include Types::PuppetObject
+  extend Forwardable
 
-      def not_found
-        throw :no_such_key
-      end
-    end
+  def self._ptype
+    @type
+  end
+
+  def self.register_ptype(loader, ir)
+    tf = Types::TypeFactory
+    key_type = tf.optional(tf.scalar)
+    @type = Pcore::create_object_type(loader, ir, self, 'Puppet::LookupContext', 'Any',
+      {
+        'environment_name' => Types::PStringType::NON_EMPTY,
+        'module_name' => {
+          Types::KEY_TYPE => tf.optional(Types::PStringType::NON_EMPTY),
+          Types::KEY_VALUE => nil
+        }
+      },
+      {
+        'not_found' => tf.callable([0, 0], tf.undef),
+        'explain' => tf.callable([0, 0, tf.callable(0,0)], tf.undef),
+        'interpolate' => tf.callable(1, 1),
+        'cache' => tf.callable([key_type, tf.any], tf.any),
+        'cache_all' => tf.callable([tf.hash_kv(key_type, tf.any)], tf.undef),
+        'cache_has_key' => tf.callable([key_type], tf.boolean),
+        'cached_value' => tf.callable([key_type], tf.any),
+        'cached_entries' => tf.variant(
+          tf.callable([0, 0, tf.callable(1,1)], tf.undef),
+          tf.callable([0, 0, tf.callable(2,2)], tf.undef),
+          tf.callable([0, 0], tf.iterable(tf.tuple([key_type, tf.any])))
+        )
+      }
+    ).resolve(Types::TypeParser.singleton, loader)
+  end
+
+  # Mainly for test purposes. Makes it possible to create a {Context} in Puppet code provided that a current {Invocation} exists.
+  def self.from_asserted_args(environment_name, module_name)
+    new(FunctionContext.new(environment_name, module_name, nil), Invocation.current)
+  end
+
+  # Public methods delegated to the {FunctionContext}
+  def_delegators :@function_context, :cache, :cache_all, :cache_has_key, :cached_value, :cached_entries, :environment_name, :module_name
+
+  def initialize(function_context, lookup_invocation)
+    @lookup_invocation = lookup_invocation
+    @function_context = function_context
+  end
+
+  # Will call the given block to obtain a textual explanation if explanation support is active.
+  #
+  def explain(&block)
+    @lookup_invocation.report_text(&block)
+    nil
+  end
+
+  # Resolve interpolation expressions in the given value
+  # @param [Object] value
+  # @return [Object] the value with all interpolation expressions resolved
+  def interpolate(value)
+    @function_context.interpolate(value, @lookup_invocation, true)
+  end
+
+  def not_found
+    throw :no_such_key
   end
+
+  # @api private
+  def invocation
+    @lookup_invocation
+  end
+end
+end
 end

data/lib/puppet/pops/lookup/data_adapter.rb

@@ -0,0 +1,27 @@
+module Puppet::Pops
+module Lookup
+# A class that adapts a Hash
+# @api private
+class DataAdapter < Adaptable::Adapter
+  def self.create_adapter(o)
+    new
+  end
+
+  def initialize
+    @data = {}
+  end
+
+  def [](name)
+    @data[name]
+  end
+
+  def include?(name)
+    @data.include? name
+  end
+
+  def []=(name, value)
+    @data[name] = value
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/data_dig_function_provider.rb

@@ -0,0 +1,55 @@
+require_relative 'function_provider'
+
+module Puppet::Pops
+module Lookup
+# @api private
+class DataDigFunctionProvider < FunctionProvider
+  TAG = 'data_dig'.freeze
+
+  # Performs a lookup with the assumption that a recursive check has been made.
+  #
+  # @param key [LookupKey] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String => Object},nil] Merge strategy, merge strategy name, strategy and options hash, or nil (implies "first found")
+  # @return [Object] the found object
+  # @throw :no_such_key when the object is not found
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    segments = key.to_a
+    lookup_invocation.with(:data_provider, self) do
+      MergeStrategy.strategy(merge).lookup(locations, lookup_invocation) do |location|
+        if location.nil?
+          value = data_dig(key, segments, lookup_invocation, nil)
+          lookup_invocation.report_found(key, validate_data_value(self, value))
+          key.undig(value)
+        else
+          lookup_invocation.with(:location, location) do
+            if location.exist?
+              value = data_dig(key, segments, lookup_invocation, location.location)
+              lookup_invocation.report_found(key, validate_data_value(self, value))
+              key.undig(value)
+            else
+              lookup_invocation.report_path_not_found
+              throw :no_such_key
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  def data_dig(key, segments, lookup_invocation, location)
+    ctx = function_context(lookup_invocation, location)
+    ctx.data_hash ||= {}
+    catch(:no_such_key) do
+      hash = ctx.data_hash
+      hash[key] = ctx.function.call(lookup_invocation.scope, segments, options(location), Context.new(ctx, lookup_invocation)) unless hash.include?(key)
+      return hash[key]
+    end
+    lookup_invocation.report_not_found(key)
+    throw :no_such_key
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/data_hash_function_provider.rb

@@ -0,0 +1,111 @@
+require_relative 'function_provider'
+require_relative 'interpolation'
+
+module Puppet::Pops
+module Lookup
+# @api private
+class DataHashFunctionProvider < FunctionProvider
+  include SubLookup
+  include Interpolation
+
+  TAG = 'data_hash'.freeze
+
+  # Performs a lookup with the assumption that a recursive check has been made.
+  #
+  # @param key [LookupKey] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String => Object},nil] Merge strategy, merge strategy name, strategy and options hash, or nil (implies "first found")
+  # @return [Object] the found object
+  # @throw :no_such_key when the object is not found
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    root_key = key.root_key
+    lookup_invocation.with(:data_provider, self) do
+      MergeStrategy.strategy(merge).lookup(locations, lookup_invocation) do |location|
+        invoke_with_location(lookup_invocation, location, root_key)
+      end
+    end
+  end
+
+  private
+
+  def invoke_with_location(lookup_invocation, location, root_key)
+    if location.nil?
+      lookup_key(lookup_invocation, nil, root_key)
+    else
+      lookup_invocation.with(:location, location) do
+        if location.exist?
+          lookup_key(lookup_invocation, location.location, root_key)
+        else
+          lookup_invocation.report_location_not_found
+          throw :no_such_key
+        end
+      end
+    end
+  end
+
+  def lookup_key(lookup_invocation, location, root_key)
+    lookup_invocation.report_found(root_key, data_value(lookup_invocation, location, root_key))
+  end
+
+  def data_value(lookup_invocation, location, root_key)
+    hash = data_hash(lookup_invocation, location)
+    value = hash[root_key]
+    if value.nil? && !hash.include?(root_key)
+      lookup_invocation.report_not_found(root_key)
+      throw :no_such_key
+    end
+    interpolate(value, lookup_invocation, true)
+  end
+
+  def data_hash(lookup_invocation, location)
+    ctx = function_context(lookup_invocation, location)
+    ctx.data_hash ||= parent_data_provider.validate_data_hash(self, call_data_hash_function(ctx, lookup_invocation, location))
+  end
+
+  def call_data_hash_function(ctx, lookup_invocation, location)
+    ctx.function.call(lookup_invocation.scope, options(location), Context.new(ctx, lookup_invocation))
+  end
+end
+
+# @api private
+class V3DataHashFunctionProvider < DataHashFunctionProvider
+  TAG = 'v3_data_hash'.freeze
+
+  def initialize(name, parent_data_provider, function_name, options, locations)
+    @datadir = options.delete(HieraConfig::KEY_DATADIR)
+    super
+  end
+
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    extra_paths = lookup_invocation.hiera_v3_location_overrides
+    if extra_paths.nil? || extra_paths.empty?
+      super
+    else
+      # Extra paths provided. Must be resolved and placed in front of known paths
+      paths = parent_data_provider.config(lookup_invocation).resolve_paths(@datadir, extra_paths, lookup_invocation, false, ".#{@name}")
+      all_locations = paths + locations
+      root_key = key.root_key
+      lookup_invocation.with(:data_provider, self) do
+        MergeStrategy.strategy(merge).lookup(all_locations, lookup_invocation) do |location|
+          invoke_with_location(lookup_invocation, location, root_key)
+        end
+      end
+    end
+  end
+end
+
+# TODO: API 5.0, remove this class
+# @api private
+class V4DataHashFunctionProvider < DataHashFunctionProvider
+  TAG = 'v4_data_hash'.freeze
+
+  def name
+    "deprecated API function \"#{function_name}\""
+  end
+
+  def call_data_hash_function(ctx, lookup_invocation, location)
+    ctx.function.call(lookup_invocation.scope)
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/data_provider.rb

@@ -0,0 +1,102 @@
+module Puppet::Pops
+module Lookup
+# @api private
+module DataProvider
+  def self.key_type
+    ensure_types_initialized
+    @key_type
+  end
+
+  def self.value_type
+    ensure_types_initialized
+    @value_type
+  end
+
+  def self.ensure_types_initialized
+    if @key_type.nil?
+      (@key_type, @value_type) = Pcore::register_aliases(
+        # The Pcore type for all keys and subkeys in a data hash.
+        'Puppet::LookupKey' => 'Variant[String,Numeric]',
+
+        # The Pcore type for all values and sub-values in a data hash. The
+        # type is self-recursive to enforce the same constraint on values contained
+        # in arrays and hashes
+        'Puppet::LookupValue' => <<-PUPPET
+          Variant[
+            Scalar,
+            Undef,
+            Sensitive,
+            Type,
+            Hash[Puppet::LookupKey, Puppet::LookupValue],
+            Array[Puppet::LookupValue]
+          ]
+        PUPPET
+      )
+    end
+  end
+
+  # Performs a lookup with an endless recursion check.
+  #
+  # @param key [LookupKey] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String=>Object},nil] Merge strategy or hash with strategy and options
+  #
+  def key_lookup(key, lookup_invocation, merge)
+    lookup_invocation.check(key.to_s) { unchecked_key_lookup(key, lookup_invocation, merge) }
+  end
+
+  def lookup(key, lookup_invocation, merge)
+    lookup_invocation.check(key.to_s) { unchecked_key_lookup(key, lookup_invocation, merge) }
+  end
+
+  # Performs a lookup with the assumption that a recursive check has been made.
+  #
+  # @param key [LookupKey] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String => Object},nil] Merge strategy, merge strategy name, strategy and options hash, or nil (implies "first found")
+  # @return [Object] the found object
+  # @throw :no_such_key when the object is not found
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    raise NotImplementedError, "Subclass of #{DataProvider.name} must implement 'unchecked_lookup' method"
+  end
+
+  # @return [String,nil] the name of the module that this provider belongs to nor `nil` if it doesn't belong to a module
+  def module_name
+    nil
+  end
+
+  # @return [String] the name of the this data provider
+  def name
+    raise NotImplementedError, "Subclass of #{DataProvider.name} must implement 'name' method"
+  end
+
+  # Asserts that _data_hash_ is a valid hash.
+  #
+  # @param data_provider [DataProvider] The data provider that produced the hash
+  # @param data_hash [Hash{String=>Object}] The data hash
+  # @return [Hash{String=>Object}] The data hash
+  def validate_data_hash(data_provider, data_hash)
+    Types::TypeAsserter.assert_instance_of(nil, Types::PHashType::DEFAULT, data_hash) { "Value returned from #{data_provider.name}" }
+    data_hash.each_pair { |k, v| validate_data_entry(data_provider, k, v) }
+    data_hash
+  end
+
+  def validate_data_value(data_provider, value, where = '')
+    Types::TypeAsserter.assert_instance_of(nil, DataProvider.value_type, value) { "Value #{where}returned from #{data_provider.name}" }
+    case value
+    when Hash
+      value.each_pair { |k, v| validate_data_entry(data_provider, k, v) }
+    when Array
+      value.each {|v| validate_data_value(data_provider, v, 'in array ') }
+    end
+    value
+  end
+
+  def validate_data_entry(data_provider, key, value)
+    Types::TypeAsserter.assert_instance_of(nil, DataProvider.key_type, key) { "Key in hash returned from #{data_provider.name}" }
+    validate_data_value(data_provider, value, 'in hash ')
+    nil
+  end
+end
+end
+end