puppet 4.1.0 → 4.10.0

data/lib/puppet/pops/loaders.rb

@@ -1,14 +1,40 @@
-class Puppet::Pops::Loaders
+module Puppet::Pops
+# This is the container for all Loader instances. Each Loader instance has a `loader_name` by which it can be uniquely
+# identified within this container.
+# A Loader can be private or public. In general, code will have access to the private loader associated with the
+# location of the code. It will be parented by a loader that in turn have access to other public loaders that
+# can load only such entries that have been publicly available. The split between public and private is not
+# yet enforced in Puppet.
+#
+# The name of a private loader should always end with ' private'
+#
+class Loaders
   class LoaderError < Puppet::Error; end
 
   attr_reader :static_loader
   attr_reader :puppet_system_loader
   attr_reader :public_environment_loader
   attr_reader :private_environment_loader
+  attr_reader :implementation_registry
+  attr_reader :environment
+
+  def self.new(environment, for_agent = false)
+    obj = environment.loaders
+    if obj.nil?
+      obj = self.allocate
+      obj.send(:initialize, environment, for_agent)
+    end
+    obj
+  end
 
-  def initialize(environment)
-    # The static loader can only be changed after a reboot
-    @@static_loader ||= Puppet::Pops::Loader::StaticLoader.new()
+  def initialize(environment, for_agent)
+    # Protect against environment havoc
+    raise ArgumentError.new("Attempt to redefine already initialized loaders for environment") unless environment.loaders.nil?
+    environment.loaders = self
+    @environment = environment
+    @loaders_by_name = {}
+
+    add_loader_by_name(self.class.static_loader)
 
     # Create the set of loaders
     # 1. Puppet, loads from the "running" puppet - i.e. bundled functions, types, extension points and extensions
@@ -23,26 +49,157 @@ class Puppet::Pops::Loaders
     #    concept of environment the same way as when running as a master (except when doing apply).
     #    The creation mechanisms should probably differ between the two.
     #
-    @private_environment_loader = create_environment_loader(environment)
+    @private_environment_loader = if for_agent
+      add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@puppet_system_loader, 'agent environment'))
+    else
+      create_environment_loader(environment)
+    end
 
-    # 3. module loaders are set up from the create_environment_loader, they register themselves
+    # 3. The implementation registry maintains mappings between Puppet types and Runtime types for
+    #    the current environment
+    @implementation_registry = Types::ImplementationRegistry.new(@private_environment_loader)
+    Pcore.init(@puppet_system_loader, @implementation_registry, for_agent)
+
+    # 4. module loaders are set up from the create_environment_loader, they register themselves
   end
 
   # Clears the cached static and puppet_system loaders (to enable testing)
   #
   def self.clear
     @@static_loader = nil
-    @puppet_system_loader = nil
+  end
+
+  # Calls {#loaders} to obtain the {{Loaders}} instance and then uses it to find the appropriate loader
+  # for the given `module_name`, or for the environment in case `module_name` is `nil` or empty.
+  #
+  # @param module_name [String,nil] the name of the module
+  # @return [Loader::Loader] the found loader
+  # @raise [Puppet::ParseError] if no loader can be found
+  # @api private
+  def self.find_loader(module_name)
+    loaders.find_loader(module_name)
+  end
+
+  def self.static_loader
+    # The static loader can only be changed after a reboot
+    @@static_loader ||= Loader::StaticLoader.new()
+  end
+
+  def self.implementation_registry
+    loaders = Puppet.lookup(:loaders) { nil }
+    loaders.nil? ? nil : loaders.implementation_registry
+  end
+
+  def register_implementations(obj_classes, name_authority)
+    loader = @private_environment_loader
+    types = obj_classes.map do |obj_class|
+      type = obj_class._ptype
+      typed_name = Loader::TypedName.new(:type, type.name.downcase, name_authority)
+      entry = loader.loaded_entry(typed_name)
+      loader.set_entry(typed_name, type, obj_class._plocation) if entry.nil? || entry.value.nil?
+      type
+    end
+    # Resolve lazy so that all types can cross reference each other
+    parser = Types::TypeParser.singleton
+    types.each { |type| type.resolve(parser, loader) }
+  end
+
+  # Register the given type with the Runtime3TypeLoader. The registration will not happen unless
+  # the type system has been initialized.
+  #
+  # @param name [String,Symbol] the name of the entity being set
+  # @param origin [URI] the origin or the source where the type is defined
+  # @api private
+  def self.register_runtime3_type(name, origin)
+    loaders = Puppet.lookup(:loaders) { nil }
+    return nil if loaders.nil?
+
+    rt3_loader = loaders.runtime3_type_loader
+    return nil if rt3_loader.nil?
+
+    name = name.to_s
+    caps_name = Types::TypeFormatter.singleton.capitalize_segments(name)
+    typed_name = Loader::TypedName.new(:type, name.downcase)
+    rt3_loader.set_entry(typed_name, Types::PResourceType.new(caps_name), origin)
+    nil
+  end
+
+  # Finds a loader to use when deserializing a catalog and then subsequenlty use user
+  # defined types found in that catalog.
+  #
+  def self.catalog_loader
+    loaders = Puppet.lookup(:loaders) { nil }
+    if loaders.nil?
+      loaders = Loaders.new(Puppet.lookup(:current_environment), true)
+      Puppet.push_context(:loaders => loaders)
+    end
+    loaders.find_loader(nil)
+  end
+
+  # Finds the `Loaders` instance by looking up the :loaders in the global Puppet context
+  #
+  # @return [Loaders] the loaders instance
+  # @raise [Puppet::ParseError] if loader has been bound to the global context
+  # @api private
+  def self.loaders
+    loaders = Puppet.lookup(:loaders) { nil }
+    raise Puppet::ParseError, "Internal Error: Puppet Context ':loaders' missing" if loaders.nil?
+    loaders
+  end
+
+  # Lookup a loader by its unique name.
+  #
+  # @param [String] loader_name the name of the loader to lookup
+  # @return [Loader] the found loader
+  # @raise [Puppet::ParserError] if no loader is found
+  def [](loader_name)
+    loader = @loaders_by_name[loader_name]
+    if loader.nil?
+      # Unable to find the module private loader. Try resolving the module
+      loader = private_loader_for_module(loader_name[0..-9]) if loader_name.end_with?(' private')
+      raise Puppet::ParseError, "Unable to find loader named '#{loader_name}'" if loader.nil?
+    end
+    loader
+  end
+
+  # Finds the appropriate loader for the given `module_name`, or for the environment in case `module_name`
+  # is `nil` or empty.
+  #
+  # @param module_name [String,nil] the name of the module
+  # @return [Loader::Loader] the found loader
+  # @raise [Puppet::ParseError] if no loader can be found
+  # @api private
+  def find_loader(module_name)
+    if module_name.nil? || module_name == ''
+      # TODO : Later when fdefinition can be private, a decision is needed regarding what that means.
+      #        A private environment loader could be used for logic outside of modules, then only that logic
+      #        would see the definition.
+      #
+      # Use the private loader, this definition may see the environment's dependencies (currently, all modules)
+      loader = private_environment_loader()
+      raise Puppet::ParseError, 'Internal Error: did not find public loader' if loader.nil?
+      loader
+    else
+      # TODO : Later check if definition is private, and then add it to private_loader_for_module
+      #
+      loader = public_loader_for_module(module_name)
+      raise Puppet::ParseError, "Internal Error: did not find public loader for module: '#{module_name}'" if loader.nil?
+      loader
+    end
   end
 
   def static_loader
-    @@static_loader
+    self.class.static_loader
   end
 
   def puppet_system_loader
     @puppet_system_loader
   end
 
+  def runtime3_type_loader
+    @runtime3_type_loader
+  end
+
   def public_loader_for_module(module_name)
     md = @module_resolver[module_name] || (return nil)
     # Note, this loader is not resolved until there is interest in the visibility of entities from the
@@ -63,10 +220,16 @@ class Puppet::Pops::Loaders
     md.private_loader
   end
 
+  def add_loader_by_name(loader)
+    name = loader.loader_name
+    raise Puppet::ParseError, "Internal Error: Attempt to redefine loader named '#{name}'" if @loaders_by_name.include?(name)
+    @loaders_by_name[name] = loader
+  end
+
   private
 
   def create_puppet_system_loader()
-    Puppet::Pops::Loader::ModuleLoaders.system_loader_from(static_loader, self)
+    Loader::ModuleLoaders.system_loader_from(static_loader, self)
   end
 
   def create_environment_loader(environment)
@@ -86,15 +249,20 @@ class Puppet::Pops::Loaders
     # Puppet binder currently reads confdir/bindings - that is bad, it should be using the new environment support.
 
     # The environment is not a namespace, so give it a nil "module_name"
-    module_name = nil
     loader_name = "environment:#{environment.name}"
+    # env_conf is setup from the environment_dir value passed into Puppet::Environments::Directories.new
     env_conf = Puppet.lookup(:environments).get_conf(environment.name)
-    if env_conf.nil? || !env_conf.is_a?(Puppet::Settings::EnvironmentConf)
+    env_path = env_conf.nil? || !env_conf.is_a?(Puppet::Settings::EnvironmentConf) ? nil : env_conf.path_to_env
+
+    # Create the 3.x resource type loader
+    @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(puppet_system_loader, self, environment, env_conf.nil? ? nil : env_path))
+
+    if env_path.nil?
       # Not a real directory environment, cannot work as a module TODO: Drop when legacy env are dropped?
-      loader = Puppet::Pops::Loader::SimpleEnvironmentLoader.new(puppet_system_loader, loader_name)
+      loader = add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@runtime3_type_loader, loader_name))
     else
       # View the environment as a module to allow loading from it - this module is always called 'environment'
-      loader = Puppet::Pops::Loader::ModuleLoaders.module_loader_from(puppet_system_loader, self, 'environment', env_conf.path_to_env)
+      loader = Loader::ModuleLoaders.environment_loader_from(@runtime3_type_loader, self, env_path)
     end
 
     # An environment has a module path even if it has a null loader
@@ -105,7 +273,7 @@ class Puppet::Pops::Loaders
     # Code in the environment gets to see all modules (since there is no metadata for the environment)
     # but since this is not given to the module loaders, they can not load global code (since they can not
     # have prior knowledge about this
-    loader = Puppet::Pops::Loader::DependencyLoader.new(loader, "environment", @module_resolver.all_module_loaders())
+    loader = add_loader_by_name(Loader::DependencyLoader.new(loader, 'environment private', @module_resolver.all_module_loaders()))
 
     # The module loader gets the private loader via a lazy operation to look up the module's private loader.
     # This does not work for an environment since it is not resolved the same way.
@@ -116,12 +284,12 @@ class Puppet::Pops::Loaders
   end
 
   def configure_loaders_for_modules(parent_loader, environment)
-    @module_resolver = mr = ModuleResolver.new()
+    @module_resolver = mr = ModuleResolver.new(self)
     environment.modules.each do |puppet_module|
       # Create data about this module
       md = LoaderModuleData.new(puppet_module)
       mr[puppet_module.name] = md
-      md.public_loader = Puppet::Pops::Loader::ModuleLoaders.module_loader_from(parent_loader, self, md.name, md.path)
+      md.public_loader = Loader::ModuleLoaders.module_loader_from(parent_loader, self, md.name, md.path)
     end
     # NOTE: Do not resolve all modules here - this is wasteful if only a subset of modules / functions are used
     #       The resolution is triggered by asking for a module's private loader, since this means there is interest
@@ -138,7 +306,6 @@ class Puppet::Pops::Loaders
   #
   class LoaderModuleData
 
-    attr_accessor :state
     attr_accessor :public_loader
     attr_accessor :private_loader
     attr_accessor :resolutions
@@ -149,7 +316,6 @@ class Puppet::Pops::Loaders
     # @param puppet_module [Puppet::Module] the module instance for the module being represented
     #
     def initialize(puppet_module)
-      @state = :initial
       @puppet_module = puppet_module
       @resolutions = []
       @public_loader = nil
@@ -169,7 +335,7 @@ class Puppet::Pops::Loaders
     end
 
     def resolved?
-      @state == :resolved
+      !@private_loader.nil?
     end
 
     def restrict_to_dependencies?
@@ -189,7 +355,8 @@ class Puppet::Pops::Loaders
   #
   class ModuleResolver
 
-    def initialize()
+    def initialize(loaders)
+      @loaders = loaders
       @index = {}
       @all_module_loaders = nil
     end
@@ -208,7 +375,7 @@ class Puppet::Pops::Loaders
 
     def resolve(module_data)
       if module_data.resolved?
-        return
+        nil
       else
         module_data.private_loader =
           if module_data.restrict_to_dependencies?
@@ -224,17 +391,28 @@ class Puppet::Pops::Loaders
     def create_loader_with_all_modules_visible(from_module_data)
       Puppet.debug{"ModuleLoader: module '#{from_module_data.name}' has unknown dependencies - it will have all other modules visible"}
 
-      Puppet::Pops::Loader::DependencyLoader.new(from_module_data.public_loader, from_module_data.name, all_module_loaders())
+      @loaders.add_loader_by_name(Loader::DependencyLoader.new(from_module_data.public_loader, "#{from_module_data.name} private", all_module_loaders()))
     end
 
     def create_loader_with_only_dependencies_visible(from_module_data)
       if from_module_data.unmet_dependencies?
-        Puppet.warning("ModuleLoader: module '#{from_module_data.name}' has unresolved dependencies"+
-          " - it will only see those that are resolved."+
-          " Use 'puppet module list --tree' to see information about modules")
+        if Puppet[:strict] != :off
+          msg = "ModuleLoader: module '#{from_module_data.name}' has unresolved dependencies" \
+              " - it will only see those that are resolved." \
+              " Use 'puppet module list --tree' to see information about modules"
+          case Puppet[:strict]
+          when :error
+              raise LoaderError.new(msg)
+          when :warning
+            Puppet.warn_once(:unresolved_module_dependencies,
+                             "unresolved_dependencies_for_module_#{from_module_data.name}",
+                             msg)
+          end
+        end
       end
       dependency_loaders = from_module_data.dependency_names.collect { |name| @index[name].public_loader }
-      Puppet::Pops::Loader::DependencyLoader.new(from_module_data.public_loader, from_module_data.name, dependency_loaders)
+      @loaders.add_loader_by_name(Loader::DependencyLoader.new(from_module_data.public_loader, "#{from_module_data.name} private", dependency_loaders))
     end
   end
 end
+end

data/lib/puppet/pops/lookup/configured_data_provider.rb

@@ -0,0 +1,93 @@
+require_relative 'hiera_config'
+require_relative 'data_provider'
+
+module Puppet::Pops
+module Lookup
+# @api private
+class ConfiguredDataProvider
+  include DataProvider
+
+  # @param config [HieraConfig,nil] the configuration
+  def initialize(config = nil)
+    @config = config.nil? ? nil : assert_config_version(config)
+  end
+
+  def config(lookup_invocation)
+    @config ||= assert_config_version(HieraConfig.create(lookup_invocation, configuration_path(lookup_invocation), self))
+  end
+
+  # Needed to assign generated version 4 config
+  # @deprecated
+  def config=(config)
+    @config = config
+  end
+
+  # @return [Pathname] the path to the configuration
+  def config_path
+    @config.nil? ? nil : @config.config_path
+  end
+
+  # @return [String] the name of this provider
+  def name
+    n = "#{place} "
+    n << '"' << module_name << '" ' unless module_name.nil?
+    n << 'Data Provider'
+    n << " (#{@config.name})" unless @config.nil?
+    n
+  end
+
+  # Performs a lookup by searching all configured locations for the given _key_. A merge will be performed if
+  # the value is found in more than one location.
+  #
+  # @param key [String] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String => Object},nil] Merge strategy, merge strategy name, strategy and options hash, or nil (implies "first found")
+  # @return [Object] the found object
+  # @throw :no_such_key when the object is not found
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    lookup_invocation.with(:data_provider, self) do
+      merge_strategy = MergeStrategy.strategy(merge)
+      dps = data_providers(lookup_invocation)
+      if dps.empty?
+        lookup_invocation.report_not_found(key)
+        throw :no_such_key
+      end
+      merge_strategy.lookup(dps, lookup_invocation) do |data_provider|
+        data_provider.unchecked_key_lookup(key, lookup_invocation, merge_strategy)
+      end
+    end
+  end
+
+  protected
+
+  # Assert that the given config version is accepted by this data provider.
+  #
+  # @param config [HieraConfig] the configuration to check
+  # @return [HieraConfig] the argument
+  # @raise [Puppet::DataBinding::LookupError] if the configuration version is unacceptable
+  def assert_config_version(config)
+    config
+  end
+
+  # Return the root of the configured entity
+  #
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @return [Pathname] Path to root of the module
+  # @raise [Puppet::DataBinding::LookupError] if the given module is can not be found
+  #
+  def provider_root(lookup_invocation)
+    raise NotImplementedError, "#{self.class.name} must implement method '#provider_root'"
+  end
+
+  def configuration_path(lookup_invocation)
+    provider_root(lookup_invocation) + HieraConfig::CONFIG_FILE_NAME
+  end
+
+  private
+
+  def data_providers(lookup_invocation)
+    config(lookup_invocation).configured_data_providers(lookup_invocation, self)
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/context.rb

@@ -0,0 +1,200 @@
+require_relative 'interpolation'
+
+module Puppet::Pops
+module Lookup
+# The EnvironmentContext is adapted to the current environment
+#
+class EnvironmentContext < Adaptable::Adapter
+  class FileData
+    attr_reader :data
+
+    def initialize(path, inode, mtime, size, data)
+      @path = path
+      @inode = inode
+      @mtime = mtime
+      @size = size
+      @data = data
+    end
+
+    def valid?(stat)
+      stat.ino == @inode && stat.mtime == @mtime && stat.size == @size
+    end
+  end
+
+  attr_reader :environment_name
+
+  def self.create_adapter(environment)
+    new(environment)
+  end
+
+  def initialize(environment)
+    @environment_name = environment.name
+    @file_data_cache = {}
+  end
+
+  # Loads the contents of the file given by _path_. The content is then yielded to the provided block in
+  # case a block is given, and the returned value from that block is cached and returned by this method.
+  # If no block is given, the content is stored instead.
+  #
+  # The cache is retained as long as the inode, mtime, and size of the file remains unchanged.
+  #
+  # @param path [String] path to the file to be read
+  # @yieldparam content [String] the content that was read from the file
+  # @yieldreturn [Object] some result based on the content
+  # @return [Object] the content, or if a block was given, the return value of the block
+  #
+  def cached_file_data(path)
+    file_data = @file_data_cache[path]
+    stat = Puppet::FileSystem.stat(path)
+    unless file_data && file_data.valid?(stat)
+      Puppet.debug("File at '#{path}' was changed, reloading") if file_data
+      content = Puppet::FileSystem.read(path, :encoding => 'utf-8')
+      file_data = FileData.new(path, stat.ino, stat.mtime, stat.size, block_given? ? yield(content) : content)
+      @file_data_cache[path] = file_data
+    end
+    file_data.data
+  end
+end
+
+# A FunctionContext is created for each unique hierarchy entry and adapted to the Compiler (and hence shares
+# the compiler's life-cycle).
+# @api private
+class FunctionContext
+  include Interpolation
+
+  attr_reader :module_name, :function
+  attr_accessor :data_hash
+
+  def initialize(environment_context, module_name, function)
+    @data_hash = nil
+    @cache = {}
+    @environment_context = environment_context
+    @module_name = module_name
+    @function = function
+  end
+
+  def cache(key, value)
+    @cache[key] = value
+  end
+
+  def cache_all(hash)
+    @cache.merge!(hash)
+    nil
+  end
+
+  def cache_has_key(key)
+    @cache.include?(key)
+  end
+
+  def cached_value(key)
+    @cache[key]
+  end
+
+  def cached_entries(&block)
+    if block_given?
+      enumerator = @cache.each_pair
+      @cache.size.times do
+        if block.arity == 2
+          yield(*enumerator.next)
+        else
+          yield(enumerator.next)
+        end
+      end
+      nil
+    else
+      Types::Iterable.on(@cache)
+    end
+  end
+
+  def cached_file_data(path, &block)
+    @environment_context.cached_file_data(path, &block)
+  end
+
+  def environment_name
+    @environment_context.environment_name
+  end
+end
+
+# The Context is created once for each call to a function. It provides a combination of the {Invocation} object needed
+# to provide explanation support and the {FunctionContext} object needed to provide the private cache.
+# The {Context} is part of the public API. It will be passed to a _data_hash_, _data_dig_, or _lookup_key_ function and its
+# attributes and methods can be used in a Puppet function as well as in a Ruby function.
+# The {Context} is maps to the Pcore type 'Puppet::LookupContext'
+#
+# @api public
+class Context
+  include Types::PuppetObject
+  extend Forwardable
+
+  def self._ptype
+    @type
+  end
+
+  def self.register_ptype(loader, ir)
+    tf = Types::TypeFactory
+    key_type = tf.optional(tf.scalar)
+    @type = Pcore::create_object_type(loader, ir, self, 'Puppet::LookupContext', 'Any',
+      {
+        'environment_name' => {
+          Types::KEY_TYPE => Types::PStringType::NON_EMPTY,
+          Types::KEY_KIND => Types::PObjectType::ATTRIBUTE_KIND_DERIVED
+        },
+        'module_name' => {
+          Types::KEY_TYPE => tf.variant(Types::PStringType::NON_EMPTY, Types::PUndefType::DEFAULT)
+        }
+      },
+      {
+        'not_found' => tf.callable([0, 0], tf.undef),
+        'explain' => tf.callable([0, 0, tf.callable(0,0)], tf.undef),
+        'interpolate' => tf.callable(1, 1),
+        'cache' => tf.callable([key_type, tf.any], tf.any),
+        'cache_all' => tf.callable([tf.hash_kv(key_type, tf.any)], tf.undef),
+        'cache_has_key' => tf.callable([key_type], tf.boolean),
+        'cached_value' => tf.callable([key_type], tf.any),
+        'cached_entries' => tf.variant(
+          tf.callable([0, 0, tf.callable(1,1)], tf.undef),
+          tf.callable([0, 0, tf.callable(2,2)], tf.undef),
+          tf.callable([0, 0], tf.iterable(tf.tuple([key_type, tf.any])))),
+        'cached_file_data' => tf.callable(tf.string, tf.optional(tf.callable([1, 1])))
+      }
+    ).resolve(Types::TypeParser.singleton, loader)
+  end
+
+  # Mainly for test purposes. Makes it possible to create a {Context} in Puppet code provided that a current {Invocation} exists.
+  def self.from_asserted_args(module_name)
+    new(FunctionContext.new(EnvironmentContext.adapt(Puppet.lookup(:environments).get(Puppet[:environment])), module_name, nil), Invocation.current)
+  end
+
+  # Public methods delegated to the {FunctionContext}
+  def_delegators :@function_context, :cache, :cache_all, :cache_has_key, :cached_value, :cached_entries, :environment_name, :module_name, :cached_file_data
+
+  def initialize(function_context, lookup_invocation)
+    @lookup_invocation = lookup_invocation
+    @function_context = function_context
+  end
+
+  # Will call the given block to obtain a textual explanation if explanation support is active.
+  #
+  def explain(&block)
+    @lookup_invocation.report_text(&block)
+    nil
+  end
+
+  # Resolve interpolation expressions in the given value
+  # @param [Object] value
+  # @return [Object] the value with all interpolation expressions resolved
+  def interpolate(value)
+    @function_context.interpolate(value, @lookup_invocation, true)
+  end
+
+  def not_found
+    throw :no_such_key
+  end
+
+  # @api private
+  def invocation
+    @lookup_invocation
+  end
+end
+end
+end

data/lib/puppet/pops/lookup/data_adapter.rb

@@ -0,0 +1,27 @@
+module Puppet::Pops
+module Lookup
+# A class that adapts a Hash
+# @api private
+class DataAdapter < Adaptable::Adapter
+  def self.create_adapter(o)
+    new
+  end
+
+  def initialize
+    @data = {}
+  end
+
+  def [](name)
+    @data[name]
+  end
+
+  def include?(name)
+    @data.include? name
+  end
+
+  def []=(name, value)
+    @data[name] = value
+  end
+end
+end
+end