puppet 4.0.0 → 4.9.1

data/lib/puppet/pops/loader/loader.rb

@@ -19,7 +19,18 @@
 #
 # @api public
 #
-class Puppet::Pops::Loader::Loader
+module Puppet::Pops
+module Loader
+class Loader
+  attr_reader :loader_name
+
+  # Describes the kinds of things that loaders can load
+  LOADABLE_KINDS = [:func_4x, :func_4xpp, :type_pp, :resource_type_pp].freeze
+
+  # @param [String] name the name of the loader. Must be unique among all loaders maintained by a {Loader} instance
+  def initialize(loader_name)
+    @loader_name = loader_name.freeze
+  end
 
   # Produces the value associated with the given name if already loaded, or available for loading
   # by this loader, one of its parents, or other loaders visible to this loader.
@@ -49,7 +60,19 @@ class Puppet::Pops::Loader::Loader
   # @api public
   #
   def load_typed(typed_name)
-    raise NotImplementedError.new
+    raise NotImplementedError, "Class #{self.class.name} must implement method #load_typed"
+  end
+
+  # Returns an already loaded entry if one exists, or nil. This does not trigger loading
+  # of the given type/name.
+  #
+  # @param typed_name [TypedName] - the type, name combination to lookup
+  # @param check_dependencies [Boolean] - if dependencies should be checked in additiona to here and parent
+  # @return [NamedEntry, nil] the entry containing the loaded value, or nil if not found
+  # @api public
+  #
+  def loaded_entry(typed_name, check_dependencies = false)
+    raise NotImplementedError, "Class #{self.class.name} must implement method #loaded_entry"
   end
 
   # Produces the value associated with the given name if defined **in this loader**, or nil if not defined.
@@ -79,7 +102,7 @@ class Puppet::Pops::Loader::Loader
   # @api private
   #
   def find(typed_name)
-    raise NotImplementedError.new
+    raise NotImplementedError, "Class #{self.class.name} must implement method #find"
   end
 
   # Returns the parent of the loader, or nil, if this is the top most loader. This implementation returns nil.
@@ -119,6 +142,31 @@ class Puppet::Pops::Loader::Loader
     raise NotImplementedError.new
   end
 
+  # A loader is by default a loader for all kinds of loadables. An implementation may override
+  # if it cannot load all kinds.
+  #
+  # @api private
+  def loadables
+    LOADABLE_KINDS
+  end
+
+  # A loader may want to implement its own version with more detailed information.
+  def to_s
+    loader_name
+  end
+
+  # Loaders may contain references to the environment they load items within.
+  # Consequently, calling Kernel#inspect may return strings that are large
+  # enough to cause OutOfMemoryErrors on some platforms.
+  #
+  # We do not call alias_method here as that would copy the content of to_s
+  # at this point to inspect (ie children would print out `loader_name`
+  # rather than their version of to_s if they chose to implement it).
+  def inspect
+    self.to_s
+  end
+
+
   # An entry for one entity loaded by the loader.
   #
   class NamedEntry
@@ -133,49 +181,6 @@ class Puppet::Pops::Loader::Loader
       freeze()
     end
   end
-
-  # A name/type combination that can be used as a compound hash key
-  #
-  class TypedName
-    DOUBLE_COLON = '::'
-
-    attr_reader :hash
-    attr_reader :type
-    attr_reader :name
-    attr_reader :name_parts
-    attr_reader :compound_name
-
-    def initialize(type, name)
-      @type = type
-      # relativize the name (get rid of leading ::), and make the split string available
-      parts = name.to_s.split(DOUBLE_COLON)
-      if parts[0].empty?
-        parts.shift
-        @name = name[2..-1]
-      else
-        @name = name
-      end
-      @name_parts = parts
-
-      # Use a frozen compound key for the hash and comparison
-      @compound_name = "#{type}/#{name}".freeze
-      @hash = @compound_name.hash
-      freeze
-    end
-
-    def ==(o)
-      o.class == self.class && o.compound_name == @compound_name
-    end
-
-    alias eql? ==
-
-    def qualified?
-      @name_parts.size > 1
-    end
-
-    def to_s
-      @compound_name
-    end
-  end
 end
-
+end
+end

data/lib/puppet/pops/loader/loader_paths.rb

@@ -4,7 +4,7 @@
 # The central loader knowledge about paths, what they represent and how to instantiate from them.
 # Contains helpers (*smart paths*) to deal with lazy resolution of paths.
 #
-# TODO: Currently only supports loading of functions (3 kinds)
+# TODO: Currently only supports loading of functions (2 kinds)
 #
 module Puppet::Pops::Loader::LoaderPaths
   # Returns an array of SmartPath, each instantiated with a reference to the given loader (for root path resolution
@@ -12,10 +12,21 @@ module Puppet::Pops::Loader::LoaderPaths
   # mutated.
   #
   def self.relative_paths_for_type(type, loader)
-    result =
+    result = []
     case type
     when :function
-        [FunctionPath4x.new(loader)]
+        # Only include support for the loadable items the loader states it can contain
+        if loader.loadables.include?(:func_4x)
+          result << FunctionPath4x.new(loader)
+        end
+        if loader.loadables.include?(:func_4xpp)
+          result << FunctionPathPP.new(loader)
+        end
+        # When wanted also add FunctionPath3x to load 3x functions
+    when :type
+      result << TypePathPP.new(loader) if loader.loadables.include?(:type_pp)
+    when :resource_type_pp
+      result << ResourceTypeImplPP.new(loader) if loader.loadables.include?(:resource_type_pp)
     else
       # unknown types, simply produce an empty result; no paths to check, nothing to find... move along...
       []
@@ -44,8 +55,12 @@ module Puppet::Pops::Loader::LoaderPaths
     def generic_path()
       return @generic_path unless @generic_path.nil?
 
-      root_path = @loader.path
-      @generic_path = (root_path.nil? ? relative_path : File.join(root_path, relative_path))
+      the_root_path = root_path() # @loader.path
+      @generic_path = (the_root_path.nil? ? relative_path : File.join(the_root_path, relative_path))
+    end
+
+    def root_path
+      @loader.path
     end
 
     # Effective path is the generic path + the name part(s) + extension.
@@ -64,8 +79,10 @@ module Puppet::Pops::Loader::LoaderPaths
   end
 
   class RubySmartPath < SmartPath
+    EXTENSION = '.rb'.freeze
+
     def extension
-      ".rb"
+      EXTENSION
     end
 
     # Duplication of extension information, but avoids one call
@@ -74,6 +91,30 @@ module Puppet::Pops::Loader::LoaderPaths
     end
   end
 
+  # A PuppetSmartPath is rooted at the loader's directory one level up from what the loader specifies as it
+  # path (which is a reference to its 'lib' directory.
+  #
+  class PuppetSmartPath < SmartPath
+    EXTENSION = '.pp'.freeze
+
+    def extension
+      EXTENSION
+    end
+
+    def root_path
+      Puppet::FileSystem.dir_string(@loader.path)
+    end
+
+    # Duplication of extension information, but avoids one call
+    def effective_path(typed_name, start_index_in_name)
+      # Puppet name to path always skips the name-space as that is part of the generic path
+      # i.e. <module>/mymodule/functions/foo.pp is the function mymodule::foo
+      parts = typed_name.name_parts
+      parts = parts[start_index_in_name..-1] if parts.size > 1
+      "#{File.join(generic_path, parts)}.pp"
+    end
+  end
+
   class FunctionPath4x < RubySmartPath
     FUNCTION_PATH_4X = File.join('puppet', 'functions')
 
@@ -86,6 +127,69 @@ module Puppet::Pops::Loader::LoaderPaths
     end
   end
 
+  class FunctionPath3x < RubySmartPath
+    FUNCTION_PATH_3X = File.join('puppet', 'parser', 'functions')
+
+    def relative_path
+      FUNCTION_PATH_3X
+    end
+
+    def instantiator()
+      Puppet::Pops::Loader::RubyLegacyFunctionInstantiator
+    end
+  end
+
+  class FunctionPathPP < PuppetSmartPath
+    # Navigate to directory where 'lib' is, then down again
+    FUNCTION_PATH_PP = File.join('functions')
+
+    def relative_path
+      FUNCTION_PATH_PP
+    end
+
+    def instantiator()
+      Puppet::Pops::Loader::PuppetFunctionInstantiator
+    end
+  end
+
+  class TypePathPP < PuppetSmartPath
+    TYPE_PATH_PP = File.join('types')
+
+    def relative_path
+      TYPE_PATH_PP
+    end
+
+    def instantiator()
+      Puppet::Pops::Loader::TypeDefinitionInstantiator
+    end
+  end
+
+  class ResourceTypeImplPP < PuppetSmartPath
+    RESOURCE_TYPES_PATH_PP = File.join('.resource_types')
+
+    def relative_path
+      RESOURCE_TYPES_PATH_PP
+    end
+
+    def root_path
+      @loader.path
+    end
+
+    def instantiator()
+      Puppet::Pops::Loader::PuppetResourceTypeImplInstantiator
+    end
+
+    # The effect paths for resource type impl is the full name
+    # since resource types are not name spaced.
+    # This overrides the default PuppetSmartPath.
+    #
+    def effective_path(typed_name, start_index_in_name)
+      # Resource type to name does not skip the name-space
+      # i.e. <module>/mymodule/resource_types/foo.pp is the reource type foo
+      "#{File.join(generic_path, typed_name.name_parts)}.pp"
+    end
+  end
+
   # SmartPaths
   # ===
   # Holds effective SmartPath instances per type

data/lib/puppet/pops/loader/module_loaders.rb

@@ -1,4 +1,5 @@
-
+module Puppet::Pops
+module Loader
 # =ModuleLoaders
 # A ModuleLoader loads items from a single module.
 # The ModuleLoaders (ruby) module contains various such loaders. There is currently one concrete
@@ -14,12 +15,14 @@
 # internal layout etc.)
 #
 # A module loader is also not aware of the mapping of name to relative paths - this is performed by the
-# included module Puppet::Pops::Loader::PathBasedInstantatorConfig which knows about the map from type/name to
+# included module PathBasedInstantatorConfig which knows about the map from type/name to
 # relative path, and the logic that can instantiate what is expected to be found in the content of that path.
 #
 # @api private
 #
-module Puppet::Pops::Loader::ModuleLoaders
+module ModuleLoaders
+  ENVIRONMENT = 'environment'.freeze
+
   def self.system_loader_from(parent_loader, loaders)
     # Puppet system may be installed in a fixed location via RPM, installed as a Gem, via source etc.
     # The only way to find this across the different ways puppet can be installed is
@@ -27,22 +30,43 @@ module Puppet::Pops::Loader::ModuleLoaders
     # puppet.
     #
     puppet_lib = File.join(File.dirname(__FILE__), '../../..')
-    Puppet::Pops::Loader::ModuleLoaders::FileBased.new(parent_loader,
+    ModuleLoaders::FileBased.new(parent_loader,
                                                        loaders,
                                                        nil,
-                                                       puppet_lib,
-                                                       'puppet_system')
+                                                       puppet_lib,   # may or may not have a 'lib' above 'puppet'
+                                                       'puppet_system',
+                                                        [:func_4x]   # only load ruby functions from "puppet"
+                                                       )
+  end
+
+  def self.environment_loader_from(parent_loader, loaders, env_path)
+    ModuleLoaders::FileBased.new(parent_loader,
+      loaders,
+      ENVIRONMENT,
+      File.join(env_path, 'lib'),
+      ENVIRONMENT
+    )
   end
 
   def self.module_loader_from(parent_loader, loaders, module_name, module_path)
-    Puppet::Pops::Loader::ModuleLoaders::FileBased.new(parent_loader,
+    ModuleLoaders::FileBased.new(parent_loader,
                                                        loaders,
                                                        module_name,
                                                        File.join(module_path, 'lib'),
-                                                       module_name)
+                                                       module_name
+                                                       )
   end
 
-  class AbstractPathBasedModuleLoader < Puppet::Pops::Loader::BaseLoader
+  def self.pcore_resource_type_loader_from(parent_loader, loaders, environment_path)
+    ModuleLoaders::FileBased.new(parent_loader,
+      loaders,
+      nil,
+      environment_path,
+      'pcore_resource_types'
+    )
+  end
+
+  class AbstractPathBasedModuleLoader < BaseLoader
 
     # The name of the module, or nil, if this is a global "component"
     attr_reader :module_name
@@ -61,27 +85,39 @@ module Puppet::Pops::Loader::ModuleLoaders
     attr_accessor :private_loader
 
     # Initialize a kind of ModuleLoader for one module
-    # @param parent_loader [Puppet::Pops::Loader] loader with higher priority
+    # @param parent_loader [Loader] loader with higher priority
+    # @param loaders [Loaders] the container for this loader
     # @param module_name [String] the name of the module (non qualified name), may be nil for a global "component"
     # @param path [String] the path to the root of the module (semantics defined by subclass)
     # @param loader_name [String] a name that is used for human identification (useful when module_name is nil)
     #
-    def initialize(parent_loader, loaders, module_name, path, loader_name)
+    def initialize(parent_loader, loaders, module_name, path, loader_name, loadables)
       super parent_loader, loader_name
 
       @module_name = module_name
       @path = path
-      @smart_paths = Puppet::Pops::Loader::LoaderPaths::SmartPaths.new(self)
+      @smart_paths = LoaderPaths::SmartPaths.new(self)
       @loaders = loaders
+      @loadables = loadables
+      unless (loadables - LOADABLE_KINDS).empty?
+        raise ArgumentError, 'given loadables are not of supported loadable kind'
+      end
+      loaders.add_loader_by_name(self)
+    end
+
+    def loadables
+      @loadables
     end
 
     # Finds typed/named entity in this module
-    # @param typed_name [Puppet::Pops::Loader::TypedName] the type/name to find
-    # @return [Puppet::Pops::Loader::Loader::NamedEntry, nil found/created entry, or nil if not found
+    # @param typed_name [TypedName] the type/name to find
+    # @return [Loader::NamedEntry, nil found/created entry, or nil if not found
     #
     def find(typed_name)
+      # This loader is tailored to only find entries in the current runtime
+      return nil unless typed_name.name_authority == Pcore::RUNTIME_NAME_AUTHORITY
+
       # Assume it is a global name, and that all parts of the name should be used when looking up
-      name_part_index = 0
       name_parts = typed_name.name_parts
 
       # Certain types and names can be disqualified up front
@@ -93,10 +129,6 @@ module Puppet::Pops::Loader::ModuleLoaders
         # ok since such a "module" cannot have namespaced content).
         #
         return nil unless name_parts[0] == module_name
-
-        # Skip the first part of the name when computing the path since the path already contains the name of the
-        # module
-        name_part_index = 1
       else
         # The name is in the global name space.
 
@@ -104,6 +136,8 @@ module Puppet::Pops::Loader::ModuleLoaders
         case typed_name.type
         when :function
         when :resource_type
+        when :resource_type_pp
+        when :type
         else
           # anything else cannot possibly be in this module
           # TODO: should not be allowed anyway... may have to revisit this decision
@@ -116,7 +150,7 @@ module Puppet::Pops::Loader::ModuleLoaders
       # Find the file to instantiate, and instantiate the entity if file is found
       origin = nil
       if (smart_path = smart_paths.effective_paths(typed_name.type).find do |sp|
-          origin = sp.effective_path(typed_name, name_part_index)
+          origin = sp.effective_path(typed_name, global? ? 0 : 1)
           existing_path(origin)
         end)
         value = smart_path.instantiator.create(self, typed_name, origin, get_contents(origin))
@@ -172,12 +206,20 @@ module Puppet::Pops::Loader::ModuleLoaders
       raise NotImplementedError.new
     end
 
+    # Answers the question if this loader represents a global component (true for resource type loader and environment loader)
+    #
+    # @return [Boolean] `true` if this loader represents a global component
+    #
+    def global?
+      module_name.nil? || module_name == ENVIRONMENT
+    end
+
     # Produces the private loader for the module. If this module is not already resolved, this will trigger resolution
     #
     def private_loader
       # The system loader has a nil module_name and it does not have a private_loader as there are no functions
-      # that can only by called by puppet runtime - if so, it acts as the privuate loader directly.
-      @private_loader ||= ((module_name.nil? && self) || @loaders.private_loader_for_module(module_name))
+      # that can only by called by puppet runtime - if so, it acts as the private loader directly.
+      @private_loader ||= (global? ? self : @loaders.private_loader_for_module(module_name))
     end
   end
 
@@ -190,12 +232,12 @@ module Puppet::Pops::Loader::ModuleLoaders
 
     # Create a kind of ModuleLoader for one module (Puppet Module, or module like)
     #
-    # @param parent_loader [Puppet::Pops::Loader::Loader] typically the loader for the environment or root
+    # @param parent_loader [Loader] typically the loader for the environment or root
     # @param module_name [String] the name of the module (non qualified name), may be nil for "modules" only containing globals
     # @param path [String] the path to the root of the module (semantics defined by subclass)
     # @param loader_name [String] a name that identifies the loader
     #
-    def initialize(parent_loader, loaders, module_name, path, loader_name)
+    def initialize(parent_loader, loaders, module_name, path, loader_name, loadables = LOADABLE_KINDS)
       super
       @path_index = Set.new()
     end
@@ -220,7 +262,7 @@ module Puppet::Pops::Loader::ModuleLoaders
     end
 
     def get_contents(effective_path)
-      Puppet::FileSystem.read(effective_path)
+      Puppet::FileSystem.read(effective_path, :encoding => 'utf-8')
     end
   end
 
@@ -235,7 +277,7 @@ module Puppet::Pops::Loader::ModuleLoaders
   # @api private
   #
   class GemBased < FileBased
-    include Puppet::Pops::Loader::GemSupport
+    include GemSupport
 
     attr_reader :gem_ref
 
@@ -246,9 +288,9 @@ module Puppet::Pops::Loader::ModuleLoaders
     # * gem_ref - [URI, String] gem reference to the root of the module (URI, gem://gemname/optional/path/in/gem), or
     #     just the gem's name as a String.
     #
-    def initialize(parent_loader, loaders, module_name, gem_ref, loader_name)
+    def initialize(parent_loader, loaders, module_name, gem_ref, loader_name, loadables = LOADABLE_KINDS)
       @gem_ref = gem_ref
-      super parent_loader, loaders, module_name, gem_dir(gem_ref), loader_name
+      super parent_loader, loaders, module_name, gem_dir(gem_ref), loader_name, loadables
     end
 
     def to_s()
@@ -256,3 +298,5 @@ module Puppet::Pops::Loader::ModuleLoaders
     end
   end
 end
+end
+end

data/lib/puppet/pops/loader/null_loader.rb

@@ -6,7 +6,7 @@ class Puppet::Pops::Loader::NullLoader < Puppet::Pops::Loader::Loader
   # Construct a NullLoader, optionally with a parent loader
   #
   def initialize(parent_loader=nil, loader_name = "null-loader")
-    @loader_name = loader_name
+    super(loader_name)
     @parent = parent_loader
   end
 
@@ -15,6 +15,14 @@ class Puppet::Pops::Loader::NullLoader < Puppet::Pops::Loader::Loader
     @parent
   end
 
+  def find(typed_name)
+    if @parent.nil?
+      nil
+    else
+      @parent.find(typed_name)
+    end
+  end
+
   def load_typed(typed_name)
     if @parent.nil?
       nil
@@ -23,6 +31,14 @@ class Puppet::Pops::Loader::NullLoader < Puppet::Pops::Loader::Loader
     end
   end
 
+  def loaded_entry(typed_name, check_dependencies = false)
+    if @parent.nil?
+      nil
+    else
+      @parent.loaded_entry(typed_name, check_dependencies)
+    end
+  end
+
   # Has no entries on its own - always nil
   def get_entry(typed_name)
     nil

data/lib/puppet/pops/loader/predefined_loader.rb

@@ -0,0 +1,29 @@
+module Puppet::Pops::Loader
+
+# A PredefinedLoader is a loader that is manually populated with loaded elements
+# before being used. It never loads anything on its own.
+# When searching for a type, it must exist or an error is raised
+#
+class PredefinedLoader < BaseLoader
+  def find(typed_name)
+    if typed_name.type == :type
+      raise Puppet::Pops::Loaders::LoaderError, "Cannot load undefined type '#{typed_name.name.capitalize}'"
+    else
+      nil
+    end
+  end
+
+  def to_s()
+    "(PredefinedLoader '#{loader_name}')"
+  end
+
+  # Allows shadowing since this loader is used internally for things like function local types
+  # And they should win as there is otherwise a risk that the local types clash with built in types
+  # that were added after the function was written, or by resource types loaded by the 3x auto loader.
+  #
+  def allow_shadowing?
+    true
+  end
+end
+
+end

data/lib/puppet/pops/loader/puppet_function_instantiator.rb

@@ -0,0 +1,84 @@
+module Puppet::Pops
+module Loader
+# The PuppetFunctionInstantiator instantiates a Puppet::Functions::PuppetFunction given a Puppet Programming language
+# source that when called evaluates the Puppet logic it contains.
+#
+class PuppetFunctionInstantiator
+  # Produces an instance of the Function class with the given typed_name, or fails with an error if the
+  # given puppet source does not produce this instance when evaluated.
+  #
+  # @param loader [Loader] The loader the function is associated with
+  # @param typed_name [TypedName] the type / name of the function to load
+  # @param source_ref [URI, String] a reference to the source / origin of the puppet code to evaluate
+  # @param pp_code_string [String] puppet code in a string
+  #
+  # @return [Functions::Function] - an instantiated function with global scope closure associated with the given loader
+  #
+  def self.create(loader, typed_name, source_ref, pp_code_string)
+    parser = Parser::EvaluatingParser.new()
+
+    # parse and validate
+    result = parser.parse_string(pp_code_string, source_ref)
+    # Only one function is allowed (and no other definitions)
+    case result.model.definitions.size
+    when 0
+      raise ArgumentError, "The code loaded from #{source_ref} does not define the function '#{typed_name.name}' - it is empty."
+    when 1
+      # ok
+    else
+      raise ArgumentError, "The code loaded from #{source_ref} must contain only the function '#{typed_name.name}' - it has additional definitions."
+    end
+    the_function_definition = result.model.definitions[0]
+
+    unless the_function_definition.is_a?(Model::FunctionDefinition)
+      raise ArgumentError, "The code loaded from #{source_ref} does not define the function '#{typed_name.name}' - no function found."
+    end
+    unless the_function_definition.name == typed_name.name
+      expected = typed_name.name
+      actual = the_function_definition.name
+      raise ArgumentError, "The code loaded from #{source_ref} produced function with the wrong name, expected #{expected}, actual #{actual}"
+    end
+    unless result.model().body == the_function_definition
+      raise ArgumentError, "The code loaded from #{source_ref} contains additional logic - can only contain the function #{typed_name.name}"
+    end
+
+    # Adapt the function definition with loader - this is used from logic contained in it body to find the
+    # loader to use when making calls to the new function API. Such logic have a hard time finding the closure (where
+    # the loader is known - hence this mechanism
+    private_loader = loader.private_loader
+    Adapters::LoaderAdapter.adapt(the_function_definition).loader_name = private_loader.loader_name
+
+    # Cannot bind loaded functions to global scope, that must be done without binding that scope as
+    # loaders survive a compilation.
+    closure_scope = nil # Puppet.lookup(:global_scope) { {} }
+
+    created = create_function_class(the_function_definition)
+    # create the function instance - it needs closure (scope), and loader (i.e. where it should start searching for things
+    # when calling functions etc.
+    # It should be bound to global scope
+
+    created.new(closure_scope, private_loader)
+  end
+
+  # Creates Function class and instantiates it based on a FunctionDefinition model
+  # @return [Array<TypedName, Functions.Function>] - array of
+  #   typed name, and an instantiated function with global scope closure associated with the given loader
+  #
+  def self.create_from_model(function_definition, loader)
+    created = create_function_class(function_definition)
+    typed_name = TypedName.new(:function, function_definition.name)
+    [typed_name, created.new(nil, loader)]
+  end
+
+  def self.create_function_class(function_definition)
+    # Create a 4x function wrapper around a named closure
+    Puppet::Functions.create_function(function_definition.name, Puppet::Functions::PuppetFunction) do
+      # TODO: should not create a new evaluator per function
+      init_dispatch(Evaluator::Closure::Named.new(
+        function_definition.name,
+        Evaluator::EvaluatorImpl.new(), function_definition))
+    end
+  end
+end
+end
+end