puppet 5.3.7 → 5.4.0

data/lib/puppet/pops/loader/module_loaders.rb

@@ -14,9 +14,7 @@ module Loader
 # modules" to determine which module loader to use for each individual module. (There could be differences in
 # internal layout etc.)
 #
-# A module loader is also not aware of the mapping of name to relative paths - this is performed by the
-# included module PathBasedInstantatorConfig which knows about the map from type/name to
-# relative path, and the logic that can instantiate what is expected to be found in the content of that path.
+# A module loader is also not aware of the mapping of name to relative paths.
 #
 # @api private
 #
@@ -27,30 +25,34 @@ module ModuleLoaders
     # to search up the path from this source file's __FILE__ location until it finds the base of
     # puppet.
     #
-    puppet_lib = File.join(File.dirname(__FILE__), '../../..')
-    ModuleLoaders::FileBased.new(parent_loader,
+    puppet_lib = File.realpath(File.join(File.dirname(__FILE__), '../../..'))
+    LibRootedFileBased.new(parent_loader,
                                                        loaders,
                                                        nil,
                                                        puppet_lib,   # may or may not have a 'lib' above 'puppet'
                                                        'puppet_system',
-                                                        [:func_4x]   # only load ruby functions from "puppet"
+                                                        [:func_4x, :datatype]   # only load ruby functions and types from "puppet"
                                                        )
   end
 
   def self.environment_loader_from(parent_loader, loaders, env_path)
-    ModuleLoaders::FileBased.new(parent_loader,
-      loaders,
-      ENVIRONMENT,
-      File.join(env_path, 'lib'),
-      ENVIRONMENT
-    )
+    if env_path.nil? || env_path.empty?
+      EmptyLoader.new(parent_loader, ENVIRONMENT)
+    else
+      FileBased.new(parent_loader,
+        loaders,
+        ENVIRONMENT,
+        env_path,
+        ENVIRONMENT
+      )
+    end
   end
 
   def self.module_loader_from(parent_loader, loaders, module_name, module_path)
     ModuleLoaders::FileBased.new(parent_loader,
                                                        loaders,
                                                        module_name,
-                                                       File.join(module_path, 'lib'),
+                                                       module_path,
                                                        module_name
                                                        )
   end
@@ -64,6 +66,20 @@ module ModuleLoaders
     )
   end
 
+  class EmptyLoader < BaseLoader
+    def find(typed_name)
+      return nil
+    end
+
+    def private_loader
+      @private_loader ||= self
+    end
+
+    def private_loader=(loader)
+      @private_loader = loader
+    end
+  end
+
   class AbstractPathBasedModuleLoader < BaseLoader
 
     # The name of the module, or nil, if this is a global "component"
@@ -92,6 +108,8 @@ module ModuleLoaders
     def initialize(parent_loader, loaders, module_name, path, loader_name, loadables)
       super parent_loader, loader_name
 
+      raise ArgumentError, 'path based loader cannot be instantiated without a path' if path.nil? || path.empty?
+
       @module_name = module_name
       @path = path
       @smart_paths = LoaderPaths::SmartPaths.new(self)
@@ -108,6 +126,33 @@ module ModuleLoaders
       @loadables
     end
 
+    def discover(type, error_collector = nil, name_authority = Pcore::RUNTIME_NAME_AUTHORITY, &block)
+      global = global?
+      if name_authority == Pcore::RUNTIME_NAME_AUTHORITY
+        smart_paths.effective_paths(type).each do |sp|
+          relative_paths(sp).each do |rp|
+            tp = sp.typed_name(type, name_authority, rp, global ? nil : @module_name)
+            next unless sp.valid_name?(tp)
+            begin
+              load_typed(tp) unless block_given? && !block.yield(tp)
+            rescue StandardError => e
+              if error_collector.nil?
+                Puppet.warn_once(:unloadable_entity, tp.to_s, e.message)
+              else
+                err = Puppet::DataTypes::Error.new(
+                  Issues::LOADER_FAILURE.format(:type => type),
+                  'PUPPET_LOADER_FAILURE',
+                  { 'original_error' => e.message },
+                  Issues::LOADER_FAILURE.issue_code)
+                error_collector << err unless error_collector.include?(err)
+              end
+            end
+          end
+        end
+      end
+      super
+    end
+
     # Finds typed/named entity in this module
     # @param typed_name [TypedName] the type/name to find
     # @return [Loader::NamedEntry, nil found/created entry, or nil if not found
@@ -131,18 +176,42 @@ module ModuleLoaders
       else
         # The name is in the global name space.
 
-        # The only globally name-spaced elements that may be loaded from modules are functions and resource types
         case typed_name.type
-        when :function
-        when :resource_type
-        when :resource_type_pp
-        when :type
+        when :function, :resource_type, :resource_type_pp
+          # Can be defined in module using a global name. No action required
+
+        when :plan
+          if !global?
+            # Global name must be the name of the module
+            return nil unless name_parts[0] == module_name
+
+            # Look for the special 'init' plan.
+            origin, smart_path = find_existing_path(init_plan_name)
+            return smart_path.nil? ? nil : instantiate(smart_path, typed_name, origin)
+          end
+
+        when :task
           if !global?
-            # Global name can only be the module typeset
+            # Global name must be the name of the module
             return nil unless name_parts[0] == module_name
 
+            # Look for the special 'init' Task
+            origin, smart_path = find_existing_path(init_task_name)
+            return smart_path.nil? ? nil : instantiate(smart_path, typed_name, origin)
+          end
+
+        when :type
+          if !global?
+            # Global name must be the name of the module
+            unless name_parts[0] == module_name
+              # Check for ruby defined data type in global namespace before giving up
+              origin, smart_path = find_existing_path(typed_name)
+              return smart_path.is_a?(LoaderPaths::DataTypePath) ? instantiate(smart_path, typed_name, origin) : nil
+            end
+
+            # Look for the special 'init_typeset' TypeSet
             origin, smart_path = find_existing_path(init_typeset_name)
-            return nil unless smart_path
+            return nil if smart_path.nil?
 
             value = smart_path.instantiator.create(self, typed_name, origin, get_contents(origin))
             if value.is_a?(Types::PTypeSetType)
@@ -165,11 +234,7 @@ module ModuleLoaders
       # The result is an array (that may be empty).
       # Find the file to instantiate, and instantiate the entity if file is found
       origin, smart_path = find_existing_path(typed_name)
-      if smart_path
-        value = smart_path.instantiator.create(self, typed_name, origin, get_contents(origin))
-        # cache the entry and return it
-        return set_entry(typed_name, value, origin)
-      end
+      return instantiate(smart_path, typed_name, origin) unless smart_path.nil?
 
       return nil unless typed_name.type == :type && typed_name.qualified?
 
@@ -191,6 +256,16 @@ module ModuleLoaders
       nil
     end
 
+    def instantiate(smart_path, typed_name, origin)
+      if origin.is_a?(Array)
+        value = smart_path.instantiator.create(self, typed_name, origin)
+      else
+        value = smart_path.instantiator.create(self, typed_name, origin, get_contents(origin))
+      end
+      # cache the entry and return it
+      set_entry(typed_name, value, origin)
+    end
+
     # Abstract method that subclasses override that checks if it is meaningful to search using a generic smart path.
     # This optimization is performed to not be tricked into searching an empty directory over and over again.
     # The implementation may perform a deep search for file content other than directories and cache this in
@@ -210,12 +285,22 @@ module ModuleLoaders
     # Abstract method that subclasses override to answer if the given relative path exists, and if so returns that path
     #
     # @param resolved_path [String] a path resolved by a smart path against the loader's root (if it has one)
-    # @return [Boolean] true if the file exists
+    # @return [String, nil] the found path or nil if no such path was found
     #
     def existing_path(resolved_path)
       raise NotImplementedError.new
     end
 
+    # Abstract method that subclasses override to return an array of paths that match the resolved path regardless of
+    # path extension.
+    #
+    # @param resolved_path [String] a path, without extension, resolved by a smart path against the loader's root (if it has one)
+    # @return [Array<String>]
+    #
+    def existing_paths(resolved_path)
+      raise NotImplementedError.new
+    end
+
     # Abstract method that subclasses override to produce the content of the effective path.
     # It should either succeed and return a String or fail with an exception.
     #
@@ -244,6 +329,16 @@ module ModuleLoaders
       module_name.nil? || module_name == ENVIRONMENT
     end
 
+    # Answers `true` if the loader used by this instance is rooted beneath 'lib'. This is
+    # typically true for the the system_loader. It will have a path relative to the parent
+    # of 'puppet' instead of the parent of 'lib/puppet' since the 'lib' directory of puppet
+    # is renamed during install. This is significant for loaders that load ruby code.
+    #
+    # @return [Boolean] a boolean answering if the loader is rooted beneath 'lib'.
+    def lib_root?
+      false
+    end
+
     # Produces the private loader for the module. If this module is not already resolved, this will trigger resolution
     #
     def private_loader
@@ -252,6 +347,15 @@ module ModuleLoaders
       @private_loader ||= (global? ? self : @loaders.private_loader_for_module(module_name))
     end
 
+    # Return all paths that matches the given smart path. The returned paths are
+    # relative to the `#generic_path` of the given smart path.
+    #
+    # @param smart_path [SmartPath] the path to find relative paths for
+    # @return [Array<String>] found paths
+    def relative_paths(smart_path)
+      raise NotImplementedError.new
+    end
+
     private
 
     # @return [TypedName] the fake typed name that maps to the init_typeset path for this module
@@ -259,16 +363,37 @@ module ModuleLoaders
       @init_typeset_name ||= TypedName.new(:type, "#{module_name}::init_typeset")
     end
 
-    # Find an existing path for the given `typed_name`. Return `nil` if no such path is found
+    # @return [TypedName] the fake typed name that maps to the path of an init[arbitrary extension]
+    #   file that represents a task named after the module
+    def init_task_name
+      @init_task_name ||= TypedName.new(:task, "#{module_name}::init")
+    end
+
+    # @return [TypedName] the fake typed name that maps to the path of an init.pp file that represents
+    #   a plan named after the module
+    def init_plan_name
+      @init_plan_name ||= TypedName.new(:plan, "#{module_name}::init")
+    end
+
+    # Find an existing path or paths for the given `typed_name`. Return `nil` if no path is found
     # @param typed_name [TypedName] the `typed_name` to find a path for
-    # @return [Array,nil] `nil`or a two element array an effective path `String` and the `SmartPath` that produced the effective path.
+    # @return [Array,nil] `nil`or a two element array where the first element is an effective path or array of paths
+    #   (depending on the `SmartPath`) and the second element is the `SmartPath` that produced the effective path or
+    #   paths. A path is a String
     def find_existing_path(typed_name)
       is_global = global?
       smart_paths.effective_paths(typed_name.type).each do |sp|
+        next unless sp.valid_name?(typed_name)
         origin = sp.effective_path(typed_name, is_global ? 0 : 1)
         unless origin.nil?
-          existing = existing_path(origin)
-          return [origin, sp] unless existing.nil?
+          if sp.match_many?
+            # Find all paths that starts with origin
+            origins = existing_paths(origin)
+            return [origins, sp] unless origins.empty?
+          else
+            existing = existing_path(origin)
+            return [origin, sp] unless existing.nil?
+          end
         end
       end
       nil
@@ -291,7 +416,7 @@ module ModuleLoaders
     #
     def initialize(parent_loader, loaders, module_name, path, loader_name, loadables = LOADABLE_KINDS)
       super
-      @path_index = Set.new()
+      @path_index = Set.new
     end
 
     def existing_path(effective_path)
@@ -299,16 +424,32 @@ module ModuleLoaders
       @path_index.include?(effective_path) ? effective_path : nil
     end
 
+    def existing_paths(effective_path)
+      dirname = File.dirname(effective_path)
+      basename = File.basename(effective_path)
+      # Select all paths matching `effective_path` up until an optional file extension
+      @path_index.select do |path|
+        File.basename(path, '.*') == basename &&
+          File.dirname(path) == dirname
+      end
+    end
+
     def meaningful_to_search?(smart_path)
       ! add_to_index(smart_path).empty?
     end
 
     def to_s()
-      "(ModuleLoader::FileBased '#{loader_name()}' '#{module_name()}')"
+      "(ModuleLoader::FileBased '#{loader_name}' '#{module_name}')"
     end
 
     def add_to_index(smart_path)
       found = Dir.glob(File.join(smart_path.generic_path, '**', "*#{smart_path.extension}"))
+
+      # The reason for not always rejecting directories here is performance (avoid extra stat calls). The
+      # false positives (directories with a matching extension) is an error in any case and will be caught
+      # later.
+      found = found.reject { |file_name| File.directory?(file_name) } if smart_path.extension.empty?
+
       @path_index.merge(found)
       found
     end
@@ -316,6 +457,34 @@ module ModuleLoaders
     def get_contents(effective_path)
       Puppet::FileSystem.read(effective_path, :encoding => 'utf-8')
     end
+
+    # Return all paths that matches the given smart path. The returned paths are
+    # relative to the `#generic_path` of the given smart path.
+    #
+    # This method relies on the cache and does not perform any file system access
+    #
+    # @param smart_path [SmartPath] the path to find relative paths for
+    # @return [Array<String>] found paths
+    def relative_paths(smart_path)
+      root = smart_path.generic_path
+      found = []
+      @path_index.each do |path|
+        found << Pathname(path).relative_path_from(Pathname(root)).to_s if smart_path.valid_path?(path)
+      end
+      found
+    end
+  end
+
+  # Specialization used by the system_loader which is limited to see what's beneath 'lib' and hence
+  # cannot be rooted in its parent. The 'lib' directory is renamed during install so any attempt
+  # to traverse into it from above would fail.
+  #
+  # @api private
+  #
+  class LibRootedFileBased < FileBased
+    def lib_root?
+      true
+    end
   end
 
   # Loads from a gem specified as a URI, gem://gemname/optional/path/in/gem, or just a String gemname.
@@ -346,7 +515,7 @@ module ModuleLoaders
     end
 
     def to_s()
-      "(ModuleLoader::GemBased '#{loader_name()}' '#{@gem_ref}' [#{module_name()}])"
+      "(ModuleLoader::GemBased '#{loader_name}' '#{@gem_ref}' [#{module_name}])"
     end
   end
 end

data/lib/puppet/pops/loader/puppet_plan_instantiator.rb

@@ -0,0 +1,84 @@
+module Puppet::Pops
+module Loader
+# The PuppetPlanInstantiator instantiates a Puppet::Functions::PuppetFunction given a Puppet Programming language
+# source that when called evaluates the Puppet logic it contains.
+#
+class PuppetPlanInstantiator
+  # Produces an instance of the Function class with the given typed_name, or fails with an error if the
+  # given puppet source does not produce this instance when evaluated.
+  #
+  # @param loader [Loader] The loader the function is associated with
+  # @param typed_name [TypedName] the type / name of the function to load
+  # @param source_ref [URI, String] a reference to the source / origin of the puppet code to evaluate
+  # @param pp_code_string [String] puppet code in a string
+  #
+  # @return [Functions::Function] - an instantiated function with global scope closure associated with the given loader
+  #
+  def self.create(loader, typed_name, source_ref, pp_code_string)
+    parser = Parser::EvaluatingParser.new()
+
+    # parse and validate
+    result = parser.parse_string(pp_code_string, source_ref)
+    # Only one function is allowed (and no other definitions)
+    case result.definitions.size
+    when 0
+      raise ArgumentError, _("The code loaded from %{source_ref} does not define the plan '%{plan_name}' - it is empty.") % { source_ref: source_ref, plan_name: typed_name.name }
+    when 1
+      # ok
+    else
+      raise ArgumentError, _("The code loaded from %{source_ref} must contain only the plan '%{plan_name}' - it has additional definitions.") % { source_ref: source_ref, plan_name: typed_name.name }
+    end
+    the_plan_definition = result.definitions[0]
+
+    unless the_plan_definition.is_a?(Model::PlanDefinition)
+      raise ArgumentError, _("The code loaded from %{source_ref} does not define the plan '%{plan_name}' - no plan found.") % { source_ref: source_ref, plan_name: typed_name.name }
+    end
+    unless the_plan_definition.name == typed_name.name
+      expected = typed_name.name
+      actual = the_plan_definition.name
+      raise ArgumentError, _("The code loaded from %{source_ref} produced plan with the wrong name, expected %{expected}, actual %{actual}") % { source_ref: source_ref, expected: expected, actual: actual }
+    end
+    unless result.body == the_plan_definition
+      raise ArgumentError, _("The code loaded from %{source} contains additional logic - can only contain the plan %{plan_name}") % { source: source_ref, plan_name: typed_name.name }
+    end
+
+    # Adapt the function definition with loader - this is used from logic contained in it body to find the
+    # loader to use when making calls to the new function API. Such logic have a hard time finding the closure (where
+    # the loader is known - hence this mechanism
+    private_loader = loader.private_loader
+    Adapters::LoaderAdapter.adapt(the_plan_definition).loader_name = private_loader.loader_name
+
+    # Cannot bind loaded functions to global scope, that must be done without binding that scope as
+    # loaders survive a compilation.
+    closure_scope = nil
+
+    created = create_function_class(the_plan_definition)
+    # create the function instance - it needs closure (scope), and loader (i.e. where it should start searching for things
+    # when calling functions etc.
+    # It should be bound to global scope
+
+    created.new(closure_scope, private_loader)
+  end
+
+  # Creates Function class and instantiates it based on a FunctionDefinition model
+  # @return [Array<TypedName, Functions.Function>] - array of
+  #   typed name, and an instantiated function with global scope closure associated with the given loader
+  #
+  def self.create_from_model(plan_definition, loader)
+    created = create_function_class(plan_definition)
+    typed_name = TypedName.new(:plan, plan_definition.name)
+    [typed_name, created.new(nil, loader)]
+  end
+
+  def self.create_function_class(plan_definition)
+    # Create a 4x function wrapper around a named closure
+    Puppet::Functions.create_function(plan_definition.name, Puppet::Functions::PuppetFunction) do
+      # TODO: should not create a new evaluator per function
+      init_dispatch(Evaluator::Closure::Named.new(
+        plan_definition.name,
+        Evaluator::EvaluatorImpl.new(), plan_definition))
+    end
+  end
+end
+end
+end