puppet 6.0.0 → 6.1.0

data/lib/puppet/module/task.rb

@@ -71,19 +71,22 @@ class Puppet::Module
 
       { "name" => name, "path" =>  path }
     end
+    private_class_method :get_file_details
 
-    # Find task's required lib files and retrieve paths
-    # for both 'files' and 'implementation:files' metadata keys
-    def self.find_files(files, mod)
-      env = mod.environment.respond_to?(:name) ? mod.environment.name : 'production'
+    # Find task's required lib files and retrieve paths for both 'files' and 'implementation:files' metadata keys
+    def self.find_extra_files(metadata, envname = nil)
+      return [] if metadata.nil?
 
-      file_list = files.flat_map do |file|
+      files = metadata.fetch('files', []) +
+        metadata.fetch('implementations', []).flat_map { |impl| impl.fetch('files', []) }
+
+      files.uniq.flat_map do |file|
         module_name, mount, endpath = file.split("/", 3)
         # If there's a mount directory with no trailing slash this will be nil
         # We want it to be empty to construct a path
         endpath ||= ''
 
-        pup_module = Puppet::Module.find(module_name, env)
+        pup_module = Puppet::Module.find(module_name, envname)
         if pup_module.nil?
           msg = _("Could not find module %{module_name} containing task file %{filename}" %
                   {module_name: module_name, filename: endpath})
@@ -114,22 +117,18 @@ class Puppet::Module
             raise InvalidMetadata.new(msg, 'puppet.tasks/invalid-metadata')
           end
           dir_files = Dir.glob("#{path}**/*").select { |f| File.file?(f) }
-          files = dir_files.map { |f| get_file_details(f, pup_module) }
+          dir_files.map { |f| get_file_details(f, pup_module) }
         else
           if last_char
             msg = _("Files specified in task metadata cannot include a trailing slash: %{file}" % { file: file } )
             raise InvalidMetadata.new(msg, 'puppet.task/invalid-metadata')
           end
-          files = get_file_details(path, pup_module)
+          get_file_details(path, pup_module)
         end
-
-        files
       end
-      return file_list
     end
+    private_class_method :find_extra_files
 
-    # Copied from TaskInstantiator so we can use the Error classes here
-    # TODO: harmonize on one implementation
     # Executables list should contain the full path of all possible implementation files
     def self.find_implementations(name, directory, metadata, executables)
       basename = name.split('::')[1] || 'init'
@@ -148,7 +147,7 @@ class Puppet::Module
             msg = _("Task metadata for task %{name} specifies missing implementation %{implementation}" % { name: name, implementation: impl['name'] })
             raise InvalidTask.new(msg, 'puppet.tasks/missing-implementation', { missing: [impl['name']] } )
           end
-          { "name" => impl['name'], "requirements" => impl.fetch('requirements', []), "path" => path }
+          { "name" => impl['name'], "path" => path }
         end
         return implementations
       end
@@ -166,7 +165,13 @@ class Puppet::Module
         raise InvalidTask.new(msg, 'puppet.tasks/multiple-implementations')
       end
 
-      [{ "name" => File.basename(implementations.first), "path" => implementations.first, "requirements" => [] }]
+      [{ "name" => File.basename(implementations.first), "path" => implementations.first }]
+    end
+    private_class_method :find_implementations
+
+    def self.find_files(name, directory, metadata, executables, envname = nil)
+      # PXP agent relies on 'impls' (which is the task file) being first if there is no metadata
+      find_implementations(name, directory, metadata, executables) + find_extra_files(metadata, envname)
     end
 
     def self.is_tasks_metadata_filename?(name)
@@ -206,7 +211,7 @@ class Puppet::Module
       @module_executables = module_executables || []
     end
 
-    def read_metadata(file)
+    def self.read_metadata(file)
       Puppet::Util::Json.load(Puppet::FileSystem.read(file, :encoding => 'utf-8')) if file
     rescue SystemCallError, IOError => err
       msg = _("Error reading metadata: %{message}" % {message: err.message})
@@ -216,34 +221,15 @@ class Puppet::Module
     end
 
     def metadata
-      @metadata ||= read_metadata(@metadata_file)
-    end
-
-    def implementations
-      @implementations ||= self.class.find_implementations(@name, @module.tasks_directory, metadata, @module_executables)
+      @metadata ||= self.class.read_metadata(@metadata_file)
     end
 
     def files
-      md = metadata
-      outer_files = []
-      impl_lib_files = []
-      lib_files = []
-
-      unless md.nil?
-        outer_files = md['files'] if md.key?('files')
-        # There's definitely a more elegant way to do this...
-        if md.key?('implementations')
-          md['implementations'].each { |impl| impl_lib_files << impl['files'] if impl.key?('files') }
-        end
-        lib_files = self.class.find_files((impl_lib_files.flatten.uniq + outer_files).uniq, @module)
-      end
-      task_file = implementations.map {|imp| { 'name' => imp['name'], 'path' => imp['path'] } }
-      # PXP agent relies on 'impls' (which is the task file) being first if there is no metadata
-      task_file + lib_files
+      @files ||= self.class.find_files(@name, @module.tasks_directory, metadata, @module_executables, environment_name)
     end
 
     def validate
-      implementations
+      files
       true
     end
 
@@ -252,6 +238,11 @@ class Puppet::Module
       self.module == other.module
     end
 
+    def environment_name
+      @module.environment.respond_to?(:name) ? @module.environment.name : 'production'
+    end
+    private :environment_name
+
     def self.new_with_files(pup_module, name, task_files, module_executables)
       metadata_file = task_files.find { |f| is_tasks_metadata_filename?(f) }
       Puppet::Module::Task.new(pup_module, name, module_executables, metadata_file)

data/lib/puppet/pal/catalog_compiler.rb

@@ -0,0 +1,96 @@
+module Puppet
+module Pal
+  # A CatalogCompiler is a compiler that builds a catalog of resources and dependencies as a side effect of
+  # evaluating puppet language code.
+  # When the compilation of the given input manifest(s)/code string/file is finished the catalog is complete
+  # for encoding and use. It is also possible to evaluate more strings within the same compilation context to
+  # add or remove things from the catalog.
+  #
+  # @api public
+  class CatalogCompiler < Compiler
+
+    # @api private
+    def catalog
+      internal_compiler.catalog
+    end
+    private :catalog
+
+    # Returns true if this is a compiler that compiles a catalog.
+    # This implementation returns `true`
+    # @return [Boolean] true
+    # @api public
+    def has_catalog?
+      true
+    end
+
+    # Calls a block of code and yields a configured `JsonCatalogEncoder` to the block.
+    # @example Get resulting catalog as pretty printed Json
+    #   Puppet::Pal.in_environment(...) do |pal|
+    #     pal.with_catalog_compiler(...) do |compiler|
+    #       compiler.with_json_encoding {| encoder | encoder.encode
+    #     end
+    #   end
+    #
+    # @api public
+    #
+    def with_json_encoding(pretty: true, exclude_virtual: true)
+      yield JsonCatalogEncoder.new(catalog, pretty: pretty, exclude_virtual: exclude_virtual)
+    end
+
+    # Evaluates an AST obtained from `parse_string` or `parse_file` in topscope.
+    # If the ast is a `Puppet::Pops::Model::Program` (what is returned from the `parse` methods, any definitions
+    # in the program (that is, any function, plan, etc. that is defined will be made available for use).
+    #
+    # @param ast [Puppet::Pops::Model::PopsObject] typically the returned `Program` from the parse methods, but can be any `Expression`
+    # @returns [Object] whatever the ast evaluates to
+    #
+    def evaluate(ast)
+      if ast.is_a?(Puppet::Pops::Model::Program)
+        bridged = Puppet::Parser::AST::PopsBridge::Program.new(ast)
+        # define all catalog types
+        internal_compiler.environment.known_resource_types.import_ast(bridged, "")
+        bridged.evaluate(internal_compiler.topscope)
+      else
+        internal_evaluator.evaluate(topscope, ast)
+      end
+    end
+
+
+    # Compiles the result of additional evaluation taking place in a PAL catalog compilation.
+    # This will evaluate all lazy constructs until all have been evaluated, and will the validate
+    # the result.
+    #
+    # This should be called if evaluating string or files of puppet logic after the initial
+    # compilation taking place by giving PAL a manifest or code-string.
+    # This method should be called when a series of evaluation should have reached a
+    # valid state (there should be no dangling relationships (to resources that does not
+    # exist).
+    #
+    # As an alternative the methods `evaluate_additions` can be called without any
+    # requirements on consistency and then calling `validate` at the end.
+    #
+    # Can be called multiple times.
+    #
+    # @return [Void]
+    def compile_additions
+      internal_compiler.compile_additions
+    end
+
+    # Validates the state of the catalog (without performing evaluation of any elements
+    # requiring lazy evaluation. Can be called multiple times.
+    #
+    def validate
+      internal_compiler.validate
+    end
+
+    # Evaluates all lazy constructs that were produced as a side effect of evaluating puppet logic.
+    # Can be called multiple times.
+    #
+    def evaluate_additions
+      internal_compiler.evaluate_additions
+    end
+
+  end
+
+end
+end

data/lib/puppet/pal/compiler.rb

@@ -0,0 +1,221 @@
+module Puppet
+module Pal
+
+  # A configured compiler as obtained in the callback from `Puppet::Pal.with_script_compiler`.
+  # (Later, there may also be a catalog compiler available.)
+  #
+  class Compiler
+    attr_reader :internal_compiler
+    protected :internal_compiler
+
+    attr_reader :internal_evaluator
+    protected :internal_evaluator
+
+    def initialize(internal_compiler)
+      @internal_compiler = internal_compiler
+      @internal_evaluator = Puppet::Pops::Parser::EvaluatingParser.new
+    end
+
+    # Calls a function given by name with arguments specified in an `Array`, and optionally accepts a code block.
+    # @param function_name [String] the name of the function to call
+    # @param args [Object] the arguments to the function
+    # @param block [Proc] an optional callable block that is given to the called function
+    # @return [Object] what the called function returns
+    #
+    def call_function(function_name, *args, &block)
+      # TRANSLATORS: do not translate variable name strings in these assertions
+      Pal::assert_non_empty_string(function_name, 'function_name', false)
+      Pal::assert_type(Pal::T_ANY_ARRAY, args, 'args', false)
+      internal_evaluator.evaluator.external_call_function(function_name, args, topscope, &block)
+    end
+
+    # Returns a Puppet::Pal::FunctionSignature object or nil if function is not found
+    # The returned FunctionSignature has information about all overloaded signatures of the function
+    #
+    # @example using function_signature
+    #   # returns true if 'myfunc' is callable with three integer arguments 1, 2, 3
+    #   compiler.function_signature('myfunc').callable_with?([1,2,3])
+    #
+    # @param function_name [String] the name of the function to get a signature for
+    # @return [Puppet::Pal::FunctionSignature] a function signature, or nil if function not found
+    #
+    def function_signature(function_name)
+      loader = internal_compiler.loaders.private_environment_loader
+      if func = loader.load(:function, function_name)
+        return FunctionSignature.new(func.class)
+      end
+      # Could not find function
+      nil
+    end
+
+    # Returns an array of TypedName objects for all functions, optionally filtered by a regular expression.
+    # The returned array has more information than just the leaf name - the typical thing is to just get
+    # the name as showing the following example.
+    #
+    # Errors that occur during function discovery will either be logged as warnings or collected by the optional
+    # `error_collector` array. When provided, it will receive {Puppet::DataTypes::Error} instances describing
+    # each error in detail and no warnings will be logged.
+    #
+    # @example getting the names of all functions
+    #   compiler.list_functions.map {|tn| tn.name }
+    #
+    # @param filter_regex [Regexp] an optional regexp that filters based on name (matching names are included in the result)
+    # @param error_collector [Array<Puppet::DataTypes::Error>] an optional array that will receive errors during load
+    # @return [Array<Puppet::Pops::Loader::TypedName>] an array of typed names
+    #
+    def list_functions(filter_regex = nil, error_collector = nil)
+      list_loadable_kind(:function, filter_regex, error_collector)
+    end
+
+    # Evaluates a string of puppet language code in top scope.
+    # A "source_file" reference to a source can be given - if not an actual file name, by convention the name should
+    # be bracketed with < > to indicate it is something symbolic; for example `<commandline>` if the string was given on the
+    # command line.
+    #
+    # If the given `puppet_code` is `nil` or an empty string, `nil` is returned, otherwise the result of evaluating the
+    # puppet language string. The given string must form a complete and valid expression/statement as an error is raised
+    # otherwise. That is, it is not possible to divide a compound expression by line and evaluate each line individually.
+    #
+    # @param puppet_code [String, nil] the puppet language code to evaluate, must be a complete expression/statement
+    # @param source_file [String, nil] an optional reference to a source (a file or symbolic name/location)
+    # @return [Object] what the `puppet_code` evaluates to
+    #
+    def evaluate_string(puppet_code, source_file = nil)
+      return nil if puppet_code.nil? || puppet_code == ''
+      unless puppet_code.is_a?(String)
+        raise ArgumentError, _("The argument 'puppet_code' must be a String, got %{type}") % { type: puppet_code.class }
+      end
+      evaluate(parse_string(puppet_code, source_file))
+    end
+
+    # Evaluates a puppet language file in top scope.
+    # The file must exist and contain valid puppet language code or an error is raised.
+    #
+    # @param file [Path, String] an absolute path to a file with puppet language code, must exist
+    # @return [Object] what the last evaluated expression in the file evaluated to
+    #
+    def evaluate_file(file)
+      evaluate(parse_file(file))
+    end
+
+    # Evaluates an AST obtained from `parse_string` or `parse_file` in topscope.
+    # If the ast is a `Puppet::Pops::Model::Program` (what is returned from the `parse` methods, any definitions
+    # in the program (that is, any function, plan, etc. that is defined will be made available for use).
+    #
+    # @param ast [Puppet::Pops::Model::PopsObject] typically the returned `Program` from the parse methods, but can be any `Expression`
+    # @returns [Object] whatever the ast evaluates to
+    #
+    def evaluate(ast)
+      if ast.is_a?(Puppet::Pops::Model::Program)
+        loaders = Puppet.lookup(:loaders)
+        loaders.instantiate_definitions(ast, loaders.public_environment_loader)
+      end
+      internal_evaluator.evaluate(topscope, ast)
+    end
+
+    # Produces a literal value if the AST obtained from `parse_string` or `parse_file` does not require any actual evaluation.
+    # This method is useful if obtaining an AST that represents literal values; string, integer, float, boolean, regexp, array, hash;
+    # for example from having read this from the command line or as values in some file.
+    #
+    # @param ast [Puppet::Pops::Model::PopsObject] typically the returned `Program` from the parse methods, but can be any `Expression`
+    # @returns [Object] whatever the literal value the ast evaluates to
+    #
+    def evaluate_literal(ast)
+      catch :not_literal do
+        return Puppet::Pops::Evaluator::LiteralEvaluator.new().literal(ast)
+      end
+      # TRANSLATORS, the 'ast' is the name of a parameter, do not translate
+      raise ArgumentError, _("The given 'ast' does not represent a literal value")
+    end
+
+    # Parses and validates a puppet language string and returns an instance of Puppet::Pops::Model::Program on success.
+    # If the content is not valid an error is raised.
+    #
+    # @param code_string [String] a puppet language string to parse and validate
+    # @param source_file [String] an optional reference to a file or other location in angled brackets
+    # @return [Puppet::Pops::Model::Program] returns a `Program` instance on success
+    #
+    def parse_string(code_string, source_file = nil)
+      unless code_string.is_a?(String)
+        raise ArgumentError, _("The argument 'code_string' must be a String, got %{type}") % { type: code_string.class }
+      end
+      internal_evaluator.parse_string(code_string, source_file)
+    end
+
+    # Parses and validates a puppet language file and returns an instance of Puppet::Pops::Model::Program on success.
+    # If the content is not valid an error is raised.
+    #
+    # @param file [String] a file with puppet language content to parse and validate
+    # @return [Puppet::Pops::Model::Program] returns a `Program` instance on success
+    #
+    def parse_file(file)
+      unless file.is_a?(String)
+        raise ArgumentError, _("The argument 'file' must be a String, got %{type}") % { type: file.class }
+      end
+      internal_evaluator.parse_file(file)
+    end
+
+    # Parses a puppet data type given in String format and returns that type, or raises an error.
+    # A type is needed in calls to `new` to create an instance of the data type, or to perform type checking
+    # of values - typically using `type.instance?(obj)` to check if `obj` is an instance of the type.
+    #
+    # @example Verify if obj is an instance of a data type
+    #   # evaluates to true
+    #   pal.type('Enum[red, blue]').instance?("blue")
+    #
+    # @example Create an instance of a data type
+    #   # using an already create type
+    #   t = pal.type('Car')
+    #   pal.create(t, 'color' => 'black', 'make' => 't-ford')
+    #
+    #   # letting 'new_object' parse the type from a string
+    #   pal.create('Car', 'color' => 'black', 'make' => 't-ford')
+    #
+    # @param type_string [String] a puppet language data type
+    # @return [Puppet::Pops::Types::PAnyType] the data type
+    #
+    def type(type_string)
+      Puppet::Pops::Types::TypeParser.singleton.parse(type_string)
+    end
+
+    # Creates a new instance of a given data type.
+    # @param data_type [String, Puppet::Pops::Types::PAnyType] the data type as a data type or in String form.
+    # @param arguments [Object] one or more arguments to the called `new` function
+    # @return [Object] an instance of the given data type,
+    #   or raises an error if it was not possible to parse data type or create an instance.
+    #
+    def create(data_type, *arguments)
+      t = data_type.is_a?(String) ? type(data_type) : data_type
+      unless t.is_a?(Puppet::Pops::Types::PAnyType)
+        raise ArgumentError, _("Given data_type value is not a data type, got '%{type}'") % {type: t.class}
+      end
+      call_function('new', t, *arguments)
+    end
+
+    # Returns true if this is a compiler that compiles a catalog.
+    # This implementation returns `false`
+    # @return Boolan false
+    def has_catalog?
+      false
+    end
+
+    protected
+
+    def list_loadable_kind(kind, filter_regex = nil, error_collector = nil)
+      loader = internal_compiler.loaders.private_environment_loader
+      if filter_regex.nil?
+        loader.discover(kind, error_collector)
+      else
+        loader.discover(kind, error_collector) {|f| f.name =~ filter_regex }
+      end
+    end
+
+    private
+
+    def topscope
+      internal_compiler.topscope
+    end
+  end
+
+end
+end

data/lib/puppet/pal/function_signature.rb

@@ -0,0 +1,52 @@
+module Puppet
+module Pal
+  # A FunctionSignature is returned from `function_signature`. Its purpose is to answer questions about the function's parameters
+  # and if it can be called with a set of parameters.
+  #
+  # It is also possible to get an array of puppet Callable data type where each callable describes one possible way
+  # the function can be called.
+  #
+  # @api public
+  #
+  class FunctionSignature
+    # @api private
+    def initialize(function_class)
+      @func = function_class
+    end
+
+    # Returns true if the function can be called with the given arguments and false otherwise.
+    # If the function is not callable, and a code block is given, it is given a formatted error message that describes
+    # the type mismatch. That error message can be quite complex if the function has multiple dispatch depending on
+    # given types.
+    #
+    # @param args [Array] The arguments as given to the function call
+    # @param callable [Proc, nil] An optional ruby Proc or puppet lambda given to the function
+    # @yield [String] a formatted error message describing a type mismatch if the function is not callable with given args + block
+    # @return [Boolean] true if the function can be called with given args + block, and false otherwise
+    # @api public
+    #
+    def callable_with?(args, callable=nil)
+      signatures = @func.dispatcher.to_type
+      callables = signatures.is_a?(Puppet::Pops::Types::PVariantType) ? signatures.types : [signatures]
+
+      return true if callables.any? {|t| t.callable_with?(args) }
+      return false unless block_given?
+      args_type = Puppet::Pops::Types::TypeCalculator.singleton.infer_set(callable.nil? ? args : args + [callable])
+      error_message = Puppet::Pops::Types::TypeMismatchDescriber.describe_signatures(@func.name, @func.signatures, args_type)
+      yield error_message
+      false
+    end
+
+    # Returns an array of Callable puppet data type
+    # @return [Array<Puppet::Pops::Types::PCallableType] one callable per way the function can be called
+    #
+    # @api public
+    #
+    def callables
+      signatures = @func.dispatcher.to_type
+      signatures.is_a?(Puppet::Pops::Types::PVariantType) ? signatures.types : [signatures]
+    end
+  end
+
+end
+end

data/lib/puppet/pal/json_catalog_encoder.rb

@@ -0,0 +1,67 @@
+# The JsonCatalogEncoder is a wrapper around a catalog produced by the Pal::CatalogCompiler.with_json_encoding
+# method.
+# It allows encoding the entire catalog or an individual resource as Rich Data Json.
+#
+# @api public
+#
+class JsonCatalogEncoder
+  # Is the resulting Json pretty printed or not.
+  attr_reader :pretty
+
+  # Should unrealized virtual resources be included in the result or not.
+  attr_reader :exclude_virtual
+
+  # The internal catalog being build - what this class wraps with a public API.
+  attr_reader :catalog
+  private :catalog
+
+  # Do not instantiate this class directly! Use the `Pal::CatalogCompiler#with_json_encoding` method
+  # instead.
+  #
+  # @param catalog [Puppet::Resource::Catalog] the internal catalog that this class wraps
+  # @param pretty [Boolean] (true), if the resulting JSON should be pretty printed or not
+  # @param exclude_virtual [Boolean] (true), if the resulting catalog should contain unrealzed virtual resources or not
+  #
+  # @api private
+  #
+  def initialize(catalog, pretty: true, exclude_virtual: true)
+    @catalog = catalog
+    @pretty = pretty
+    @exclude_virtual = exclude_virtual
+  end
+
+  # Encodes the entire catalog as a rich-data Json catalog.
+  # @return String The catalog in Json format using rich data format
+  # @api public
+  #
+  def encode
+    possibly_filtered_catalog.to_json(:pretty => pretty)
+  end
+
+  # Returns one particular resource as a Json string, or returns nil if resource was not found.
+  # @param type [String] the name of the puppet type (case independent)
+  # @param title [String] the title of the wanted resource
+  # @return [String] the resulting Json text
+  # @api public
+  #
+  def encode_resource(type, title)
+    # Ensure that both type and title are given since the underlying API will do mysterious things
+    # if 'title' is nil. (Other assertions are made by the catalog when looking up the resource).
+    #
+    # TRANSLATORS 'type' and 'title' are internal parameter names - do not translate
+    raise ArgumentError, _("Both type and title must be given") if type.nil? or title.nil?
+    r = possibly_filtered_catalog.resource(type, title)
+    return nil if r.nil?
+    r.to_data_hash.to_json(:pretty => pretty)
+  end
+
+  # Applies a filter for virtual resources and returns filtered catalog
+  # or the catalog itself if filtering was not needed.
+  # The result is cached.
+  # @api private
+  #
+  def possibly_filtered_catalog
+    @filtered ||= (exclude_virtual ? catalog.filter { |r| r.virtual? } : catalog)
+  end
+  private :possibly_filtered_catalog
+end

data/lib/puppet/pal/pal_api.rb

@@ -0,0 +1,15 @@
+module Puppet
+  require 'puppet/parser/script_compiler'
+  require 'puppet/parser/catalog_compiler'
+
+  module Pal
+    require 'puppet/pal/json_catalog_encoder'
+    require 'puppet/pal/function_signature'
+    require 'puppet/pal/task_signature'
+    require 'puppet/pal/plan_signature'
+    require 'puppet/pal/compiler'
+    require 'puppet/pal/script_compiler'
+    require 'puppet/pal/catalog_compiler'
+    require 'puppet/pal/pal_impl'
+  end
+end