puppet 6.17.0 → 6.21.0

data/lib/puppet/parser/ast/leaf.rb

@@ -50,8 +50,9 @@ class Puppet::Parser::AST::HostName < Puppet::Parser::AST::Leaf
 end
 
 class Puppet::Parser::AST::Regex < Puppet::Parser::AST::Leaf
-  def initialize(hash)
-    super(**hash)
+  def initialize(value: nil, file: nil, line: nil, pos: nil)
+    super(value: value, file: file, line: line, pos: pos)
+
     # transform value from hash options unless it is already a regular expression
     @value = Regexp.new(@value) unless @value.is_a?(Regexp)
   end

data/lib/puppet/parser/functions.rb

@@ -77,6 +77,26 @@ module Puppet::Parser::Functions
   # @api private
   class AnonymousModuleAdapter < Puppet::Pops::Adaptable::Adapter
     attr_accessor :module
+
+    def self.create_adapter(env)
+      adapter = super(env)
+      adapter.module = Module.new do
+        @metadata = {}
+
+        def self.all_function_info
+          @metadata
+        end
+
+        def self.get_function_info(name)
+          @metadata[name]
+        end
+
+        def self.add_function_info(name, info)
+          @metadata[name] = info
+        end
+      end
+      adapter
+    end
   end
 
   @environment_module_lock = Puppet::Concurrent::Lock.new
@@ -87,23 +107,7 @@ module Puppet::Parser::Functions
   # @api private
   def self.environment_module(env)
     @environment_module_lock.synchronize do
-      AnonymousModuleAdapter.adapt(env) do |a|
-        a.module ||= Module.new do
-          @metadata = {}
-
-          def self.all_function_info
-            @metadata
-          end
-
-          def self.get_function_info(name)
-            @metadata[name]
-          end
-
-          def self.add_function_info(name, info)
-            @metadata[name] = info
-          end
-        end
-      end.module
+      AnonymousModuleAdapter.adapt(env).module
     end
   end
 

data/lib/puppet/parser/functions/create_resources.rb

@@ -1,5 +1,9 @@
 Puppet::Parser::Functions::newfunction(:create_resources, :arity => -3, :doc => <<-'ENDHEREDOC') do |args|
     Converts a hash into a set of resources and adds them to the catalog.
+    
+    **Note**: Use this function selectively. It's generally better to write resources in
+     [Puppet](https://puppet.com/docs/puppet/latest/lang_resources.html), as
+     resources created with `create_resource` are difficult to read and troubleshoot.
 
     This function takes two mandatory arguments: a resource type, and a hash describing
     a set of resources. The hash should be in the form `{title => {parameters} }`:
@@ -33,7 +37,7 @@ Puppet::Parser::Functions::newfunction(:create_resources, :arity => -3, :doc =>
     as native resources.
 
     Virtual and Exported resources may be created by prefixing the type name
-    with @ or @@ respectively.  For example, the $myusers hash may be exported
+    with @ or @@ respectively. For example, the $myusers hash may be exported
     in the following manner:
 
         create_resources("@@user", $myusers)
@@ -42,9 +46,9 @@ Puppet::Parser::Functions::newfunction(:create_resources, :arity => -3, :doc =>
 
         create_resources("@user", $myusers)
 
-    Note that `create_resources` will filter out parameter values that are `undef` so that normal
-    data binding and puppet default value expressions are considered (in that order) for the
-    final value of a parameter (just as when setting a parameter to `undef` in a puppet language
+    Note that `create_resources` filters out parameter values that are `undef` so that normal
+    data binding and Puppet default value expressions are considered (in that order) for the
+    final value of a parameter (just as when setting a parameter to `undef` in a Puppet language
     resource declaration).
   ENDHEREDOC
   if Puppet[:tasks]
@@ -63,9 +67,9 @@ Puppet::Parser::Functions::newfunction(:create_resources, :arity => -3, :doc =>
   defaults ||= {}
   type_name = type.sub(/^@{1,2}/, '').downcase
 
-  # Get file/line information from the puppet stack (where call comes from in puppet source)
-  # If relayed via other puppet functions in ruby that do not nest their calls, the source position
-  # will be in the original puppet source.
+  # Get file/line information from the Puppet stack (where call comes from in Puppet source)
+  # If relayed via other Puppet functions in ruby that do not nest their calls, the source position
+  # will be in the original Puppet source.
   #
   file, line = Puppet::Pops::PuppetStack.top_of_stack
 

data/lib/puppet/parser/templatewrapper.rb

@@ -50,7 +50,7 @@ class Puppet::Parser::TemplateWrapper
   # @return [Array<String>] The tags defined in the current scope
   # @api public
   def tags
-    scope.tags
+    raise NotImplementedError, "Call 'all_tags' instead."
   end
 
   # @return [Array<String>] All the defined tags

data/lib/puppet/parser/type_loader.rb

@@ -81,7 +81,7 @@ class Puppet::Parser::TypeLoader
   end
 
   def parse_file(file)
-    Puppet.debug("importing '#{file}' in environment #{environment}")
+    Puppet.debug { "importing '#{file}' in environment #{environment}" }
     parser = Puppet::Parser::ParserFactory.parser
     parser.file = file
     return parser.parse
@@ -118,7 +118,7 @@ class Puppet::Parser::TypeLoader
         # still be parsed. Mark this file as loaded so that
         # it would not be parsed next time (handle it as if
         # it was successfully parsed).
-        Puppet.debug("Unable to parse '#{file}': #{e.message}")
+        Puppet.debug { "Unable to parse '#{file}': #{e.message}" }
       end
     else
       loaded_asts << parse_file(file)

data/lib/puppet/pops/adaptable.rb

@@ -69,11 +69,7 @@ module Adaptable
     #
     def self.get(o)
       attr_name = self_attr_name
-      if o.instance_variable_defined?(attr_name)
-        o.instance_variable_get(attr_name)
-      else
-        nil
-      end
+      o.instance_variable_get(attr_name)
     end
 
     # Returns an existing adapter for the given object, or creates a new adapter if the
@@ -94,14 +90,13 @@ module Adaptable
     #
     def self.adapt(o, &block)
       attr_name = self_attr_name
-      value = o.instance_variable_get(attr_name) if o.instance_variable_defined?(attr_name)
+      value = o.instance_variable_get(attr_name)
       adapter = value || associate_adapter(create_adapter(o), o)
       if block_given?
-        case block.arity
-          when 1
-            block.call(adapter)
-          else
-            block.call(adapter, o)
+        if block.arity == 1
+          block.call(adapter)
+        else
+          block.call(adapter, o)
         end
       end
       adapter
@@ -127,8 +122,7 @@ module Adaptable
     def self.adapt_new(o, &block)
       adapter = associate_adapter(create_adapter(o), o)
       if block_given?
-        case block.arity
-        when 1
+        if block.arity == 1
           block.call(adapter)
         else
           block.call(adapter, o)

data/lib/puppet/pops/adapters.rb

@@ -99,6 +99,13 @@ module Adapters
 
     class PathsAndNameCacheAdapter < Puppet::Pops::Adaptable::Adapter
       attr_accessor :cache, :paths
+
+      def self.create_adapter(env)
+        adapter = super(env)
+        adapter.paths = env.modulepath.map { |p| Pathname.new(p) }
+        adapter.cache = {}
+        adapter
+      end
     end
 
     # Attempts to find the module that `instance` originates from by looking at it's {SourcePosAdapter} and
@@ -117,10 +124,7 @@ module Adapters
     def self.loader_name_by_source(environment, instance, file)
       file = instance.file if file.nil?
       return nil if file.nil? || EMPTY_STRING == file
-      pn_adapter = PathsAndNameCacheAdapter.adapt(environment) do |a|
-        a.paths ||= environment.modulepath.map { |p| Pathname.new(p) }
-        a.cache ||= {}
-      end
+      pn_adapter = PathsAndNameCacheAdapter.adapt(environment)
       dir = File.dirname(file)
       pn_adapter.cache.fetch(dir) do |key|
         mod = find_module_for_dir(environment, pn_adapter.paths, dir)

data/lib/puppet/pops/evaluator/collectors/abstract_collector.rb

@@ -45,9 +45,7 @@ class Puppet::Pops::Evaluator::Collectors::AbstractCollector
     return false if objects.empty?
 
     if @overrides and !objects.empty?
-      overrides[:source].meta_def(:child_of?) do |klass|
-        true
-      end
+      overrides[:source].override = true
 
       objects.each do |res|
         unless @collected.include?(res.ref)

data/lib/puppet/pops/evaluator/deferred_resolver.rb

@@ -16,10 +16,12 @@ class DeferredResolver
   #
   # @param facts [Puppet::Node::Facts] the facts object for the node
   # @param catalog [Puppet::Resource::Catalog] the catalog where all deferred values should be replaced
+  # @param environment [Puppet::Node::Environment] the environment whose anonymous module methods
+  #  are to be mixed into the scope
   # @return [nil] does not return anything - the catalog is modified as a side effect
   #
-  def self.resolve_and_replace(facts, catalog)
-    compiler = Puppet::Parser::ScriptCompiler.new(catalog.environment_instance, catalog.name, true)
+  def self.resolve_and_replace(facts, catalog, environment = catalog.environment_instance)
+    compiler = Puppet::Parser::ScriptCompiler.new(environment, catalog.name, true)
     resolver = new(compiler)
     resolver.set_facts_variable(facts)
     # TODO:
@@ -108,7 +110,7 @@ class DeferredResolver
     # If any of the arguments to a future is a future it needs to be resolved first
     func_name = f.name
     mapped_arguments = map_arguments(f.arguments)
-    # if name starts with $ then this is a call to dig 
+    # if name starts with $ then this is a call to dig
     if func_name[0] == DOLLAR
       var_name = func_name[1..-1]
       func_name = DIG

data/lib/puppet/pops/evaluator/evaluator_impl.rb

@@ -462,10 +462,24 @@ class EvaluatorImpl
   end
 
   def eval_EppExpression(o, scope)
+    contains_sensitive = false
+
     scope["@epp"] = []
     evaluate(o.body, scope)
-    result = scope["@epp"].join
-    result
+    result = scope["@epp"].map do |r|
+      if r.instance_of?(Puppet::Pops::Types::PSensitiveType::Sensitive)
+        contains_sensitive = true
+        string(r.unwrap, scope)
+      else
+        r
+      end
+    end.join
+
+    if contains_sensitive
+      Puppet::Pops::Types::PSensitiveType::Sensitive.new(result)
+    else
+      result
+    end
   end
 
   def eval_RenderStringExpression(o, scope)
@@ -474,7 +488,12 @@ class EvaluatorImpl
   end
 
   def eval_RenderExpression(o, scope)
-    scope["@epp"] << string(evaluate(o.expr, scope), scope)
+    result = evaluate(o.expr, scope)
+    if result.instance_of?(Puppet::Pops::Types::PSensitiveType::Sensitive)
+      scope["@epp"] << result
+    else
+      scope["@epp"] << string(result, scope)
+    end
     nil
   end
 

data/lib/puppet/pops/evaluator/runtime3_converter.rb

@@ -79,9 +79,9 @@ class Runtime3Converter
 
   def convert_String(o, scope, undef_value)
     # Although wasteful, a dup is needed because user code may mutate these strings when applying
-    # Resources. This does not happen when in master mode since it only uses Resources that are
+    # Resources. This does not happen when in server mode since it only uses Resources that are
     # in puppet core and those are all safe.
-    o.frozen? && !Puppet.run_mode.master? ? o.dup : o
+    o.frozen? && !Puppet.run_mode.server? ? o.dup : o
   end
 
   def convert_Object(o, scope, undef_value)

data/lib/puppet/pops/loader/runtime3_type_loader.rb

@@ -7,10 +7,12 @@ module Loader
 #
 # @api private
 class Runtime3TypeLoader < BaseLoader
-  def initialize(parent_loader, loaders, environment, env_path)
+  attr_reader :resource_3x_loader
+
+  def initialize(parent_loader, loaders, environment, resource_3x_loader)
     super(parent_loader, environment.name)
     @environment = environment
-    @resource_3x_loader = env_path.nil? ? nil : ModuleLoaders.pcore_resource_type_loader_from(parent_loader, loaders, env_path)
+    @resource_3x_loader = resource_3x_loader
   end
 
   def discover(type, error_collector = nil, name_authority = Pcore::RUNTIME_NAME_AUTHORITY, &block)

data/lib/puppet/pops/loaders.rb

@@ -18,18 +18,18 @@ class Loaders
   attr_reader :private_environment_loader
   attr_reader :environment
 
-  def self.new(environment, for_agent = false)
+  def self.new(environment, for_agent = false, load_from_pcore = true)
     environment.lock.synchronize do
       obj = environment.loaders
       if obj.nil?
         obj = self.allocate
-        obj.send(:initialize, environment, for_agent)
+        obj.send(:initialize, environment, for_agent, load_from_pcore)
       end
       obj
     end
   end
 
-  def initialize(environment, for_agent)
+  def initialize(environment, for_agent, load_from_pcore = true)
     # Protect against environment havoc
     raise ArgumentError.new(_("Attempt to redefine already initialized loaders for environment")) unless environment.loaders.nil?
     environment.loaders = self
@@ -51,12 +51,13 @@ class Loaders
     #    TODO: loaders need to work when also running in an agent doing catalog application. There is no
     #    concept of environment the same way as when running as a master (except when doing apply).
     #    The creation mechanisms should probably differ between the two.
-    @private_environment_loader = if for_agent
-      @puppet_cache_loader = create_puppet_cache_loader()
-      create_environment_loader(environment, @puppet_cache_loader)
-    else
-      create_environment_loader(environment, @puppet_system_loader)
-    end
+    @private_environment_loader =
+      if for_agent
+        @puppet_cache_loader = create_puppet_cache_loader
+        create_environment_loader(environment, @puppet_cache_loader, load_from_pcore)
+      else
+        create_environment_loader(environment, @puppet_system_loader, load_from_pcore)
+      end
 
     Pcore.init_env(@private_environment_loader)
 
@@ -362,7 +363,7 @@ class Loaders
     Loader::ModuleLoaders.cached_loader_from(puppet_system_loader, self)
   end
 
-  def create_environment_loader(environment, parent_loader)
+  def create_environment_loader(environment, parent_loader, load_from_pcore = true)
     # This defines where to start parsing/evaluating - the "initial import" (to use 3x terminology)
     # Is either a reference to a single .pp file, or a directory of manifests. If the environment becomes
     # a module and can hold functions, types etc. then these are available across all other modules without
@@ -387,7 +388,13 @@ class Loaders
     else
       # Create the 3.x resource type loader
       static_loader.runtime_3_init
-      @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(parent_loader, self, environment, env_conf.nil? ? nil : env_path))
+      # Create pcore resource type loader, if applicable
+      pcore_resource_type_loader = if load_from_pcore && env_path
+                                     Loader::ModuleLoaders.pcore_resource_type_loader_from(parent_loader, self, env_path)
+                                   else
+                                     nil
+                                   end
+      @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(parent_loader, self, environment, pcore_resource_type_loader))
 
       if env_path.nil?
         # Not a real directory environment, cannot work as a module TODO: Drop when legacy env are dropped?

data/lib/puppet/pops/lookup/context.rb

@@ -47,7 +47,7 @@ class EnvironmentContext < Adaptable::Adapter
     file_data = @file_data_cache[path]
     stat = Puppet::FileSystem.stat(path)
     unless file_data && file_data.valid?(stat)
-      Puppet.debug("File at '#{path}' was changed, reloading") if file_data
+      Puppet.debug { "File at '#{path}' was changed, reloading" } if file_data
       content = Puppet::FileSystem.read(path, :encoding => 'utf-8')
       file_data = FileData.new(path, stat.ino, stat.mtime, stat.size, block_given? ? yield(content) : content)
       @file_data_cache[path] = file_data

data/lib/puppet/pops/lookup/hiera_config.rb

@@ -46,6 +46,7 @@ class HieraConfig
   KEY_DATADIR = 'datadir'.freeze
   KEY_DEFAULT_HIERARCHY = 'default_hierarchy'.freeze
   KEY_HIERARCHY = 'hierarchy'.freeze
+  KEY_PLAN_HIERARCHY = 'plan_hierarchy'.freeze
   KEY_LOGGER = 'logger'.freeze
   KEY_OPTIONS = 'options'.freeze
   KEY_PATH = 'path'.freeze
@@ -580,6 +581,7 @@ class HieraConfigV5 < HieraConfig
           tf.optional(KEY_OPTIONS) => tf.hash_kv(option_name_t, tf.data),
         }),
       tf.optional(KEY_HIERARCHY) => hierarchy_t,
+      tf.optional(KEY_PLAN_HIERARCHY) => hierarchy_t,
       tf.optional(KEY_DEFAULT_HIERARCHY) => hierarchy_t
     })
   end
@@ -600,7 +602,15 @@ class HieraConfigV5 < HieraConfig
       return data_providers
     end
 
-    @config[use_default_hierarchy ? KEY_DEFAULT_HIERARCHY : KEY_HIERARCHY].each do |he|
+    compiler = Puppet.lookup(:pal_compiler) { nil }
+    config_key = if compiler.is_a?(Puppet::Pal::ScriptCompiler) && !@config[KEY_PLAN_HIERARCHY].nil?
+          KEY_PLAN_HIERARCHY
+        elsif use_default_hierarchy
+          KEY_DEFAULT_HIERARCHY
+        else
+          KEY_HIERARCHY
+        end
+    @config[config_key].each do |he|
       name = he[KEY_NAME]
       if data_providers.include?(name)
         first_line = find_line_matching(/\s+name:\s+['"]?#{name}(?:[^\w]|$)/)
@@ -691,6 +701,9 @@ class HieraConfigV5 < HieraConfig
     defaults = config[KEY_DEFAULTS]
     validate_defaults(defaults) unless defaults.nil?
     config[KEY_HIERARCHY].each { |he| validate_hierarchy(he, defaults, owner) }
+    if config.include?(KEY_PLAN_HIERARCHY)
+      config[KEY_PLAN_HIERARCHY].each { |he| validate_hierarchy(he, defaults, owner) }
+    end
 
     if config.include?(KEY_DEFAULT_HIERARCHY)
       unless owner.is_a?(ModuleDataProvider)

data/lib/puppet/pops/model/ast_transformer.rb

@@ -31,7 +31,7 @@ class Puppet::Pops::Model::AstTransformer
   def ast(o, klass, hash={})
     # create and pass hash with file and line information
     # PUP-3274 - still needed since hostname transformation requires AST::HostName, and AST::Regexp
-    klass.new(merge_location(hash, o))
+    klass.new(**merge_location(hash, o))
   end
 
   # THIS IS AN EXPENSIVE OPERATION

data/lib/puppet/pops/types/iterable.rb

@@ -23,13 +23,16 @@ module Puppet::Pops::Types
     #
     # An `ArgumentError` is raised for all other objects.
     #
-    # @param o [Object] The object to produce an `Iterable` for
+    # @param my_caller [Object] The calling object to reference in errors
+    # @param obj [Object] The object to produce an `Iterable` for
+    # @param infer_elements [Boolean] Whether or not to recursively infer all elements of obj. Optional
+    #
     # @return [Iterable,nil] The produced `Iterable`
     # @raise [ArgumentError] In case an `Iterable` cannot be produced
     # @api public
-    def self.asserted_iterable(caller, obj)
-      iter = self.on(obj)
-      raise ArgumentError, "#{caller.class}(): wrong argument type (#{obj.class}; is not Iterable." if iter.nil?
+    def self.asserted_iterable(my_caller, obj, infer_elements = false)
+      iter = self.on(obj, nil, infer_elements)
+      raise ArgumentError, "#{my_caller.class}(): wrong argument type (#{obj.class}; is not Iterable." if iter.nil?
       iter
     end
 
@@ -48,11 +51,14 @@ module Puppet::Pops::Types
     # The value `nil` is returned for all other objects.
     #
     # @param o [Object] The object to produce an `Iterable` for
-    # @param element_type [PAnyType] the element type for the iterator. Optional (inferred if not provided)
+    # @param element_type [PAnyType] the element type for the iterator. Optional
+    # @param infer_elements [Boolean] if element_type is nil, whether or not to recursively
+    #   infer types for the entire collection. Optional
+    #
     # @return [Iterable,nil] The produced `Iterable` or `nil` if it couldn't be produced
     #
     # @api public
-    def self.on(o, element_type = nil)
+    def self.on(o, element_type = nil, infer_elements = true)
       case o
       when IteratorProducer
         o.iterator
@@ -64,7 +70,7 @@ module Puppet::Pops::Types
         if o.empty?
           Iterator.new(PUnitType::DEFAULT, o.each)
         else
-          if element_type.nil?
+          if element_type.nil? && infer_elements
             tc = TypeCalculator.singleton
             element_type = PVariantType.maybe_create(o.map {|e| tc.infer_set(e) })
           end
@@ -75,7 +81,7 @@ module Puppet::Pops::Types
         if o.empty?
           HashIterator.new(PHashType::DEFAULT_KEY_PAIR_TUPLE, o.each)
         else
-          if element_type.nil?
+          if element_type.nil? && infer_elements
             tc = TypeCalculator.singleton
             element_type = PTupleType.new([
               PVariantType.maybe_create(o.keys.map {|e| tc.infer_set(e) }),
@@ -202,6 +208,26 @@ module Puppet::Pops::Types
       @enumeration.send(name, *arguments, &block)
     end
 
+    def next
+      @enumeration.next
+    end
+
+    def map(*args, &block)
+      @enumeration.map(*args, &block)
+    end
+
+    def reduce(*args, &block)
+      @enumeration.reduce(*args, &block)
+    end
+
+    def all?(&block)
+      @enumeration.all?(&block)
+    end
+
+    def any?(&block)
+      @enumeration.any?(&block)
+    end
+
     def step(step, &block)
       raise ArgumentError if step <= 0
       r = self