puppet 5.3.4 → 6.29.0

data/lib/puppet/pops/lookup/global_data_provider.rb

@@ -1,3 +1,4 @@
+require 'hiera/scope'
 require_relative 'configured_data_provider'
 
 module Puppet::Pops
@@ -11,7 +12,6 @@ class GlobalDataProvider < ConfiguredDataProvider
   def unchecked_key_lookup(key, lookup_invocation, merge)
     config = config(lookup_invocation)
     if(config.version == 3)
-      require 'hiera/scope'
       # Hiera version 3 needs access to special scope variables
       scope = lookup_invocation.scope
       unless scope.is_a?(Hiera::Scope)

data/lib/puppet/pops/lookup/hiera_config.rb

@@ -46,6 +46,7 @@ class HieraConfig
   KEY_DATADIR = 'datadir'.freeze
   KEY_DEFAULT_HIERARCHY = 'default_hierarchy'.freeze
   KEY_HIERARCHY = 'hierarchy'.freeze
+  KEY_PLAN_HIERARCHY = 'plan_hierarchy'.freeze
   KEY_LOGGER = 'logger'.freeze
   KEY_OPTIONS = 'options'.freeze
   KEY_PATH = 'path'.freeze
@@ -132,7 +133,7 @@ class HieraConfig
       if config_path.exist?
         env_context = EnvironmentContext.adapt(lookup_invocation.scope.compiler.environment)
         loaded_config = env_context.cached_file_data(config_path) do |content|
-          parsed = YAML.load(content, config_path)
+          parsed = Puppet::Util::Yaml.safe_load(content, [Symbol], config_path)
 
           # For backward compatibility, we must treat an empty file, or a yaml that doesn't
           # produce a Hash as Hiera version 3 default.
@@ -339,6 +340,7 @@ class HieraConfig
   def self.not_implemented(impl, method_name)
     raise NotImplementedError, "The class #{impl.class.name} should have implemented the method #{method_name}()"
   end
+  private_class_method :not_implemented
 end
 
 # @api private
@@ -579,6 +581,7 @@ class HieraConfigV5 < HieraConfig
           tf.optional(KEY_OPTIONS) => tf.hash_kv(option_name_t, tf.data),
         }),
       tf.optional(KEY_HIERARCHY) => hierarchy_t,
+      tf.optional(KEY_PLAN_HIERARCHY) => hierarchy_t,
       tf.optional(KEY_DEFAULT_HIERARCHY) => hierarchy_t
     })
   end
@@ -599,7 +602,15 @@ class HieraConfigV5 < HieraConfig
       return data_providers
     end
 
-    @config[use_default_hierarchy ? KEY_DEFAULT_HIERARCHY : KEY_HIERARCHY].each do |he|
+    compiler = Puppet.lookup(:pal_compiler) { nil }
+    config_key = if compiler.is_a?(Puppet::Pal::ScriptCompiler) && !@config[KEY_PLAN_HIERARCHY].nil?
+          KEY_PLAN_HIERARCHY
+        elsif use_default_hierarchy
+          KEY_DEFAULT_HIERARCHY
+        else
+          KEY_HIERARCHY
+        end
+    @config[config_key].each do |he|
       name = he[KEY_NAME]
       if data_providers.include?(name)
         first_line = find_line_matching(/\s+name:\s+['"]?#{name}(?:[^\w]|$)/)
@@ -690,6 +701,9 @@ class HieraConfigV5 < HieraConfig
     defaults = config[KEY_DEFAULTS]
     validate_defaults(defaults) unless defaults.nil?
     config[KEY_HIERARCHY].each { |he| validate_hierarchy(he, defaults, owner) }
+    if config.include?(KEY_PLAN_HIERARCHY)
+      config[KEY_PLAN_HIERARCHY].each { |he| validate_hierarchy(he, defaults, owner) }
+    end
 
     if config.include?(KEY_DEFAULT_HIERARCHY)
       unless owner.is_a?(ModuleDataProvider)

data/lib/puppet/pops/lookup/interpolation.rb

@@ -125,7 +125,7 @@ module Interpolation
     interpolate_method
   end
 
-  # Because the semanitcs of Puppet::Parser::Scope#include? differs from Hash#include?
+  # Because the semantics of Puppet::Parser::Scope#include? differs from Hash#include?
   def nil_in_scope?(scope, key)
     if scope.is_a?(Hash)
       scope.include?(key)
@@ -135,7 +135,8 @@ module Interpolation
   end
 
   def get_method_and_data(data, allow_methods)
-    if match = data.match(/^(\w+)\((?:["]([^"]+)["]|[']([^']+)['])\)$/)
+    match = data.match(/^(\w+)\((?:["]([^"]+)["]|[']([^']+)['])\)$/)
+    if match
       fail(Issues::HIERA_INTERPOLATION_METHOD_SYNTAX_NOT_ALLOWED) unless allow_methods
       key = match[1].to_sym
       data = match[2] || match[3] # double or single qouted

data/lib/puppet/pops/lookup/invocation.rb

@@ -1,3 +1,5 @@
+require 'puppet/thread_local'
+
 require_relative 'explainer'
 
 module Puppet::Pops
@@ -7,7 +9,11 @@ class Invocation
   attr_reader :scope, :override_values, :default_values, :explainer, :module_name, :top_key, :adapter_class
 
   def self.current
-    @current
+    (@current ||= Puppet::ThreadLocal.new(nil)).value
+  end
+
+  def self.current=(new_value)
+    @current.value = new_value
   end
 
   # Creates a new instance with same settings as this instance but with a new given scope
@@ -37,6 +43,7 @@ class Invocation
     @default_values = default_values
 
     parent_invocation = self.class.current
+
     if parent_invocation && (adapter_class.nil? || adapter_class == parent_invocation.adapter_class)
       # Inherit from parent invocation (track recursion)
       @name_stack = parent_invocation.name_stack
@@ -71,10 +78,10 @@ class Invocation
       yield
     else
       begin
-        self.class.instance_variable_set(:@current, self)
+        self.class.current = self
         yield
       ensure
-        self.class.instance_variable_set(:@current, save_current)
+        self.class.current = save_current
       end
     end
   end

data/lib/puppet/pops/lookup/key_recorder.rb

@@ -0,0 +1,18 @@
+# This class defines the private API of the Lookup Key Recorder support.
+# @api private
+#
+class Puppet::Pops::Lookup::KeyRecorder
+
+  def initialize()
+  end
+
+  def self.singleton
+    @null_recorder ||= self.new
+  end
+
+  # Records a key
+  # (This implementation does nothing)
+  #
+  def record(key)
+  end
+end

data/lib/puppet/pops/lookup/lookup_adapter.rb

@@ -16,6 +16,8 @@ class LookupAdapter < DataAdapter
 
   HASH = 'hash'.freeze
   MERGE = 'merge'.freeze
+  CONVERT_TO = 'convert_to'.freeze
+  NEW = 'new'.freeze
 
   def self.create_adapter(compiler)
     new(compiler)
@@ -25,6 +27,8 @@ class LookupAdapter < DataAdapter
     super()
     @compiler = compiler
     @lookup_options = {}
+    # Get a KeyRecorder from context, and set a "null recorder" if not defined
+    @key_recorder = Puppet.lookup(:lookup_key_recorder) { KeyRecorder.singleton }
   end
 
   # Performs a lookup using global, environment, and module data providers. Merge the result using the given
@@ -46,25 +50,76 @@ class LookupAdapter < DataAdapter
       end
     end
 
+    # Record that the key was looked up. This will record all keys for which a lookup is performed
+    # except 'lookup_options' (since that is illegal from a user perspective,
+    # and from an impact perspective is always looked up).
+    @key_recorder.record(key)
+
     key = LookupKey.new(key)
     lookup_invocation.lookup(key, key.module_name) do
-      merge_explained = false
       if lookup_invocation.only_explain_options?
         catch(:no_such_key) { do_lookup(LookupKey::LOOKUP_OPTIONS, lookup_invocation, HASH) }
         nil
       else
+        lookup_options = lookup_lookup_options(key, lookup_invocation) || {}
+
         if merge.nil?
           # Used cached lookup_options
-          merge = lookup_merge_options(key, lookup_invocation)
+          # merge = lookup_merge_options(key, lookup_invocation)
+          merge = lookup_options[MERGE]
           lookup_invocation.report_merge_source(LOOKUP_OPTIONS) unless merge.nil?
         end
-        lookup_invocation.with(:data, key.to_s) do
-          catch(:no_such_key) { return do_lookup(key, lookup_invocation, merge) }
-          throw :no_such_key if lookup_invocation.global_only?
-          key.dig(lookup_invocation, lookup_default_in_module(key, lookup_invocation))
-        end
+        convert_result(key.to_s, lookup_options, lookup_invocation, lambda do
+          lookup_invocation.with(:data, key.to_s) do
+            catch(:no_such_key) { return do_lookup(key, lookup_invocation, merge) }
+            throw :no_such_key if lookup_invocation.global_only?
+            key.dig(lookup_invocation, lookup_default_in_module(key, lookup_invocation))
+          end
+        end)
+      end
+    end
+  end
+
+  # Performs a possible conversion of the result of calling `the_lookup` lambda
+  # The conversion takes place if there is a 'convert_to' key in the lookup_options
+  # If there is no conversion, the result of calling `the_lookup` is returned
+  # otherwise the successfully converted value.
+  # Errors are raised if the convert_to is faulty (bad type string, or if a call to
+  # new(T, <args>) fails.
+  #
+  # @param key [String] The key to lookup
+  # @param lookup_options [Hash] a hash of options
+  # @param lookup_invocation [Invocation] the lookup invocation
+  # @param the_lookup [Lambda] zero arg lambda that performs the lookup of a value
+  # @return [Object] the looked up value, or converted value if there was conversion
+  # @throw :no_such_key when the object is not found (if thrown by `the_lookup`)
+  #
+  def convert_result(key, lookup_options, lookup_invocation, the_lookup)
+    result = the_lookup.call
+    convert_to = lookup_options[CONVERT_TO]
+    return result if convert_to.nil?
+
+    convert_to = convert_to.is_a?(Array) ? convert_to : [convert_to]
+    if convert_to[0].is_a?(String)
+      begin
+        convert_to[0] = Puppet::Pops::Types::TypeParser.singleton.parse(convert_to[0])
+      rescue StandardError => e
+        raise Puppet::DataBinding::LookupError,
+          _("Invalid data type in lookup_options for key '%{key}' could not parse '%{source}', error: '%{msg}") %
+            { key: key, source: convert_to[0], msg: e.message}
       end
     end
+    begin
+      result = lookup_invocation.scope.call_function(NEW, [convert_to[0], result, *convert_to[1..-1]])
+      # TRANSLATORS 'lookup_options', 'convert_to' and args_string variable should not be translated,
+      args_string = Puppet::Pops::Types::StringConverter.singleton.convert(convert_to)
+      lookup_invocation.report_text { _("Applying convert_to lookup_option with arguments %{args}") % { args: args_string } }
+    rescue StandardError => e
+      raise Puppet::DataBinding::LookupError,
+        _("The convert_to lookup_option for key '%{key}' raised error: %{msg}") %
+          { key: key, msg: e.message}
+    end
+    result
   end
 
   def lookup_global(key, lookup_invocation, merge_strategy)
@@ -237,7 +292,8 @@ class LookupAdapter < DataAdapter
   PROVIDER_STACK = [:lookup_global, :lookup_in_environment, :lookup_in_module].freeze
 
   def validate_lookup_options(options, module_name)
-    raise Puppet::DataBinding::LookupError.new(_("value of %{opts} must be a hash") % { opts: LOOKUP_OPTIONS }) unless options.is_a?(Hash) unless options.nil?
+    return nil if options.nil?
+    raise Puppet::DataBinding::LookupError.new(_("value of %{opts} must be a hash") % { opts: LOOKUP_OPTIONS }) unless options.is_a?(Hash)
     return options if module_name.nil?
 
     pfx = "#{module_name}::"
@@ -297,7 +353,7 @@ class LookupAdapter < DataAdapter
               module_opts = validate_lookup_options(lookup_in_module(LookupKey::LOOKUP_OPTIONS, meta_invocation, merge_strategy), module_name)
               opts = if opts.nil?
                 module_opts
-              else
+              elsif module_opts
                 merge_strategy.lookup([GLOBAL_ENV_MERGE, "Module #{lookup_invocation.module_name}"], meta_invocation) do |n|
                   meta_invocation.with(:scope, n) { meta_invocation.report_found(LOOKUP_OPTIONS,  n == GLOBAL_ENV_MERGE ? opts : module_opts) }
                 end

data/lib/puppet/pops/lookup/sub_lookup.rb

@@ -30,7 +30,7 @@ module SubLookup
       raise yield('Syntax error') unless segments.size * 2 == count + 1
       segments.map! do |segment|
         segment.strip!
-        if segment.start_with?('"') || segment.start_with?("'")
+        if segment.start_with?('"', "'")
           segment[1..-2]
         elsif segment =~ /^(:?[+-]?[0-9]+)$/
           segment.to_i

data/lib/puppet/pops/lookup.rb

@@ -94,3 +94,4 @@ end
 end
 
 require_relative 'lookup/lookup_adapter'
+require_relative 'lookup/key_recorder'

data/lib/puppet/pops/merge_strategy.rb

@@ -173,12 +173,14 @@ module Puppet::Pops
 
     protected
 
-    # Returns the type used to validate the options hash
-    #
-    # @return [Types::PStructType] the puppet type
-    #
-    def self.options_t
-      @options_t ||=Types::TypeParser.singleton.parse("Struct[{strategy=>Optional[Pattern[/#{key}/]]}]")
+    class << self
+      # Returns the type used to validate the options hash
+      #
+      # @return [Types::PStructType] the puppet type
+      #
+      def options_t
+        @options_t ||=Types::TypeParser.singleton.parse("Struct[{strategy=>Optional[Pattern[/#{key}/]]}]")
+      end
     end
 
     # Returns the type used to validate the options hash
@@ -383,18 +385,20 @@ module Puppet::Pops
 
     protected
 
-    # Returns a type that allows all deep_merge options except 'preserve_unmergeables' since we force
-    # the setting of that option to false
-    #
-    # @return [Types::PAnyType] the puppet type used when validating the options hash
-    def self.options_t
-      @options_t ||= Types::TypeParser.singleton.parse('Struct[{'\
-          "strategy=>Optional[Pattern[#{key}]],"\
-          'knockout_prefix=>Optional[String],'\
-          'merge_debug=>Optional[Boolean],'\
-          'merge_hash_arrays=>Optional[Boolean],'\
-          'sort_merged_arrays=>Optional[Boolean],'\
-          '}]')
+    class << self
+      # Returns a type that allows all deep_merge options except 'preserve_unmergeables' since we force
+      # the setting of that option to false
+      #
+      # @return [Types::PAnyType] the puppet type used when validating the options hash
+      def options_t
+        @options_t ||= Types::TypeParser.singleton.parse('Struct[{'\
+                                                         "strategy=>Optional[Pattern[#{key}]],"\
+                                                         'knockout_prefix=>Optional[String],'\
+                                                         'merge_debug=>Optional[Boolean],'\
+                                                         'merge_hash_arrays=>Optional[Boolean],'\
+                                                         'sort_merged_arrays=>Optional[Boolean],'\
+                                                         '}]')
+      end
     end
 
     def value_t

data/lib/puppet/pops/model/ast.pp

@@ -163,6 +163,9 @@ type Puppet::AST = TypeSet[{
         }
       }
     }],
+    ApplyBlockExpression => Object[{
+      parent => BlockExpression,
+    }],
     CaseOption => Object[{
       parent => Expression,
       attributes => {
@@ -377,6 +380,9 @@ type Puppet::AST = TypeSet[{
         }
       }
     }],
+    PlanDefinition => Object[{
+      parent => FunctionDefinition,
+    }],
     LambdaExpression => Object[{
       parent => Expression,
       attributes => {
@@ -394,6 +400,19 @@ type Puppet::AST = TypeSet[{
         }
       }
     }],
+    ApplyExpression => Object[{
+      parent => Expression,
+      attributes => {
+        'arguments' => {
+          type => Array[Expression],
+          value => []
+        },
+        'body' => {
+          type => Optional[Expression],
+          value => undef
+        }
+      }
+    }],
     IfExpression => Object[{
       parent => Expression,
       attributes => {

data/lib/puppet/pops/model/ast.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # # Generated by Puppet::Pops::Types::RubyGenerator from TypeSet Puppet::AST on -4712-01-01
 
 module Puppet
@@ -25,7 +26,7 @@ class PopsObject
   attr_reader :hash
 
   def initialize
-    @hash = -1631630444118989922
+    @hash = 2270595461303489901
   end
   def _pcore_init_hash
     {}
@@ -1216,6 +1217,14 @@ class BlockExpression < Expression
   alias == eql?
 end
 
+class ApplyBlockExpression < BlockExpression
+  def self._pcore_type
+    @_pcore_type ||= Types::PObjectType.new('Puppet::AST::ApplyBlockExpression', {
+      'parent' => BlockExpression._pcore_type,
+    })
+  end
+end
+
 class CaseOption < Expression
   def self._pcore_type
     @_pcore_type ||= Types::PObjectType.new('Puppet::AST::CaseOption', {
@@ -2779,6 +2788,37 @@ class HostClassDefinition < NamedDefinition
   alias == eql?
 end
 
+class PlanDefinition < FunctionDefinition
+  def self._pcore_type
+    @_pcore_type ||= Types::PObjectType.new('Puppet::AST::PlanDefinition', {
+      'parent' => FunctionDefinition._pcore_type
+    })
+  end
+
+  def _pcore_contents
+    @parameters.each { |value| yield(value) }
+    yield(@body) unless @body.nil?
+    yield(@return_type) unless @return_type.nil?
+  end
+
+  def _pcore_all_contents(path, &block)
+    path << self
+    @parameters.each do |value|
+      block.call(value, path)
+      value._pcore_all_contents(path, &block)
+    end
+    unless @body.nil?
+      block.call(@body, path)
+      @body._pcore_all_contents(path, &block)
+    end
+    unless @return_type.nil?
+      block.call(@return_type, path)
+      @return_type._pcore_all_contents(path, &block)
+    end
+    path.pop
+  end
+end
+
 class LambdaExpression < Expression
   def self._pcore_type
     @_pcore_type ||= Types::PObjectType.new('Puppet::AST::LambdaExpression', {
@@ -2878,6 +2918,90 @@ class LambdaExpression < Expression
   alias == eql?
 end
 
+class ApplyExpression < Expression
+  def self._pcore_type
+    @_pcore_type ||= Types::PObjectType.new('Puppet::AST::ApplyExpression', {
+      'parent' => Expression._pcore_type,
+      'attributes' => {
+        'arguments' => {
+          'type' => Types::PArrayType.new(Expression._pcore_type),
+          'value' => []
+        },
+        'body' => {
+          'type' => Types::POptionalType.new(Expression._pcore_type),
+          'value' => nil
+        }
+      }
+    })
+  end
+
+  def self.from_hash(init_hash)
+    from_asserted_hash(Types::TypeAsserter.assert_instance_of('Puppet::AST::ApplyExpression initializer', _pcore_type.init_hash_type, init_hash))
+  end
+
+  def self.from_asserted_hash(init_hash)
+    new(
+      init_hash['locator'],
+      init_hash['offset'],
+      init_hash['length'],
+      init_hash.fetch('arguments') { _pcore_type['arguments'].value },
+      init_hash['body'])
+  end
+
+  def self.create(locator, offset, length, arguments = _pcore_type['arguments'].value, body = nil)
+    ta = Types::TypeAsserter
+    attrs = _pcore_type.attributes(true)
+    ta.assert_instance_of('Puppet::AST::Positioned[locator]', attrs['locator'].type, locator)
+    ta.assert_instance_of('Puppet::AST::Positioned[offset]', attrs['offset'].type, offset)
+    ta.assert_instance_of('Puppet::AST::Positioned[length]', attrs['length'].type, length)
+    ta.assert_instance_of('Puppet::AST::ApplyExpression[arguments]', attrs['arguments'].type, arguments)
+    ta.assert_instance_of('Puppet::AST::ApplyExpression[body]', attrs['body'].type, body)
+    new(locator, offset, length, arguments, body)
+  end
+
+  attr_reader :arguments
+  attr_reader :body
+
+  def initialize(locator, offset, length, arguments = _pcore_type['arguments'].value, body = nil)
+    super(locator, offset, length)
+    @hash = @hash ^ arguments.hash ^ body.hash
+    @arguments = arguments
+    @body = body
+  end
+
+  def _pcore_init_hash
+    result = super
+    result['arguments'] = @arguments unless _pcore_type['arguments'].default_value?(@arguments)
+    result['body'] = @body unless @body == nil
+    result
+  end
+
+  def _pcore_contents
+    @arguments.each { |value| yield(value) }
+    yield(@body) unless @body.nil?
+  end
+
+  def _pcore_all_contents(path, &block)
+    path << self
+    @arguments.each do |value|
+      block.call(value, path)
+      value._pcore_all_contents(path, &block)
+    end
+    unless @body.nil?
+      block.call(@body, path)
+      @body._pcore_all_contents(path, &block)
+    end
+    path.pop
+  end
+
+  def eql?(o)
+    super &&
+    @arguments.eql?(o.arguments) &&
+    @body.eql?(o.body)
+  end
+  alias == eql?
+end
+
 class IfExpression < Expression
   def self._pcore_type
     @_pcore_type ||= Types::PObjectType.new('Puppet::AST::IfExpression', {
@@ -4725,6 +4849,7 @@ class Program < PopsObject
 end
 
 end
+
 module Model
 @@pcore_ast_initialized = false
 def self.register_pcore_types
@@ -4756,6 +4881,7 @@ def self.register_pcore_types
   Model::KeyedEntry,
   Model::LiteralHash,
   Model::BlockExpression,
+  Model::ApplyBlockExpression,
   Model::CaseOption,
   Model::CaseExpression,
   Model::QueryExpression,
@@ -4780,7 +4906,9 @@ def self.register_pcore_types
   Model::SubLocatedExpression,
   Model::HeredocExpression,
   Model::HostClassDefinition,
+  Model::PlanDefinition,
   Model::LambdaExpression,
+  Model::ApplyExpression,
   Model::IfExpression,
   Model::UnlessExpression,
   Model::CallExpression,

data/lib/puppet/pops/model/ast_transformer.rb

@@ -31,7 +31,7 @@ class Puppet::Pops::Model::AstTransformer
   def ast(o, klass, hash={})
     # create and pass hash with file and line information
     # PUP-3274 - still needed since hostname transformation requires AST::HostName, and AST::Regexp
-    klass.new(merge_location(hash, o))
+    klass.new(**merge_location(hash, o))
   end
 
   # THIS IS AN EXPENSIVE OPERATION