puppet 4.9.4 → 4.10.0

data/lib/puppet/pops/lookup/interpolation.rb

@@ -55,7 +55,7 @@ module Interpolation
           is_alias = method_key == :alias
 
           # Alias is only permitted if the entire string is equal to the interpolate expression
-          raise Puppet::DataBinding::LookupError, "'alias' interpolation is only permitted if the expression is equal to the entire string" if is_alias && subject != match
+          fail(Issues::HIERA_INTERPOLATION_ALIAS_NOT_ENTIRE_STRING) if is_alias && subject != match
           value = interpolate_method(method_key).call(key, lookup_invocation, subject)
           value = lookup_invocation.check(method_key == :scope ? "scope:#{key}" : key) { interpolate(value, lookup_invocation, allow_methods) }
 
@@ -69,7 +69,17 @@ module Interpolation
 
   def interpolate_method(method_key)
     @@interpolate_methods ||= begin
-      global_lookup = lambda { |key, lookup_invocation, _| Lookup.lookup(key, nil, '', true, nil, lookup_invocation) }
+      global_lookup = lambda do |key, lookup_invocation, _|
+        if lookup_invocation.scope.is_a?(Hiera::Scope) && !lookup_invocation.global_only?
+          # "unwrap" the Hiera::Scope
+          lookup_invocation = Invocation.new(
+            lookup_invocation.scope.real,
+            lookup_invocation.override_values,
+            lookup_invocation.default_values,
+            lookup_invocation.explainer)
+        end
+        Lookup.lookup(key, nil, '', true, nil, lookup_invocation)
+      end
       scope_lookup = lambda do |key, lookup_invocation, subject|
         segments = split_key(key) { |problem| Puppet::DataBinding::LookupError.new("#{problem} in string: #{subject}") }
         root_key = segments.shift
@@ -92,8 +102,8 @@ module Interpolation
             end
           end
         end
-        unless segments.empty?
-          found = '';
+        unless value.nil? || segments.empty?
+          found = nil;
           catch(:no_such_key) { found = sub_lookup(key, lookup_invocation, segments, value) }
           value = found;
         end
@@ -110,7 +120,7 @@ module Interpolation
       }.freeze
     end
     interpolate_method = @@interpolate_methods[method_key]
-    raise Puppet::DataBinding::LookupError, "Unknown interpolation method '#{method_key}'" unless interpolate_method
+    fail(Issues::HIERA_INTERPOLATION_UNKNOWN_INTERPOLATION_METHOD, :name => method_key) unless interpolate_method
     interpolate_method
   end
 
@@ -125,7 +135,7 @@ module Interpolation
 
   def get_method_and_data(data, allow_methods)
     if match = data.match(/^(\w+)\((?:["]([^"]+)["]|[']([^']+)['])\)$/)
-      raise Puppet::DataBinding::LookupError, 'Interpolation using method syntax is not allowed in this context' unless allow_methods
+      fail(Issues::HIERA_INTERPOLATION_METHOD_SYNTAX_NOT_ALLOWED) unless allow_methods
       key = match[1].to_sym
       data = match[2] || match[3] # double or single qouted
     else
@@ -133,6 +143,11 @@ module Interpolation
     end
     [key, data]
   end
+
+  def fail(issue, args = EMPTY_HASH)
+    raise Puppet::DataBinding::LookupError.new(
+      issue.format(args), nil, nil, nil, nil, issue.issue_code)
+  end
 end
 end
 end

data/lib/puppet/pops/lookup/invocation.rb

@@ -22,28 +22,33 @@ class Invocation
   # @param override_values [Hash<String,Object>|nil] A map to use as override. Values found here are returned immediately (no merge)
   # @param default_values [Hash<String,Object>] A map to use as the last resort (but before default)
   # @param explainer [boolean,Explanainer] An boolean true to use the default explanation acceptor or an explainer instance that will receive information about the lookup
-  def initialize(scope, override_values = EMPTY_HASH, default_values = EMPTY_HASH, explainer = nil, adapter_class = LookupAdapter)
+  def initialize(scope, override_values = EMPTY_HASH, default_values = EMPTY_HASH, explainer = nil, adapter_class = nil)
     @scope = scope
     @override_values = override_values
     @default_values = default_values
 
     parent_invocation = self.class.current
-    if parent_invocation.nil?
-      @name_stack = []
-      @adapter_class = adapter_class
-      unless explainer.is_a?(Explainer)
-        explainer = explainer == true ? Explainer.new : nil
-      end
-      explainer = DebugExplainer.new(explainer) if Puppet[:debug] && !explainer.is_a?(DebugExplainer)
-    else
+    if parent_invocation && (adapter_class.nil? || adapter_class == parent_invocation.adapter_class)
+      # Inherit from parent invocation (track recursion)
       @name_stack = parent_invocation.name_stack
       @adapter_class = parent_invocation.adapter_class
+
+      # Inherit Hiera 3 legacy properties
       set_hiera_xxx_call if parent_invocation.hiera_xxx_call?
       set_hiera_v3_merge_behavior if parent_invocation.hiera_v3_merge_behavior?
       set_global_only if parent_invocation.global_only?
       povr = parent_invocation.hiera_v3_location_overrides
       set_hiera_v3_location_overrides(povr) unless povr.empty?
+
+      # Inherit explainer unless a new explainer is given or disabled using false
       explainer = explainer == false ? nil : parent_invocation.explainer
+    else
+      @name_stack = []
+      @adapter_class = adapter_class.nil? ? LookupAdapter : adapter_class
+      unless explainer.is_a?(Explainer)
+        explainer = explainer == true ? Explainer.new : nil
+      end
+      explainer = DebugExplainer.new(explainer) if Puppet[:debug] && !explainer.is_a?(DebugExplainer)
     end
     @explainer = explainer
   end

data/lib/puppet/pops/lookup/location_resolver.rb

@@ -25,6 +25,11 @@ module Lookup
     def exist?
       @exist
     end
+
+    # @return the resolved location as a string
+    def to_s
+      @location.to_s
+    end
   end
 
   # Helper methods to resolve interpolated locations
@@ -67,6 +72,28 @@ module Lookup
         ResolvedLocation.new(declared_uri, uri, true)
       end
     end
+
+    def expand_mapped_paths(datadir, mapped_path_triplet, lookup_invocation)
+      # The scope interpolation method is used directly to avoid unnecessary parsing of the string that otherwise
+      # would need to be generated
+      mapped_vars = interpolate_method(:scope).call(mapped_path_triplet[0], lookup_invocation, 'mapped_path[0]')
+
+      # No paths here unless the scope lookup returned something
+      return EMPTY_ARRAY if mapped_vars.nil? || mapped_vars.empty?
+
+      mapped_vars = [mapped_vars] if mapped_vars.is_a?(String)
+      var_key = mapped_path_triplet[1]
+      template = mapped_path_triplet[2]
+      scope = lookup_invocation.scope
+      lookup_invocation.with_local_memory_eluding(var_key) do
+        mapped_vars.map do |var|
+          # Need to use parent lookup invocation to avoid adding 'var' to the set of variables to track for changes. The
+          # variable that 'var' stems from is added above.
+          path = scope.with_local_scope(var_key => var) {  datadir + interpolate(template, lookup_invocation, false) }
+          ResolvedLocation.new(template, path, path.exist?)
+        end
+      end
+    end
   end
 end
 end

data/lib/puppet/pops/lookup/lookup_adapter.rb

@@ -58,7 +58,11 @@ class LookupAdapter < DataAdapter
           merge = lookup_merge_options(key, lookup_invocation)
           lookup_invocation.report_merge_source(LOOKUP_OPTIONS) unless merge.nil?
         end
-        lookup_invocation.with(:data, key.to_s) { do_lookup(key, lookup_invocation, merge) }
+        lookup_invocation.with(:data, key.to_s) do
+          catch(:no_such_key) { return do_lookup(key, lookup_invocation, merge) }
+          throw :no_such_key if lookup_invocation.global_only?
+          key.dig(lookup_invocation, lookup_default_in_module(key, lookup_invocation))
+        end
       end
     end
   end
@@ -85,6 +89,7 @@ class LookupAdapter < DataAdapter
       end
     end
   rescue Puppet::DataBinding::LookupError => detail
+    raise detail unless detail.issue_code.nil?
     error = Puppet::Error.new("Lookup of key '#{lookup_invocation.top_key}' failed: #{detail.message}")
     error.set_backtrace(detail.backtrace)
     raise error
@@ -114,6 +119,43 @@ class LookupAdapter < DataAdapter
     provider.key_lookup(key, lookup_invocation, merge_strategy)
   end
 
+  def lookup_default_in_module(key, lookup_invocation)
+    module_name = lookup_invocation.module_name
+
+    # Do not attempt to do a lookup in a module unless the name is qualified.
+    throw :no_such_key if module_name.nil?
+
+    provider = module_provider(lookup_invocation, module_name)
+    throw :no_such_key if provider.nil? || !provider.config(lookup_invocation).has_default_hierarchy?
+
+    lookup_invocation.with(:scope, "Searching default_hierarchy of module \"#{module_name}\"") do
+      merge_strategy = nil
+      if merge_strategy.nil?
+        @module_default_lookup_options ||= {}
+        options = @module_default_lookup_options.fetch(module_name) do |k|
+          meta_invocation = Invocation.new(lookup_invocation.scope)
+          meta_invocation.lookup(LookupKey::LOOKUP_OPTIONS, k) do
+            opts = nil
+            lookup_invocation.with(:scope, "Searching for \"#{LookupKey::LOOKUP_OPTIONS}\"") do
+              catch(:no_such_key) do
+              opts = compile_patterns(
+                validate_lookup_options(
+                  provider.key_lookup_in_default(LookupKey::LOOKUP_OPTIONS, meta_invocation, MergeStrategy.strategy(HASH)), k))
+              end
+            end
+            @module_default_lookup_options[k] = opts
+          end
+        end
+        lookup_options = extract_lookup_options_for_key(key, options)
+        merge_strategy = lookup_options[MERGE] unless lookup_options.nil?
+      end
+
+      lookup_invocation.with(:scope, "Searching for \"#{key}\"") do
+        provider.key_lookup_in_default(key, lookup_invocation, merge_strategy)
+      end
+    end
+  end
+
   # Retrieve the merge options that match the given `name`.
   #
   # @param key [LookupKey] The key for which we want merge options
@@ -141,11 +183,18 @@ class LookupAdapter < DataAdapter
     else
       options = @lookup_options[module_name]
     end
+    extract_lookup_options_for_key(key, options)
+  end
+
+  def extract_lookup_options_for_key(key, options)
     return nil if options.nil?
 
     rk = key.root_key
-    key_opts = options[0][rk]
-    return key_opts unless key_opts.nil?
+    key_opts = options[0]
+    unless key_opts.nil?
+      key_opt = key_opts[rk]
+      return key_opt unless key_opt.nil?
+    end
 
     patterns = options[1]
     patterns.each_pair { |pattern, value| return value if pattern =~ rk } unless patterns.nil?
@@ -237,7 +286,7 @@ class LookupAdapter < DataAdapter
     meta_invocation.lookup(LookupKey::LOOKUP_OPTIONS, lookup_invocation.module_name) do
       meta_invocation.with(:meta, LOOKUP_OPTIONS) do
         if meta_invocation.global_only?
-          global_lookup_options(meta_invocation, merge_strategy)
+          compile_patterns(global_lookup_options(meta_invocation, merge_strategy))
         else
           opts = env_lookup_options(meta_invocation, merge_strategy)
           catch(:no_such_key) do
@@ -356,7 +405,9 @@ class LookupAdapter < DataAdapter
       when 'hiera'
         mp || ModuleDataProvider.new(module_name)
       when 'function'
-        ModuleDataProvider.new(module_name, HieraConfig.v4_function_config(Pathname(mod.path), "#{module_name}::data"))
+        mp = ModuleDataProvider.new(module_name)
+        mp.config = HieraConfig.v4_function_config(Pathname(mod.path), "#{module_name}::data", mp)
+        mp
       else
         injector = Puppet.lookup(:injector) { nil }
         provider = injector.lookup(nil,
@@ -424,7 +475,9 @@ class LookupAdapter < DataAdapter
         # Use hiera.yaml or default settings if it is missing
         ep || EnvironmentDataProvider.new
       when 'function'
-        EnvironmentDataProvider.new(HieraConfigV5.v4_function_config(env_path, 'environment::data'))
+        ep = EnvironmentDataProvider.new
+        ep.config = HieraConfigV5.v4_function_config(env_path, 'environment::data', ep)
+        ep
       else
          injector = Puppet.lookup(:injector) { nil }
 

data/lib/puppet/pops/lookup/lookup_key_function_provider.rb

@@ -25,11 +25,11 @@ class LookupKeyFunctionProvider < FunctionProvider
   def invoke_with_location(lookup_invocation, location, root_key, merge)
     if location.nil?
       value = lookup_key(root_key, lookup_invocation, nil, merge)
-      lookup_invocation.report_found(root_key, validate_data_value(self, value))
+      lookup_invocation.report_found(root_key, value)
     else
       lookup_invocation.with(:location, location) do
-        value = lookup_key(root_key, lookup_invocation, location.location, merge)
-        lookup_invocation.report_found(root_key, validate_data_value(self, value))
+        value = lookup_key(root_key, lookup_invocation, location, merge)
+        lookup_invocation.report_found(root_key, value)
       end
     end
   end
@@ -49,7 +49,12 @@ class LookupKeyFunctionProvider < FunctionProvider
     ctx.data_hash ||= {}
     catch(:no_such_key) do
       hash = ctx.data_hash
-      hash[key] = ctx.function.call(lookup_invocation.scope, key, options(location), Context.new(ctx, lookup_invocation)) unless hash.include?(key)
+      unless hash.include?(key)
+        hash[key] = validate_data_value(ctx.function.call(lookup_invocation.scope, key, options(location), Context.new(ctx, lookup_invocation))) do
+          msg = "Value for key '#{key}', returned from #{full_name}"
+          location.nil? ? msg : "#{msg}, when using location '#{location}',"
+        end
+      end
       return hash[key]
     end
     lookup_invocation.report_not_found(key)
@@ -83,78 +88,5 @@ class V3LookupKeyFunctionProvider < LookupKeyFunctionProvider
     end
   end
 end
-
-class V3BackendFunctionProvider < LookupKeyFunctionProvider
-  TAG = 'hiera3_backend'.freeze
-
-  def lookup_key(key, lookup_invocation, location, merge)
-    @backend ||= instantiate_backend(lookup_invocation)
-    config = parent_data_provider.config(lookup_invocation)
-
-    # A merge_behavior retrieved from hiera.yaml must not be converted here. Instead, passing the symbol :hash
-    # tells the V3 backend to pick it up from the config.
-    resolution_type = lookup_invocation.hiera_v3_merge_behavior? ? :hash : convert_merge(merge)
-    @backend.lookup(key, lookup_invocation.scope, lookup_invocation.hiera_v3_location_overrides, resolution_type, context = {:recurse_guard => nil})
-  end
-
-  private
-
-  def instantiate_backend(lookup_invocation)
-    backend_name = options[HieraConfig::KEY_BACKEND]
-    begin
-      require 'hiera/backend'
-      require "hiera/backend/#{backend_name.downcase}_backend"
-      backend = Hiera::Backend.const_get("#{backend_name.capitalize}_backend").new
-      return backend.method(:lookup).arity == 4 ? Hiera::Backend::Backend1xWrapper.new(backend) : backend
-    rescue LoadError => e
-      lookup_invocation.report_text { "Unable to load backend '#{backend_name}': #{e.message}" }
-      throw :no_such_key
-    rescue NameError => e
-      lookup_invocation.report_text { "Unable to instantiate backend '#{backend_name}': #{e.message}" }
-      throw :no_such_key
-    end
-  end
-
-  # Converts a lookup 'merge' parameter argument into a Hiera 'resolution_type' argument.
-  #
-  # @param merge [String,Hash,nil] The lookup 'merge' argument
-  # @return [Symbol,Hash,nil] The Hiera 'resolution_type'
-  def convert_merge(merge)
-    case merge
-    when nil
-    when 'first', 'default'
-      # Nil is OK. Defaults to Hiera :priority
-      nil
-    when Puppet::Pops::MergeStrategy
-      convert_merge(merge.configuration)
-    when 'unique'
-      # Equivalent to Hiera :array
-      :array
-    when 'hash'
-      # Equivalent to Hiera :hash with default :native merge behavior. A Hash must be passed here
-      # to override possible Hiera deep merge config settings.
-      { :behavior => :native }
-    when 'deep', 'unconstrained_deep'
-      # Equivalent to Hiera :hash with :deeper merge behavior.
-      { :behavior => :deeper }
-    when 'reverse_deep'
-      # Equivalent to Hiera :hash with :deep merge behavior.
-      { :behavior => :deep }
-    when Hash
-      strategy = merge['strategy']
-      case strategy
-      when 'deep', 'unconstrained_deep', 'reverse_deep'
-        result = { :behavior => strategy == 'reverse_deep' ? :deep : :deeper }
-        # Remaining entries must have symbolic keys
-        merge.each_pair { |k,v| result[k.to_sym] = v unless k == 'strategy' }
-        result
-      else
-        convert_merge(strategy)
-      end
-    else
-      raise Puppet::DataBinding::LookupError, "Unrecognized value for request 'merge' parameter: '#{merge}'"
-    end
-  end
-end
 end
 end

data/lib/puppet/pops/lookup/module_data_provider.rb

@@ -16,28 +16,51 @@ class ModuleDataProvider < ConfiguredDataProvider
     'Module'
   end
 
+  # Performs a lookup using a module default hierarchy with an endless recursion check.
+  #
+  # @param key [LookupKey] The key to lookup
+  # @param lookup_invocation [Invocation] The current lookup invocation
+  # @param merge [MergeStrategy,String,Hash{String=>Object},nil] Merge strategy or hash with strategy and options
+  #
+  def key_lookup_in_default(key, lookup_invocation, merge)
+    dps = config(lookup_invocation).configured_data_providers(lookup_invocation, self, true)
+    if dps.empty?
+      lookup_invocation.report_not_found(key)
+      throw :no_such_key
+    end
+    merge_strategy = MergeStrategy.strategy(merge)
+    lookup_invocation.check(key.to_s) do
+      lookup_invocation.with(:data_provider, self) do
+        merge_strategy.lookup(dps, lookup_invocation) do |data_provider|
+          data_provider.unchecked_key_lookup(key, lookup_invocation, merge_strategy)
+        end
+      end
+    end
+  end
+
   # Asserts that all keys in the given _data_hash_ are prefixed with the configured _module_name_. Removes entries
   # that does not follow the convention and logs a warning.
   #
   # @param data_hash [Hash] The data hash
   # @return [Hash] The possibly pruned hash
-  def validate_data_hash(data_provider, data_hash)
+  def validate_data_hash(data_hash)
     super
     module_prefix = "#{module_name}::"
     data_hash.each_key.reduce(data_hash) do |memo, k|
       next memo if k == LOOKUP_OPTIONS || k.start_with?(module_prefix)
-      msg = 'must use keys qualified with the name of the module'
+      msg = "#{yield} must use keys qualified with the name of the module"
       memo = memo.clone if memo.equal?(data_hash)
       memo.delete(k)
-      Puppet.warning("Module '#{module_name}': #{data_provider.name} #{msg}")
+      Puppet.warning("Module '#{module_name}': #{msg}")
       memo
     end
+    data_hash
   end
 
   protected
 
   def assert_config_version(config)
-    raise Puppet::DataBinding::LookupError, "#{config.name} cannot be used in a module" unless config.version > 3
+    config.fail(Issues::HIERA_VERSION_3_NOT_GLOBAL, :where => 'module') unless config.version > 3
     config
   end
 

data/lib/puppet/pops/parser/lexer2.rb

@@ -179,7 +179,7 @@ class Lexer2
   PATTERN_COMMENT   = %r{#.*\r?}
   PATTERN_MLCOMMENT = %r{/\*(.*?)\*/}m
 
-  PATTERN_REGEX     = %r{/[^/\n]*/}
+  PATTERN_REGEX     = %r{/[^/]*/}
   PATTERN_REGEX_END = %r{/}
   PATTERN_REGEX_A   = %r{\A/} # for replacement to ""
   PATTERN_REGEX_Z   = %r{/\Z} # for replacement to ""

data/lib/puppet/pops/pcore.rb

@@ -41,6 +41,7 @@ module Pcore
 
       Resource.register_ptypes(loader, ir)
       Lookup::Context.register_ptype(loader, ir);
+      Lookup::DataProvider.register_types(loader)
     end
   end
 
@@ -81,11 +82,10 @@ module Pcore
   end
 
   def self.register_implementations(impls, name_authority = RUNTIME_NAME_AUTHORITY)
-    Loaders.loaders.register_implementations(impls, name_authority = RUNTIME_NAME_AUTHORITY)
+    Loaders.loaders.register_implementations(impls, name_authority)
   end
 
-  def self.register_aliases(aliases, name_authority = RUNTIME_NAME_AUTHORITY)
-    loader = Loaders.loaders.private_environment_loader
+  def self.register_aliases(aliases, name_authority = RUNTIME_NAME_AUTHORITY, loader = Loaders.loaders.private_environment_loader)
     aliases.each do |name, type_string|
       add_type(Types::PTypeAliasType.new(name, Types::TypeFactory.type_reference(type_string), nil), loader, name_authority)
     end