puppet 4.9.2 → 4.9.3

data/lib/hiera/puppet_function.rb

@@ -65,6 +65,7 @@ class Hiera::PuppetFunction < Puppet::Functions::InternalFunction
     end
     lookup_invocation = Puppet::Pops::Lookup::Invocation.new(scope, {}, {})
     adapter = lookup_invocation.lookup_adapter
+    lookup_invocation.set_hiera_xxx_call
     lookup_invocation.set_global_only unless adapter.global_only? || adapter.has_environment_data_provider?(lookup_invocation)
     lookup_invocation.set_hiera_v3_location_overrides(override) unless override.nil? || override.is_a?(Array) && override.empty?
     Puppet::Pops::Lookup.lookup(key, nil, default, has_default, merge_type, lookup_invocation, &default_block)

data/lib/puppet/feature/hiera_eyaml.rb

@@ -0,0 +1,3 @@
+require 'puppet/util/feature'
+
+Puppet.features.add(:hiera_eyaml, :libs => ['hiera/backend/eyaml/parser/parser'])

data/lib/puppet/functions/eyaml_lookup_key.rb

@@ -0,0 +1,73 @@
+# @since 5.0.0
+#
+Puppet::Functions.create_function(:eyaml_lookup_key) do
+  unless Puppet.features.hiera_eyaml?
+    raise Puppet::DataBinding::LookupError, 'Lookup using eyaml lookup_key function is only supported when the hiera_eyaml library is present'
+  end
+
+  require 'hiera/backend/eyaml/encryptor'
+  require 'hiera/backend/eyaml/utils'
+  require 'hiera/backend/eyaml/options'
+  require 'hiera/backend/eyaml/parser/parser'
+
+  dispatch :eyaml_lookup_key do
+    param 'String[1]', :key
+    param 'Hash[String[1],Any]', :options
+    param 'Puppet::LookupContext', :context
+  end
+
+  def eyaml_lookup_key(key, options, context)
+    return context.cached_value(key) if context.cache_has_key(key)
+
+    # nil key is used to indicate that the cache contains the raw content of the eyaml file
+    raw_data = context.cached_value(nil)
+    if raw_data.nil?
+      options.each_pair do |k, v|
+        unless k == 'path'
+          Hiera::Backend::Eyaml::Options[k.to_sym] = v
+          context.explain { "Setting Eyaml option '#{k}' to '#{v}'" }
+        end
+      end
+      raw_data = load_data_hash(options)
+      context.cache(nil, raw_data)
+    end
+    context.not_found unless raw_data.include?(key)
+    context.cache(key, decrypt_value(raw_data[key], context))
+  end
+
+  def load_data_hash(options)
+    begin
+      data = YAML.load_file(options['path'])
+      Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data.is_a?(Hash) ? data : {})
+    rescue YAML::SyntaxError => ex
+      # Psych errors includes the absolute path to the file, so no need to add that
+      # to the message
+      raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
+    end
+  end
+
+  def decrypt_value(value, context)
+    case value
+    when String
+      decrypt(value, context)
+    when Hash
+      result = {}
+      value.each_pair { |k, v| result[k] = decrypt_value(v, context) }
+      result
+    when Array
+      value.map { |v| decrypt_value(v, context) }
+    else
+      value
+    end
+  end
+
+  def decrypt(data, context)
+    return context.interpolate(data) unless encrypted?(data)
+    tokens = Hiera::Backend::Eyaml::Parser::ParserFactory.hiera_backend_parser.parse(data)
+    tokens.map(&:to_plain_text).join.chomp
+  end
+
+  def encrypted?(data)
+    /.*ENC\[.*?\]/ =~ data ? true : false
+  end
+end

data/lib/puppet/pops/lookup/global_data_provider.rb

@@ -21,7 +21,18 @@ class GlobalDataProvider < ConfiguredDataProvider
           lookup_invocation.default_values,
           lookup_invocation.explainer)
       end
-      merge = config.merge_strategy if merge.is_a?(DefaultMergeStrategy)
+
+      unless config.merge_strategy.is_a?(DefaultMergeStrategy)
+        if lookup_invocation.hiera_xxx_call?
+          # Merge strategy of the hiera_xxx call should only be applied when no merge strategy is defined in the hiera config
+          merge = config.merge_strategy
+          lookup_invocation.set_hiera_v3_merge_behavior
+        elsif merge.is_a?(DefaultMergeStrategy)
+          # For all other calls, the strategy of the call overrides the strategy defined in the hiera config
+          merge = config.merge_strategy
+          lookup_invocation.set_hiera_v3_merge_behavior
+        end
+      end
     end
     super(key, lookup_invocation, merge)
   end

data/lib/puppet/pops/lookup/hiera_config.rb

@@ -44,6 +44,7 @@ class HieraConfig
   KEY_LOOKUP_KEY = LookupKeyFunctionProvider::TAG
   KEY_DATA_DIG = DataDigFunctionProvider::TAG
   KEY_V3_DATA_HASH = V3DataHashFunctionProvider::TAG
+  KEY_V3_LOOKUP_KEY = V3LookupKeyFunctionProvider::TAG
   KEY_V3_BACKEND = V3BackendFunctionProvider::TAG
   KEY_V4_DATA_HASH = V4DataHashFunctionProvider::TAG
   KEY_BACKEND = 'backend'.freeze
@@ -57,6 +58,7 @@ class HieraConfig
     KEY_LOOKUP_KEY => LookupKeyFunctionProvider,
     KEY_V3_DATA_HASH => V3DataHashFunctionProvider,
     KEY_V3_BACKEND => V3BackendFunctionProvider,
+    KEY_V3_LOOKUP_KEY => V3LookupKeyFunctionProvider,
     KEY_V4_DATA_HASH => V4DataHashFunctionProvider
   }
 
@@ -294,7 +296,7 @@ class HieraConfigV3 < HieraConfig
 
     [@config[KEY_BACKENDS]].flatten.each do |backend|
       raise Puppet::DataBinding::LookupError, "#{@config_path}: Backend '#{backend}' defined more than once" if data_providers.include?(backend)
-      original_paths = @config[KEY_HIERARCHY]
+      original_paths = [@config[KEY_HIERARCHY]].flatten
       backend_config = @config[backend] || EMPTY_HASH
       datadir = Pathname(interpolate(backend_config[KEY_DATADIR] || default_datadir, lookup_invocation, false))
       ext = backend == 'hocon' ? '.conf' : ".#{backend}"
@@ -304,6 +306,8 @@ class HieraConfigV3 < HieraConfig
         create_data_provider(backend, parent_data_provider, KEY_V3_DATA_HASH, "#{backend}_data", { KEY_DATADIR => datadir }, paths)
       when backend == 'hocon' && Puppet.features.hocon?
         create_data_provider(backend, parent_data_provider, KEY_V3_DATA_HASH, 'hocon_data', { KEY_DATADIR => datadir }, paths)
+      when backend == 'eyaml' && Puppet.features.hiera_eyaml?
+        create_data_provider(backend, parent_data_provider, KEY_V3_LOOKUP_KEY, 'eyaml_lookup_key', backend_config.merge(KEY_DATADIR => datadir), paths)
       else
         create_hiera3_backend_provider(backend, backend, parent_data_provider, datadir, paths, @loaded_config)
       end
@@ -352,10 +356,8 @@ class HieraConfigV3 < HieraConfig
   def create_merge_strategy
     key = @config[KEY_MERGE_BEHAVIOR]
     case key
-    when nil
+    when nil, 'native'
       MergeStrategy.strategy(nil)
-    when 'native'
-      MergeStrategy.strategy(:first)
     when 'array'
       MergeStrategy.strategy(:unique)
     when 'deep', 'deeper'

data/lib/puppet/pops/lookup/invocation.rb

@@ -38,6 +38,8 @@ class Invocation
     else
       @name_stack = parent_invocation.name_stack
       @adapter_class = parent_invocation.adapter_class
+      set_hiera_xxx_call if parent_invocation.hiera_xxx_call?
+      set_hiera_v3_merge_behavior if parent_invocation.hiera_v3_merge_behavior?
       set_global_only if parent_invocation.global_only?
       povr = parent_invocation.hiera_v3_location_overrides
       set_hiera_v3_location_overrides(povr) unless povr.empty?
@@ -193,6 +195,24 @@ class Invocation
     lookup_adapter.global_hiera_config_path
   end
 
+  # @return [Boolean] `true` if the invocation stems from the hiera_xxx function family
+  def hiera_xxx_call?
+    instance_variable_defined?(:@hiera_xxx_call)
+  end
+
+  def set_hiera_xxx_call
+    @hiera_xxx_call = true
+  end
+
+  # @return [Boolean] `true` if the invocation stems from the hiera_xxx function family
+  def hiera_v3_merge_behavior?
+    instance_variable_defined?(:@hiera_v3_merge_behavior)
+  end
+
+  def set_hiera_v3_merge_behavior
+    @hiera_v3_merge_behavior = true
+  end
+
   # Overrides passed from hiera_xxx functions down to V3DataHashFunctionProvider
   def set_hiera_v3_location_overrides(overrides)
     @hiera_v3_location_overrides = [overrides].flatten unless overrides.nil?

data/lib/puppet/pops/lookup/location_resolver.rb

@@ -35,8 +35,8 @@ module Lookup
 
     def expand_globs(datadir, declared_globs, lookup_invocation)
       declared_globs.map do |declared_glob|
-        glob = interpolate(declared_glob, lookup_invocation, false)
-        Pathname.glob(datadir, glob).reject { |path| path.directory? }.map { |path| ResolvedLocation.new(glob, path, true) }
+        glob = datadir + interpolate(declared_glob, lookup_invocation, false)
+        Pathname.glob(glob).reject { |path| path.directory? }.map { |path| ResolvedLocation.new(glob.to_s, path, true) }
       end.flatten
     end
 

data/lib/puppet/pops/lookup/lookup_key_function_provider.rb

@@ -14,17 +14,22 @@ class LookupKeyFunctionProvider < FunctionProvider
   # @return [Object] the found object
   # @throw :no_such_key when the object is not found
   def unchecked_key_lookup(key, lookup_invocation, merge)
+    root_key = key.root_key
     lookup_invocation.with(:data_provider, self) do
       MergeStrategy.strategy(merge).lookup(locations, lookup_invocation) do |location|
-        if location.nil?
-          value = lookup_key(key.root_key, lookup_invocation, nil, merge)
-          lookup_invocation.report_found(key.root_key, validate_data_value(self, value))
-        else
-          lookup_invocation.with(:location, location) do
-            value = lookup_key(key.root_key, lookup_invocation, location.location, merge)
-            lookup_invocation.report_found(key.root_key, validate_data_value(self, value))
-          end
-        end
+        invoke_with_location(lookup_invocation, location, root_key, merge)
+      end
+    end
+  end
+
+  def invoke_with_location(lookup_invocation, location, root_key, merge)
+    if location.nil?
+      value = lookup_key(root_key, lookup_invocation, nil, merge)
+      lookup_invocation.report_found(root_key, validate_data_value(self, value))
+    else
+      lookup_invocation.with(:location, location) do
+        value = lookup_key(root_key, lookup_invocation, location.location, merge)
+        lookup_invocation.report_found(root_key, validate_data_value(self, value))
       end
     end
   end
@@ -52,12 +57,43 @@ class LookupKeyFunctionProvider < FunctionProvider
   end
 end
 
+# @api private
+class V3LookupKeyFunctionProvider < LookupKeyFunctionProvider
+  TAG = 'v3_lookup_key'.freeze
+
+  def initialize(name, parent_data_provider, function_name, options, locations)
+    @datadir = options.delete(HieraConfig::KEY_DATADIR)
+    super
+  end
+
+  def unchecked_key_lookup(key, lookup_invocation, merge)
+    extra_paths = lookup_invocation.hiera_v3_location_overrides
+    if extra_paths.nil? || extra_paths.empty?
+      super
+    else
+      # Extra paths provided. Must be resolved and placed in front of known paths
+      paths = parent_data_provider.config(lookup_invocation).resolve_paths(@datadir, extra_paths, lookup_invocation, false, ".#{@name}")
+      all_locations = paths + locations
+      root_key = key.root_key
+      lookup_invocation.with(:data_provider, self) do
+        MergeStrategy.strategy(merge).lookup(all_locations, lookup_invocation) do |location|
+          invoke_with_location(lookup_invocation, location, root_key, merge)
+        end
+      end
+    end
+  end
+end
+
 class V3BackendFunctionProvider < LookupKeyFunctionProvider
   TAG = 'hiera3_backend'.freeze
 
   def lookup_key(key, lookup_invocation, location, merge)
     @backend ||= instantiate_backend(lookup_invocation)
-    @backend.lookup(key, lookup_invocation.scope, lookup_invocation.hiera_v3_location_overrides, convert_merge(merge), context = {:recurse_guard => nil})
+    config = parent_data_provider.config(lookup_invocation)
+
+    # Never pass hiera.yaml defined merge_behavior down to the backend. It will pick it up from the config
+    resolution_type = lookup_invocation.hiera_v3_merge_behavior? ? nil : convert_merge(merge)
+    @backend.lookup(key, lookup_invocation.scope, lookup_invocation.hiera_v3_location_overrides, resolution_type, context = {:recurse_guard => nil})
   end
 
   private

data/lib/puppet/provider/package/rpm.rb

@@ -19,7 +19,7 @@ Puppet::Type.type(:package).provide :rpm, :source => :rpm, :parent => Puppet::Pr
   # eventually become this Puppet::Type::Package::ProviderRpm class.
   # The query format by which we identify installed packages
   self::NEVRA_FORMAT = %Q{%{NAME} %|EPOCH?{%{EPOCH}}:{0}| %{VERSION} %{RELEASE} %{ARCH}\\n}
-  self::NEVRA_REGEX  = %r{^(\S+) (\S+) (\S+) (\S+) (\S+)$}
+  self::NEVRA_REGEX  = %r{^'?(\S+) (\S+) (\S+) (\S+) (\S+)$}
   self::NEVRA_FIELDS = [:name, :epoch, :version, :release, :arch]
 
   ARCH_LIST = [

data/lib/puppet/version.rb

@@ -7,7 +7,7 @@
 
 
 module Puppet
-  PUPPETVERSION = '4.9.2'
+  PUPPETVERSION = '4.9.3'
 
   ##
   # version is a public API method intended to always provide a fast and

data/lib/semver.rb

@@ -29,10 +29,10 @@ class SemVer < Numeric
         SemVer.new(pre(r), suppress_deprecation_warning) .. SemVer.new(r, suppress_deprecation_warning)
       when SemVer::SIMPLE_RANGE
         r += ".0" unless SemVer.valid?(r.gsub(/x/i, '0'))
-        SemVer.new(r.gsub(/x/i, '0'))...SemVer.new(r.gsub(/(\d+)\.x/i) { "#{$1.to_i + 1}.0" } + '-', suppress_deprecation_warning)
+        SemVer.new(r.gsub(/x/i, '0'), suppress_deprecation_warning)...SemVer.new(r.gsub(/(\d+)\.x/i) { "#{$1.to_i + 1}.0" } + '-', suppress_deprecation_warning)
       when /\s+-\s+/
         a, b = r.split(/\s+-\s+/)
-        SemVer.new(pre(a)) .. SemVer.new(b, suppress_deprecation_warning)
+        SemVer.new(pre(a), suppress_deprecation_warning) .. SemVer.new(b, suppress_deprecation_warning)
       when /^~/
         ver = r.sub(/~/, '').split('.').map(&:to_i)
         start = (ver + [0] * (3 - ver.length)).join('.')

data/spec/unit/functions/hiera_spec.rb

@@ -8,107 +8,123 @@ describe 'when calling' do
 
   let(:global_dir) { tmpdir('global') }
   let(:env_config) { {} }
-  let(:global_files) do
+  let(:hiera_yaml) { <<-YAML.unindent }
+    ---
+    :backends:
+      - yaml
+      - custom
+    :yaml:
+      :datadir: #{global_dir}/hieradata
+    :hierarchy:
+      - first
+      - second
+    YAML
+
+  let(:ruby_stuff_files) do
+    {
+      'hiera' => {
+        'backend' => {
+          'custom_backend.rb' => <<-RUBY.unindent
+            class Hiera::Backend::Custom_backend
+              def initialize(cache = nil)
+                Hiera.debug('Custom_backend starting')
+              end
+
+              def lookup(key, scope, order_override, resolution_type, context)
+                case key
+                when 'datasources'
+                  Hiera::Backend.datasources(scope, order_override) { |source| source }
+                when 'resolution_type'
+                  "resolution_type=\#{resolution_type}"
+                else
+                  throw :no_such_key
+                end
+              end
+            end
+            RUBY
+        }
+      }
+    }
+  end
+
+  let(:hieradata_files) do
     {
-      'hiera.yaml' => <<-YAML.unindent,
+      'first.yaml' => <<-YAML.unindent,
         ---
-        :backends:
-          - yaml
-          - custom
-        :yaml:
-          :datadir: #{global_dir}/hieradata
-        :hierarchy:
-          - first
-          - second
+        a: first a
+        class_name: "-- %{calling_class} --"
+        class_path: "-- %{calling_class_path} --"
+        module: "-- %{calling_module} --"
+        mod_name: "-- %{module_name} --"
+        database_user:
+          name: postgres
+          uid: 500
+          gid: 500
+        b:
+          b1: first b1
+          b2: first b2
+        fbb:
+          - mod::foo
+          - mod::bar
+          - mod::baz
+        empty_array: []
         YAML
-      'ruby_stuff' => {
-        'hiera' => {
-          'backend' => {
-            'custom_backend.rb' => <<-RUBY.unindent
-                    class Hiera::Backend::Custom_backend
-                      def initialize(cache = nil)
-                        Hiera.debug('Custom_backend starting')
-                      end
-      
-                      def lookup(key, scope, order_override, resolution_type, context)
-                        case key
-                        when 'datasources'
-                          Hiera::Backend.datasources(scope, order_override) { |source| source }
-                        else
-                          throw :no_such_key
-                        end
-                      end
-                    end
-              RUBY
-          }
-        }
-      },
-      'hieradata' => {
-        'first.yaml' => <<-YAML.unindent,
-                ---
-                a: first a
-                class_name: "-- %{calling_class} --"
-                class_path: "-- %{calling_class_path} --"
-                module: "-- %{calling_module} --"
-                mod_name: "-- %{module_name} --"
-                database_user:
-                  name: postgres
-                  uid: 500
-                  gid: 500
-                b:
-                  b1: first b1
-                  b2: first b2
-                fbb:
-                  - mod::foo
-                  - mod::bar
-                  - mod::baz
-                empty_array: []
-          YAML
-        'second.yaml' => <<-YAML.unindent,
-          ---
-          a: second a
-          b:
-            b1: second b1
-            b3: second b3
-          YAML
-        'the_override.yaml' => <<-YAML.unindent
-          ---
-          key: foo_result
-          YAML
-      },
-      'environments' => {
-        'test' => {
-          'modules' => {
-            'mod' => {
-              'manifests' => {
-                'foo.pp' => <<-PUPPET.unindent,
-                    class mod::foo {
-                      notice(hiera('class_name'))
-                      notice(hiera('class_path'))
-                      notice(hiera('module'))
-                      notice(hiera('mod_name'))
-                    }
-                  PUPPET
-                'bar.pp' => <<-PUPPET.unindent,
-                    class mod::bar {}
-                  PUPPET
-                'baz.pp' => <<-PUPPET.unindent
-                     class mod::baz {}
-                  PUPPET
+      'second.yaml' => <<-YAML.unindent,
+        ---
+        a: second a
+        b:
+          b1: second b1
+          b3: second b3
+        YAML
+      'the_override.yaml' => <<-YAML.unindent
+        ---
+        key: foo_result
+        YAML
+    }
+  end
+
+  let(:environment_files) do
+    {
+      'test' => {
+        'modules' => {
+          'mod' => {
+            'manifests' => {
+              'foo.pp' => <<-PUPPET.unindent,
+                class mod::foo {
+                  notice(hiera('class_name'))
+                  notice(hiera('class_path'))
+                  notice(hiera('module'))
+                  notice(hiera('mod_name'))
+                }
+                PUPPET
+              'bar.pp' => <<-PUPPET.unindent,
+                class mod::bar {}
+                PUPPET
+              'baz.pp' => <<-PUPPET.unindent
+                class mod::baz {}
+                PUPPET
               },
-              'hiera.yaml' => <<-YAML.unindent,
-                ---
-                version: 5
+            'hiera.yaml' => <<-YAML.unindent,
+              ---
+              version: 5
+              YAML
+            'data' => {
+              'common.yaml' => <<-YAML.unindent
+                mod::c: mod::c (from module)
                 YAML
-              'data' => {
-                 'common.yaml' => <<-YAML.unindent
-                    mod::c: mod::c (from module)
-                  YAML
-              }
             }
           }
         }
-      }
+      }.merge(env_config)
+    }
+  end
+
+  let(:global_files) do
+    {
+      'hiera.yaml' => hiera_yaml,
+      'ruby_stuff' => ruby_stuff_files,
+      'hieradata' => hieradata_files,
+      'environments' => environment_files
     }
   end
 
@@ -129,7 +145,7 @@ describe 'when calling' do
   around(:each) do |example|
     # Faking the load path to enable 'require' to load from 'ruby_stuff'. It removes the need for a static fixture
     # for the custom backend
-    dir_contained_in(global_dir, DeepMerge.deep_merge!(global_files, { 'environments' => { 'test' => env_config } } ))
+    dir_contained_in(global_dir, global_files)
     $LOAD_PATH.unshift(File.join(global_dir, 'ruby_stuff'))
     begin
       Puppet.override(:environments => environments, :current_environment => env) do
@@ -332,4 +348,70 @@ describe 'when calling' do
       expect(func('foo') { |k| ['mod::bar', "mod::#{k}"] }.map { |c| c.class_name }).to eql(%w[mod::bar mod::foo])
     end
   end
+
+  context 'with merge_behavior declared in hiera.yaml' do
+    let(:hiera_yaml) do
+      <<-YAML.unindent
+        ---
+        :backends:
+          - yaml
+          - custom
+        :yaml:
+          :datadir: #{global_dir}/hieradata
+        :hierarchy:
+          - other
+          - common
+        :merge_behavior: deeper
+        YAML
+    end
+
+    let(:global_files) do
+      {
+        'hiera.yaml' => hiera_yaml,
+        'hieradata' => {
+          'common.yaml' => <<-YAML.unindent,
+            dm:
+              dm1:
+                dm11: value of dm11 (from common)
+                dm12: value of dm12 (from common)
+              dm2:
+                dm21: value of dm21 (from common)
+            YAML
+          'other.yaml' => <<-YAML.unindent,
+            dm:
+              dm1:
+                dm11: value of dm11 (from other)
+                dm13: value of dm13 (from other)
+              dm3:
+                dm31: value of dm31 (from other)
+            YAML
+        },
+        'ruby_stuff' => ruby_stuff_files
+      }
+    end
+
+    context 'hiera_hash' do
+      let(:the_func) { Puppet.lookup(:loaders).puppet_system_loader.load(:function, 'hiera_hash') }
+
+      it 'the imposed hash strategy does not override declared merge_behavior' do
+        expect(func('dm')).to eql({
+          'dm1' => {
+            'dm11' => 'value of dm11 (from other)',
+            'dm12' => 'value of dm12 (from common)',
+            'dm13' => 'value of dm13 (from other)'
+          },
+          'dm2' => {
+            'dm21' => 'value of dm21 (from common)'
+          },
+          'dm3' => {
+            'dm31' => 'value of dm31 (from other)'
+          }
+        })
+      end
+
+      it "the merge behavior is not propagated to a custom backend as 'resolution_type'" do
+        expect(func('resolution_type')).to eql('resolution_type=')
+      end
+    end
+  end
 end