hiera 1.3.4 → 2.0.0

data/lib/hiera/backend/json_backend.rb

@@ -9,8 +9,9 @@ class Hiera
         @cache = cache || Filecache.new
       end
 
-      def lookup(key, scope, order_override, resolution_type)
+      def lookup(key, scope, order_override, resolution_type, context)
         answer = nil
+        found = false
 
         Hiera.debug("Looking up #{key} in JSON backend")
 
@@ -27,28 +28,29 @@ class Hiera
 
           next if data.empty?
           next unless data.include?(key)
+          found = true
 
           # for array resolution we just append to the array whatever
           # we find, we then goes onto the next file and keep adding to
           # the array
           #
           # for priority searches we break after the first found data item
-          new_answer = Backend.parse_answer(data[key], scope)
-          case resolution_type
+          new_answer = Backend.parse_answer(data[key], scope, {}, context)
+          case resolution_type.is_a?(Hash) ? :hash : resolution_type
           when :array
-            raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+            raise Exception, "Hiera type mismatch for key '#{key}': expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
             answer ||= []
             answer << new_answer
           when :hash
-            raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+            raise Exception, "Hiera type mismatch for key '#{key}': expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
             answer ||= {}
-            answer = Backend.merge_answer(new_answer,answer)
+            answer = Backend.merge_answer(new_answer, answer, resolution_type)
           else
             answer = new_answer
             break
           end
         end
-
+        throw :no_such_key unless found
         return answer
       end
     end

data/lib/hiera/backend/yaml_backend.rb

@@ -8,8 +8,9 @@ class Hiera
         @cache = cache || Filecache.new
       end
 
-      def lookup(key, scope, order_override, resolution_type)
+      def lookup(key, scope, order_override, resolution_type, context)
         answer = nil
+        found = false
 
         Hiera.debug("Looking up #{key} in YAML backend")
 
@@ -20,6 +21,7 @@ class Hiera
 
           next if data.empty?
           next unless data.include?(key)
+          found = true
 
           # Extra logging that we found the key. This can be outputted
           # multiple times if the resolution type is array or hash but that
@@ -32,22 +34,22 @@ class Hiera
           # the array
           #
           # for priority searches we break after the first found data item
-          new_answer = Backend.parse_answer(data[key], scope)
-          case resolution_type
+          new_answer = Backend.parse_answer(data[key], scope, {}, context)
+          case resolution_type.is_a?(Hash) ? :hash : resolution_type
           when :array
-            raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+            raise Exception, "Hiera type mismatch for key '#{key}': expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
             answer ||= []
             answer << new_answer
           when :hash
-            raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+            raise Exception, "Hiera type mismatch for key '#{key}': expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
             answer ||= {}
-            answer = Backend.merge_answer(new_answer,answer)
+            answer = Backend.merge_answer(new_answer, answer, resolution_type)
           else
             answer = new_answer
             break
           end
         end
-
+        throw :no_such_key unless found
         return answer
       end
 

data/lib/hiera/config.rb

@@ -54,9 +54,7 @@ class Hiera::Config
         begin
           require "deep_merge"
         rescue LoadError
-          Hiera.warn "Ignoring configured merge_behavior"
-          Hiera.warn "Must have 'deep_merge' gem installed."
-          @config[:merge_behavior] = :native
+          raise Hiera::Error, "Must have 'deep_merge' gem installed for the configured merge_behavior."
         end
       end
     end

data/lib/hiera/interpolate.rb

@@ -1,30 +1,57 @@
 require 'hiera/backend'
 require 'hiera/recursive_guard'
 
+
+class Hiera::InterpolationInvalidValue < StandardError; end
+
 class Hiera::Interpolate
   class << self
     INTERPOLATION = /%\{([^\}]*)\}/
-    METHOD_INTERPOLATION = /%\{(scope|hiera)\(['"]([^"']*)["']\)\}/
+    METHOD_INTERPOLATION = /%\{(scope|hiera|literal|alias)\(['"]([^"']*)["']\)\}/
 
-    def interpolate(data, scope, extra_data)
+    def interpolate(data, scope, extra_data, context)
       if data.is_a?(String)
         # Wrapping do_interpolation in a gsub block ensures we process
         # each interpolation site in isolation using separate recursion guards.
+        context ||= {}
+        new_context = context.clone
+        new_context[:recurse_guard] ||= Hiera::RecursiveGuard.new
         data.gsub(INTERPOLATION) do |match|
-          do_interpolation(match, Hiera::RecursiveGuard.new, scope, extra_data)
+          interp_val = do_interpolation(match, scope, extra_data, new_context)
+
+          # Get interp method in case we are aliasing
+          if data.is_a?(String) && (match = data.match(INTERPOLATION))
+            interpolate_method, key = get_interpolation_method_and_key(data)
+          else
+            interpolate_method = nil
+          end
+
+          if ( (interpolate_method == :alias_interpolate) and (!interp_val.is_a?(String)) )
+            if data.match("^#{INTERPOLATION}$")
+              return interp_val
+            else
+              raise Hiera::InterpolationInvalidValue, "Cannot call alias in the string context"
+            end
+          else
+            interp_val
+          end
         end
       else
         data
       end
     end
 
-    def do_interpolation(data, recurse_guard, scope, extra_data)
+    def do_interpolation(data, scope, extra_data, context)
       if data.is_a?(String) && (match = data.match(INTERPOLATION))
         interpolation_variable = match[1]
-        recurse_guard.check(interpolation_variable) do
+        context[:recurse_guard].check(interpolation_variable) do
           interpolate_method, key = get_interpolation_method_and_key(data)
-          interpolated_data = send(interpolate_method, data, key, scope, extra_data)
-          do_interpolation(interpolated_data, recurse_guard, scope, extra_data)
+          interpolated_data = send(interpolate_method, data, key, scope, extra_data, context)
+
+          # Halt recursion if we encounter a literal.
+          return interpolated_data if interpolate_method == :literal_interpolate
+
+          do_interpolation(interpolated_data, scope, extra_data, context)
         end
       else
         data
@@ -37,6 +64,8 @@ class Hiera::Interpolate
         case match[1]
         when 'hiera' then [:hiera_interpolate, match[2]]
         when 'scope' then [:scope_interpolate, match[2]]
+        when 'literal' then [:literal_interpolate, match[2]]
+        when 'alias' then [:alias_interpolate, match[2]]
         end
       elsif (match = data.match(INTERPOLATION))
         [:scope_interpolate, match[1]]
@@ -44,19 +73,26 @@ class Hiera::Interpolate
     end
     private :get_interpolation_method_and_key
 
-    def scope_interpolate(data, key, scope, extra_data)
-      value = scope[key]
-      if value.nil? || value == :undefined
-        value = extra_data[key]
-      end
-
-      value
+    def scope_interpolate(data, key, scope, extra_data, context)
+      segments = key.split('.')
+      catch(:no_such_key) { return Hiera::Backend.qualified_lookup(segments, scope) }
+      catch(:no_such_key) { Hiera::Backend.qualified_lookup(segments, extra_data) }
     end
     private :scope_interpolate
 
-    def hiera_interpolate(data, key, scope, extra_data)
-      Hiera::Backend.lookup(key, nil, scope, nil, :priority)
+    def hiera_interpolate(data, key, scope, extra_data, context)
+      Hiera::Backend.lookup(key, nil, scope, context[:order_override], :priority, context)
     end
     private :hiera_interpolate
+
+    def literal_interpolate(data, key, scope, extra_data, context)
+      key
+    end
+    private :literal_interpolate
+
+    def alias_interpolate(data, key, scope, extra_data, context)
+      Hiera::Backend.lookup(key, nil, scope, context[:order_override], :priority, context)
+    end
+    private :alias_interpolate
   end
 end

data/lib/hiera/util.rb

@@ -9,7 +9,7 @@ class Hiera
 
     def microsoft_windows?
       return false unless file_alt_separator
-      
+
       begin
         require 'win32/dir'
         true
@@ -21,17 +21,17 @@ class Hiera
 
     def config_dir
       if microsoft_windows?
-         File.join(common_appdata, 'PuppetLabs', 'hiera', 'etc')
+         File.join(common_appdata, 'PuppetLabs', 'code')
       else
-        '/etc'
+        '/etc/puppetlabs/code'
       end
     end
 
     def var_dir
       if microsoft_windows?
-        File.join(common_appdata, 'PuppetLabs', 'hiera', 'var')
+        File.join(common_appdata, 'PuppetLabs', 'code', 'hieradata')
       else
-        '/var/lib/hiera'
+        '/etc/puppetlabs/code/hieradata'
       end
     end
 

data/lib/hiera/version.rb

@@ -7,7 +7,7 @@
 
 
 class Hiera
-  VERSION = "1.3.4"
+  VERSION = "2.0.0"
 
   ##
   # version is a public API method intended to always provide a fast and

data/spec/spec_helper.rb

@@ -9,6 +9,13 @@ require 'tmpdir'
 RSpec.configure do |config|
   config.mock_with :mocha
 
+  if Hiera::Util.microsoft_windows? && RUBY_VERSION =~ /^1\./
+    require 'win32console'
+    config.output_stream = $stdout
+    config.error_stream = $stderr
+    config.formatters.each { |f| f.instance_variable_set(:@output, $stdout) }
+  end
+
   config.after :suite do
     # Log the spec order to a file, but only if the LOG_SPEC_ORDER environment variable is
     #  set.  This should be enabled on Jenkins runs, as it can be used with Nick L.'s bisect
@@ -21,6 +28,11 @@ RSpec.configure do |config|
   end
 end
 
+# So everyone else doesn't have to include this base constant.
+module HieraSpec
+  FIXTURE_DIR = File.join(dir = File.expand_path(File.dirname(__FILE__)), 'unit', 'fixtures') unless defined?(FIXTURE_DIR)
+end
+
 # In ruby 1.8.5 Dir does not have mktmpdir defined, so this monkey patches
 # Dir to include the 1.8.7 definition of that method if it isn't already defined.
 # Method definition borrowed from ruby-1.8.7-p357/lib/ruby/1.8/tmpdir.rb

data/spec/unit/backend/json_backend_spec.rb

@@ -25,7 +25,7 @@ class Hiera
           Backend.expects(:datafile).with(:json, {}, "one", "json").returns(nil)
           Backend.expects(:datafile).with(:json, {}, "two", "json").returns(nil)
 
-          @backend.lookup("key", {}, nil, :priority)
+          expect { @backend.lookup("key", {}, nil, :priority, nil) }.to throw_symbol(:no_such_key)
         end
 
         it "should retain the data types found in data files" do
@@ -35,9 +35,9 @@ class Hiera
 
           @cache.expects(:read_file).with("/nonexisting/one.json", Hash).returns({"stringval" => "string", "boolval" => true, "numericval" => 1}).times(3)
 
-          @backend.lookup("stringval", {}, nil, :priority).should == "string"
-          @backend.lookup("boolval", {}, nil, :priority).should == true
-          @backend.lookup("numericval", {}, nil, :priority).should == 1
+          @backend.lookup("stringval", {}, nil, :priority, nil).should == "string"
+          @backend.lookup("boolval", {}, nil, :priority, nil).should == true
+          @backend.lookup("numericval", {}, nil, :priority, nil).should == 1
         end
 
         it "should pick data earliest source that has it for priority searches" do
@@ -49,12 +49,12 @@ class Hiera
           File.stubs(:exist?).with("/nonexisting/one.json").returns(true)
           @cache.expects(:read_file).with("/nonexisting/one.json", Hash).returns({"key" => "test_%{rspec}"})
 
-          @backend.lookup("key", scope, nil, :priority).should == "test_test"
+          @backend.lookup("key", scope, nil, :priority, nil).should == "test_test"
         end
 
         it "should build an array of all data sources for array searches" do
           Hiera::Backend.stubs(:empty_answer).returns([])
-          Backend.stubs(:parse_answer).with('answer', {}).returns("answer")
+          Backend.stubs(:parse_answer).with('answer', {}, {}, anything).returns("answer")
           Backend.expects(:datafile).with(:json, {}, "one", "json").returns("/nonexisting/one.json")
           Backend.expects(:datafile).with(:json, {}, "two", "json").returns("/nonexisting/two.json")
 
@@ -66,18 +66,18 @@ class Hiera
           @cache.expects(:read_file).with("/nonexisting/one.json", Hash).returns({"key" => "answer"})
           @cache.expects(:read_file).with("/nonexisting/two.json", Hash).returns({"key" => "answer"})
 
-          @backend.lookup("key", {}, nil, :array).should == ["answer", "answer"]
+          @backend.lookup("key", {}, nil, :array, nil).should == ["answer", "answer"]
         end
 
         it "should parse the answer for scope variables" do
-          Backend.stubs(:parse_answer).with('test_%{rspec}', {'rspec' => 'test'}).returns("test_test")
+          Backend.stubs(:parse_answer).with('test_%{rspec}', {'rspec' => 'test'}, {}, anything).returns("test_test")
           Backend.expects(:datasources).yields("one")
           Backend.expects(:datafile).with(:json, {"rspec" => "test"}, "one", "json").returns("/nonexisting/one.json")
 
           File.expects(:exist?).with("/nonexisting/one.json").returns(true)
           @cache.expects(:read_file).with("/nonexisting/one.json", Hash).returns({"key" => "test_%{rspec}"})
 
-          @backend.lookup("key", {"rspec" => "test"}, nil, :priority).should == "test_test"
+          @backend.lookup("key", {"rspec" => "test"}, nil, :priority, nil).should == "test_test"
         end
       end
     end

data/spec/unit/backend/yaml_backend_spec.rb

@@ -41,7 +41,7 @@ class Hiera
           Backend.expects(:datasourcefiles).with(:yaml, {}, "yaml", nil).yields(["one", "/nonexisting/one.yaml"])
           @cache.value = "---\nkey: answer"
 
-          @backend.lookup("key", {}, nil, :priority).should == "answer"
+          @backend.lookup("key", {}, nil, :priority, nil).should == "answer"
         end
 
         describe "handling unexpected YAML values" do
@@ -49,19 +49,24 @@ class Hiera
             Backend.expects(:datasourcefiles).with(:yaml, {}, "yaml", nil).yields(["one", "/nonexisting/one.yaml"])
           end
 
-          it "returns nil when the YAML value is nil" do
+          it "throws :no_such_key when key is missing in YAML" do
             @cache.value = "---\n"
-            @backend.lookup("key", {}, nil, :priority).should be_nil
+            expect { @backend.lookup("key", {}, nil, :priority, nil) }.to throw_symbol(:no_such_key)
+          end
+
+          it "returns nil when the YAML value is nil" do
+            @cache.value = "key: ~\n"
+            @backend.lookup("key", {}, nil, :priority, nil).should be_nil
           end
 
-          it "returns nil when the YAML file is false" do
+          it "throws :no_such_key when the YAML file is false" do
             @cache.value = ""
-            @backend.lookup("key", {}, nil, :priority).should be_nil
+            expect { @backend.lookup("key", {}, nil, :priority, nil) }.to throw_symbol(:no_such_key)
           end
 
           it "raises a TypeError when the YAML value is not a hash" do
             @cache.value = "---\n[one, two, three]"
-            expect { @backend.lookup("key", {}, nil, :priority) }.to raise_error(TypeError)
+            expect { @backend.lookup("key", {}, nil, :priority, nil) }.to raise_error(TypeError)
           end
         end
 
@@ -70,7 +75,7 @@ class Hiera
           @cache.expects(:read_file).with("/nonexisting/one.yaml", Hash).returns({"key"=>"answer"})
           @cache.expects(:read_file).with("/nonexisting/two.yaml", Hash).returns({"key"=>"answer"})
 
-          @backend.lookup("key", {}, nil, :array).should == ["answer", "answer"]
+          @backend.lookup("key", {}, nil, :array, nil).should == ["answer", "answer"]
         end
 
         it "should ignore empty hash of data sources for hash searches" do
@@ -79,7 +84,7 @@ class Hiera
           @cache.expects(:read_file).with("/nonexisting/one.yaml", Hash).returns({})
           @cache.expects(:read_file).with("/nonexisting/two.yaml", Hash).returns({"key"=>{"a"=>"answer"}})
 
-          @backend.lookup("key", {}, nil, :hash).should == {"a" => "answer"}
+          @backend.lookup("key", {}, nil, :hash, nil).should == {"a" => "answer"}
         end
 
         it "should build a merged hash of data sources for hash searches" do
@@ -88,7 +93,7 @@ class Hiera
           @cache.expects(:read_file).with("/nonexisting/one.yaml", Hash).returns({"key"=>{"a"=>"answer"}})
           @cache.expects(:read_file).with("/nonexisting/two.yaml", Hash).returns({"key"=>{"b"=>"answer", "a"=>"wrong"}})
 
-          @backend.lookup("key", {}, nil, :hash).should == {"a" => "answer", "b" => "answer"}
+          @backend.lookup("key", {}, nil, :hash, nil).should == {"a" => "answer", "b" => "answer"}
         end
 
         it "should fail when trying to << a Hash" do
@@ -97,7 +102,7 @@ class Hiera
           @cache.expects(:read_file).with("/nonexisting/one.yaml", Hash).returns({"key"=>["a", "answer"]})
           @cache.expects(:read_file).with("/nonexisting/two.yaml", Hash).returns({"key"=>{"a"=>"answer"}})
 
-          expect {@backend.lookup("key", {}, nil, :array)}.to raise_error(Exception, "Hiera type mismatch: expected Array and got Hash")
+          expect {@backend.lookup("key", {}, nil, :array, nil)}.to raise_error(Exception, "Hiera type mismatch for key 'key': expected Array and got Hash")
         end
 
         it "should fail when trying to merge an Array" do
@@ -106,7 +111,7 @@ class Hiera
           @cache.expects(:read_file).with("/nonexisting/one.yaml", Hash).returns({"key"=>{"a"=>"answer"}})
           @cache.expects(:read_file).with("/nonexisting/two.yaml", Hash).returns({"key"=>["a", "wrong"]})
 
-          expect { @backend.lookup("key", {}, nil, :hash) }.to raise_error(Exception, "Hiera type mismatch: expected Hash and got Array")
+          expect { @backend.lookup("key", {}, nil, :hash, nil) }.to raise_error(Exception, "Hiera type mismatch for key 'key': expected Hash and got Array")
         end
 
         it "should parse the answer for scope variables" do
@@ -114,7 +119,7 @@ class Hiera
 
           @cache.expects(:read_file).with("/nonexisting/one.yaml", Hash).returns({"key"=>"test_%{rspec}"})
 
-          @backend.lookup("key", {"rspec" => "test"}, nil, :priority).should == "test_test"
+          @backend.lookup("key", {"rspec" => "test"}, nil, :priority, nil).should == "test_test"
         end
 
         it "should retain datatypes found in yaml files" do
@@ -123,9 +128,9 @@ class Hiera
 
           @cache.value = "---\nstringval: 'string'\nboolval: true\nnumericval: 1"
 
-          @backend.lookup("stringval", {}, nil, :priority).should == "string"
-          @backend.lookup("boolval", {}, nil, :priority).should == true
-          @backend.lookup("numericval", {}, nil, :priority).should == 1
+          @backend.lookup("stringval", {}, nil, :priority, nil).should == "string"
+          @backend.lookup("boolval", {}, nil, :priority, nil).should == true
+          @backend.lookup("numericval", {}, nil, :priority, nil).should == 1
         end
       end
     end

data/spec/unit/backend_spec.rb

@@ -2,6 +2,14 @@ require 'spec_helper'
 require 'hiera/util'
 
 class Hiera
+  module Backend
+    class Backend1x_backend
+      def lookup(key, scope, order_override, resolution_type)
+        ["a", "b"]
+      end
+    end
+  end
+
   describe Backend do
     describe "#datadir" do
       it "interpolates any values in the configured value" do
@@ -84,7 +92,7 @@ class Hiera
       end
 
       it "parses the names of the hierarchy levels using the given scope" do
-        Backend.expects(:parse_string).with("common", {:rspec => :tests})
+        Backend.expects(:parse_string).with("common", {:rspec => :tests}, {}, {:order_override => nil})
         Backend.datasources({:rspec => :tests}) { }
       end
 
@@ -181,17 +189,6 @@ class Hiera
         "test_%{scope('rspec')}_test" => "test__test"
       }
 
-      @interprets_undefined_in_scope_tests.each do |input, expected|
-        it "interprets :undefined in scope as a non-value" do
-          Backend.parse_string(input, {"rspec" => :undefined}).should == expected
-        end
-      end
-
-      it "uses the value from extra_data when scope is :undefined" do
-        input = "test_%{rspec}_test"
-        Backend.parse_string(input, {"rspec" => :undefined}, { "rspec" => "extra" }).should == "test_extra_test"
-      end
-
       @exact_lookup_tests = {
         "test_%{::rspec::data}_test" => "test_value_test",
         "test_%{scope('::rspec::data')}_test" => "test_value_test"
@@ -264,10 +261,21 @@ class Hiera
         scope = {}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("key1", scope, nil, :priority).returns("answer")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("key1", scope, nil, :priority, instance_of(Hash)).returns("answer")
 
         Backend.parse_string(input, scope).should == "answer"
       end
+
+      it "interpolation passes the order_override back into the backend" do
+        Backend.expects(:lookup).with("lookup::key", nil, {}, "order_override_datasource", :priority, instance_of(Hash))
+        Backend.parse_string("%{hiera('lookup::key')}", {}, {}, {:order_override => "order_override_datasource"})
+      end
+
+      it "replaces literal interpolations with their argument" do
+        scope = {}
+        input = "%{literal('%')}{rspec::data}"
+        Backend.parse_string(input, scope).should == "%{rspec::data}"
+      end
     end
 
     describe "#parse_answer" do
@@ -306,16 +314,47 @@ class Hiera
         scope = {}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("test")
         Backend.parse_answer(input, scope).should == "test_test_test"
       end
 
+      it "interpolates alias lookups with non-string types" do
+        input = "%{alias('rspec')}"
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns(['test', 'test'])
+        Backend.parse_answer(input, scope).should == ['test', 'test']
+      end
+
+      it 'fails if alias interpolation is attempted in a string context with a prefix' do
+        input = "stuff_before%{alias('rspec')}"
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns(['test', 'test'])
+        expect do
+          Backend.parse_answer(input, scope).should == ['test', 'test']
+        end.to raise_error(Hiera::InterpolationInvalidValue, 'Cannot call alias in the string context')
+      end
+
+      it 'fails if alias interpolation is attempted in a string context with a postfix' do
+        input = "%{alias('rspec')}_stiff after"
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns(['test', 'test'])
+        expect do
+          Backend.parse_answer(input, scope).should == ['test', 'test']
+        end.to raise_error(Hiera::InterpolationInvalidValue, 'Cannot call alias in the string context')
+      end
+
       it "interpolates hiera lookups in each string in an array" do
         input = ["test_%{hiera('rspec')}_test", "test_%{hiera('rspec')}_test", ["test_%{hiera('rspec')}_test"]]
         scope = {}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("test")
         Backend.parse_answer(input, scope).should == ["test_test_test", "test_test_test", ["test_test_test"]]
       end
 
@@ -324,7 +363,7 @@ class Hiera
         scope = {}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("test")
         Backend.parse_answer(input, scope).should == {"foo"=>"test_test_test", "bar"=>"test_test_test"}
       end
 
@@ -333,7 +372,7 @@ class Hiera
         scope = {}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("foo")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("foo")
         Backend.parse_answer(input, scope).should == {"foo"=>"test"}
       end
 
@@ -342,7 +381,7 @@ class Hiera
         scope = {}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("foo")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("foo")
         Backend.parse_answer(input, scope).should == {"topkey"=>{"foo" => "test"}}
       end
 
@@ -351,7 +390,7 @@ class Hiera
         scope = {}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("test")
         Backend.parse_answer(input, scope).should == {"foo"=>"test_test_test", "bar"=>["test_test_test", "test_test_test"]}
       end
 
@@ -360,7 +399,7 @@ class Hiera
         scope = {"rspec2" => "scope_rspec"}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("hiera_rspec")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("hiera_rspec")
         Backend.parse_answer(input, scope).should == {"foo"=>"test_hiera_rspec_test", "bar"=>"test_scope_rspec_test"}
       end
 
@@ -369,7 +408,7 @@ class Hiera
         scope = {"rspec" => "scope_rspec"}
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("hiera_rspec")
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority, instance_of(Hash)).returns("hiera_rspec")
         Backend.parse_answer(input, scope).should == "test_hiera_rspec_test_scope_rspec"
       end
 
@@ -431,7 +470,7 @@ class Hiera
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
 
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, nil).returns("answer")
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, nil, instance_of(Hash)).returns("answer")
 
         Backend.lookup("key", "default", {}, nil, nil).should == "answer"
       end
@@ -440,9 +479,9 @@ class Hiera
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
 
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("stringval", {}, nil, nil).returns("string")
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("boolval", {}, nil, nil).returns(false)
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("numericval", {}, nil, nil).returns(1)
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("stringval", {}, nil, nil, instance_of(Hash)).returns("string")
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("boolval", {}, nil, nil, instance_of(Hash)).returns(false)
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("numericval", {}, nil, nil, instance_of(Hash)).returns(1)
 
         Backend.lookup("stringval", "default", {}, nil, nil).should == "string"
         Backend.lookup("boolval", "default", {}, nil, nil).should == false
@@ -524,7 +563,7 @@ class Hiera
         Config.load_backends
 
         Backend.expects(:resolve_answer).with("test_test", :priority).returns("parsed")
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {"rspec" => "test"}, nil, :priority).returns("test_test")
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {"rspec" => "test"}, nil, :priority, instance_of(Hash)).returns("test_test")
 
         Backend.lookup("key", "test_%{rspec}", {"rspec" => "test"}, nil, :priority).should == "parsed"
       end
@@ -533,31 +572,129 @@ class Hiera
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
 
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {"rspec" => "test"}, nil, nil)
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {"rspec" => "test"}, nil, nil, instance_of(Hash)).throws(:no_such_key)
 
         Backend.lookup("key", "test_%{rspec}", {"rspec" => "test"}, nil, nil).should == "test_test"
       end
 
+      it "returns nil instead of the default when key is found with a nil value" do
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {"rspec" => "test"}, nil, nil, instance_of(Hash)).returns(nil)
+
+        Backend.lookup("key", "test_%{rspec}", {"rspec" => "test"}, nil, nil).should == nil
+      end
+
       it "keeps string default data as a string" do
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, nil)
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, nil, instance_of(Hash)).throws(:no_such_key)
         Backend.lookup("key", "test", {}, nil, nil).should == "test"
       end
 
       it "keeps array default data as an array" do
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, :array)
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, :array, instance_of(Hash)).throws(:no_such_key)
         Backend.lookup("key", ["test"], {}, nil, :array).should == ["test"]
       end
 
       it "keeps hash default data as a hash" do
         Config.load({:yaml => {:datadir => "/tmp"}})
         Config.load_backends
-        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, :hash)
+        Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {}, nil, :hash, instance_of(Hash)).throws(:no_such_key)
         Backend.lookup("key", {"test" => "value"}, {}, nil, :hash).should == {"test" => "value"}
       end
+
+      it 'can use qualified key to lookup value in hash' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', {}, nil, nil, instance_of(Hash)).returns({ 'test' => 'value'})
+        Backend.lookup('key.test', 'dflt', {}, nil, nil).should == 'value'
+      end
+
+      it 'can use qualified key to lookup value in array' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', {}, nil, nil, instance_of(Hash)).returns([ 'first', 'second'])
+        Backend.lookup('key.1', 'dflt', {}, nil, nil).should == 'second'
+      end
+
+      it 'will fail when qualified key is partially found but not expected hash' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', {}, nil, nil, instance_of(Hash)).returns(['value 1', 'value 2'])
+        expect do
+          Backend.lookup('key.test', 'dflt', {}, nil, nil)
+        end.to raise_error(Exception, /^Hiera type mismatch:/)
+      end
+
+      it 'will fail when qualified key used with resolution_type :hash' do
+        expect do
+          Backend.lookup('key.test', 'dflt', {}, nil, :hash)
+        end.to raise_error(ArgumentError, /^Resolution type :hash is illegal/)
+      end
+
+      it 'will fail when qualified key used with resolution_type :array' do
+        expect do
+          Backend.lookup('key.test', 'dflt', {}, nil, :array)
+        end.to raise_error(ArgumentError, /^Resolution type :array is illegal/)
+      end
+
+      it 'will succeed when qualified key used with resolution_type :priority' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', {}, nil, :priority, instance_of(Hash)).returns({ 'test' => 'value'})
+        Backend.lookup('key.test', 'dflt', {}, nil, :priority).should == 'value'
+      end
+
+      it 'will fail when qualified key is partially found but not expected array' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', {}, nil, nil, instance_of(Hash)).returns({ 'test' => 'value'})
+        expect do
+          Backend.lookup('key.2', 'dflt', {}, nil, nil)
+        end.to raise_error(Exception, /^Hiera type mismatch:/)
+      end
+
+      it 'will not fail when qualified key is partially not found' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', {}, nil, nil, instance_of(Hash)).returns(nil)
+        Backend.lookup('key.test', 'dflt', {}, nil, nil).should == 'dflt'
+      end
+
+      it 'will not fail when qualified key is array index out of bounds' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', {}, nil, nil, instance_of(Hash)).returns(['value 1', 'value 2'])
+        Backend.lookup('key.33', 'dflt', {}, nil, nil).should == 'dflt'
+      end
+
+      it 'can use qualified key in interpolation to lookup value in hash' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        Hiera::Backend.stubs(:datasourcefiles).yields('foo', 'bar')
+        Hiera::Filecache.any_instance.expects(:read_file).at_most(2).returns({'key' => '%{hiera(\'some.subkey\')}', 'some' => { 'subkey' => 'value' }})
+        Backend.lookup('key', 'dflt', {}, nil, nil).should == 'value'
+      end
+
+      it 'can use qualified key in interpolated default and scope' do
+        Config.load({:yaml => {:datadir => '/tmp'}})
+        Config.load_backends
+        scope = { 'some' => { 'test' => 'value'}}
+        Backend::Yaml_backend.any_instance.expects(:lookup).with('key', scope, nil, nil, instance_of(Hash))
+        Backend.lookup('key.notfound', '%{some.test}', scope, nil, nil).should == 'value'
+      end
+
+      it "handles older backend with 4 argument lookup" do
+        Config.load({})
+        Config.instance_variable_set("@config", {:backends => ["Backend1x"]})
+
+        Hiera.expects(:debug).at_least_once.with(regexp_matches /Using Hiera 1.x backend/)
+        Backend.lookup("key", {}, {"rspec" => "test"}, nil, :priority).should == ["a", "b"]
+      end
     end
 
     describe '#merge_answer' do
@@ -575,13 +712,30 @@ class Hiera
 
       it "uses deep_merge! when configured with :merge_behavior => :deeper" do
         Config.load({:merge_behavior => :deeper})
-        Hash.any_instance.expects('deep_merge!').with({"b" => "bnswer"}).returns({"a" => "answer", "b" => "bnswer"})
+        Hash.any_instance.expects('deep_merge!').with({"b" => "bnswer"}, {}).returns({"a" => "answer", "b" => "bnswer"})
         Backend.merge_answer({"a" => "answer"},{"b" => "bnswer"}).should == {"a" => "answer", "b" => "bnswer"}
       end
 
       it "uses deep_merge when configured with :merge_behavior => :deep" do
         Config.load({:merge_behavior => :deep})
-        Hash.any_instance.expects('deep_merge').with({"b" => "bnswer"}).returns({"a" => "answer", "b" => "bnswer"})
+        Hash.any_instance.expects('deep_merge').with({"b" => "bnswer"}, {}).returns({"a" => "answer", "b" => "bnswer"})
+        Backend.merge_answer({"a" => "answer"},{"b" => "bnswer"}).should == {"a" => "answer", "b" => "bnswer"}
+      end
+
+      it "disregards configuration when 'merge' parameter is given as a Hash" do
+        Config.load({:merge_behavior => :deep})
+        Hash.any_instance.expects('deep_merge!').with({"b" => "bnswer"}, {}).returns({"a" => "answer", "b" => "bnswer"})
+        Backend.merge_answer({"a" => "answer"},{"b" => "bnswer"}, {:behavior => 'deeper' }).should == {"a" => "answer", "b" => "bnswer"}
+      end
+
+      it "propagates deep merge options when given Hash 'merge' parameter" do
+        Hash.any_instance.expects('deep_merge!').with({"b" => "bnswer"}, { :knockout_prefix => '-' }).returns({"a" => "answer", "b" => "bnswer"})
+        Backend.merge_answer({"a" => "answer"},{"b" => "bnswer"}, {:behavior => 'deeper', :knockout_prefix => '-'}).should == {"a" => "answer", "b" => "bnswer"}
+      end
+
+      it "passes Config[:deep_merge_options] into calls to deep_merge" do
+        Config.load({:merge_behavior => :deep, :deep_merge_options => { :knockout_prefix => '-' } })
+        Hash.any_instance.expects('deep_merge').with({"b" => "bnswer"}, {:knockout_prefix => '-'}).returns({"a" => "answer", "b" => "bnswer"})
         Backend.merge_answer({"a" => "answer"},{"b" => "bnswer"}).should == {"a" => "answer", "b" => "bnswer"}
       end
     end