hiera 0.1.1 → 0.2.0

data/bin/hiera

@@ -15,8 +15,7 @@ require 'rubygems'
 require 'hiera'
 require 'optparse'
 
-
-options = {:default => nil, :config => "/etc/hiera.yaml", :scope => {}, :key => nil, :verbose => false}
+options = {:default => nil, :config => "/etc/hiera.yaml", :scope => {}, :key => nil, :verbose => false, :resolution_type => :priority}
 
 class Hiera::Noop_logger
     class << self
@@ -26,11 +25,29 @@ class Hiera::Noop_logger
 end
 
 # Loads the scope from YAML or JSON files
-def load_scope(file, type=:yaml)
-    raise "Cannot find scope #{type} file #{file}" unless File.exist?(file)
-
+def load_scope(source, type=:yaml)
     case type
+    when :mcollective
+        begin
+            require 'mcollective'
+
+            include MCollective::RPC
+
+            util = rpcclient("rpcutil")
+            util.progress = false
+            nodestats = util.custom_request("inventory", {}, source, {"identity" => source}).first
+
+            raise "Failed to retrieve facts for node #{source}: #{nodestats[:statusmsg]}" unless nodestats[:statuscode] == 0
+
+            scope = nodestats[:data][:facts]
+        rescue Exception => e
+            STDERR.puts "MCollective lookup failed: #{e.class}: #{e}"
+            exit 1
+        end
+
     when :yaml
+        raise "Cannot find scope #{type} file #{source}" unless File.exist?(source)
+
         require 'yaml'
 
         # Attempt to load puppet in case we're going to be fed
@@ -40,19 +57,21 @@ def load_scope(file, type=:yaml)
         rescue
         end
 
-        scope = YAML.load_file(file)
+        scope = YAML.load_file(source)
 
         # Puppet makes dumb yaml files that do not promote data reuse.
         scope = scope.values if scope.is_a?(Puppet::Node::Facts)
     when :json
+        raise "Cannot find scope #{type} file #{source}" unless File.exist?(source)
+
         require 'json'
 
-        scope = JSON.load(File.read(file))
+        scope = JSON.load(File.read(source))
     else
         raise "Don't know how to load data type #{type}"
     end
 
-    raise "Scope from #{type} file #{file} should be a Hash" unless scope.is_a?(Hash)
+    raise "Scope from #{type} file #{source} should be a Hash" unless scope.is_a?(Hash)
 
     scope
 end
@@ -67,6 +86,10 @@ OptionParser.new do |opts|
         options[:verbose] = true
     end
 
+    opts.on("--array", "-a", "Array search") do
+        options[:resolution_type] = :array
+    end
+
     opts.on("--config CONFIG", "-c", "Configuration file") do |v|
         if File.exist?(v)
             options[:config] = v
@@ -87,12 +110,21 @@ OptionParser.new do |opts|
 
     opts.on("--yaml SCOPE", "-y", "YAML format file to load scope from") do |v|
         begin
-            options[:scope] = load_scope(v)
+            options[:scope] = load_scope(v, :json)
         rescue Exception => e
             STDERR.puts "Could not load YAML scope: #{e.class}: #{e}"
             exit 1
         end
     end
+
+    opts.on("--mcollective IDENTITY", "-m", "Retrieve facts from a node via mcollective as scope") do |v|
+        begin
+            options[:scope] = load_scope(v, :mcollective)
+        rescue Exception => e
+            STDERR.puts "Could not load MCollective scope: #{e.class}: #{e}"
+            exit 1
+        end
+    end
 end.parse!
 
 # arguments can be:
@@ -134,7 +166,7 @@ unless options[:verbose]
     Hiera.logger = "noop"
 end
 
-ans = hiera.lookup(options[:key], options[:default], options[:scope])
+ans = hiera.lookup(options[:key], options[:default], options[:scope], nil, options[:resolution_type])
 
 if ans.is_a?(String)
     puts ans

data/lib/hiera/backend.rb

@@ -15,6 +15,32 @@ class Hiera
                 end
             end
 
+            # Finds the path to a datafile based on the Backend#datadir
+            # and extension
+            #
+            # If the file is not found nil is returned
+            def datafile(backend, scope, source, extension)
+                file = File.join([datadir(backend, scope), "#{source}.#{extension}"])
+
+                unless File.exist?(file)
+                    Hiera.debug("Cannot find datafile #{file}, skipping")
+
+                    return nil
+                end
+
+                return file
+            end
+
+            # Returns an appropriate empty answer dependant on resolution type
+            def empty_answer(resolution_type)
+                case resolution_type
+                when :array
+                    return []
+                else
+                    return nil
+                end
+            end
+
             # Constructs a list of data sources to search
             #
             # If you give it a specific hierarchy it will just use that
@@ -62,6 +88,9 @@ class Hiera
                         var = $1
                         val = scope[var] || extra_data[var] || ""
 
+                        # Puppet can return this for unknown scope vars
+                        val = "" if val == :undefined
+
                         tdata.gsub!(/%\{#{var}\}/, val)
                     end
                 end
@@ -69,6 +98,39 @@ class Hiera
                 return tdata
             end
 
+            # Parses a answer received from data files
+            #
+            # Ultimately it just pass the data through parse_string but
+            # it makes some effort to handle arrays of strings as well
+            def parse_answer(data, scope, extra_data={})
+                if data.is_a?(String)
+                    return parse_string(data, scope, extra_data)
+                elsif data.is_a?(Hash)
+                    answer = {}
+                    data.each_pair do |key, val|
+                        answer[key] = parse_answer(val, scope, extra_data)
+                    end
+
+                    return answer
+                elsif data.is_a?(Array)
+                    answer = []
+                    data.each do |item|
+                        answer << parse_answer(item, scope, extra_data)
+                    end
+
+                    return answer
+                end
+            end
+
+            def resolve_answer(answer, resolution_type)
+                case resolution_type
+                when :array
+                    [answer].flatten.uniq.sort
+                else
+                    answer
+                end
+            end
+
             # Calls out to all configured backends in the order they
             # were specified.  The first one to answer will win.
             #
@@ -95,7 +157,7 @@ class Hiera
                     end
                 end
 
-                answer || parse_string(default, scope)
+                resolve_answer(answer, resolution_type) || parse_string(default, scope)
             end
         end
     end

data/lib/hiera/backend/yaml_backend.rb

@@ -8,37 +8,35 @@ class Hiera
             end
 
             def lookup(key, scope, order_override, resolution_type)
-                answer = nil
+                answer = Backend.empty_answer(resolution_type)
 
                 Hiera.debug("Looking up #{key} in YAML backend")
 
-                datadir = Backend.datadir(:yaml, scope)
-
-                raise "Cannot find data directory #{datadir}" unless File.directory?(datadir)
-
                 Backend.datasources(scope, order_override) do |source|
-                    unless answer
-                        Hiera.debug("Looking for data source #{source}")
-
-                        datafile = File.join([datadir, "#{source}.yaml"])
+                    Hiera.debug("Looking for data source #{source}")
 
-                        unless File.exist?(datafile)
-                            Hiera.warn("Cannot find datafile #{datafile}, skipping")
-                            next
-                        end
+                    yamlfile = Backend.datafile(:yaml, scope, source, "yaml") || next
 
-                        data = YAML.load_file(datafile)
+                    data = YAML.load_file(yamlfile)
 
-                        next if data.empty?
-                        next unless data.include?(key)
+                    next if data.empty?
+                    next unless data.include?(key)
 
-                        answer = Backend.parse_string(data[key], scope)
+                    # for array resolution we just append to the array whatever
+                    # we find, we then goes onto the next file and keep adding to
+                    # the array
+                    #
+                    # for priority searches we break after the first found data item
+                    case resolution_type
+                    when :array
+                        answer << Backend.parse_answer(data[key], scope)
                     else
+                        answer = Backend.parse_answer(data[key], scope)
                         break
                     end
                 end
 
-                answer
+                return answer
             end
         end
     end

data/lib/hiera/puppet_logger.rb

@@ -0,0 +1,8 @@
+class Hiera
+    module Puppet_logger
+        class << self
+            def warn(msg); Puppet.notice("hiera(): #{msg}"); end
+            def debug(msg); Puppet.debug("hiera(): #{msg}"); end
+        end
+    end
+end

data/spec/spec_helper.rb

@@ -1,4 +1,4 @@
-$:.insert(0, File.join([File.dirname(__FILE__), "..", "..", "lib"]))
+$:.insert(0, File.join([File.dirname(__FILE__), "..", "lib"]))
 
 require 'rubygems'
 require 'rspec'

data/spec/unit/backend/yaml_backend_spec.rb

@@ -5,6 +5,12 @@ require 'hiera/backend/yaml_backend'
 class Hiera
     module Backend
         describe Yaml_backend do
+            before do
+                Hiera.stubs(:debug)
+                Hiera.stubs(:warn)
+                @backend = Yaml_backend.new
+            end
+
             describe "#initialize" do
                 it "should announce its creation" do # because other specs checks this
                     Hiera.expects(:debug).with("Hiera YAML backend starting")
@@ -13,49 +19,42 @@ class Hiera
             end
 
             describe "#lookup" do
-                before do
-                    Hiera.stubs(:debug)
-                    Hiera.stubs(:warn)
-                    @backend = Yaml_backend.new
-                end
-
-                it "should fail for missing datadir" do
-                    Backend.expects(:datadir).with(:yaml, {}).returns("/nonexisting")
+                it "should look for data in all sources" do
+                    Backend.expects(:datasources).multiple_yields(["one"], ["two"])
+                    Backend.expects(:datafile).with(:yaml, {}, "one", "yaml").returns(nil)
+                    Backend.expects(:datafile).with(:yaml, {}, "two", "yaml").returns(nil)
 
-                    expect {
-                        @backend.lookup("key", {}, nil, nil)
-                    }.to raise_error("Cannot find data directory /nonexisting")
+                    @backend.lookup("key", {}, nil, :priority)
                 end
 
-                it "should look for data in all sources" do
-                    Backend.expects(:datadir).with(:yaml, {}).returns("/nonexisting")
-                    File.expects(:directory?).with("/nonexisting").returns(true)
+                it "should pick data earliest source that has it for priority searches" do
                     Backend.expects(:datasources).multiple_yields(["one"], ["two"])
-                    File.expects(:exist?).with("/nonexisting/one.yaml").returns(false)
-                    File.expects(:exist?).with("/nonexisting/two.yaml").returns(false)
-                    @backend.lookup("key", {}, nil, nil)
+                    Backend.expects(:datafile).with(:yaml, {}, "one", "yaml").returns("/nonexisting/one.yaml")
+                    Backend.expects(:datafile).with(:yaml, {}, "two", "yaml").returns(nil).never
+                    YAML.expects(:load_file).with("/nonexisting/one.yaml").returns({"key" => "answer"})
+
+                    @backend.lookup("key", {}, nil, :priority).should == "answer"
                 end
 
-                it "should pick data earliest source that has it" do
-                    Backend.expects(:datadir).with(:yaml, {}).returns("/nonexisting")
-                    File.expects(:directory?).with("/nonexisting").returns(true)
+                it "should build an array of all data sources for array searches" do
+                    Backend.expects(:datafile).with(:yaml, {}, "one", "yaml").returns("/nonexisting/one.yaml")
+                    Backend.expects(:datafile).with(:yaml, {}, "two", "yaml").returns("/nonexisting/two.yaml")
+
                     Backend.expects(:datasources).multiple_yields(["one"], ["two"])
-                    File.expects(:exist?).with("/nonexisting/one.yaml").returns(true)
-                    YAML.expects(:load_file).with("/nonexisting/one.yaml").returns({"key" => "answer"})
 
-                    File.expects(:exist?).with("/nonexisting/two.yaml").never
+                    YAML.expects(:load_file).with("/nonexisting/one.yaml").returns({"key" => "answer"})
+                    YAML.expects(:load_file).with("/nonexisting/two.yaml").returns({"key" => "answer"})
 
-                    @backend.lookup("key", {}, nil, nil).should == "answer"
+                    @backend.lookup("key", {}, nil, :array).should == ["answer", "answer"]
                 end
 
                 it "should parse the answer for scope variables" do
-                    Backend.expects(:datadir).with(:yaml, {"rspec" => "test"}).returns("/nonexisting")
-                    File.expects(:directory?).with("/nonexisting").returns(true)
                     Backend.expects(:datasources).yields("one")
-                    File.expects(:exist?).with("/nonexisting/one.yaml").returns(true)
+                    Backend.expects(:datafile).with(:yaml, {"rspec" => "test"}, "one", "yaml").returns("/nonexisting/one.yaml")
                     YAML.expects(:load_file).with("/nonexisting/one.yaml").returns({"key" => "test_%{rspec}"})
 
-                    @backend.lookup("key", {"rspec" => "test"}, nil, nil).should == "test_test"
+
+                    @backend.lookup("key", {"rspec" => "test"}, nil, :priority).should == "test_test"
                 end
             end
         end

data/spec/unit/backend_spec.rb

@@ -16,6 +16,30 @@ class Hiera
             end
         end
 
+        describe "#empty_anwer" do
+            it "should return [] for array searches" do
+                Backend.empty_answer(:array).should == []
+            end
+
+            it "should return nil otherwise" do
+                Backend.empty_answer(:meh).should == nil
+            end
+        end
+
+        describe "#datafile" do
+            it "should check if the file exist and return nil if not" do
+                Hiera.expects(:debug).with("Cannot find datafile /nonexisting/test.yaml, skipping")
+                Backend.expects(:datadir).returns("/nonexisting")
+                Backend.datafile(:yaml, {}, "test", "yaml").should == nil
+            end
+
+            it "should return the correct file name" do
+                Backend.expects(:datadir).returns("/nonexisting")
+                File.expects(:exist?).with("/nonexisting/test.yaml").returns(true)
+                Backend.datafile(:yaml, {}, "test", "yaml").should == "/nonexisting/test.yaml"
+            end
+        end
+
         describe "#datasources" do
             it "should use the supplied hierarchy" do
                 expected = ["one", "two"]
@@ -101,6 +125,43 @@ class Hiera
                 input = "test_%{rspec}_test"
                 Backend.parse_string(input, {"rspec" => "test"}, {"rspec" => "fail"}).should == "test_test_test"
             end
+
+            it "should treat :undefined in scope as empty" do
+                input = "test_%{rspec}_test"
+                Backend.parse_string(input, {"rspec" => :undefined}).should == "test__test"
+            end
+        end
+
+        describe "#parse_answer" do
+            it "should parse strings correctly" do
+                input = "test_%{rspec}_test"
+                Backend.parse_answer(input, {"rspec" => "test"}).should == "test_test_test"
+            end
+
+            it "should parse each string in an array" do
+                input = ["test_%{rspec}_test", "test_%{rspec}_test", ["test_%{rspec}_test"]]
+                Backend.parse_answer(input, {"rspec" => "test"}).should == ["test_test_test", "test_test_test", ["test_test_test"]]
+            end
+
+            it "should parse each string in a hash" do
+                input = {"foo" => "test_%{rspec}_test", "bar" => "test_%{rspec}_test"}
+                Backend.parse_answer(input, {"rspec" => "test"}).should == {"foo"=>"test_test_test", "bar"=>"test_test_test"}
+            end
+
+            it "should parse mixed arrays and hashes" do
+                input = {"foo" => "test_%{rspec}_test", "bar" => ["test_%{rspec}_test", "test_%{rspec}_test"]}
+                Backend.parse_answer(input, {"rspec" => "test"}).should == {"foo"=>"test_test_test", "bar"=>["test_test_test", "test_test_test"]}
+            end
+        end
+
+        describe "#resolve_answer" do
+            it "should correctly parse array data" do
+                Backend.resolve_answer(["foo", ["foo", "foo"], "bar"], :array).should == ["bar", "foo"]
+            end
+
+            it "should just return the answer for non array data" do
+                Backend.resolve_answer(["foo", ["foo", "foo"], "bar"], :priority).should == ["foo", ["foo", "foo"], "bar"]
+            end
         end
 
         describe "#lookup" do
@@ -110,7 +171,6 @@ class Hiera
             end
 
             it "should cache backends" do
-                Hiera.stubs(:debug)
                 Hiera.expects(:debug).with(regexp_matches(/Hiera YAML backend starting/)).once
 
                 Config.load({:yaml => {:datadir => "/tmp"}})
@@ -121,7 +181,6 @@ class Hiera
             end
 
             it "should return the answer from the backend" do
-                Hiera.stubs(:debug)
                 Config.load({:yaml => {:datadir => "/tmp"}})
                 Config.load_backends
 
@@ -133,7 +192,6 @@ class Hiera
             it "should call to all backends till an answer is found" do
                 backend = mock
                 backend.expects(:lookup).returns("answer")
-                Hiera.stubs(:debug)
                 Config.load({})
                 Config.instance_variable_set("@config", {:backends => ["yaml", "rspec"]})
                 Backend.instance_variable_set("@backends", {"rspec" => backend})
@@ -144,8 +202,17 @@ class Hiera
 
             end
 
+            it "should parse the answers based on resolution_type" do
+                Config.load({:yaml => {:datadir => "/tmp"}})
+                Config.load_backends
+
+                Backend.expects(:resolve_answer).with("test_test", :priority).returns("parsed")
+                Backend::Yaml_backend.any_instance.expects(:lookup).with("key", {"rspec" => "test"}, nil, :priority).returns("test_test")
+
+                Backend.lookup("key", "test_%{rspec}", {"rspec" => "test"}, nil, :priority).should == "parsed"
+            end
+
             it "should return the default with variables parsed if nothing is found" do
-                Hiera.stubs(:debug)
                 Config.load({:yaml => {:datadir => "/tmp"}})
                 Config.load_backends
 

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 1
-  version: 0.1.1
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - R.I.Pienaar
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-05 00:00:00 +01:00
+date: 2011-06-12 00:00:00 +01:00
 default_executable: hiera
 dependencies: []
 
@@ -29,6 +29,7 @@ extra_rdoc_files: []
 files: 
 - bin/hiera
 - lib/hiera.rb
+- lib/hiera/puppet_logger.rb
 - lib/hiera/backend.rb
 - lib/hiera/config.rb
 - lib/hiera/console_logger.rb