hiera 1.2.0.rc1 → 1.2.0.rc2

data/README.md

@@ -1,5 +1,7 @@
 # Hiera
 
+[![Build Status](https://travis-ci.org/puppetlabs/hiera.png?branch=master)](https://travis-ci.org/puppetlabs/hiera)
+
 A simple pluggable Hierarchical Database.
 
 -

data/lib/hiera.rb

@@ -1,14 +1,16 @@
 require 'yaml'
 
 class Hiera
-  VERSION = "1.2.0-rc1"
+  VERSION = "1.2.0-rc2"
 
-  autoload :Config,         "hiera/config"
-  autoload :Util,           "hiera/util"
-  autoload :Backend,        "hiera/backend"
-  autoload :Console_logger, "hiera/console_logger"
-  autoload :Puppet_logger,  "hiera/puppet_logger"
-  autoload :Noop_logger,    "hiera/noop_logger"
+  require "hiera/config"
+  require "hiera/util"
+  require "hiera/backend"
+  require "hiera/console_logger"
+  require "hiera/puppet_logger"
+  require "hiera/noop_logger"
+  require "hiera/fallback_logger"
+  require "hiera/filecache"
 
   class << self
     attr_reader :logger
@@ -23,13 +25,13 @@ class Hiera
     # See hiera-puppet for an example that uses the Puppet
     # loging system instead of our own
     def logger=(logger)
-      loggerclass = "#{logger.capitalize}_logger"
+      require "hiera/#{logger}_logger"
 
-      require "hiera/#{logger}_logger" unless constants.include?(loggerclass)
-
-      @logger = const_get(loggerclass)
+      @logger = Hiera::FallbackLogger.new(
+        Hiera.const_get("#{logger.capitalize}_logger"),
+        Hiera::Console_logger)
     rescue Exception => e
-      @logger = Console_logger
+      @logger = Hiera::Console_logger
       warn("Failed to load #{logger} logger: #{e.class}: #{e}")
     end
 

data/lib/hiera/backend.rb

@@ -1,4 +1,6 @@
 require 'hiera/util'
+require 'hiera/recursive_lookup'
+
 begin
   require 'deep_merge'
 rescue LoadError
@@ -6,6 +8,8 @@ end
 
 class Hiera
   module Backend
+    INTERPOLATION = /%\{([^\}]*)\}/
+
     class << self
       # Data lives in /var/lib/hiera by default.  If a backend
       # supplies a datadir in the config it will be used and
@@ -65,44 +69,38 @@ class Hiera
         end
       end
 
-      # Parse a string like '%{foo}' against a supplied
+      # Parse a string like <code>'%{foo}'</code> against a supplied
       # scope and additional scope.  If either scope or
-      # extra_scope includes the varaible 'foo' it will
+      # extra_scope includes the variable 'foo', then it will
       # be replaced else an empty string will be placed.
       #
-      # If both scope and extra_data has "foo" scope
-      # will win.  See hiera-puppet for an example of
-      # this to make hiera aware of additional non scope
-      # variables
+      # If both scope and extra_data has "foo", then the value in scope
+      # will be used.
+      #
+      # @param data [String] The string to perform substitutions on.
+      #   This will not be modified, instead a new string will be returned.
+      # @param scope [#[]] The primary source of data for substitutions.
+      # @param extra_data [#[]] The secondary source of data for substitutions.
+      # @return [String] A copy of the data with all instances of <code>%{...}</code> replaced.
+      #
+      # @api public
       def parse_string(data, scope, extra_data={})
-        return nil unless data
-
-        tdata = data.clone
-
-        if tdata.is_a?(String)
-          while tdata =~ /%\{(.+?)\}/
-            begin
-              var = $1
-
-              val = ""
-
-              # Puppet can return :undefined for unknown scope vars,
-              # If it does then we still need to evaluate extra_data
-              # before returning an empty string.
-              scope_val = scope[var]
-              if !scope_val.nil? && scope_val != :undefined
-                  val = scope_val
-              elsif extra_data[var]
-                  val = extra_data[var]
-              end
-            end until val != "" || var !~ /::(.+)/
+        interpolate(data, Hiera::RecursiveLookup.new(scope, extra_data))
+      end
 
-            tdata.gsub!(/%\{(::)?#{var}\}/, val)
+      def interpolate(data, values)
+        if data.is_a?(String)
+          data.gsub(INTERPOLATION) do
+            name = $1
+            values.lookup(name) do |value|
+              interpolate(value, values)
+            end
           end
+        else
+          data
         end
-
-        return tdata
       end
+      private :interpolate
 
       # Parses a answer received from data files
       #

data/lib/hiera/backend/json_backend.rb

@@ -1,10 +1,12 @@
 class Hiera
   module Backend
     class Json_backend
-      def initialize
+      def initialize(cache=nil)
         require 'json'
 
         Hiera.debug("Hiera JSON backend starting")
+
+        @cache = cache || Filecache.new
       end
 
       def lookup(key, scope, order_override, resolution_type)
@@ -17,7 +19,11 @@ class Hiera
 
           jsonfile = Backend.datafile(:json, scope, source, "json") || next
 
-          data = JSON.parse(File.read(jsonfile))
+          next unless File.exist?(jsonfile)
+
+          data = @cache.read(jsonfile, Hash, {}) do |data|
+            JSON.parse(data)
+          end
 
           next if data.empty?
           next unless data.include?(key)

data/lib/hiera/backend/yaml_backend.rb

@@ -1,11 +1,11 @@
 class Hiera
   module Backend
     class Yaml_backend
-      def initialize
+      def initialize(cache=nil)
         require 'yaml'
         Hiera.debug("Hiera YAML backend starting")
-        @data  = Hash.new
-        @cache = Hash.new
+
+        @cache = cache || Filecache.new
       end
 
       def lookup(key, scope, order_override, resolution_type)
@@ -17,20 +17,14 @@ class Hiera
           Hiera.debug("Looking for data source #{source}")
           yamlfile = Backend.datafile(:yaml, scope, source, "yaml") || next
 
-          # If you call stale? BEFORE you do encounter the YAML.load_file line
-          # it will populate the @cache variable and return true. The second
-          # time you call it, it will return false because @cache has been
-          # populated. Because of this there are two conditions to check:
-          # is @data[yamlfile] populated AND is the cache stale.
-          if @data[yamlfile]
-            @data[yamlfile] = YAML.load_file(yamlfile) if stale?(yamlfile)
-          else
-            @data[yamlfile] = YAML.load_file(yamlfile)
+          next unless File.exist?(yamlfile)
+
+          data = @cache.read(yamlfile, Hash, {}) do |data|
+            YAML.load(data)
           end
 
-          next if ! @data[yamlfile]
-          next if @data[yamlfile].empty?
-          next unless @data[yamlfile].include?(key)
+          next if data.empty?
+          next unless data.include?(key)
 
           # Extra logging that we found the key. This can be outputted
           # multiple times if the resolution type is array or hash but that
@@ -43,7 +37,7 @@ class Hiera
           # the array
           #
           # for priority searches we break after the first found data item
-          new_answer = Backend.parse_answer(@data[yamlfile][key], scope)
+          new_answer = Backend.parse_answer(data[key], scope)
           case resolution_type
           when :array
             raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
@@ -61,18 +55,6 @@ class Hiera
 
         return answer
       end
-
-      def stale?(yamlfile)
-        # NOTE: The mtime change in a file MUST be > 1 second before being
-        #       recognized as stale. File mtime changes within 1 second will
-        #       not be recognized.
-        stat    = File.stat(yamlfile)
-        current = { 'inode' => stat.ino, 'mtime' => stat.mtime, 'size' => stat.size }
-        return false if @cache[yamlfile] == current
-
-        @cache[yamlfile] = current
-        return true
-      end
     end
   end
 end

data/lib/hiera/fallback_logger.rb

@@ -0,0 +1,41 @@
+# Select from a given list of loggers the first one that
+# it suitable and use that as the actual logger
+#
+# @api private
+class Hiera::FallbackLogger
+  # Chooses the first suitable logger. For all of the loggers that are
+  # unsuitable it will issue a warning using the suitable logger stating that
+  # the unsuitable logger is not being used.
+  #
+  # @param implementations [Array<Hiera::Logger>] the implementations to choose from
+  # @raises when there are no suitable loggers
+  def initialize(*implementations)
+    warnings = []
+    @implementation = implementations.find do |impl|
+      if impl.respond_to?(:suitable?)
+        if impl.suitable?
+          true
+        else
+          warnings << "Not using #{impl.name}. It does not report itself to be suitable."
+          false
+        end
+      else
+        true
+      end
+    end
+
+    if @implementation.nil?
+      raise "No suitable logging implementation found."
+    end
+
+    warnings.each { |message| warn(message) }
+  end
+
+  def warn(message)
+    @implementation.warn(message)
+  end
+
+  def debug(message)
+    @implementation.debug(message)
+  end
+end

data/lib/hiera/filecache.rb

@@ -0,0 +1,74 @@
+class Hiera
+  class Filecache
+    def initialize
+      @cache = {}
+    end
+
+    # Reads a file, optionally parse it in some way check the
+    # output type and set a default
+    #
+    # Simply invoking it this way will return the file contents
+    #
+    #    data = read("/some/file")
+    #
+    # But as most cases of file reading in hiera involves some kind
+    # of parsing through a serializer there's some help for those
+    # cases:
+    #
+    #    data = read("/some/file", Hash, {}) do |data|
+    #       JSON.parse(data)
+    #    end
+    #
+    # In this case it will read the file, parse it using JSON then
+    # check that the end result is a Hash, if it's not a hash or if
+    # reading/parsing fails it will return {} instead
+    #
+    # Prior to calling this method you should be sure the file exist
+    def read(path, expected_type=nil, default=nil)
+      @cache[path] ||= {:data => nil, :meta => path_metadata(path)}
+
+      if File.exist?(path) && !@cache[path][:data] || stale?(path)
+        if block_given?
+          begin
+            @cache[path][:data] = yield(File.read(path))
+          rescue => e
+            Hiera.debug("Reading data from %s failed: %s: %S" % [path, e.class, e.to_s])
+            @cache[path][:data] = default
+          end
+        else
+          @cache[path][:data] = File.read(path)
+        end
+      end
+
+      if block_given? && !expected_type.nil?
+        unless @cache[path][:data].is_a?(expected_type)
+          Hiera.debug("Data retrieved from %s is not a %s, setting defaults" % [path, expected_type])
+          @cache[path][:data] = default
+        end
+      end
+
+      @cache[path][:data]
+    end
+
+    def stale?(path)
+      meta = path_metadata(path)
+
+      @cache[path] ||= {:data => nil, :meta => nil}
+
+      if @cache[path][:meta] == meta
+        return false
+      else
+        @cache[path][:meta] = meta
+        return true
+      end
+    end
+
+    # This is based on the old caching in the YAML backend and has a
+    # resolution of 1 second, changes made within the same second of
+    # a previous read will be ignored
+    def path_metadata(path)
+      stat = File.stat(path)
+      {:inode => stat.ino, :mtime => stat.mtime, :size => stat.size}
+    end
+  end
+end

data/lib/hiera/puppet_logger.rb

@@ -1,6 +1,10 @@
 class Hiera
   module Puppet_logger
     class << self
+      def suitable?
+        Kernel.const_defined?(:Puppet)
+      end
+
       def warn(msg)
         Puppet.notice("hiera(): #{msg}")
       end

data/lib/hiera/recursive_lookup.rb

@@ -0,0 +1,31 @@
+# Allow for safe recursive lookup of values during variable interpolation.
+#
+# @api private
+class Hiera::RecursiveLookup
+  def initialize(scope, extra_data)
+    @seen = []
+    @scope = scope
+    @extra_data = extra_data
+  end
+
+  def lookup(name, &block)
+    if @seen.include?(name)
+      raise Exception, "Interpolation loop detected in [#{@seen.join(', ')}]"
+    end
+    @seen.push(name)
+    ret = yield(current_value)
+    @seen.pop
+    ret
+  end
+
+  def current_value
+    name = @seen.last
+
+    scope_val = @scope[name]
+    if scope_val.nil? || scope_val == :undefined
+      @extra_data[name]
+    else
+      scope_val
+    end
+  end
+end

data/spec/unit/backend/json_backend_spec.rb

@@ -8,7 +8,8 @@ class Hiera
         Hiera.stubs(:debug)
         Hiera.stubs(:warn)
         Hiera::Backend.stubs(:empty_answer).returns(nil)
-        @backend = Json_backend.new
+        @cache = mock
+        @backend = Json_backend.new(@cache)
       end
 
       describe "#initialize" do
@@ -30,13 +31,9 @@ class Hiera
         it "should retain the data types found in data files" do
           Backend.expects(:datasources).yields("one").times(3)
           Backend.expects(:datafile).with(:json, {}, "one", "json").returns("/nonexisting/one.json").times(3)
-          File.expects(:read).with("/nonexisting/one.json").returns('{"stringval":"string",
-                                                                      "boolval":true,
-                                                                      "numericval":1}').times(3)
+          File.stubs(:exist?).with("/nonexisting/one.json").returns(true)
 
-          Backend.stubs(:parse_answer).with('string', {}).returns('string')
-          Backend.stubs(:parse_answer).with(true, {}).returns(true)
-          Backend.stubs(:parse_answer).with(1, {}).returns(1)
+          @cache.expects(:read).with("/nonexisting/one.json", Hash, {}).returns({"stringval" => "string", "boolval" => true, "numericval" => 1}).times(3)
 
           @backend.lookup("stringval", {}, nil, :priority).should == "string"
           @backend.lookup("boolval", {}, nil, :priority).should == true
@@ -45,13 +42,12 @@ class Hiera
 
         it "should pick data earliest source that has it for priority searches" do
           scope = {"rspec" => "test"}
-          Backend.stubs(:parse_answer).with('answer', scope).returns("answer")
-          Backend.stubs(:parse_answer).with('test_%{rspec}', scope).returns("test_test")
           Backend.expects(:datasources).multiple_yields(["one"], ["two"])
           Backend.expects(:datafile).with(:json, scope, "one", "json").returns("/nonexisting/one.json")
-          Backend.expects(:datafile).with(:json, scope, "two", "json").returns(nil).never
-          File.expects(:read).with("/nonexisting/one.json").returns("one.json")
-          JSON.expects(:parse).with("one.json").returns({"key" => "test_%{rspec}"})
+          Backend.expects(:datafile).with(:json, scope, "two", "json").never
+
+          File.stubs(:exist?).with("/nonexisting/one.json").returns(true)
+          @cache.expects(:read).with("/nonexisting/one.json", Hash, {}).returns({"key" => "test_%{rspec}"})
 
           @backend.lookup("key", scope, nil, :priority).should == "test_test"
         end
@@ -64,11 +60,11 @@ class Hiera
 
           Backend.expects(:datasources).multiple_yields(["one"], ["two"])
 
-          File.expects(:read).with("/nonexisting/one.json").returns("one.json")
-          File.expects(:read).with("/nonexisting/two.json").returns("two.json")
+          File.expects(:exist?).with("/nonexisting/one.json").returns(true)
+          File.expects(:exist?).with("/nonexisting/two.json").returns(true)
 
-          JSON.expects(:parse).with("one.json").returns({"key" => "answer"})
-          JSON.expects(:parse).with("two.json").returns({"key" => "answer"})
+          @cache.expects(:read).with("/nonexisting/one.json", Hash, {}).returns({"key" => "answer"})
+          @cache.expects(:read).with("/nonexisting/two.json", Hash, {}).returns({"key" => "answer"})
 
           @backend.lookup("key", {}, nil, :array).should == ["answer", "answer"]
         end
@@ -78,8 +74,8 @@ class Hiera
           Backend.expects(:datasources).yields("one")
           Backend.expects(:datafile).with(:json, {"rspec" => "test"}, "one", "json").returns("/nonexisting/one.json")
 
-          File.expects(:read).with("/nonexisting/one.json").returns("one.json")
-          JSON.expects(:parse).with("one.json").returns({"key" => "test_%{rspec}"})
+          File.expects(:exist?).with("/nonexisting/one.json").returns(true)
+          @cache.expects(:read).with("/nonexisting/one.json", Hash, {}).returns({"key" => "test_%{rspec}"})
 
           @backend.lookup("key", {"rspec" => "test"}, nil, :priority).should == "test_test"
         end