hiera 1.3.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54743f04196eaecdaffcb0162606f26843f35a03
-  data.tar.gz: c83bf33b0c25e508380073c67b94796493d01124
+  metadata.gz: dd4a69f9d2131165eba3e8767b8731dcc11f3a6d
+  data.tar.gz: fd038d50edc2f8c3dd6d21d21d30cc9463b11c4c
 SHA512:
-  metadata.gz: 50281c08a2494cce3c6e2d59ec94a47eb14dbad34f0b0b20915fe2b96f0bc5ce1c687984fb74426a61760018eaef64f911eb1cf677820b8ca2261f32549f4bba
-  data.tar.gz: 570c1a8cff80579c90afd19451075bc52a4c035d77852d665980b1b3c2c0fd6075a5da11c947ad9eac2f9ce24399cd797d2ae23beed63bef372db4cd5fb9aa83
+  metadata.gz: 6d669f8a2ea3cea56f2c0d66959e2aa2df7a84f74940ae51e8c192929f45e102d99b85c7530084c87294594557229060de56b093e87c5a96076c7849343efb15
+  data.tar.gz: 10d478df00b859bddf583631ae9fe9f2f1028b3394fda6dfd38d3e0a2a57ce32a1b41a681b814552594421d9b9b373b94c6300eccdd2291827abacf8f9c3a566

data/LICENSE

@@ -1,6 +1,6 @@
 Puppet - Automating Configuration Management.
 
-Copyright (C) 2012 Puppet Labs Inc
+Copyright (C) 2012-2014 Puppet Labs Inc
 
 Puppet Labs can be contacted at: info@puppetlabs.com
 

data/README.md

@@ -84,6 +84,23 @@ Hiera can search through all the tiers in a hierarchy and merge the result into
 array.  This is used in the hiera-puppet project to replace External Node Classifiers by
 creating a Hiera compatible include function.
 
+### Qualified Key Lookup
+You can use a qualified key to lookup a value that is contained inside a hash or array:
+
+<pre>
+$ hiera user
+{"name"=>"kim", "home"=>"/home/kim"}
+$ hiera user.name
+kim
+</pre>
+
+<pre>
+$ hiera ssh_users
+["root", "jeff", "gary", "hunter"]
+$ hiera ssh_users.0
+root
+</pre>
+
 ## Future Enhancements
 
  * More backends should be created
@@ -121,29 +138,29 @@ A sample configuration file can be seen here:
   - common
 
 :yaml:
-   :datadir: /etc/puppet/hieradata
+   :datadir: /etc/puppetlabs/code/hieradata
 
 :puppet:
    :datasource: data
 </pre>
 
-This configuration will require YAML files in  _/etc/puppet/hieradata_ these need to contain
+This configuration will require YAML files in  _/etc/puppetlabs/code/hieradata_ these need to contain
 Hash data, sample files matching the hierarchy described in the _Why?_ section are below:
 
-_/etc/puppet/hieradata/sites/dc1.yaml_:
+_/etc/puppetlabs/code/hieradata/sites/dc1.yaml_:
 <pre>
 ---
 ntpserver: ntp1.dc1.example.com
 sysadmin: dc1noc@example.com
 </pre>
 
-_/etc/puppet/hieradata/sites/dc2.yaml_:
+_/etc/puppetlabs/code/hieradata/sites/dc2.yaml_:
 <pre>
 ---
 ntpserver: ntp1.dc2.example.com
 </pre>
 
-_/etc/puppet/hieradata/common.yaml_:
+_/etc/puppetlabs/code/hieradata/common.yaml_:
 <pre>
 ---
 sysadmin: "sysadmin@%{domain}"
@@ -161,13 +178,13 @@ store as found on your Puppet Masters.
 If no data is found and the facts had a location=dc1 fact the default would be _sites/dc1_
 
 <pre>
-$ hiera acme_version 'sites/%{location}' --yaml /var/lib/puppet/yaml/facts/example.com.yaml
+$ hiera acme_version 'sites/%{location}' --yaml /opt/puppetlabs/puppet/cache/yaml/facts/example.com.yaml
 </pre>
 
 You can also supply extra facts on the CLI, assuming Puppet facts did not have a location fact:
 
 <pre>
-$ hiera acme_version 'sites/%{location}' location=dc1 --yaml /var/lib/puppet/yaml/facts/example.com.yaml
+$ hiera acme_version 'sites/%{location}' location=dc1 --yaml /opt/puppetlabs/puppet/cache/yaml/facts/example.com.yaml
 </pre>
 
 Or if you use MCollective you can fetch the scope from a remote node's facts:
@@ -193,10 +210,10 @@ require 'hiera'
 require 'puppet'
 
 # load the facts for example.com
-scope = YAML.load_file("/var/lib/puppet/yaml/facts/example.com.yaml").values
+scope = YAML.load_file("/opt/puppetlabs/puppet/cache/yaml/facts/example.com.yaml").values
 
 # create a new instance based on config file
-hiera = Hiera.new(:config => "/etc/puppet/hiera.yaml")
+hiera = Hiera.new(:config => "/etc/puppetlabs/code/hiera.yaml")
 
 # resolve the 'acme_version' variable based on scope
 #
@@ -235,5 +252,25 @@ See LICENSE file.
 
 ## Support
 
-Please log tickets and issues at our [Projects site](http://projects.puppetlabs.com)
+Please log tickets and issues at our [JIRA tracker](http://tickets.puppetlabs.com).  A [mailing
+list](https://groups.google.com/forum/?fromgroups#!forum/puppet-users) is
+available for asking questions and getting help from others. In addition there
+is an active #puppet channel on Freenode.
+
+We use semantic version numbers for our releases, and recommend that users stay
+as up-to-date as possible by upgrading to patch releases and minor releases as
+they become available.
+
+Bugfixes and ongoing development will occur in minor releases for the current
+major version. Security fixes will be backported to a previous major version on
+a best-effort basis, until the previous major version is no longer maintained.
+
+
+For example: If a security vulnerability is discovered in Hiera 1.3.0, we
+would fix it in the 1 series, most likely as 1.3.1. Maintainers would then make
+a best effort to backport that fix onto the latest Hiera release they carry.
+
+Long-term support, including security patches and bug fixes, is available for
+commercial customers. Please see the following page for more details:
 
+[Puppet Enterprise Support Lifecycle](http://puppetlabs.com/misc/puppet-enterprise-lifecycle)

data/bin/hiera

@@ -1,9 +1,5 @@
 #!/usr/bin/env ruby
 
-# For security reasons, ensure that '.' is not on the load path
-# This is primarily for 1.8.7 since 1.9.2+ doesn't put '.' on the load path
-$LOAD_PATH.delete '.'
-
 # CLI client for Hiera.
 #
 # To lookup the 'release' key for a node given Puppet YAML facts:
@@ -36,9 +32,12 @@ options = {
   :scope   => {},
   :key     => nil,
   :verbose => false,
-  :resolution_type => :priority
+  :resolution_type => :priority,
+  :format => :ruby
 }
 
+initial_scopes = Array.new
+
 # Loads the scope from YAML or JSON files
 def load_scope(source, type=:yaml)
   case type
@@ -117,6 +116,26 @@ def load_scope(source, type=:yaml)
   scope
 end
 
+def output_answer(ans, format)
+  case format
+  when :json
+    require 'json'
+    puts JSON.dump(ans)
+  when :ruby
+    if ans.is_a?(String)
+      puts ans
+    else
+      pp ans
+    end
+  when :yaml
+    require 'yaml'
+    puts ans.to_yaml
+  else
+    STDERR.puts "Unknown output format: #{v}"
+    exit 1
+  end
+end
+
 OptionParser.new do |opts|
   opts.banner = "Usage: hiera [options] key [default value] [variable='text'...]\n\nThe default value will be used if no value is found for the key. Scope variables\nwill be interpolated into %{variable} placeholders in the hierarchy and in\nreturned values.\n\n"
 
@@ -147,41 +166,43 @@ OptionParser.new do |opts|
   end
 
   opts.on("--json SCOPE", "-j", "JSON format file to load scope from") do |v|
-    begin
-      options[:scope] = load_scope(v, :json)
-    rescue Exception => e
-      STDERR.puts "Could not load JSON scope: #{e.class}: #{e}"
-      exit 1
-    end
+    initial_scopes << { :type => :json, :value => v, :name => "JSON" }
   end
 
   opts.on("--yaml SCOPE", "-y", "YAML format file to load scope from") do |v|
-    begin
-      options[:scope] = load_scope(v)
-    rescue Exception => e
-      STDERR.puts "Could not load YAML scope: #{e.class}: #{e}"
-      exit 1
-    end
+    initial_scopes << { :type => :yaml, :value => v, :name => "YAML" }
   end
 
   opts.on("--mcollective IDENTITY", "-m", "Use facts from a node (via mcollective) as scope") do |v|
-    begin
-      options[:scope] = load_scope(v, :mcollective)
-    rescue Exception => e
-      STDERR.puts "Could not load MCollective scope: #{e.class}: #{e}"
+    initial_scopes << { :type => :mcollective, :value => v, :name => "Mcollective" }
+  end
+
+  opts.on("--inventory_service IDENTITY", "-i", "Use facts from a node (via Puppet's inventory service) as scope") do |v|
+    initial_scopes << { :type => :inventory_service, :value => v, :name => "Puppet inventory service" }
+  end
+
+  opts.on("--format TYPE", "-f", "Output the result in a specific format (ruby, yaml or json); default is 'ruby'") do |v|
+    options[:format] = case v
+    when 'json', 'ruby', 'yaml'
+      v.to_sym
+    else
+      STDERR.puts "Unknown output format: #{v}"
       exit 1
     end
   end
+end.parse!
 
-  opts.on("--inventory_service IDENTITY", "-i", "Use facts from a node (via Puppet's inventory service) as scope") do |v|
+unless initial_scopes.empty?
+  initial_scopes.each { |this_scope|
+    # Load initial scope
     begin
-      options[:scope] = load_scope(v, :inventory_service)
+      options[:scope] = load_scope(this_scope[:value], this_scope[:type])
     rescue Exception => e
-      STDERR.puts "Could not load Puppet inventory service scope: #{e.class}: #{e}"
+      STDERR.puts "Could not load #{this_scope[:name]} scope: #{e.class}: #{e}"
       exit 1
     end
-  end
-end.parse!
+  }
+end
 
 # arguments can be:
 #
@@ -224,8 +245,4 @@ end
 
 ans = hiera.lookup(options[:key], options[:default], options[:scope], nil, options[:resolution_type])
 
-if ans.is_a?(String)
-  puts ans
-else
-  pp ans
-end
+output_answer(ans, options[:format])

data/lib/hiera.rb

@@ -49,15 +49,67 @@ class Hiera
 
   # Calls the backends to do the actual lookup.
   #
-  # The scope can be anything that responds to [], if you have input
+  # The _scope_ can be anything that responds to `[]`, if you have input
   # data like a Puppet Scope that does not you can wrap that data in a
-  # class that has a [] method that fetches the data from your source.
+  # class that has a `[]` method that fetches the data from your source.
   # See hiera-puppet for an example of this.
   #
   # The order-override will insert as first in the hierarchy a data source
   # of your choice.
+  #
+  # Possible values for the _resolution_type_ parameter:
+  #
+  # - _:priority_ - This is the default. First found value is returned and no merge is performed
+  # - _:array_ - An array merge lookup assembles a value from every matching level of the hierarchy. It retrieves all
+  #     of the (string or array) values for a given key, then flattens them into a single array of unique values.
+  #     If _priority_ lookup can be thought of as a “default with overrides” pattern, _array_ merge lookup can be though
+  #     of as “default with additions.”
+  # - _:hash_ - A hash merge lookup assembles a value from every matching level of the hierarchy. It retrieves all of
+  #     the (hash) values for a given key, then merges the hashes into a single hash. Hash merge lookups will fail with
+  #     an error if any of the values found in the data sources are strings or arrays. It only works when every value
+  #     found is a hash. The actual merge behavior is determined by looking up the keys `:merge_behavior` and
+  #     `:deep_merge_options` in the Hiera config. `:merge_behavior` can be set to `:deep`, :deeper` or `:native`
+  #     (explained in detail below).
+  # - _{ deep merge options }_ - Configured values for `:merge_behavior` and `:deep_merge_options`will be completely
+  #     ignored. Instead the _resolution_type_ will be a `:hash` merge where the `:merge_behavior` will be the value
+  #     keyed by `:behavior` in the given hash and the `:deep_merge_options` will be the remaining top level entries of
+  #     that same hash.
+  #
+  # Valid behaviors for the _:hash_ resolution type:
+  #
+  # - _native_ - Performs a simple hash-merge by overwriting keys of lower lookup priority.
+  # - _deeper_ - In a deeper hash merge, Hiera recursively merges keys and values in each source hash. For each key,
+  #     if the value is:
+  #        - only present in one source hash, it goes into the final hash.
+  #        - a string/number/boolean and exists in two or more source hashes, the highest priority value goes into
+  #          the final hash.
+  #        - an array and exists in two or more source hashes, the values from each source are merged into a single
+  #          array and de-duplicated (but not automatically flattened, as in an array merge lookup).
+  #        - a hash and exists in two or more source hashes, the values from each source are recursively merged, as
+  #          though they were source hashes.
+  #        - mismatched between two or more source hashes, we haven’t validated the behavior. It should act as
+  #          described in the deep_merge gem documentation.
+  # - _deep_ - In a deep hash merge, Hiera behaves the same as for _deeper_, except that when a string/number/boolean
+  #     exists in two or more source hashes, the lowest priority value goes into the final hash. This is considered
+  #     largely useless and should be avoided. Use _deeper_ instead.
+  #
+  # The _merge_ can be given as a hash with the mandatory key `:strategy` to denote the actual strategy. This
+  # is useful for the `:deeper` and `:deep` strategy since they can use additional options to control the behavior.
+  # The options can be passed as top level keys in the `merge` parameter when it is a given as a hash. Recognized
+  # options are:
+  #
+  #  - 'knockout_prefix' Set to string value to signify prefix which deletes elements from existing element. Defaults is _undef_
+  #  - 'sort_merged_arrays' Set to _true_ to sort all arrays that are merged together. Default is _false_
+  #  - 'unpack_arrays' Set to string value used as a deliminator to join all array values and then split them again. Default is _undef_
+  #  - 'merge_hash_arrays' Set to _true_ to merge hashes within arrays. Default is _false_
+  #
+  # @param key [String] The key to lookup
+  # @param default [Object,nil] The value to return when there is no match for _key_
+  # @param scope [#[],nil] The scope to use for the lookup
+  # @param order_override [#[]] An override that will considered the first source of lookup
+  # @param resolution_type [String,Hash<Symbol,String>] Symbolic resolution type or deep merge configuration
+  # @return [Object] The found value or the given _default_ value
   def lookup(key, default, scope, order_override=nil, resolution_type=:priority)
     Backend.lookup(key, default, scope, order_override, resolution_type)
   end
 end
-

data/lib/hiera/backend.rb

@@ -8,6 +8,22 @@ end
 
 class Hiera
   module Backend
+    class Backend1xWrapper
+      def initialize(wrapped)
+        @wrapped = wrapped
+      end
+
+      def lookup(key, scope, order_override, resolution_type, context)
+        Hiera.debug("Using Hiera 1.x backend API to access instance of class #{@wrapped.class.name}. Lookup recursion will not be detected")
+        value = @wrapped.lookup(key, scope, order_override, resolution_type.is_a?(Hash) ? :hash : resolution_type)
+
+        # The most likely cause when an old backend returns nil is that the key was not found. In any case, it is
+        # impossible to know the difference between that and a found nil. The throw here preserves the old behavior.
+        throw (:no_such_key) if value.nil?
+        value
+      end
+    end
+
     class << self
       # Data lives in /var/lib/hiera by default.  If a backend
       # supplies a datadir in the config it will be used and
@@ -72,7 +88,7 @@ class Hiera
         hierarchy.insert(0, override) if override
 
         hierarchy.flatten.map do |source|
-          source = parse_string(source, scope)
+          source = parse_string(source, scope, {}, :order_override => override)
           yield(source) unless source == "" or source =~ /(^\/|\/\/|\/$)/
         end
       end
@@ -118,34 +134,35 @@ class Hiera
       #   This will not be modified, instead a new string will be returned.
       # @param scope [#[]] The primary source of data for substitutions.
       # @param extra_data [#[]] The secondary source of data for substitutions.
+      # @param context [#[]] Context can include :recurse_guard and :order_override.
       # @return [String] A copy of the data with all instances of <code>%{...}</code> replaced.
       #
       # @api public
-      def parse_string(data, scope, extra_data={})
-        Hiera::Interpolate.interpolate(data, scope, extra_data)
+      def parse_string(data, scope, extra_data={}, context={:recurse_guard => nil, :order_override => nil})
+        Hiera::Interpolate.interpolate(data, scope, extra_data, context)
       end
 
       # Parses a answer received from data files
       #
       # Ultimately it just pass the data through parse_string but
       # it makes some effort to handle arrays of strings as well
-      def parse_answer(data, scope, extra_data={})
+      def parse_answer(data, scope, extra_data={}, context={:recurse_guard => nil, :order_override => nil})
         if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass)
           return data
         elsif data.is_a?(String)
-          return parse_string(data, scope, extra_data)
+          return parse_string(data, scope, extra_data, context)
         elsif data.is_a?(Hash)
           answer = {}
           data.each_pair do |key, val|
-            interpolated_key = parse_string(key, scope, extra_data)
-            answer[interpolated_key] = parse_answer(val, scope, extra_data)
+            interpolated_key = parse_string(key, scope, extra_data, context)
+            answer[interpolated_key] = parse_answer(val, scope, extra_data, context)
           end
 
           return answer
         elsif data.is_a?(Array)
           answer = []
           data.each do |item|
-            answer << parse_answer(item, scope, extra_data)
+            answer << parse_answer(item, scope, extra_data, context)
           end
 
           return answer
@@ -163,21 +180,37 @@ class Hiera
         end
       end
 
-      # Merges two hashes answers with the configured merge behavior.
-      #         :merge_behavior: {:native|:deep|:deeper}
+      # Merges two hashes answers with the given or configured merge behavior. Behavior can be given
+      # by passing _resolution_type_ as a Hash
+      #
+      #  :merge_behavior: {:native|:deep|:deeper}
       #
-      # Deep merge options use the Hash utility function provided by [deep_merge](https://github.com/peritor/deep_merge)
+      # Deep merge options use the Hash utility function provided by [deep_merge](https://github.com/danielsdeleo/deep_merge)
       #
       #  :native => Native Hash.merge
       #  :deep   => Use Hash.deep_merge
       #  :deeper => Use Hash.deep_merge!
       #
-      def merge_answer(left,right)
-        case Config[:merge_behavior]
+      # @param left [Hash] left side of the merge
+      # @param right [Hash] right side of the merge
+      # @param resolution_type [String,Hash] The merge type, or if hash, the merge behavior and options
+      # @return [Hash] The merged result
+      # @see Hiera#lookup
+      #
+      def merge_answer(left,right,resolution_type=nil)
+        behavior, options =
+          if resolution_type.is_a?(Hash)
+            merge = resolution_type.clone
+            [merge.delete(:behavior), merge]
+          else
+            [Config[:merge_behavior], Config[:deep_merge_options] || {}]
+          end
+
+        case behavior
         when :deeper,'deeper'
-          left.deep_merge!(right)
+          left.deep_merge!(right, options)
         when :deep,'deep'
-          left.deep_merge(right)
+          left.deep_merge(right, options)
         else # Native and undefined
           left.merge(right)
         end
@@ -196,43 +229,97 @@ class Hiera
       # Backend instances are cached so if you need to connect to any
       # databases then do so in your constructor, future calls to your
       # backend will not create new instances
-      def lookup(key, default, scope, order_override, resolution_type)
+
+      # @param key [String] The key to lookup
+      # @param scope [#[]] The primary source of data for substitutions.
+      # @param order_override [#[],nil] An override that will be pre-pended to the hierarchy definition.
+      # @param resolution_type [Symbol,Hash,nil] One of :hash, :array,:priority or a Hash with deep merge behavior and options
+      # @param context [#[]] Context used for internal processing
+      # @return [Object] The value that corresponds to the given key or nil if no such value cannot be found
+      #
+      def lookup(key, default, scope, order_override, resolution_type, context = {:recurse_guard => nil})
         @backends ||= {}
         answer = nil
 
+        # order_override is kept as an explicit argument for backwards compatibility, but should be specified
+        # in the context for internal handling.
+        context ||= {}
+        order_override ||= context[:order_override]
+        context[:order_override] ||= order_override
+
+        strategy = resolution_type.is_a?(Hash) ? :hash : resolution_type
+
+        segments = key.split('.')
+        subsegments = nil
+        if segments.size > 1
+          raise ArgumentError, "Resolution type :#{strategy} is illegal when doing segmented key lookups" unless strategy.nil? || strategy == :priority
+          subsegments = segments.drop(1)
+        end
+
+        found = false
         Config[:backends].each do |backend|
-          if constants.include?("#{backend.capitalize}_backend") || constants.include?("#{backend.capitalize}_backend".to_sym)
-            @backends[backend] ||= Backend.const_get("#{backend.capitalize}_backend").new
-            new_answer = @backends[backend].lookup(key, scope, order_override, resolution_type)
-
-            if not new_answer.nil?
-              case resolution_type
-              when :array
-                raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
-                answer ||= []
-                answer << new_answer
-              when :hash
-                raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
-                answer ||= {}
-                answer = merge_answer(new_answer,answer)
-              else
-                answer = new_answer
-                break
-              end
+          backend_constant = "#{backend.capitalize}_backend"
+          if constants.include?(backend_constant) || constants.include?(backend_constant.to_sym)
+            backend = (@backends[backend] ||= find_backend(backend_constant))
+            found_in_backend = false
+            new_answer = catch(:no_such_key) do
+              value = backend.lookup(segments[0], scope, order_override, resolution_type, context)
+              value = qualified_lookup(subsegments, value) unless subsegments.nil?
+              found_in_backend = true
+              value
+            end
+            next unless found_in_backend
+            found = true
+
+            case strategy
+            when :array
+              raise Exception, "Hiera type mismatch for key '#{key}': expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+              answer ||= []
+              answer << new_answer
+            when :hash
+              raise Exception, "Hiera type mismatch for key '#{key}': expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+              answer ||= {}
+              answer = merge_answer(new_answer, answer, resolution_type)
+            else
+              answer = new_answer
+              break
             end
           end
         end
 
-        answer = resolve_answer(answer, resolution_type) unless answer.nil?
-        answer = parse_string(default, scope) if answer.nil? and default.is_a?(String)
+        answer = resolve_answer(answer, strategy) unless answer.nil?
+        answer = parse_string(default, scope, {}, context) if !found && default.is_a?(String)
 
-        return default if answer.nil?
+        return default if !found && answer.nil?
         return answer
       end
 
       def clear!
         @backends = {}
       end
+
+      def qualified_lookup(segments, hash)
+        value = hash
+        segments.each do |segment|
+          throw :no_such_key if value.nil?
+          if segment =~ /^[0-9]+$/
+            segment = segment.to_i
+            raise Exception, "Hiera type mismatch: Got #{value.class.name} when Array was expected enable lookup using key '#{segment}'" unless value.instance_of?(Array)
+            throw :no_such_key unless segment < value.size
+          else
+            raise Exception, "Hiera type mismatch: Got #{value.class.name} when a non Array object that responds to '[]' was expected to enable lookup using key '#{segment}'" unless value.respond_to?(:'[]') && !value.instance_of?(Array);
+            throw :no_such_key unless value.include?(segment)
+          end
+          value = value[segment]
+        end
+        value
+      end
+
+      def find_backend(backend_constant)
+        backend = Backend.const_get(backend_constant).new
+        return backend.method(:lookup).arity == 4 ? Backend1xWrapper.new(backend) : backend
+      end
+      private :find_backend
     end
   end
 end