hiera 2.0.0-x86-mingw32

data/bin/hiera

@@ -0,0 +1,248 @@
+#!/usr/bin/env ruby
+
+# CLI client for Hiera.
+#
+# To lookup the 'release' key for a node given Puppet YAML facts:
+#
+# $ hiera release 'rel/%{location}' --yaml some.node.yaml
+#
+# If the node yaml had a location fact the default would match that
+# else you can supply scope values on the command line
+#
+# $ hiera release 'rel/%{location}' location=dc2 --yaml some.node.yaml
+
+# Bundler and rubygems maintain a set of directories from which to
+# load gems. If Bundler is loaded, let it determine what can be
+# loaded. If it's not loaded, then use rubygems. But do this before
+# loading any hiera code, so that our gem loading system is sane.
+if not defined? ::Bundler
+  begin
+    require 'rubygems'
+  rescue LoadError
+  end
+end
+require 'hiera'
+require 'hiera/util'
+require 'optparse'
+require 'pp'
+
+options = {
+  :default => nil,
+  :config  => File.join(Hiera::Util.config_dir, 'hiera.yaml'),
+  :scope   => {},
+  :key     => nil,
+  :verbose => false,
+  :resolution_type => :priority,
+  :format => :ruby
+}
+
+initial_scopes = Array.new
+
+# Loads the scope from YAML or JSON files
+def load_scope(source, type=:yaml)
+  case type
+  when :mcollective
+    begin
+      require 'mcollective'
+
+      include MCollective::RPC
+
+      util = rpcclient("rpcutil")
+      util.progress = false
+      nodestats = util.custom_request("inventory", {}, source, {"identity" => source}).first
+
+      raise "Failed to retrieve facts for node #{source}: #{nodestats[:statusmsg]}" unless nodestats[:statuscode] == 0
+
+      scope = nodestats[:data][:facts]
+    rescue Exception => e
+      STDERR.puts "MCollective lookup failed: #{e.class}: #{e}"
+      exit 1
+    end
+
+  when :yaml
+    raise "Cannot find scope #{type} file #{source}" unless File.exist?(source)
+
+    require 'yaml'
+
+    # Attempt to load puppet in case we're going to be fed
+    # Puppet yaml files
+    begin
+        require 'puppet'
+    rescue
+    end
+
+    scope = YAML.load_file(source)
+
+    # Puppet makes dumb yaml files that do not promote data reuse.
+    scope = scope.values if scope.is_a?(Puppet::Node::Facts)
+
+  when :json
+    raise "Cannot find scope #{type} file #{source}" unless File.exist?(source)
+
+    require 'json'
+
+    scope = JSON.load(File.read(source))
+
+  when :inventory_service
+    # For this to work the machine running the hiera command needs access to
+    # /facts REST endpoint on your inventory server.  This access is
+    # controlled in auth.conf and identification is by the certname of the
+    # machine running hiera commands.
+    #
+    # Another caveat is that if your inventory server isn't at the short dns
+    # name of 'puppet' you will need to set the inventory_sever option in
+    # your puppet.conf.  Set it in either the master or main sections.  It
+    # is fine to have the inventory_server option set even if the config
+    # doesn't have the fact_terminus set to rest.
+    begin
+      require 'puppet/util/run_mode'
+      $puppet_application_mode = Puppet::Util::RunMode[:master]
+      require 'puppet'
+      Puppet.settings.parse
+      Puppet::Node::Facts.indirection.terminus_class = :rest
+      scope = YAML.load(Puppet::Node::Facts.indirection.find(source).to_yaml)
+      # Puppet makes dumb yaml files that do not promote data reuse.
+      scope = scope.values if scope.is_a?(Puppet::Node::Facts)
+    rescue Exception => e
+        STDERR.puts "Puppet inventory service lookup failed: #{e.class}: #{e}"
+        exit 1
+    end
+  else
+    raise "Don't know how to load data type #{type}"
+  end
+
+  raise "Scope from #{type} file #{source} should be a Hash" unless scope.is_a?(Hash)
+
+  scope
+end
+
+def output_answer(ans, format)
+  case format
+  when :json
+    require 'json'
+    puts JSON.dump(ans)
+  when :ruby
+    if ans.is_a?(String)
+      puts ans
+    else
+      pp ans
+    end
+  when :yaml
+    require 'yaml'
+    puts ans.to_yaml
+  else
+    STDERR.puts "Unknown output format: #{v}"
+    exit 1
+  end
+end
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: hiera [options] key [default value] [variable='text'...]\n\nThe default value will be used if no value is found for the key. Scope variables\nwill be interpolated into %{variable} placeholders in the hierarchy and in\nreturned values.\n\n"
+
+  opts.on("--version", "-V", "Version information") do
+    puts Hiera.version
+    exit
+  end
+
+  opts.on("--debug", "-d", "Show debugging information") do
+    options[:verbose] = true
+  end
+
+  opts.on("--array", "-a", "Return all values as an array") do
+    options[:resolution_type] = :array
+  end
+
+  opts.on("--hash", "-h", "Return all values as a hash") do
+    options[:resolution_type] = :hash
+  end
+
+  opts.on("--config CONFIG", "-c", "Configuration file") do |v|
+    if File.exist?(v)
+      options[:config] = v
+    else
+      STDERR.puts "Cannot find config file: #{v}"
+      exit 1
+    end
+  end
+
+  opts.on("--json SCOPE", "-j", "JSON format file to load scope from") do |v|
+    initial_scopes << { :type => :json, :value => v, :name => "JSON" }
+  end
+
+  opts.on("--yaml SCOPE", "-y", "YAML format file to load scope from") do |v|
+    initial_scopes << { :type => :yaml, :value => v, :name => "YAML" }
+  end
+
+  opts.on("--mcollective IDENTITY", "-m", "Use facts from a node (via mcollective) as scope") do |v|
+    initial_scopes << { :type => :mcollective, :value => v, :name => "Mcollective" }
+  end
+
+  opts.on("--inventory_service IDENTITY", "-i", "Use facts from a node (via Puppet's inventory service) as scope") do |v|
+    initial_scopes << { :type => :inventory_service, :value => v, :name => "Puppet inventory service" }
+  end
+
+  opts.on("--format TYPE", "-f", "Output the result in a specific format (ruby, yaml or json); default is 'ruby'") do |v|
+    options[:format] = case v
+    when 'json', 'ruby', 'yaml'
+      v.to_sym
+    else
+      STDERR.puts "Unknown output format: #{v}"
+      exit 1
+    end
+  end
+end.parse!
+
+unless initial_scopes.empty?
+  initial_scopes.each { |this_scope|
+    # Load initial scope
+    begin
+      options[:scope] = load_scope(this_scope[:value], this_scope[:type])
+    rescue Exception => e
+      STDERR.puts "Could not load #{this_scope[:name]} scope: #{e.class}: #{e}"
+      exit 1
+    end
+  }
+end
+
+# arguments can be:
+#
+# key default var=val another=val
+#
+# The var=val's assign scope
+unless ARGV.empty?
+  options[:key] = ARGV.delete_at(0)
+
+  ARGV.each do |arg|
+    if arg =~ /^(.+?)=(.+?)$/
+      options[:scope][$1] = $2
+    else
+      unless options[:default]
+        options[:default] = arg.dup
+      else
+        STDERR.puts "Don't know how to parse scope argument: #{arg}"
+      end
+    end
+  end
+else
+  STDERR.puts "Please supply a data item to look up"
+  exit 1
+end
+
+begin
+  hiera = Hiera.new(:config => options[:config])
+rescue Exception => e
+  if options[:verbose]
+    raise
+  else
+    STDERR.puts "Failed to start Hiera: #{e.class}: #{e}"
+    exit 1
+  end
+end
+
+unless options[:verbose]
+  Hiera.logger = "noop"
+end
+
+ans = hiera.lookup(options[:key], options[:default], options[:scope], nil, options[:resolution_type])
+
+output_answer(ans, options[:format])

data/lib/hiera.rb

@@ -0,0 +1,115 @@
+require 'yaml'
+
+class Hiera
+  require "hiera/error"
+  require "hiera/version"
+  require "hiera/config"
+  require "hiera/util"
+  require "hiera/backend"
+  require "hiera/console_logger"
+  require "hiera/puppet_logger"
+  require "hiera/noop_logger"
+  require "hiera/fallback_logger"
+  require "hiera/filecache"
+
+  class << self
+    attr_reader :logger
+
+    # Loggers are pluggable, just provide a class called
+    # Hiera::Foo_logger and respond to :warn and :debug
+    #
+    # See hiera-puppet for an example that uses the Puppet
+    # loging system instead of our own
+    def logger=(logger)
+      require "hiera/#{logger}_logger"
+
+      @logger = Hiera::FallbackLogger.new(
+        Hiera.const_get("#{logger.capitalize}_logger"),
+        Hiera::Console_logger)
+    rescue Exception => e
+      @logger = Hiera::Console_logger
+      warn("Failed to load #{logger} logger: #{e.class}: #{e}")
+    end
+
+    def warn(msg); @logger.warn(msg); end
+    def debug(msg); @logger.debug(msg); end
+  end
+
+  attr_reader :options, :config
+
+  # If the config option is a string its assumed to be a filename,
+  # else a hash of what would have been in the YAML config file
+  def initialize(options={})
+    options[:config] ||= File.join(Util.config_dir, 'hiera.yaml')
+
+    @config = Config.load(options[:config])
+
+    Config.load_backends
+  end
+
+  # Calls the backends to do the actual lookup.
+  #
+  # The _scope_ can be anything that responds to `[]`, if you have input
+  # data like a Puppet Scope that does not you can wrap that data in a
+  # class that has a `[]` method that fetches the data from your source.
+  # See hiera-puppet for an example of this.
+  #
+  # The order-override will insert as first in the hierarchy a data source
+  # of your choice.
+  #
+  # Possible values for the _resolution_type_ parameter:
+  #
+  # - _:priority_ - This is the default. First found value is returned and no merge is performed
+  # - _:array_ - An array merge lookup assembles a value from every matching level of the hierarchy. It retrieves all
+  #     of the (string or array) values for a given key, then flattens them into a single array of unique values.
+  #     If _priority_ lookup can be thought of as a “default with overrides” pattern, _array_ merge lookup can be though
+  #     of as “default with additions.”
+  # - _:hash_ - A hash merge lookup assembles a value from every matching level of the hierarchy. It retrieves all of
+  #     the (hash) values for a given key, then merges the hashes into a single hash. Hash merge lookups will fail with
+  #     an error if any of the values found in the data sources are strings or arrays. It only works when every value
+  #     found is a hash. The actual merge behavior is determined by looking up the keys `:merge_behavior` and
+  #     `:deep_merge_options` in the Hiera config. `:merge_behavior` can be set to `:deep`, :deeper` or `:native`
+  #     (explained in detail below).
+  # - _{ deep merge options }_ - Configured values for `:merge_behavior` and `:deep_merge_options`will be completely
+  #     ignored. Instead the _resolution_type_ will be a `:hash` merge where the `:merge_behavior` will be the value
+  #     keyed by `:behavior` in the given hash and the `:deep_merge_options` will be the remaining top level entries of
+  #     that same hash.
+  #
+  # Valid behaviors for the _:hash_ resolution type:
+  #
+  # - _native_ - Performs a simple hash-merge by overwriting keys of lower lookup priority.
+  # - _deeper_ - In a deeper hash merge, Hiera recursively merges keys and values in each source hash. For each key,
+  #     if the value is:
+  #        - only present in one source hash, it goes into the final hash.
+  #        - a string/number/boolean and exists in two or more source hashes, the highest priority value goes into
+  #          the final hash.
+  #        - an array and exists in two or more source hashes, the values from each source are merged into a single
+  #          array and de-duplicated (but not automatically flattened, as in an array merge lookup).
+  #        - a hash and exists in two or more source hashes, the values from each source are recursively merged, as
+  #          though they were source hashes.
+  #        - mismatched between two or more source hashes, we haven’t validated the behavior. It should act as
+  #          described in the deep_merge gem documentation.
+  # - _deep_ - In a deep hash merge, Hiera behaves the same as for _deeper_, except that when a string/number/boolean
+  #     exists in two or more source hashes, the lowest priority value goes into the final hash. This is considered
+  #     largely useless and should be avoided. Use _deeper_ instead.
+  #
+  # The _merge_ can be given as a hash with the mandatory key `:strategy` to denote the actual strategy. This
+  # is useful for the `:deeper` and `:deep` strategy since they can use additional options to control the behavior.
+  # The options can be passed as top level keys in the `merge` parameter when it is a given as a hash. Recognized
+  # options are:
+  #
+  #  - 'knockout_prefix' Set to string value to signify prefix which deletes elements from existing element. Defaults is _undef_
+  #  - 'sort_merged_arrays' Set to _true_ to sort all arrays that are merged together. Default is _false_
+  #  - 'unpack_arrays' Set to string value used as a deliminator to join all array values and then split them again. Default is _undef_
+  #  - 'merge_hash_arrays' Set to _true_ to merge hashes within arrays. Default is _false_
+  #
+  # @param key [String] The key to lookup
+  # @param default [Object,nil] The value to return when there is no match for _key_
+  # @param scope [#[],nil] The scope to use for the lookup
+  # @param order_override [#[]] An override that will considered the first source of lookup
+  # @param resolution_type [String,Hash<Symbol,String>] Symbolic resolution type or deep merge configuration
+  # @return [Object] The found value or the given _default_ value
+  def lookup(key, default, scope, order_override=nil, resolution_type=:priority)
+    Backend.lookup(key, default, scope, order_override, resolution_type)
+  end
+end

data/lib/hiera/backend.rb

@@ -0,0 +1,325 @@
+require 'hiera/util'
+require 'hiera/interpolate'
+
+begin
+  require 'deep_merge'
+rescue LoadError
+end
+
+class Hiera
+  module Backend
+    class Backend1xWrapper
+      def initialize(wrapped)
+        @wrapped = wrapped
+      end
+
+      def lookup(key, scope, order_override, resolution_type, context)
+        Hiera.debug("Using Hiera 1.x backend API to access instance of class #{@wrapped.class.name}. Lookup recursion will not be detected")
+        value = @wrapped.lookup(key, scope, order_override, resolution_type.is_a?(Hash) ? :hash : resolution_type)
+
+        # The most likely cause when an old backend returns nil is that the key was not found. In any case, it is
+        # impossible to know the difference between that and a found nil. The throw here preserves the old behavior.
+        throw (:no_such_key) if value.nil?
+        value
+      end
+    end
+
+    class << self
+      # Data lives in /var/lib/hiera by default.  If a backend
+      # supplies a datadir in the config it will be used and
+      # subject to variable expansion based on scope
+      def datadir(backend, scope)
+        backend = backend.to_sym
+
+        if Config[backend] && Config[backend][:datadir]
+          dir = Config[backend][:datadir]
+        else
+          dir = Hiera::Util.var_dir
+        end
+
+        if !dir.is_a?(String)
+          raise(Hiera::InvalidConfigurationError,
+                "datadir for #{backend} cannot be an array")
+        end
+
+        parse_string(dir, scope)
+      end
+
+      # Finds the path to a datafile based on the Backend#datadir
+      # and extension
+      #
+      # If the file is not found nil is returned
+      def datafile(backend, scope, source, extension)
+        datafile_in(datadir(backend, scope), source, extension)
+      end
+
+      # @api private
+      def datafile_in(datadir, source, extension)
+        file = File.join(datadir, "#{source}.#{extension}")
+
+        if File.exist?(file)
+          file
+        else
+          Hiera.debug("Cannot find datafile #{file}, skipping")
+          nil
+        end
+      end
+
+      # Constructs a list of data sources to search
+      #
+      # If you give it a specific hierarchy it will just use that
+      # else it will use the global configured one, failing that
+      # it will just look in the 'common' data source.
+      #
+      # An override can be supplied that will be pre-pended to the
+      # hierarchy.
+      #
+      # The source names will be subject to variable expansion based
+      # on scope
+      def datasources(scope, override=nil, hierarchy=nil)
+        if hierarchy
+          hierarchy = [hierarchy]
+        elsif Config.include?(:hierarchy)
+          hierarchy = [Config[:hierarchy]].flatten
+        else
+          hierarchy = ["common"]
+        end
+
+        hierarchy.insert(0, override) if override
+
+        hierarchy.flatten.map do |source|
+          source = parse_string(source, scope, {}, :order_override => override)
+          yield(source) unless source == "" or source =~ /(^\/|\/\/|\/$)/
+        end
+      end
+
+      # Constructs a list of data files to search
+      #
+      # If you give it a specific hierarchy it will just use that
+      # else it will use the global configured one, failing that
+      # it will just look in the 'common' data source.
+      #
+      # An override can be supplied that will be pre-pended to the
+      # hierarchy.
+      #
+      # The source names will be subject to variable expansion based
+      # on scope
+      #
+      # Only files that exist will be returned. If the file is missing, then
+      # the block will not receive the file.
+      #
+      # @yield [String, String] the source string and the name of the resulting file
+      # @api public
+      def datasourcefiles(backend, scope, extension, override=nil, hierarchy=nil)
+        datadir = Backend.datadir(backend, scope)
+        Backend.datasources(scope, override, hierarchy) do |source|
+          Hiera.debug("Looking for data source #{source}")
+          file = datafile_in(datadir, source, extension)
+
+          if file
+            yield source, file
+          end
+        end
+      end
+
+      # Parse a string like <code>'%{foo}'</code> against a supplied
+      # scope and additional scope.  If either scope or
+      # extra_scope includes the variable 'foo', then it will
+      # be replaced else an empty string will be placed.
+      #
+      # If both scope and extra_data has "foo", then the value in scope
+      # will be used.
+      #
+      # @param data [String] The string to perform substitutions on.
+      #   This will not be modified, instead a new string will be returned.
+      # @param scope [#[]] The primary source of data for substitutions.
+      # @param extra_data [#[]] The secondary source of data for substitutions.
+      # @param context [#[]] Context can include :recurse_guard and :order_override.
+      # @return [String] A copy of the data with all instances of <code>%{...}</code> replaced.
+      #
+      # @api public
+      def parse_string(data, scope, extra_data={}, context={:recurse_guard => nil, :order_override => nil})
+        Hiera::Interpolate.interpolate(data, scope, extra_data, context)
+      end
+
+      # Parses a answer received from data files
+      #
+      # Ultimately it just pass the data through parse_string but
+      # it makes some effort to handle arrays of strings as well
+      def parse_answer(data, scope, extra_data={}, context={:recurse_guard => nil, :order_override => nil})
+        if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass)
+          return data
+        elsif data.is_a?(String)
+          return parse_string(data, scope, extra_data, context)
+        elsif data.is_a?(Hash)
+          answer = {}
+          data.each_pair do |key, val|
+            interpolated_key = parse_string(key, scope, extra_data, context)
+            answer[interpolated_key] = parse_answer(val, scope, extra_data, context)
+          end
+
+          return answer
+        elsif data.is_a?(Array)
+          answer = []
+          data.each do |item|
+            answer << parse_answer(item, scope, extra_data, context)
+          end
+
+          return answer
+        end
+      end
+
+      def resolve_answer(answer, resolution_type)
+        case resolution_type
+        when :array
+          [answer].flatten.uniq.compact
+        when :hash
+          answer # Hash structure should be preserved
+        else
+          answer
+        end
+      end
+
+      # Merges two hashes answers with the given or configured merge behavior. Behavior can be given
+      # by passing _resolution_type_ as a Hash
+      #
+      #  :merge_behavior: {:native|:deep|:deeper}
+      #
+      # Deep merge options use the Hash utility function provided by [deep_merge](https://github.com/danielsdeleo/deep_merge)
+      #
+      #  :native => Native Hash.merge
+      #  :deep   => Use Hash.deep_merge
+      #  :deeper => Use Hash.deep_merge!
+      #
+      # @param left [Hash] left side of the merge
+      # @param right [Hash] right side of the merge
+      # @param resolution_type [String,Hash] The merge type, or if hash, the merge behavior and options
+      # @return [Hash] The merged result
+      # @see Hiera#lookup
+      #
+      def merge_answer(left,right,resolution_type=nil)
+        behavior, options =
+          if resolution_type.is_a?(Hash)
+            merge = resolution_type.clone
+            [merge.delete(:behavior), merge]
+          else
+            [Config[:merge_behavior], Config[:deep_merge_options] || {}]
+          end
+
+        case behavior
+        when :deeper,'deeper'
+          left.deep_merge!(right, options)
+        when :deep,'deep'
+          left.deep_merge(right, options)
+        else # Native and undefined
+          left.merge(right)
+        end
+      end
+
+      # Calls out to all configured backends in the order they
+      # were specified.  The first one to answer will win.
+      #
+      # This lets you declare multiple backends, a possible
+      # use case might be in Puppet where a Puppet module declares
+      # default data using in-module data while users can override
+      # using JSON/YAML etc.  By layering the backends and putting
+      # the Puppet one last you can override module author data
+      # easily.
+      #
+      # Backend instances are cached so if you need to connect to any
+      # databases then do so in your constructor, future calls to your
+      # backend will not create new instances
+
+      # @param key [String] The key to lookup
+      # @param scope [#[]] The primary source of data for substitutions.
+      # @param order_override [#[],nil] An override that will be pre-pended to the hierarchy definition.
+      # @param resolution_type [Symbol,Hash,nil] One of :hash, :array,:priority or a Hash with deep merge behavior and options
+      # @param context [#[]] Context used for internal processing
+      # @return [Object] The value that corresponds to the given key or nil if no such value cannot be found
+      #
+      def lookup(key, default, scope, order_override, resolution_type, context = {:recurse_guard => nil})
+        @backends ||= {}
+        answer = nil
+
+        # order_override is kept as an explicit argument for backwards compatibility, but should be specified
+        # in the context for internal handling.
+        context ||= {}
+        order_override ||= context[:order_override]
+        context[:order_override] ||= order_override
+
+        strategy = resolution_type.is_a?(Hash) ? :hash : resolution_type
+
+        segments = key.split('.')
+        subsegments = nil
+        if segments.size > 1
+          raise ArgumentError, "Resolution type :#{strategy} is illegal when doing segmented key lookups" unless strategy.nil? || strategy == :priority
+          subsegments = segments.drop(1)
+        end
+
+        found = false
+        Config[:backends].each do |backend|
+          backend_constant = "#{backend.capitalize}_backend"
+          if constants.include?(backend_constant) || constants.include?(backend_constant.to_sym)
+            backend = (@backends[backend] ||= find_backend(backend_constant))
+            found_in_backend = false
+            new_answer = catch(:no_such_key) do
+              value = backend.lookup(segments[0], scope, order_override, resolution_type, context)
+              value = qualified_lookup(subsegments, value) unless subsegments.nil?
+              found_in_backend = true
+              value
+            end
+            next unless found_in_backend
+            found = true
+
+            case strategy
+            when :array
+              raise Exception, "Hiera type mismatch for key '#{key}': expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+              answer ||= []
+              answer << new_answer
+            when :hash
+              raise Exception, "Hiera type mismatch for key '#{key}': expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+              answer ||= {}
+              answer = merge_answer(new_answer, answer, resolution_type)
+            else
+              answer = new_answer
+              break
+            end
+          end
+        end
+
+        answer = resolve_answer(answer, strategy) unless answer.nil?
+        answer = parse_string(default, scope, {}, context) if !found && default.is_a?(String)
+
+        return default if !found && answer.nil?
+        return answer
+      end
+
+      def clear!
+        @backends = {}
+      end
+
+      def qualified_lookup(segments, hash)
+        value = hash
+        segments.each do |segment|
+          throw :no_such_key if value.nil?
+          if segment =~ /^[0-9]+$/
+            segment = segment.to_i
+            raise Exception, "Hiera type mismatch: Got #{value.class.name} when Array was expected enable lookup using key '#{segment}'" unless value.instance_of?(Array)
+            throw :no_such_key unless segment < value.size
+          else
+            raise Exception, "Hiera type mismatch: Got #{value.class.name} when a non Array object that responds to '[]' was expected to enable lookup using key '#{segment}'" unless value.respond_to?(:'[]') && !value.instance_of?(Array);
+            throw :no_such_key unless value.include?(segment)
+          end
+          value = value[segment]
+        end
+        value
+      end
+
+      def find_backend(backend_constant)
+        backend = Backend.const_get(backend_constant).new
+        return backend.method(:lookup).arity == 4 ? Backend1xWrapper.new(backend) : backend
+      end
+      private :find_backend
+    end
+  end
+end