puppet 4.0.0 → 4.5.0

data/lib/puppet/data_providers/hiera_config.rb

@@ -0,0 +1,129 @@
+require 'pathname'
+require 'puppet/pops/lookup/interpolation'
+
+module Puppet::DataProviders
+  class HieraConfig
+    include Puppet::Plugins::DataProviders
+    include Puppet::Pops::Lookup::Interpolation
+
+    DEFAULT_CONFIG = {
+      'version' => 4,
+      'datadir' => 'data',
+      'hierarchy' => [
+        {
+          'name' => 'common',
+          'backend' => 'yaml'
+        }
+      ]
+    }.freeze
+
+    def self.config_type
+      @@CONFIG_TYPE ||= create_config_type
+    end
+
+    def self.symkeys_to_string(struct)
+      case(struct)
+      when Hash
+        Hash[struct.map { |k,v| [k.to_s, symkeys_to_string(v)] }]
+      when Array
+        struct.map { |v| symkeys_to_string(v) }
+      else
+        struct
+      end
+    end
+
+    def self.create_config_type
+      hierarchy_elem_type_base = 'Struct[{'\
+        'backend=>String[1],'\
+        'name=>String[1],'\
+        'datadir=>Optional[String[1]]'
+
+      hierarchy_elem_type_v1 = hierarchy_elem_type_base + ',path=>String[1]}]'
+      hierarchy_elem_type_v2 = hierarchy_elem_type_base + ',paths=>Array[String[1]]}]'
+      hierarchy_elem_type_v3 = hierarchy_elem_type_base + '}]'
+
+      Puppet::Pops::Types::TypeParser.new.parse('Struct[{'\
+        'version=>Integer[4],'\
+        "hierarchy=>Optional[Array[Variant[#{hierarchy_elem_type_v1},#{hierarchy_elem_type_v2},#{hierarchy_elem_type_v3}]]],"\
+        'datadir=>Optional[String[1]]}]')
+    end
+    private_class_method :create_config_type
+
+    attr_reader :config_path, :version
+
+    # Creates a new HieraConfig from the given _config_root_. This is where the 'hiera.yaml' is expected to be found
+    # and is also the base location used when resolving relative paths.
+    #
+    # @param config_root [Pathname] Absolute path to the configuration root
+    # @api public
+    def initialize(config_root)
+      @config_root = config_root
+      @config_path = config_root + 'hiera.yaml'
+      if @config_path.exist?
+        @config = validate_config(HieraConfig.symkeys_to_string(YAML.load_file(@config_path)))
+        @config['hierarchy'] ||= DEFAULT_CONFIG['hierarchy']
+        @config['datadir'] ||= DEFAULT_CONFIG['datadir']
+      else
+        @config = DEFAULT_CONFIG
+      end
+      @version = @config['version']
+    end
+
+    def create_data_providers(lookup_invocation)
+      Puppet.deprecation_warning('Method HieraConfig#create_data_providers is deprecated. Use create_configured_data_providers instead')
+      create_configured_data_providers(lookup_invocation, nil)
+    end
+
+    # Creates the data providers for this config
+    #
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] Invocation data containing scope, overrides, and defaults
+    # @param parent_data_provider [Puppet::Plugins::DataProviders::DataProvider] The data provider that loaded this configuration
+    # @return [Array[DataProvider]] the created providers
+    #
+    # @api private
+    def create_configured_data_providers(lookup_invocation, parent_data_provider)
+      injector = Puppet.lookup(:injector)
+      service_type = Registry.hash_of_path_based_data_provider_factories
+      default_datadir = @config['datadir']
+
+      # Hashes enumerate their values in the order that the corresponding keys were inserted so it's safe to use
+      # a hash for the data_providers.
+      data_providers = {}
+      @config['hierarchy'].each do |he|
+        name = he['name']
+        raise Puppet::DataBinding::LookupError, "#{path}: Name '#{name}' defined more than once" if data_providers.include?(name)
+        original_paths = he['paths']
+        if original_paths.nil?
+          single_path = he['path']
+          single_path = name if single_path.nil?
+          original_paths = [single_path]
+        end
+        paths = original_paths.map { |path| interpolate(path, lookup_invocation, false)}
+        provider_name = he['backend']
+        provider_factory = injector.lookup(nil, service_type, PATH_BASED_DATA_PROVIDER_FACTORIES_KEY)[provider_name]
+        raise Puppet::DataBinding::LookupError, "#{@config_path}: No data provider is registered for backend '#{provider_name}' " unless provider_factory
+
+        datadir = @config_root + (he['datadir'] || default_datadir)
+        data_providers[name] = create_data_provider(parent_data_provider, provider_factory, name,
+          provider_factory.resolve_paths(datadir, original_paths, paths, lookup_invocation))
+      end
+      data_providers.values
+    end
+
+    def create_data_provider(parent_data_provider, provider_factory, name, resolved_paths)
+      provider_factory_version = provider_factory.respond_to?(:version) ? provider_factory.version : 1
+      if provider_factory_version == 1
+        # Version 1 is not aware of the parent provider
+        provider_factory.create(name, resolved_paths)
+      else
+        provider_factory.create(name, resolved_paths, parent_data_provider)
+      end
+    end
+    private :create_data_provider
+
+    def validate_config(hiera_config)
+      Puppet::Pops::Types::TypeAsserter.assert_instance_of(["The Hiera Configuration at '%s'", @config_path], self.class.config_type, hiera_config)
+    end
+    private :validate_config
+  end
+end

data/lib/puppet/data_providers/hiera_env_data_provider.rb

@@ -0,0 +1,18 @@
+# This file is loaded by the autoloader, and it does not find the hiera support unless required relative
+require_relative 'hiera_support'
+
+module Puppet::DataProviders
+  class HieraEnvDataProvider < Puppet::Plugins::DataProviders::EnvironmentDataProvider
+    include HieraSupport
+
+    # Return the root of the environment found in the given _scope_
+    #
+    # @param data_key [String] not used
+    # @param scope [Puppet::Parser::Scope] the parser scope where the environment is found
+    # @return [Pathname] Path to root of the environment
+    def provider_root(_, scope)
+      Pathname.new(scope.environment.configuration.path_to_env)
+    end
+    protected :provider_root
+  end
+end

data/lib/puppet/data_providers/hiera_interpolate.rb

@@ -0,0 +1,19 @@
+# @deprecated Moved to Puppet::Pops::Lookup::Interpolation
+module Puppet::DataProviders::HieraInterpolate
+  include Puppet::Pops::Lookup::Interpolation
+
+  # For backward compatibility
+  # @deprecated
+  def qualified_lookup(segments, value)
+    if Puppet[:strict] != :off
+      msg = 'Puppet::DataProviders::HieraInterpolate#qualified_lookup is deprecated, use Puppet::Pops::Lookup::SubLookup#sub_lookup'
+      case Puppet[:strict]
+      when :error
+        raise Puppet::DataBinding::LookupError.new(msg)
+      when :warning
+        Puppet.warn_once(:deprecation, 'HieraInterpolate#qualified_lookup', msg)
+      end
+    end
+    sub_lookup('<unknown key>', Puppet::Pops::Lookup::Invocation.new(nil), segments, value)
+  end
+end

data/lib/puppet/data_providers/hiera_module_data_provider.rb

@@ -0,0 +1,23 @@
+# This file is loaded by the autoloader, and it does not find the hiera support unless required relative
+require_relative 'hiera_support'
+
+module Puppet::DataProviders
+  class HieraModuleDataProvider < Puppet::Plugins::DataProviders::ModuleDataProvider
+    include HieraSupport
+
+    # Return the root of the module with the name equal to _data_key_ found in the environment of the given _scope_
+    #
+    # @param data_key [String] the name of the module
+    # @param scope [Puppet::Parser::Scope] the parser scope where the environment is found
+    # @return [Pathname] Path to root of the environment
+    # @raise [Puppet::DataBinder::LookupError] if the given module is can not be found
+    #
+    def provider_root(module_name, scope)
+      env = scope.environment
+      mod = env.modules.find { |m| m.name == module_name }
+      raise Puppet::DataBinder::LookupError, "Environment '#{env.name}', cannot find module '#{module_name}'" unless mod
+      Pathname.new(mod.path)
+    end
+    protected :provider_root
+  end
+end

data/lib/puppet/data_providers/hiera_support.rb

@@ -0,0 +1,37 @@
+require_relative 'hiera_config'
+
+module Puppet::DataProviders::HieraSupport
+  def config_path
+    @hiera_config.nil? ? 'not yet configured' : @hiera_config.config_path
+  end
+
+  def name
+    'Hiera Data Provider' + (@hiera_config.nil? ? '' : ", version #{@hiera_config.version}")
+  end
+
+  # Performs a lookup by searching all given paths for the given _key_. A merge will be performed if
+  # the value is found in more than one location and _merge_ is not nil.
+  #
+  # @param key [String] The key to lookup
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+  # @param merge [Puppet::Pops::MergeStrategy,String,Hash<String,Object>,nil] Merge strategy or hash with strategy and options
+  #
+  # @api public
+  def unchecked_lookup(key, lookup_invocation, merge)
+    lookup_invocation.with(:data_provider, self) do
+      merge_strategy = Puppet::Pops::MergeStrategy.strategy(merge)
+      lookup_invocation.with(:merge, merge_strategy) do
+        merged_result = merge_strategy.merge_lookup(data_providers(data_key(key, lookup_invocation), lookup_invocation)) do |data_provider|
+          data_provider.unchecked_lookup(key, lookup_invocation, merge_strategy)
+        end
+        lookup_invocation.report_result(merged_result)
+      end
+    end
+  end
+
+  def data_providers(data_key, lookup_invocation)
+    @hiera_config ||= Puppet::DataProviders::HieraConfig.new(provider_root(data_key, lookup_invocation.scope))
+    @data_providers ||= @hiera_config.create_configured_data_providers(lookup_invocation, self)
+  end
+  private :data_providers
+end

data/lib/puppet/data_providers/json_data_provider_factory.rb

@@ -0,0 +1,24 @@
+# This file is loaded by the autoloader, and it does not find the data function support unless required relative
+#
+require 'json'
+
+module Puppet::DataProviders
+  class JsonDataProviderFactory < Puppet::Plugins::DataProviders::FileBasedDataProviderFactory
+    def create(name, paths, parent_data_provider)
+      JsonDataProvider.new(name, paths, parent_data_provider)
+    end
+
+    def path_extension
+      '.json'
+    end
+  end
+
+  class JsonDataProvider < Puppet::Plugins::DataProviders::PathBasedDataProvider
+    def initialize_data(path, lookup_invocation)
+      JSON.parse(Puppet::FileSystem.read(path, :encoding => 'utf-8'))
+    rescue JSON::ParserError => ex
+      # Filename not included in message, so we add it here.
+      raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+    end
+  end
+end

data/lib/puppet/data_providers/lookup_adapter.rb

@@ -0,0 +1,247 @@
+# A LookupAdapter is a specialized DataAdapter that uses its hash to store module providers. It also remembers the environment
+# that it is attached to and maintains a cache of _lookup options_ retrieved from its data providers.
+#
+class Puppet::DataProviders::LookupAdapter < Puppet::DataProviders::DataAdapter
+
+  LOOKUP_OPTIONS = Puppet::Pops::Lookup::LOOKUP_OPTIONS
+  LOOKUP_OPTIONS_PREFIX = LOOKUP_OPTIONS + '.'
+  LOOKUP_OPTIONS_PREFIX.freeze
+  HASH = 'hash'.freeze
+  MERGE = 'merge'.freeze
+
+  def self.create_adapter(env)
+    new(env)
+  end
+
+  def initialize(env)
+    super()
+    @env = env
+    @lookup_options = {}
+  end
+
+  # Performs a lookup using global, environment, and module data providers. Merge the result using the given
+  # _merge_ strategy. If the merge strategy is nil, then an attempt is made to find merge options in the
+  # `lookup_options` hash for an entry associated with the key. If no options are found, the no merge is performed
+  # and the first found entry is returned.
+  #
+  # @param key [String] The key to lookup
+  # @param merge [Puppet::Pops::MergeStrategy,String,Hash<String,Object>,nil] Merge strategy or hash with strategy and options
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] Invocation data containing scope, overrides, and defaults
+  # @return [Object] The found value
+  # @throws :no_such_key if the given key is not found
+  #
+  def lookup(key, lookup_invocation, merge)
+    # The 'lookup_options' key is reserved and not found as normal data
+    if key == LOOKUP_OPTIONS || key.start_with?(LOOKUP_OPTIONS_PREFIX)
+      lookup_invocation.with(:invalid_key, LOOKUP_OPTIONS) do
+        throw :no_such_key
+      end
+    end
+
+    lookup_invocation.top_key ||= key
+    merge_explained = false
+    if lookup_invocation.explain_options?
+      catch(:no_such_key) do
+        module_name = extract_module_name(key) unless key == Puppet::Pops::Lookup::GLOBAL
+        lookup_invocation.module_name = module_name
+        if lookup_invocation.only_explain_options?
+          do_lookup(LOOKUP_OPTIONS, lookup_invocation, HASH)
+          return nil
+        end
+
+        # Bypass cache and do a "normal" lookup of the lookup_options
+        lookup_invocation.with(:meta, LOOKUP_OPTIONS) do
+          key_options = do_lookup(LOOKUP_OPTIONS, lookup_invocation, HASH)[key]
+          merge = key_options[MERGE] unless key_options.nil?
+          merge_explained = true
+        end
+      end
+    elsif merge.nil?
+      # Used cached lookup_options
+      merge = lookup_merge_options(key, lookup_invocation)
+      lookup_invocation.report_merge_source('lookup_options') unless merge.nil?
+    end
+
+    if merge_explained
+      # Merge lookup is explained in detail so we need to explain the data in a section
+      # on the same level to avoid confusion
+      lookup_invocation.with(:data, key) { do_lookup(key, lookup_invocation, merge) }
+    else
+      do_lookup(key, lookup_invocation, merge)
+    end
+  end
+
+  # @api private
+  def lookup_global(name, lookup_invocation, merge_strategy)
+    terminus = Puppet[:data_binding_terminus]
+    lookup_invocation.with(:global, terminus) do
+      catch(:no_such_key) do
+        return lookup_invocation.report_found(name, Puppet::DataBinding.indirection.find(name,
+            { :environment => @env, :variables => lookup_invocation.scope, :merge => merge_strategy }))
+      end
+      lookup_invocation.report_not_found(name)
+      throw :no_such_key
+    end
+  rescue Puppet::DataBinding::LookupError => detail
+    error = Puppet::Error.new("Lookup of key '#{lookup_invocation.top_key}' failed: #{detail.message}")
+    error.set_backtrace(detail.backtrace)
+    raise error
+  end
+
+  # @api private
+  def lookup_in_environment(name, lookup_invocation, merge_strategy)
+    env_provider.lookup(name, lookup_invocation, merge_strategy)
+  end
+
+  # @api private
+  def lookup_in_module(name, lookup_invocation, merge_strategy)
+    module_name = lookup_invocation.module_name || extract_module_name(name)
+
+    # Do not attempt to do a lookup in a module unless the name is qualified.
+    throw :no_such_key if module_name.nil?
+
+    lookup_invocation.with(:module, module_name) do
+      if @env.module(module_name).nil?
+        lookup_invocation.report_module_not_found
+        throw :no_such_key
+      end
+      module_provider(module_name).lookup(name, lookup_invocation, merge_strategy)
+    end
+  end
+
+  # Retrieve the merge options that match the given `name`.
+  #
+  # @param name [String] The key for which we want merge options
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] the lookup invocation
+  # @return [String,Hash,nil] The found merge options or nil
+  #
+  def lookup_merge_options(name, lookup_invocation)
+    lookup_options = lookup_lookup_options(name, lookup_invocation)
+    lookup_options.nil? ? nil : lookup_options[MERGE]
+  end
+
+  # Retrieve the lookup options that match the given `name`.
+  #
+  # @param name [String] The key for which we want lookup options
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] the lookup invocation
+  # @return [String,Hash,nil] The found lookup options or nil
+  #
+  def lookup_lookup_options(name, lookup_invocation)
+    module_name = extract_module_name(name)
+
+    # Retrieve the options for the module. We use nil as a key in case we have none
+    if !@lookup_options.include?(module_name)
+      options = retrieve_lookup_options(module_name, lookup_invocation, Puppet::Pops::MergeStrategy.strategy(HASH))
+      raise Puppet::DataBinding::LookupError.new("value of #{LOOKUP_OPTIONS} must be a hash") unless options.nil? || options.is_a?(Hash)
+      @lookup_options[module_name] = options
+    else
+      options = @lookup_options[module_name]
+    end
+    options.nil? ? nil : options[name]
+  end
+
+  private
+
+  def do_lookup(key, lookup_invocation, merge)
+    merge_strategy = Puppet::Pops::MergeStrategy.strategy(merge)
+    lookup_invocation.with(:merge, merge_strategy) do
+      result = merge_strategy.merge_lookup([:lookup_global, :lookup_in_environment, :lookup_in_module]) { |m| send(m, key, lookup_invocation, merge_strategy) }
+      lookup_invocation.report_result(result)
+      result
+    end
+  end
+
+  # Retrieve lookup options that applies when using a specific module (i.e. a merge of the pre-cached
+  # `env_lookup_options` and the module specific data)
+  def retrieve_lookup_options(module_name, lookup_invocation, merge_strategy)
+    meta_invocation = Puppet::Pops::Lookup::Invocation.new(lookup_invocation.scope)
+    meta_invocation.top_key = lookup_invocation.top_key
+    env_opts = env_lookup_options(meta_invocation, merge_strategy)
+    unless module_name.nil? || @env.module(module_name).nil?
+      catch(:no_such_key) do
+        meta_invocation.module_name = module_name
+        options = module_provider(module_name).lookup(LOOKUP_OPTIONS, meta_invocation, merge_strategy)
+        options = merge_strategy.merge(env_opts, options) unless env_opts.nil?
+        return options
+      end
+    end
+    env_opts
+  end
+
+  # Retrieve and cache lookup options specific to the environment that this adapter is attached to (i.e. a merge
+  # of global and environment lookup options).
+  def env_lookup_options(meta_invocation, merge_strategy)
+    if !instance_variable_defined?(:@env_lookup_options)
+      @env_lookup_options = nil
+      catch(:no_such_key) do
+        @env_lookup_options = merge_strategy.merge_lookup([:lookup_global, :lookup_in_environment]) do |m|
+          send(m, LOOKUP_OPTIONS, meta_invocation, merge_strategy)
+        end
+      end
+    end
+    @env_lookup_options
+  end
+
+  def env_provider
+    @env_provider ||= initialize_env_provider
+  end
+
+  def module_provider(module_name)
+    # Test if the key is present for the given module_name. It might be there even if the
+    # value is nil (which indicates that no module provider is configured for the given name)
+    unless data.include?(module_name)
+      data[module_name] = initialize_module_provider(module_name)
+    end
+    data[module_name]
+  end
+
+  def initialize_module_provider(module_name)
+    injector = Puppet.lookup(:injector) { nil }
+
+    # Support running tests without an injector being configured == using a null implementation
+    return ModuleDataProvider.new() unless injector
+
+    # Get the registry of module to provider implementation name
+    module_service_type = Registry.hash_of_per_module_data_provider
+    module_service_name = PER_MODULE_DATA_PROVIDER_KEY
+    module_service = injector.lookup(nil, module_service_type, module_service_name)
+    provider_name = module_service[module_name] || 'none'
+
+    service_type = Registry.hash_of_module_data_providers
+    service_name = MODULE_DATA_PROVIDERS_KEY
+
+    # Get the service (registry of known implementations)
+    service = injector.lookup(nil, service_type, service_name)
+    provider = service[provider_name]
+    unless provider
+      raise Puppet::Error.new("Environment '#{@env.name}', cannot find module_data_provider '#{provider_name}'")
+    end
+    # Provider is configured per module but cached using environment life cycle so it must be cloned
+    provider.clone
+  end
+
+  def initialize_env_provider
+    injector = Puppet.lookup(:injector) { nil }
+
+    # Support running tests without an injector being configured == using a null implementation
+    return EnvironmentDataProvider.new() unless injector
+
+    # Get the name of the data provider from the environment's configuration and find the bound implementation
+    provider_name = @env.configuration.environment_data_provider
+    service_type = Registry.hash_of_environment_data_providers
+    service_name = ENV_DATA_PROVIDERS_KEY
+
+    # Get the service (registry of known implementations)
+    service = injector.lookup(nil, service_type, service_name)
+    provider = service[provider_name]
+    unless provider
+      raise Puppet::Error.new("Environment '#{@env.name}', cannot find environment_data_provider '#{provider_name}'")
+    end
+    provider
+  end
+
+  def extract_module_name(name)
+    qual_index = name.index('::')
+    qual_index.nil? ? nil : name[0..qual_index-1]
+  end
+end

data/lib/puppet/data_providers/yaml_data_provider_factory.rb

@@ -0,0 +1,26 @@
+# This file is loaded by the autoloader, and it does not find the data function support unless required relative
+#
+require 'yaml'
+
+module Puppet::DataProviders
+  class YamlDataProviderFactory < Puppet::Plugins::DataProviders::FileBasedDataProviderFactory
+    def create(name, paths, parent_data_provider)
+      YamlDataProvider.new(name, paths, parent_data_provider)
+    end
+
+    def path_extension
+      '.yaml'
+    end
+  end
+
+  class YamlDataProvider < Puppet::Plugins::DataProviders::PathBasedDataProvider
+    def initialize_data(path, lookup_invocation)
+      data = YAML.load_file(path)
+      HieraConfig.symkeys_to_string(data.nil? ? {} : data)
+    rescue YAML::SyntaxError => ex
+      # Psych errors includes the absolute path to the file, so no need to add that
+      # to the message
+      raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
+    end
+  end
+end

data/lib/puppet/data_providers.rb

@@ -4,28 +4,27 @@ module Puppet::DataProviders
     unless @loaded
       require 'puppet/pops'
       require 'puppet/data_providers/data_adapter'
+      require 'puppet/data_providers/lookup_adapter'
     end
     @loaded = true
   end
 
-  def self.lookup_in_environment(name, scope, merge)
-    assert_loaded()
-    adapter = Puppet::DataProviders::DataAdapter.adapt(Puppet.lookup(:current_environment))
-    adapter.env_provider.lookup(name, scope, merge)
+  # @deprecated use `lookup_adapter(lookup_invocation).lookup` instead
+  def self.lookup_in_environment(name, lookup_invocation, merge)
+    Puppet.deprecation_warning('The method Puppet::DataProviders.lookup_in_environment is deprecated and will be removed in the next major release of Puppet.')
+    lookup_adapter(lookup_invocation).lookup_in_environment(name, lookup_invocation, Puppet::Pops::MergeStrategy.strategy(merge))
   end
 
   MODULE_NAME = 'module_name'.freeze
 
-  def self.lookup_in_module(name, scope, merge)
-    # Do not attempt to do a lookup in a module unless the name is qualified.
-    qual_index = name.index('::')
-    throw :no_such_key if qual_index.nil?
-    module_name = name[0..qual_index-1]
+  # @deprecated use `adapter(lookup_invocation).lookup` instead
+  def self.lookup_in_module(name, lookup_invocation, merge)
+    Puppet.deprecation_warning('The method Puppet::DataProviders.lookup_in_module is deprecated and will be removed in the next major release of Puppet.')
+    lookup_adapter(lookup_invocation).lookup_in_module(name, lookup_invocation, Puppet::Pops::MergeStrategy.strategy(merge))
+  end
 
+  def self.lookup_adapter(lookup_invocation)
     assert_loaded()
-    adapter = Puppet::DataProviders::DataAdapter.adapt(Puppet.lookup(:current_environment))
-    data_provider = adapter.module_provider(module_name)
-    throw :no_such_key if data_provider.nil?
-    data_provider.lookup(name, scope, merge)
+    LookupAdapter.adapt(lookup_invocation.scope.environment)
   end
 end