puppet 4.2.3 → 4.3.0

data/lib/puppet/data_providers/hiera_interpolate.rb

@@ -0,0 +1,97 @@
+require_relative 'hiera_config'
+
+# Add support for Hiera-like interpolation expressions. The expressions may contain keys that uses dot-notation
+# to further navigate into hashes and arrays
+#
+module Puppet::DataProviders::HieraInterpolate
+  def interpolate(subject, lookup_invocation, allow_methods)
+    return subject unless subject.is_a?(String) && !subject.index('%{').nil?
+
+    lookup_invocation.with(:interpolate, subject) do
+      subject.gsub(/%\{([^\}]+)\}/) do |match|
+        method_key, key = get_method_and_data($1, allow_methods)
+        is_alias = method_key == 'alias'
+
+        # Alias is only permitted if the entire string is equal to the interpolate expression
+        raise Puppet::DataBinding::LookupError, "'alias' interpolation is only permitted if the expression is equal to the entire string" if is_alias && subject != match
+
+        segments = key.split('.')
+        value = interpolate_method(method_key).call(segments[0], lookup_invocation)
+        value = qualified_lookup(segments.drop(1), value) if segments.size > 1
+        value = lookup_invocation.check(key) { interpolate(value, lookup_invocation, allow_methods) } if value.is_a?(String)
+
+        # break gsub and return value immediately if this was an alias substitution. The value might be something other than a String
+        return value if is_alias
+
+        value || ''
+      end
+    end
+  end
+
+  private
+
+  def interpolate_method(method_key)
+    @@interpolate_methods ||= begin
+      global_lookup = lambda { |key, lookup_invocation| Puppet::Pops::Lookup.lookup(key, nil, '', true, nil, lookup_invocation) }
+      scope_lookup = lambda do |key, lookup_invocation|
+        lookup_invocation.with(:scope, nil) do
+          ovr = lookup_invocation.override_values
+          if ovr.include?(key)
+            lookup_invocation.report_found_in_overrides(key, ovr[key])
+          else
+            scope = lookup_invocation.scope
+            if scope.include?(key)
+              lookup_invocation.report_found(key, scope[key])
+            else
+              defaults = lookup_invocation.default_values
+              if defaults.include?(key)
+                lookup_invocation.report_found_in_defaults(key, defaults[key])
+              else
+                nil
+              end
+            end
+          end
+        end
+      end
+
+
+      {
+        'lookup' => global_lookup,
+        'hiera' => global_lookup, # this is just an alias for 'lookup'
+        'alias' => global_lookup, # same as 'lookup' but expression must be entire string. The result that is not subject to string substitution
+        'scope' => scope_lookup,
+        'literal' => lambda { |key, _| key }
+      }
+    end
+    interpolate_method = @@interpolate_methods[method_key]
+    raise Puppet::DataBinding::LookupError, "Unknown interpolation method '#{method_key}'" unless interpolate_method
+    interpolate_method
+  end
+
+  def qualified_lookup(segments, value)
+    segments.each do |segment|
+      throw :no_such_key if value.nil?
+      if segment =~ /^[0-9]+$/
+        segment = segment.to_i
+        raise Puppet::DataBinding::LookupError, "Data provider type mismatch: Got #{value.class.name} when Array was expected to enable lookup using key '#{segment}'" unless value.instance_of?(Array)
+        throw :no_such_key unless segment < value.size
+      else
+        raise Puppet::DataBinding::LookupError, "Data provider type mismatch: Got #{value.class.name} when a non Array object that responds to '[]' was expected to enable lookup using key '#{segment}'" unless value.respond_to?(:'[]') && !value.instance_of?(Array)
+        throw :no_such_key unless value.include?(segment)
+      end
+      value = value[segment]
+    end
+    value
+  end
+
+  def get_method_and_data(data, allow_methods)
+    if match = data.match(/^(\w+)\((?:["]([^"]+)["]|[']([^']+)['])\)$/)
+      raise Puppet::DataBinding::LoookupError, 'Interpolation using method syntax is not allowed in this context' unless allow_methods
+      key = match[1]
+      data = match[2] || match[3] # double or single qouted
+    else
+      key = 'scope'
+    end
+    [key, data]
+  end
+end

data/lib/puppet/data_providers/hiera_module_data_provider.rb

@@ -0,0 +1,23 @@
+# This file is loaded by the autoloader, and it does not find the hiera support unless required relative
+require_relative 'hiera_support'
+
+module Puppet::DataProviders
+  class HieraModuleDataProvider < Puppet::Plugins::DataProviders::ModuleDataProvider
+    include HieraSupport
+
+    # Return the root of the module with the name equal to _data_key_ found in the environment of the given _scope_
+    #
+    # @param data_key [String] the name of the module
+    # @param scope [Puppet::Parser::Scope] the parser scope where the environment is found
+    # @return [Pathname] Path to root of the environment
+    # @raise [Puppet::DataBinder::LookupError] if the given module is can not be found
+    #
+    def provider_root(module_name, scope)
+      env = scope.environment
+      mod = env.modules.find { |m| m.name == module_name }
+      raise Puppet::DataBinder::LookupError, "Environment '#{env.name}', cannot find module '#{module_name}'" unless mod
+      Pathname.new(mod.path)
+    end
+    protected :provider_root
+  end
+end

data/lib/puppet/data_providers/hiera_support.rb

@@ -0,0 +1,37 @@
+require_relative 'hiera_config'
+
+module Puppet::DataProviders::HieraSupport
+  def config_path
+    @hiera_config.nil? ? 'not yet configured' : @hiera_config.config_path
+  end
+
+  def name
+    'Hiera Data Provider' + (@hiera_config.nil? ? '' : ", version #{@hiera_config.version}")
+  end
+
+  # Performs a lookup by searching all given paths for the given _key_. A merge will be performed if
+  # the value is found in more than one location and _merge_ is not nil.
+  #
+  # @param key [String] The key to lookup
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+  # @param merge [Puppet::Pops::MergeStrategy,String,Hash<String,Object>,nil] Merge strategy or hash with strategy and options
+  #
+  # @api public
+  def unchecked_lookup(key, lookup_invocation, merge)
+    lookup_invocation.with(:data_provider, self) do
+      merge_strategy = Puppet::Pops::MergeStrategy.strategy(merge)
+      lookup_invocation.with(:merge, merge_strategy) do
+        merged_result = merge_strategy.merge_lookup(data_providers(data_key(key, lookup_invocation), lookup_invocation)) do |data_provider|
+          data_provider.unchecked_lookup(key, lookup_invocation, merge_strategy)
+        end
+        lookup_invocation.report_result(merged_result)
+      end
+    end
+  end
+
+  def data_providers(data_key, lookup_invocation)
+    @hiera_config ||= Puppet::DataProviders::HieraConfig.new(provider_root(data_key, lookup_invocation.scope))
+    @data_providers ||= @hiera_config.create_data_providers(lookup_invocation)
+  end
+  private :data_providers
+end

data/lib/puppet/data_providers/json_data_provider_factory.rb

@@ -0,0 +1,31 @@
+# This file is loaded by the autoloader, and it does not find the data function support unless required relative
+#
+require 'json'
+require_relative 'hiera_interpolate'
+
+module Puppet::DataProviders
+  class JsonDataProviderFactory < Puppet::Plugins::DataProviders::FileBasedDataProviderFactory
+    def create(name, paths)
+      JsonDataProvider.new(name, paths)
+    end
+
+    def path_extension
+      '.json'
+    end
+  end
+
+  class JsonDataProvider < Puppet::Plugins::DataProviders::PathBasedDataProvider
+    include HieraInterpolate
+
+    def initialize_data(path, lookup_invocation)
+      JSON.parse(File.read(path))
+    rescue JSON::ParserError => ex
+      # Filename not included in message, so we add it here.
+      raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+    end
+
+    def post_process(value, lookup_invocation)
+      interpolate(value, lookup_invocation, true)
+    end
+  end
+end

data/lib/puppet/data_providers/lookup_adapter.rb

@@ -0,0 +1,200 @@
+# A LookupAdapter is a specialized DataAdapter that uses its hash to store module providers. It also remembers the environment
+# that it is attached to and maintains a cache of _lookup options_ retrieved from its data providers.
+#
+class Puppet::DataProviders::LookupAdapter < Puppet::DataProviders::DataAdapter
+
+  LOOKUP_OPTIONS = 'lookup_options'.freeze
+  HASH = 'hash'.freeze
+  MERGE = 'merge'.freeze
+
+  def self.create_adapter(env)
+    new(env)
+  end
+
+  def initialize(env)
+    super()
+    @env = env
+    @lookup_options = {}
+  end
+
+  # Performs a lookup using global, environment, and module data providers. Merge the result using the given
+  # _merge_ strategy. If the merge strategy is nil, then an attempt is made to find merge options in the
+  # `lookup_options` hash for an entry associated with the key. If no options are found, the no merge is performed
+  # and the first found entry is returned.
+  #
+  # @param key [String] The key to lookup
+  # @param merge [Puppet::Pops::MergeStrategy,String,Hash<String,Object>,nil] Merge strategy or hash with strategy and options
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] Invocation data containing scope, overrides, and defaults
+  # @return [Object] The found value
+  # @throws :no_such_key if the given key is not found
+  #
+  def lookup(key, lookup_invocation, merge)
+    merge = lookup_merge_options(key, lookup_invocation) if merge.nil?
+    merge_strategy = Puppet::Pops::MergeStrategy.strategy(merge)
+    lookup_invocation.with(:merge, merge_strategy) do
+      result = merge_strategy.merge_lookup([:lookup_global, :lookup_in_environment, :lookup_in_module]) { |m| send(m, key, lookup_invocation, merge_strategy) }
+      lookup_invocation.report_result(result)
+      result
+    end
+  end
+
+  # @api private
+  def lookup_global(name, lookup_invocation, merge_strategy)
+    terminus = Puppet[:data_binding_terminus]
+    lookup_invocation.with(:global, terminus) do
+      catch(:no_such_key) do
+        return lookup_invocation.report_found(name, Puppet::DataBinding.indirection.find(name,
+            { :environment => @env.to_s, :variables => lookup_invocation.scope, :merge => merge_strategy }))
+      end
+      lookup_invocation.report_not_found(name)
+      throw :no_such_key
+    end
+  rescue Puppet::DataBinding::LookupError => e
+    raise Puppet::Error.new("Error from DataBinding '#{terminus}' while looking up '#{name}': #{e.message}", e)
+  end
+
+  # @api private
+  def lookup_in_environment(name, lookup_invocation, merge_strategy)
+    env_provider.lookup(name, lookup_invocation, merge_strategy)
+  end
+
+  # @api private
+  def lookup_in_module(name, lookup_invocation, merge_strategy)
+    module_name = extract_module_name(name)
+
+    # Do not attempt to do a lookup in a module unless the name is qualified.
+    throw :no_such_key if module_name.nil?
+
+    lookup_invocation.with(:module, module_name) do
+      if @env.module(module_name).nil?
+        lookup_invocation.report_module_not_found
+        throw :no_such_key
+      end
+      module_provider(module_name).lookup(name, lookup_invocation, merge_strategy)
+    end
+  end
+
+  # Retrieve the merge options that match the given `name`.
+  #
+  # @param name [String] The key for which we want merge options
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] the lookup invocation
+  # @return [String,Hash,nil] The found merge options or nil
+  #
+  def lookup_merge_options(name, lookup_invocation)
+    lookup_options = lookup_lookup_options(name, lookup_invocation)
+    lookup_options.nil? ? nil : lookup_options[MERGE]
+  end
+
+  # Retrieve the lookup options that match the given `name`.
+  #
+  # @param name [String] The key for which we want lookup options
+  # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] the lookup invocation
+  # @return [String,Hash,nil] The found lookup options or nil
+  #
+  def lookup_lookup_options(name, lookup_invocation)
+    module_name = extract_module_name(name)
+
+    # Retrieve the options for the module. We use nil as a key in case we have none
+    options = @lookup_options[module_name]
+    if options.nil? && !@lookup_options.include?(module_name)
+      options = retrieve_lookup_options(module_name, lookup_invocation, Puppet::Pops::MergeStrategy.strategy(HASH))
+      raise Puppet::DataBinding::LookupError.new("value of #{LOOKUP_OPTIONS} must be a hash") unless options.nil? || options.is_a?(Hash)
+      @lookup_options[module_name] = options
+    end
+    options.nil? ? nil : options[name]
+  end
+
+  private
+
+  # Retrieve lookup options that applies when using a specific module (i.e. a merge of the pre-cached
+  # `env_lookup_options` and the module specific data)
+  def retrieve_lookup_options(module_name, lookup_invocation, merge_strategy)
+    lookup_invocation.with(:meta, module_name) do
+      env_opts = env_lookup_options(lookup_invocation, merge_strategy)
+      options = nil
+      unless module_name.nil?
+        catch(:no_such_key) do
+          options = module_provider(module_name).lookup(LOOKUP_OPTIONS, lookup_invocation, merge_strategy)
+          options = merge_strategy.merge(env_opts, options) unless env_opts.nil?
+        end
+      end
+      options.nil? ? env_opts : options
+    end
+  end
+
+  # Retrieve and cache lookup options specific to the environment that this adapter is attached to (i.e. a merge
+  # of global and environment lookup options).
+  def env_lookup_options(lookup_invocation, merge_strategy)
+    unless instance_variable_defined?(:@env_lookup_options)
+      @env_lookup_options = nil
+      catch(:no_such_key) do
+        @env_lookup_options = merge_strategy.merge_lookup([:lookup_global, :lookup_in_environment]) do |m|
+          send(m, LOOKUP_OPTIONS, lookup_invocation, merge_strategy)
+        end
+      end
+    end
+    @env_lookup_options
+  end
+
+  def env_provider
+    @env_provider ||= initialize_env_provider
+  end
+
+  def module_provider(module_name)
+    # Test if the key is present for the given module_name. It might be there even if the
+    # value is nil (which indicates that no module provider is configured for the given name)
+    unless data.include?(module_name)
+      data[module_name] = initialize_module_provider(module_name)
+    end
+    data[module_name]
+  end
+
+  def initialize_module_provider(module_name)
+    injector = Puppet.lookup(:injector) { nil }
+
+    # Support running tests without an injector being configured == using a null implementation
+    return ModuleDataProvider.new() unless injector
+
+    # Get the registry of module to provider implementation name
+    module_service_type = Registry.hash_of_per_module_data_provider
+    module_service_name = PER_MODULE_DATA_PROVIDER_KEY
+    module_service = injector.lookup(nil, module_service_type, module_service_name)
+    provider_name = module_service[module_name] || 'none'
+
+    service_type = Registry.hash_of_module_data_providers
+    service_name = MODULE_DATA_PROVIDERS_KEY
+
+    # Get the service (registry of known implementations)
+    service = injector.lookup(nil, service_type, service_name)
+    provider = service[provider_name]
+    unless provider
+      raise Puppet::Error.new("Environment '#{@env.name}', cannot find module_data_provider '#{provider_name}'")
+    end
+    provider
+  end
+
+  def initialize_env_provider
+    injector = Puppet.lookup(:injector) { nil }
+
+    # Support running tests without an injector being configured == using a null implementation
+    return EnvironmentDataProvider.new() unless injector
+
+    # Get the name of the data provider from the environment's configuration and find the bound implementation
+    provider_name = @env.configuration.environment_data_provider
+    service_type = Registry.hash_of_environment_data_providers
+    service_name = ENV_DATA_PROVIDERS_KEY
+
+    # Get the service (registry of known implementations)
+    service = injector.lookup(nil, service_type, service_name)
+    provider = service[provider_name]
+    unless provider
+      raise Puppet::Error.new("Environment '#{@env.name}', cannot find environment_data_provider '#{provider_name}'")
+    end
+    provider
+  end
+
+  def extract_module_name(name)
+    qual_index = name.index('::')
+    qual_index.nil? ? nil : name[0..qual_index-1]
+  end
+end

data/lib/puppet/data_providers/yaml_data_provider_factory.rb

@@ -0,0 +1,32 @@
+# This file is loaded by the autoloader, and it does not find the data function support unless required relative
+#
+require 'yaml'
+require_relative 'hiera_interpolate'
+
+module Puppet::DataProviders
+  class YamlDataProviderFactory < Puppet::Plugins::DataProviders::FileBasedDataProviderFactory
+    def create(name, paths)
+      YamlDataProvider.new(name, paths)
+    end
+
+    def path_extension
+      '.yaml'
+    end
+  end
+
+  class YamlDataProvider < Puppet::Plugins::DataProviders::PathBasedDataProvider
+    include HieraInterpolate
+
+    def initialize_data(path, lookup_invocation)
+      HieraConfig.symkeys_to_string(YAML.load_file(path))
+    rescue YAML::SyntaxError => ex
+      # Psych errors includes the absolute path to the file, so no need to add that
+      # to the message
+      raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
+    end
+
+    def post_process(value, lookup_invocation)
+      interpolate(value, lookup_invocation, true)
+    end
+  end
+end

data/lib/puppet/defaults.rb

@@ -21,7 +21,8 @@ module Puppet
         :default => false,
         :type    => :boolean,
         :desc    => 'Whether to enable a pre-Facter 3.0 release of native Facter (distributed as
-          the "cfacter" package). This is not necessary if Facter 3.0 or later is installed.',
+          the "cfacter" package). This is not necessary if Facter 3.0 or later is installed.
+          This setting is deprecated, as Facter 3 is now the default in puppet-agent.',
         :hook    => proc do |value|
           return unless value
           raise ArgumentError, 'facter has already evaluated facts.' if Facter.instance_variable_get(:@collection)
@@ -526,6 +527,15 @@ module Puppet
       :desc    => "The directory where catalog previews per node are generated."
     }
   )
+
+  define_settings(:main,
+      :app_management => {
+          :default  => false,
+          :type     => :boolean,
+          :desc     => "Whether the application management feature is on or off. You must restart Puppet Server after changing this setting.",
+      }
+  )
+
   Puppet.define_settings(:module_tool,
     :module_repository  => {
       :default  => 'https://forgeapi.puppetlabs.com',
@@ -545,7 +555,7 @@ module Puppet
     },
     :module_groups => {
         :default  => nil,
-        :desc     => "Extra module groups to request from the Puppet Forge",
+        :desc     => "Extra module groups to request from the Puppet Forge. This is an internal setting, and users should never change it.",
     }
   )