puppet 4.2.3-x64-mingw32 → 4.3.0-x64-mingw32

data/lib/puppet/parser/functions/with.rb

@@ -3,17 +3,25 @@ Puppet::Parser::Functions::newfunction(
   :type => :rvalue,
   :arity => -1,
   :doc => <<-DOC
-Call a lambda code block with the given arguments. Since the parameters of the lambda
-are local to the lambda's scope, this can be used to create private sections
-of logic in a class so that the variables are not visible outside of the
-class.
+Call a [lambda](https://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html)
+with the given arguments and return the result. Since a lambda's scope is
+[local](https://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html#lambda-scope)
+to the lambda, you can use the `with` function to create private blocks of code within a
+class using variables whose values cannot be accessed outside of the lambda.
 
-Example:
+**Example**: Using `with`
 
-     # notices the array [1, 2, 'foo']
-     with(1, 2, 'foo') |$x, $y, $z| { notice [$x, $y, $z] }
+~~~ puppet
+# Concatenate three strings into a single string formatted as a list.
+$fruit = with("apples", "oranges", "bananas") |$x, $y, $z| { 
+  "${x}, ${y}, and ${z}" 
+}
+$check_var = $x
+# $fruit contains "apples, oranges, and bananas"
+# $check_var is undefined, as the value of $x is local to the lambda.
+~~~
 
-- since 4.0.0
+- Since 4.0.0
 DOC
 ) do |args|
   function_fail(["with() is only available when parser/evaluator future is in effect"])

data/lib/puppet/parser/resource.rb

@@ -73,18 +73,15 @@ class Puppet::Parser::Resource < Puppet::Resource
   # Retrieve the associated definition and evaluate it.
   def evaluate
     return if evaluated?
-
     Puppet::Util::Profiler.profile("Evaluated resource #{self}", [:compiler, :evaluate_resource, self]) do
       @evaluated = true
-      if klass = resource_type and ! builtin_type?
-        finish
-        evaluated_code = klass.evaluate_code(self)
-
-        return evaluated_code
-      elsif builtin?
+      if builtin?
         devfail "Cannot evaluate a builtin type (#{type})"
-      else
+      elsif resource_type.nil?
         self.fail "Cannot find definition #{type}"
+      else
+        finish(false) # Call finish but do not validate
+        resource_type.evaluate_code(self)
       end
     end
   end
@@ -101,13 +98,18 @@ class Puppet::Parser::Resource < Puppet::Resource
   end
 
   # Do any finishing work on this object, called before evaluation or
-  # before storage/translation.
-  def finish
+  # before storage/translation. The method does nothing the second time
+  # it is called on the same resource.
+  #
+  # @param do_validate [Boolean] true if validation should be performed
+  #
+  # @api private
+  def finish(do_validate = true)
     return if finished?
     @finished = true
     add_defaults
     add_scope_tags
-    validate
+    validate if do_validate
   end
 
   # Has this resource already been finished?
@@ -115,9 +117,9 @@ class Puppet::Parser::Resource < Puppet::Resource
     @finished
   end
 
-  def initialize(*args)
-    raise ArgumentError, "Resources require a hash as last argument" unless args.last.is_a? Hash
-    raise ArgumentError, "Resources require a scope" unless args.last[:scope]
+  def initialize(type, title, attributes)
+    raise ArgumentError, 'Resources require a hash as last argument' unless attributes.is_a? Hash
+    raise ArgumentError, 'Resources require a scope' unless attributes[:scope]
     super
 
     @source ||= scope.source
@@ -132,6 +134,15 @@ class Puppet::Parser::Resource < Puppet::Resource
     end
   end
 
+  def is_unevaluated_consumer?
+    # We don't declare a new variable here just to test. Saves memory
+    instance_variable_defined?(:@unevaluated_consumer)
+  end
+
+  def mark_unevaluated_consumer
+    @unevaluated_consumer = true
+  end
+
   # Merge an override resource in.  This will throw exceptions if
   # any overrides aren't allowed.
   def merge(resource)
@@ -202,6 +213,69 @@ class Puppet::Parser::Resource < Puppet::Resource
     super || ((scope_resource = scope.resource) && !scope_resource.equal?(self) && scope_resource.raw_tagged?(tag_array))
   end
 
+  # Fills resource params from a capability
+  #
+  # This backs 'consumes => Sql[one]'
+  # @api private
+  def add_parameters_from_consume
+    return if self[:consume].nil?
+
+    map = {}
+    [ self[:consume] ].flatten.map do |ref|
+      # Assert that the ref really is a resource reference
+      raise Puppet::Error, "Invalid consume in #{self.ref}: #{ref} is not a resource" unless ref.is_a?(Puppet::Resource)
+
+      # Resolve references
+      cap = catalog.resource(ref.type, ref.title)
+      if cap.nil?
+        raise "Resource #{ref} could not be found; it might not have been produced yet"
+      end
+
+      # Ensure that the found resource is a capability resource
+      raise Puppet::Error, "Invalid consume in #{ref}: #{cap} is not a capability resource" unless cap.resource_type.is_capability?
+      cap
+    end.each do |cns|
+      # Establish mappings
+      blueprint = resource_type.consumes.find do |bp|
+        bp[:capability] == cns.type
+      end
+      # @todo lutter 2015-08-03: catch this earlier, can we do this during
+      # static analysis ?
+      raise "Resource #{self} tries to consume #{cns} but no 'consumes' mapping exists for #{self.resource_type} and #{cns.type}" unless blueprint
+
+      # setup scope that has, for each attr of cns, a binding to cns[attr]
+      cns_scope = scope.find_global_scope.newscope(:source => self, :resource => self)
+      cns.to_hash.each { |name, value| cns_scope[name.to_s] = value }
+
+      # evaluate mappings in that scope
+      resource_type.arguments.keys.each do |name|
+        if expr = blueprint[:mappings][name]
+          # Explicit mapping
+          value = expr.safeevaluate(cns_scope)
+        else
+          value = cns[name]
+        end
+        unless value.nil?
+          # @todo lutter 2015-07-01: this should be caught by the checker
+          # much earlier. We consume several capres, at least two of which
+          # want to map to the same parameter (PUP-5080)
+          raise "Attempt to reassign attribute '#{name}' in '#{self}' caused by multiple consumed mappings to the same attribute" if map[name]
+          map[name] = value
+        end
+      end
+    end
+
+    map.each { |name, value| self[name] = value if self[name].nil? }
+  end
+
+  def offset
+    nil
+  end
+
+  def pos
+    nil
+  end
+
   private
 
   # Add default values from our definition.
@@ -216,7 +290,8 @@ class Puppet::Parser::Resource < Puppet::Resource
   end
 
   def add_scope_tags
-    if scope_resource = scope.resource
+    scope_resource = scope.resource
+    unless scope_resource.nil? || scope_resource.equal?(self)
       merge_tags(scope_resource)
     end
   end
@@ -259,11 +334,15 @@ class Puppet::Parser::Resource < Puppet::Resource
 
   # Make sure the resource's parameters are all valid for the type.
   def validate
-    @parameters.each do |name, param|
-      validate_parameter(name)
+    if builtin_type?
+      begin
+        @parameters.each { |name, value| validate_parameter(name) }
+      rescue => detail
+        self.fail Puppet::ParseError, detail.to_s + " on #{self}", detail
+      end
+    else
+      resource_type.validate_resource(self)
     end
-  rescue => detail
-    self.fail Puppet::ParseError, detail.to_s + " on #{self}", detail
   end
 
   def extract_parameters(params)

data/lib/puppet/plugins/configuration.rb

@@ -11,6 +11,7 @@ module Puppet::Plugins::Configuration
     require 'puppet/plugins/binding_schemes'
     require 'puppet/plugins/syntax_checkers'
     require 'puppet/plugins/data_providers'
+    require 'puppet/plugins/data_providers/registry'
 
     # Extension-points are registered here:
     #
@@ -33,7 +34,7 @@ module Puppet::Plugins::Configuration
 
     extensions.multibind(checkers_name).name(checkers_name).hash_of(checkers_type)
     extensions.multibind(schemes_name).name(schemes_name).hash_of(schemes_type)
-    Puppet::Plugins::DataProviders::register_extensions(extensions)
+    Puppet::Plugins::DataProviders::Registry.register_extensions(extensions)
 
     # Register injector boot bindings
     # -------------------------------
@@ -64,5 +65,5 @@ module Puppet::Plugins::Configuration
       in_multibind(checkers_name)
       to_instance('Puppet::SyntaxCheckers::Json')
     end
-    Puppet::Plugins::DataProviders::register_defaults(bindings)
+    Puppet::Plugins::DataProviders::Registry.register_defaults(bindings)
 end

data/lib/puppet/plugins/data_providers.rb

@@ -1,6 +1,6 @@
 module Puppet::Plugins; end
 
-class Puppet::Plugins::DataProviders
+module Puppet::Plugins::DataProviders
 
   # The lookup **key** for the multibind containing data provider name per module
   # @api public
@@ -26,65 +26,17 @@ class Puppet::Plugins::DataProviders
   # @api public
   MODULE_DATA_PROVIDERS_TYPE         = 'Puppet::Plugins::DataProviders::ModuleDataProvider'
 
-  def self.register_extensions(extensions)
-    extensions.multibind(PER_MODULE_DATA_PROVIDER_KEY).name(PER_MODULE_DATA_PROVIDER_KEY).hash_of(PER_MODULE_DATA_PROVIDER_TYPE)
-    extensions.multibind(ENV_DATA_PROVIDERS_KEY).name(ENV_DATA_PROVIDERS_KEY).hash_of(ENV_DATA_PROVIDERS_TYPE)
-    extensions.multibind(MODULE_DATA_PROVIDERS_KEY).name(MODULE_DATA_PROVIDERS_KEY).hash_of(MODULE_DATA_PROVIDERS_TYPE)
-  end
-
-  def self.hash_of_per_module_data_provider
-    @@HASH_OF_PER_MODULE_DATA_PROVIDERS ||= Puppet::Pops::Types::TypeFactory.hash_of(PER_MODULE_DATA_PROVIDER_TYPE)
-  end
-
-  def self.hash_of_module_data_providers
-    @@HASH_OF_MODULE_DATA_PROVIDERS ||= Puppet::Pops::Types::TypeFactory.hash_of(
-      Puppet::Pops::Types::TypeFactory.type_of(MODULE_DATA_PROVIDERS_TYPE))
-  end
-
-  def self.hash_of_environment_data_providers
-    @@HASH_OF_ENV_DATA_PROVIDERS ||= Puppet::Pops::Types::TypeFactory.hash_of(
-      Puppet::Pops::Types::TypeFactory.type_of(ENV_DATA_PROVIDERS_TYPE))
-  end
-
-  # Registers a 'none' environment data provider, and a 'none' module data provider as the defaults.
-  # This is only done to allow that something binds to 'none' rather than removing the entire binding (which
-  # has the same effect).
-  #
-  def self.register_defaults(default_bindings)
-    default_bindings.bind do
-      name('none')
-      in_multibind(ENV_DATA_PROVIDERS_KEY)
-      to_instance(ENV_DATA_PROVIDERS_TYPE)
-    end
-
-    default_bindings.bind do
-      name('function')
-      in_multibind(ENV_DATA_PROVIDERS_KEY)
-      to_instance('Puppet::DataProviders::FunctionEnvDataProvider')
-    end
-
-    default_bindings.bind do
-      name('none')
-      in_multibind(MODULE_DATA_PROVIDERS_KEY)
-      to_instance(MODULE_DATA_PROVIDERS_TYPE)
-    end
-
-    default_bindings.bind do
-      name('function')
-      in_multibind(MODULE_DATA_PROVIDERS_KEY)
-      to_instance('Puppet::DataProviders::FunctionModuleDataProvider')
-    end
-  end
+  # The lookup **key** for the multibind containing map of provider name to path based data provider factory
+  # implementation.
+  # @api public
+  PATH_BASED_DATA_PROVIDER_FACTORIES_KEY  = 'puppet::path_based_data_provider_factories'
 
-  class ModuleDataProvider
-    def lookup(name, scope, merge)
-      throw :no_such_key
-    end
-  end
+  # The lookup **type** for the multibind containing map of provider name to path based data provider factory
+  # implementation.
+  # @api public
+  PATH_BASED_DATA_PROVIDER_FACTORIES_TYPE = 'Puppet::Plugins::DataProviders::PathBasedDataProviderFactory'
 
-  class EnvironmentDataProvider
-    def lookup(name, scope, merge)
-      throw :no_such_key
-    end
-  end
 end
+
+require_relative 'data_providers/data_provider'
+require_relative 'data_providers/registry'

data/lib/puppet/plugins/data_providers/data_provider.rb

@@ -0,0 +1,283 @@
+module Puppet::Plugins::DataProviders
+  module DataProvider
+    # Performs a lookup with an endless recursion check.
+    #
+    # @param key [String] The key to lookup
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @param merge [String|Hash<String,Object>|nil] Merge strategy or hash with strategy and options
+    #
+    # @api public
+    def lookup(name, lookup_invocation, merge)
+      lookup_invocation.check(name) { unchecked_lookup(name, lookup_invocation, merge) }
+    end
+
+    # Performs a lookup with the assumption that a recursive check has been made.
+    #
+    # @param key [String] The key to lookup
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @param merge [String|Hash<String,Object>|nil] Merge strategy or hash with strategy and options
+    #
+    # @api public
+    def unchecked_lookup(key, lookup_invocation, merge)
+      lookup_invocation.with(:data_provider, self) do
+        hash = data(data_key(key, lookup_invocation), lookup_invocation)
+        value = hash[key]
+        if value || hash.include?(key)
+          lookup_invocation.report_found(key, post_process(value, lookup_invocation))
+        else
+          lookup_invocation.report_not_found(key)
+          throw :no_such_key
+        end
+      end
+    end
+
+    # Perform optional post processing of found value. This hook is used by the hiera style
+    # providers to perform interpolation. The default method simply returns the given _value_.
+    #
+    # @param value [Object] The value to perform post processing on
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @return [Object] The result of post processing the value.
+    #
+    # @api public
+    def post_process(value, lookup_invocation)
+      value
+    end
+
+    # Gets the data from the compiler, or initializes it by calling #initialize_data if not present in the compiler.
+    # This means, that data is initialized once per compilation, and the data is cached for as long as the compiler
+    # lives (which is for one catalog production). This makes it possible to return data that is tailored for the
+    # request.
+    #
+    # If data is obtained using the #initialize_data method it will be sent to the #validate_data for validation
+    #
+    # @param data_key [String] The data key such as the name of a module or the constant 'environment'
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @param merge [String|Hash<String,Object>|nil] Merge strategy or hash with strategy and options
+    # @return [Hash] The data hash for the given _key_
+    #
+    # @api public
+    def data(data_key, lookup_invocation)
+      compiler = lookup_invocation.scope.compiler
+      adapter = Puppet::DataProviders::DataAdapter.get(compiler) || Puppet::DataProviders::DataAdapter.adapt(compiler)
+      adapter.data[data_key] ||= validate_data(initialize_data(data_key, lookup_invocation), data_key)
+    end
+    protected :data
+
+    # Obtain an optional key to use when retrieving the data.
+    #
+    # @param key [String] The key to lookup
+    # @return [String,nil] The data key or nil if not applicable
+    #
+    # @api public
+    def data_key(key, lookup_invocation)
+      nil
+    end
+    protected :data_key
+
+    # Should be reimplemented by subclass to provide the hash that corresponds to the given name.
+    #
+    # @param data_key [String] The data key such as the name of a module or the constant 'environment'
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @return [Hash] The hash of values
+    #
+    # @api public
+    def initialize_data(data_key, lookup_invocation)
+      {}
+    end
+    protected :initialize_data
+
+    def name
+      cname = self.class.name
+      cname[cname.rindex(':')+1..-1]
+    end
+
+    def validate_data(data, data_key)
+      data
+    end
+    protected :validate_data
+  end
+
+  class ModuleDataProvider
+    include DataProvider
+
+    # Retrieve the first segment of the qualified name _key_. This method will throw
+    # :no_such_key unless the segment can be extracted.
+    #
+    # @param key [String] The key
+    # @return [String] The first segment of the given key
+    def data_key(key, lookup_invocation)
+      return lookup_invocation.module_name if key == 'lookup_options'
+      qual_index = key.index('::')
+      throw :no_such_key if qual_index.nil?
+      key[0..qual_index-1]
+    end
+    protected :data_key
+
+    # Assert that all keys in the given _data_ are prefixed with the given _module_name_.
+    #
+    # @param data [Hash] The data hash
+    # @param module_name [String] The name of the module where the data was found
+    # @return [Hash] The data_hash unaltered
+    def validate_data(data, module_name)
+      module_prefix = "#{module_name}::"
+      data.each_key do |k|
+        unless k.is_a?(String) && (k == 'lookup_options' || k.start_with?(module_prefix))
+          raise Puppet::DataBinding::LookupError, "Module data for module '#{module_name}' must use keys qualified with the name of the module"
+        end
+      end
+    end
+    protected :validate_data
+  end
+
+  class EnvironmentDataProvider
+    include DataProvider
+
+    def data_key(key, lookup_invocation)
+      'environment'
+    end
+    protected :data_key
+  end
+
+  # Class that keeps track of the original path (as it appears in the declaration, before interpolation),
+  # the fully resolved path, and whether or the resolved path exists.
+  #
+  # @api public
+  class ResolvedPath
+    attr_reader :original_path, :path
+
+    # @param original_path [String] path as found in declaration. May contain interpolation expressions
+    # @param path [Pathname] the expanded absolue path
+    # @api public
+    def initialize(original_path, path)
+      @original_path = original_path
+      @path = path
+      @exists = nil
+    end
+
+    # @return [Boolean] cached info if the path exists or not
+    # @api public
+    def exists?
+      @exists = @path.exist? if @exists.nil?
+      @exists
+    end
+  end
+
+  # A data provider that is initialized with a set of _paths_. When performing lookup, each
+  # path is search in the order they appear. If a value is found in more than one location it
+  # will be merged according to a given (optional) merge strategy.
+  #
+  # @abstract
+  # @api public
+  class PathBasedDataProvider
+    include DataProvider
+
+    attr_reader :name
+
+    # @param name [String] The name of the data provider
+    # @param paths [Array<ResolvedPath>] Paths used by this provider
+    #
+    # @api public
+    def initialize(name, paths)
+      @name = name
+      @paths = paths
+    end
+
+    # Performs a lookup by searching all given paths for the given _key_. A merge will be performed if
+    # the value is found in more than one location and _merge_ is not nil.
+    #
+    # @param key [String] The key to lookup
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @param merge [Puppet::Pops::MergeStrategy,String,Hash<String,Object>,nil] Merge strategy or hash with strategy and options
+    #
+    # @api public
+    def unchecked_lookup(key, lookup_invocation, merge)
+      lookup_invocation.with(:data_provider, self) do
+        merge_strategy = Puppet::Pops::MergeStrategy.strategy(merge)
+        lookup_invocation.with(:merge, merge_strategy) do
+          merged_result = merge_strategy.merge_lookup(@paths) do |path|
+            lookup_invocation.with(:path, path) do
+              if path.exists?
+                hash = data(path.path, lookup_invocation)
+                value = hash[key]
+                if value || hash.include?(key)
+                  lookup_invocation.report_found(key, post_process(value, lookup_invocation))
+                else
+                  lookup_invocation.report_not_found(key)
+                  throw :no_such_key
+                end
+              else
+                lookup_invocation.report_path_not_found
+                throw :no_such_key
+              end
+            end
+          end
+          lookup_invocation.report_result(merged_result)
+        end
+      end
+    end
+  end
+
+  # Factory for creating path based data providers
+  #
+  # @abstract
+  # @api public
+  class PathBasedDataProviderFactory
+    # Create a path based data provider with the given _name_ and _paths_
+    #
+    # @param name [String] the name of the created provider (for logging and debugging)
+    # @param paths [Array<String>] array of resolved paths
+    # @return [DataProvider] The created data provider
+    #
+    # @api public
+    def create(name, paths)
+      raise NotImplementedError, "Subclass of PathBasedDataProviderFactory must implement 'create' method"
+    end
+
+    # Resolve the given _paths_ to something that is meaningful as a _paths_ argument when creating
+    # a provider using the #create call.
+    #
+    # In order to increase efficiency, the implementors of this method should ensure that resolved
+    # paths that exists are included in the result.
+    #
+    # @param datadir [Pathname] The base when creating absolute paths
+    # @param declared_paths [Array<String>] paths as found in declaration. May contain interpolation expressions
+    # @param paths [Array<String>] paths that have been preprocessed (interpolations resolved)
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @return [Array<ResolvedPath>] Array of resolved paths
+    #
+    # @api public
+    def resolve_paths(datadir, declared_paths, paths, lookup_invocation)
+      []
+    end
+  end
+
+  # Factory for creating file based data providers. This is an extension of the path based
+  # factory where it is required that each resolved path appoints an existing file in the local
+  # file system.
+  #
+  # @abstract
+  # @api public
+  class FileBasedDataProviderFactory < PathBasedDataProviderFactory
+    # @param datadir [Pathname] The base when creating absolute paths
+    # @param declared_paths [Array<String>] paths as found in declaration. May contain interpolation expressions
+    # @param paths [Array<String>] paths that have been preprocessed (interpolations resolved)
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @return [Array<ResolvedPath>] Array of resolved paths
+    def resolve_paths(datadir, declared_paths, paths, lookup_invocation)
+      resolved_paths = []
+      unless paths.nil? || datadir.nil?
+        ext = path_extension
+        paths.each_with_index do |path, idx|
+          path = path + ext unless path.end_with?(ext)
+          resolved_paths << ResolvedPath.new(declared_paths[idx], datadir + path)
+        end
+      end
+      resolved_paths
+    end
+
+    def path_extension
+      raise NotImplementedError, "Subclass of FileBasedProviderFactory must implement 'path_extension' method"
+    end
+    protected :path_extension
+  end
+end