puppet 4.3.2 → 4.4.0

data/lib/puppet/parser/functions/fqdn_rand.rb

@@ -16,6 +16,6 @@ Puppet::Parser::Functions::newfunction(:fqdn_rand, :arity => -2, :type => :rvalu
   node. (For example, `fqdn_rand(30)`, `fqdn_rand(30, 'expensive job 1')`, and
   `fqdn_rand(30, 'expensive job 2')` will produce totally different numbers.)") do |args|
     max = args.shift.to_i
-    seed = Digest::MD5.hexdigest([self['::fqdn'],args].join(':')).hex
+    seed = Digest::MD5.hexdigest([self['::fqdn'],max,args].join(':')).hex
     Puppet::Util.deterministic_rand_int(seed,max)
 end

data/lib/puppet/parser/functions/inline_template.rb

@@ -1,6 +1,6 @@
 Puppet::Parser::Functions::newfunction(:inline_template, :type => :rvalue, :arity => -2, :doc =>
   "Evaluate a template string and return its value.  See
-  [the templating docs](http://docs.puppetlabs.com/puppet/latest/reference/lang_template.html) for
+  [the templating docs](https://docs.puppetlabs.com/puppet/latest/reference/lang_template.html) for
   more information. Note that if multiple template strings are specified, their
   output is all concatenated and returned as the output of the function.") do |vals|
 

data/lib/puppet/parser/functions/map.rb

@@ -3,7 +3,7 @@ Puppet::Parser::Functions::newfunction(
   :type => :rvalue,
   :arity => -3,
   :doc => <<-DOC
-Applies a [lambda](http://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html)
+Applies a [lambda](https://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html)
 to every value in a data structure and returns an array containing the results.
 
 This function takes two mandatory arguments, in this order:

data/lib/puppet/parser/functions/scanf.rb

@@ -5,37 +5,26 @@ Puppet::Parser::Functions::newfunction(
   :type => :rvalue,
   :arity => 2,
   :doc => <<-DOC
-Scans a string and returns an array of one or more converted values as directed by a given format string.args
-See the documenation of Ruby's String::scanf method for details about the supported formats (which
-are similar but not identical to the formats used in Puppet's `sprintf` function.
+Scans a string and returns an array of one or more converted values based on the given format string.
+See the documenation of Ruby's String#scanf method for details about the supported formats (which
+are similar but not identical to the formats used in Puppet's `sprintf` function.)
 
-This function takes two mandatory arguments: the first is the String to
-convert, and the second the format String. The result of the scan is an Array,
-with each sucessfully scanned and transformed value.args The scanning stops if
-a scan is unsuccesful and the scanned result up to that point is returned. If
-there was no succesful scan at all, the result is an empty Array.
+This function takes two mandatory arguments: the first is the string to convert, and the second is
+the format string. The result of the scan is an array, with each sucessfully scanned and transformed value.
+The scanning stops if a scan is unsuccesful, and the scanned result up to that point is returned. If there
+was no succesful scan, the result is an empty array.
 
-      scanf("42", "%i")[0] == 42
+   "42".scanf("%i")
 
+You can also optionally pass a lambda to scanf, to do additional validation or processing.
 
-When used with the future parser, an optional parameterized block may be given.
-The block is called with the result that is produced by scanf if no block is
-present, the result of the block is then returned by the function.
-
-The optional code block is typically used to assert that the scan was
-succesful, and either produce the same input, or perform unwrapping of
-the result:
-
-      "42".scanf("%i")
-      "42".scanf("%i") |$x| {
-        unless $x[0] =~ Integer {
-          fail "Expected a well formed integer value, got '$x[0]'"
-        }
-        $x[0]
+    "42".scanf("%i") |$x| {
+      unless $x[0] =~ Integer {
+        fail "Expected a well formed integer value, got '$x[0]'"
       }
-
-- since 3.7.4 with `parser = future`
-- since 3.7.5 with classic parser
+      $x[0]
+    }
+- since 4.0.0
 DOC
 ) do |args|
   data = args[0]

data/lib/puppet/parser/functions/slice.rb

@@ -3,42 +3,35 @@ Puppet::Parser::Functions::newfunction(
   :type => :rvalue,
   :arity => -3,
   :doc => <<-DOC
-Applies a parameterized block to each _slice_ of elements in a sequence of selected entries from the first
-argument and returns the first argument, or if no block is given returns a new array with a concatenation of
-the slices.
+This function takes two mandatory arguments: the first should be an array or hash, and the second specifies
+the number of elements to include in each slice.
 
-This function takes two mandatory arguments: the first, `$a`, should be an Array, Hash, or something of
-enumerable type (integer, Integer range, or String), and the second, `$n`, the number of elements to include
-in each slice. The optional third argument should be a a parameterized block as produced by the puppet syntax:
-
-    $a.slice($n) |$x| { ... }
-    slice($a) |$x| { ... }
-
-The parameterized block should have either one parameter (receiving an array with the slice), or the same number
-of parameters as specified by the slice size (each parameter receiving its part of the slice).
-In case there are fewer remaining elements than the slice size for the last slice it will contain the remaining
-elements. When the block has multiple parameters, excess parameters are set to undef for an array or
-enumerable type, and to empty arrays for a Hash.
-
-    $a.slice(2) |$first, $second| { ... }
-
-When the first argument is a Hash, each `key,value` entry is counted as one, e.g, a slice size of 2 will produce
+When the first argument is a hash, each key value pair is counted as one. For example, a slice size of 2 will produce
 an array of two arrays with key, and value.
 
-Example Using slice with Hash
-
     $a.slice(2) |$entry|          { notice "first ${$entry[0]}, second ${$entry[1]}" }
     $a.slice(2) |$first, $second| { notice "first ${first}, second ${second}" }
 
-When called without a block, the function produces a concatenated result of the slices.
-
-Example Using slice without a block
+The function produces a concatenated result of the slices.
 
     slice([1,2,3,4,5,6], 2) # produces [[1,2], [3,4], [5,6]]
     slice(Integer[1,6], 2)  # produces [[1,2], [3,4], [5,6]]
     slice(4,2)              # produces [[0,1], [2,3]]
     slice('hello',2)        # produces [[h, e], [l, l], [o]]
 
+You can also optionally pass a lambda to slice.
+
+    $a.slice($n) |$x| { ... }
+    slice($a) |$x| { ... }
+
+The lambda should have either one parameter (receiving an array with the slice), or the same number
+of parameters as specified by the slice size (each parameter receiving its part of the slice).
+If there are fewer remaining elements than the slice size for the last slice, it will contain the remaining
+elements. If the lambda has multiple parameters, excess parameters are set to undef for an array, or
+to empty arrays for a hash.
+
+    $a.slice(2) |$first, $second| { ... }
+
 - Since 4.0.0
 DOC
 ) do |args|

data/lib/puppet/parser/functions/split.rb

@@ -12,7 +12,7 @@ Split a string variable into an array using the specified split regexp.
     $string     = 'v1.v2:v3.v4'
     $array_var1 = split($string, ':')
     $array_var2 = split($string, '[.]')
-    $array_var3 = split($string, '[.:]')
+    $array_var3 = split($string, Regexp['[.:]'])
 
 `$array_var1` now holds the result `['v1.v2', 'v3.v4']`,
 while `$array_var2` holds `['v1', 'v2:v3', 'v4']`, and

data/lib/puppet/parser/resource.rb

@@ -244,23 +244,25 @@ class Puppet::Parser::Resource < Puppet::Resource
       raise "Resource #{self} tries to consume #{cns} but no 'consumes' mapping exists for #{self.resource_type} and #{cns.type}" unless blueprint
 
       # setup scope that has, for each attr of cns, a binding to cns[attr]
-      cns_scope = scope.find_global_scope.newscope(:source => self, :resource => self)
-      cns.to_hash.each { |name, value| cns_scope[name.to_s] = value }
-
-      # evaluate mappings in that scope
-      resource_type.arguments.keys.each do |name|
-        if expr = blueprint[:mappings][name]
-          # Explicit mapping
-          value = expr.safeevaluate(cns_scope)
-        else
-          value = cns[name]
-        end
-        unless value.nil?
-          # @todo lutter 2015-07-01: this should be caught by the checker
-          # much earlier. We consume several capres, at least two of which
-          # want to map to the same parameter (PUP-5080)
-          raise "Attempt to reassign attribute '#{name}' in '#{self}' caused by multiple consumed mappings to the same attribute" if map[name]
-          map[name] = value
+      scope.with_global_scope do |global_scope|
+        cns_scope = global_scope.newscope(:source => self, :resource => self)
+        cns.to_hash.each { |name, value| cns_scope[name.to_s] = value }
+  
+        # evaluate mappings in that scope
+        resource_type.arguments.keys.each do |name|
+          if expr = blueprint[:mappings][name]
+            # Explicit mapping
+            value = expr.safeevaluate(cns_scope)
+          else
+            value = cns[name]
+          end
+          unless value.nil?
+            # @todo lutter 2015-07-01: this should be caught by the checker
+            # much earlier. We consume several capres, at least two of which
+            # want to map to the same parameter (PUP-5080)
+            raise "Attempt to reassign attribute '#{name}' in '#{self}' caused by multiple consumed mappings to the same attribute" if map[name]
+            map[name] = value
+          end
         end
       end
     end

data/lib/puppet/parser/scope.rb

@@ -174,13 +174,104 @@ class Puppet::Parser::Scope
 
   end
 
+  # @api private
+  class ParameterScope < Ephemeral
+    class Access
+      attr_accessor :value
+
+      def assigned?
+        instance_variable_defined?(:@value)
+      end
+    end
+
+    # A parameter default must be evaluated using a special scope. The scope that is given to this method must
+    # have a `ParameterScope` as its last ephemeral scope. This method will then push a `MatchScope` while the
+    # given `expression` is evaluated. The method will catch any throw of `:unevaluated_parameter` and produce
+    # an error saying that the evaluated parameter X tries to access the unevaluated parameter Y.
+    #
+    # @param name [String] the name of the currently evaluated parameter
+    # @param expression [Puppet::Parser::AST] the expression to evaluate
+    # @param scope [Puppet::Parser::Scope] a scope where a `ParameterScope` has been pushed
+    # @return [Object] the result of the evaluation
+    #
+    # @api private
+    def evaluate3x(name, expression, scope)
+      scope.with_guarded_scope do
+        bad = catch(:unevaluated_parameter) do
+          scope.new_match_scope(nil)
+          return as_read_only { expression.safeevaluate(scope) }
+        end
+        raise Puppet::Error, "default expression for $#{name} tries to illegally access not yet evaluated $#{bad}"
+      end
+    end
+
+    def evaluate(name, expression, scope, evaluator)
+      scope.with_guarded_scope do
+        bad = catch(:unevaluated_parameter) do
+          scope.new_match_scope(nil)
+          return as_read_only { evaluator.evaluate(expression, scope) }
+        end
+        raise Puppet::Error, "default expression for $#{name} tries to illegally access not yet evaluated $#{bad}"
+      end
+    end
+
+    def initialize(parent, param_names)
+      super(parent)
+      @params = {}
+      param_names.each { |name| @params[name] = Access.new }
+    end
+
+    def [](name)
+      access = @params[name]
+      return super if access.nil?
+      throw(:unevaluated_parameter, name) unless access.assigned?
+      access.value
+    end
+
+    def []=(name, value)
+      raise Puppet::Error, "Attempt to assign variable #{name} when evaluating parameters" if @read_only
+      @params[name] ||= Access.new
+      @params[name].value = value
+    end
+
+    def bound?(name)
+      @params.include?(name)
+    end
+
+    def include?(name)
+      @params.include?(name) || super
+    end
+
+    def is_local_scope?
+      true
+    end
+
+    def as_read_only
+      read_only = @read_only
+      @read_only = true
+      begin
+        yield
+      ensure
+        @read_only = read_only
+      end
+    end
+
+    def to_hash
+      Hash[@params.select {|_, access| access.assigned? }.map { |name, access| [name, access.value] }]
+    end
+  end
+
+
   # Returns true if the variable of the given name has a non nil value.
   # TODO: This has vague semantics - does the variable exist or not?
   #       use ['name'] to get nil or value, and if nil check with exist?('name')
   #       this include? is only useful because of checking against the boolean value false.
   #
   def include?(name)
-    ! self[name].nil?
+    catch(:undefined_variable) {
+      return ! self[name].nil?
+    }
+    false
   end
 
   # Returns true if the variable of the given name is set to any value (including nil)
@@ -412,14 +503,28 @@ class Puppet::Parser::Scope
     end
   end
 
+  UNDEFINED_VARIABLES_KIND = 'undefined_variables'.freeze
+
   def variable_not_found(name, reason=nil)
     # Built in variables always exist
     if BUILT_IN_VARS.include?(name)
       return nil
     end
     if Puppet[:strict_variables]
-      throw :undefined_variable
+      throw(:undefined_variable, reason)
     else
+      # Always warn, unfortunately without location (unless given in "reason") since
+      # a location is in most cases not given to scope (operator [], and lookupvar), and
+      # would be too expensive to always give.
+      # The ideal solution would be to always throw :undefined_variable, but that has to
+      # wait until a major release. It would then force all callers of scope to deal with
+      # the case of :undefined_variable. (Should check with include? first or catch the throw).
+      # Use deprecation warning to enable turning off these warnings, and to ensure each variable
+      # is only logged once.
+      unless name =~ Puppet::Pops::Patterns::NUMERIC_VAR_NAME
+        Puppet.warn_once(UNDEFINED_VARIABLES_KIND, "Variable: #{name}",
+          "Undefined variable '#{name}'; #{reason}" )
+      end
       nil
     end
   end
@@ -511,7 +616,8 @@ class Puppet::Parser::Scope
                  else
                    ""
                  end
-      warning "Could not look up qualified variable '#{class_name}::#{variable_name}'; #{reason}#{location}"
+      variable_not_found("#{class_name}::#{variable_name}", "#{reason}#{location}")
+      return nil
     end
     variable_not_found("#{class_name}::#{variable_name}", reason)
   end
@@ -738,10 +844,12 @@ class Puppet::Parser::Scope
 
   alias_method :inspect, :to_s
 
-  # remove ephemeral scope up to level
-  # TODO: Who uses :all ? Remove ??
+  # Pop ephemeral scopes up to level and return them
   #
+  # @deprecated use #pop_epehemeral
+  # @api private
   def unset_ephemeral_var(level=:all)
+    Puppet.deprecation_warning('Method Parser::Scope#unset_ephemeral_var() is deprecated')
     if level == :all
       @ephemeral = [ MatchScope.new(@symtable, nil)]
     else
@@ -749,6 +857,22 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Pop ephemeral scopes up to level and return them
+  #
+  # @param level [Fixnum] a positive integer
+  # @return [Array] the removed ephemeral scopes
+  # @api private
+  def pop_ephemerals(level)
+    @ephemeral.pop(@ephemeral.size - level)
+  end
+
+  # Push ephemeral scopes onto the ephemeral scope stack
+  # @param ephemeral_scopes [Array]
+  # @api private
+  def push_ephemerals(ephemeral_scopes)
+    ephemeral_scopes.each { |ephemeral_scope| @ephemeral.push(ephemeral_scope) } unless ephemeral_scopes.nil?
+  end
+
   def ephemeral_level
     @ephemeral.size
   end
@@ -762,6 +886,53 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Execute given block in global scope with no ephemerals present
+  #
+  # @yieldparam [Scope] global_scope the global and ephemeral less scope
+  # @return [Object] the return of the block
+  #
+  # @api private
+  def with_global_scope(&block)
+    find_global_scope.without_ephemeral_scopes(&block)
+  end
+
+  # Execute given block with all ephemeral popped from the ephemeral stack
+  #
+  # @api private
+  def without_ephemeral_scopes
+    save_ephemeral = @ephemeral
+    begin
+      @ephemeral = [ @symtable ]
+      yield(self)
+    ensure
+      @ephemeral = save_ephemeral
+    end
+  end
+
+  # Nests a parameter scope
+  # @api private
+  def with_parameter_scope(param_names)
+    param_scope = ParameterScope.new(@ephemeral.last, param_names)
+    with_guarded_scope do
+      @ephemeral.push(param_scope)
+      yield(param_scope)
+    end
+  end
+
+  # Execute given block and ensure that ephemeral level is restored
+  #
+  # @return [Object] the return of the block
+  #
+  # @api private
+  def with_guarded_scope
+    elevel = ephemeral_level
+    begin
+      yield
+    ensure
+      pop_ephemerals(elevel)
+    end
+  end
+
   # Sets match data in the most nested scope (which always is a MatchScope), it clobbers match data already set there
   #
   def set_match_data(match_data)

data/lib/puppet/plugins/data_providers/data_provider.rb

@@ -72,7 +72,6 @@ module Puppet::Plugins::DataProviders
     def data_key(key, lookup_invocation)
       nil
     end
-    protected :data_key
 
     # Should be reimplemented by subclass to provide the hash that corresponds to the given name.
     #
@@ -94,7 +93,6 @@ module Puppet::Plugins::DataProviders
     def validate_data(data, data_key)
       data
     end
-    protected :validate_data
   end
 
   class ModuleDataProvider
@@ -112,22 +110,29 @@ module Puppet::Plugins::DataProviders
       throw :no_such_key if qual_index.nil?
       key[0..qual_index-1]
     end
-    protected :data_key
 
-    # Assert that all keys in the given _data_ are prefixed with the given _module_name_.
+    # Asserts that all keys in the given _data_ are prefixed with the given _module_name_. Remove entries
+    # that does not follow the convention and log a warning.
     #
     # @param data [Hash] The data hash
     # @param module_name [String] The name of the module where the data was found
-    # @return [Hash] The data_hash unaltered
+    # @return [Hash] The possibly pruned hash
+    # @api public
     def validate_data(data, module_name)
       module_prefix = "#{module_name}::"
-      data.each_key do |k|
-        unless k.is_a?(String) && (k == LOOKUP_OPTIONS || k.start_with?(module_prefix))
-          raise Puppet::DataBinding::LookupError, "Module data for module '#{module_name}' must use keys qualified with the name of the module"
+      data.each_key.reduce(data) do |memo, k|
+        if k.is_a?(String)
+          next memo if k == LOOKUP_OPTIONS || k.start_with?(module_prefix)
+          msg = 'must use keys qualified with the name of the module'
+        else
+          msg = "must use keys of type String, got #{k.class.name}"
         end
+        memo = memo.clone if memo.equal?(data)
+        memo.delete(k)
+        Puppet.warning("Module data for module '#{module_name}' #{msg}")
+        memo
       end
     end
-    protected :validate_data
   end
 
   class EnvironmentDataProvider
@@ -136,7 +141,6 @@ module Puppet::Plugins::DataProviders
     def data_key(key, lookup_invocation)
       'environment'
     end
-    protected :data_key
   end
 
   # Class that keeps track of the original path (as it appears in the declaration, before interpolation),
@@ -176,11 +180,38 @@ module Puppet::Plugins::DataProviders
 
     # @param name [String] The name of the data provider
     # @param paths [Array<ResolvedPath>] Paths used by this provider
+    # @param parent_data_provider [DataProvider] The data provider that is the container of this data provider
     #
     # @api public
-    def initialize(name, paths)
+    def initialize(name, paths, parent_data_provider = nil)
       @name = name
       @paths = paths
+      @parent_data_provider = parent_data_provider
+    end
+
+    # Gets the data from the compiler, or initializes it by calling #initialize_data if not present in the compiler.
+    # This means, that data is initialized once per compilation, and the data is cached for as long as the compiler
+    # lives (which is for one catalog production). This makes it possible to return data that is tailored for the
+    # request.
+    #
+    # If data is obtained using the #initialize_data method it will be sent to the #validate_data for validation
+    #
+    # @param path [String] The path to the data to be loaded (passed to #initialize_data)
+    # @param data_key [String] The data key such as the name of a module or the constant 'environment'
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] The current lookup invocation
+    # @param merge [String|Hash<String,Object>|nil] Merge strategy or hash with strategy and options
+    # @return [Hash] The data hash for the given _key_
+    #
+    # @api public
+    def load_data(path, data_key, lookup_invocation)
+      compiler = lookup_invocation.scope.compiler
+      adapter = Puppet::DataProviders::DataAdapter.get(compiler) || Puppet::DataProviders::DataAdapter.adapt(compiler)
+      adapter.data[path] ||= validate_data(initialize_data(path, lookup_invocation), data_key)
+    end
+    protected :data
+
+    def validate_data(data, module_name)
+      @parent_data_provider.nil? ? data : @parent_data_provider.validate_data(data, module_name)
     end
 
     # Performs a lookup by searching all given paths for the given _key_. A merge will be performed if
@@ -192,13 +223,14 @@ module Puppet::Plugins::DataProviders
     #
     # @api public
     def unchecked_lookup(key, lookup_invocation, merge)
+      module_name = @parent_data_provider.nil? ? nil : @parent_data_provider.data_key(key, lookup_invocation)
       lookup_invocation.with(:data_provider, self) do
         merge_strategy = Puppet::Pops::MergeStrategy.strategy(merge)
         lookup_invocation.with(:merge, merge_strategy) do
           merged_result = merge_strategy.merge_lookup(@paths) do |path|
             lookup_invocation.with(:path, path) do
               if path.exists?
-                hash = data(path.path, lookup_invocation)
+                hash = load_data(path.path, module_name, lookup_invocation)
                 value = hash[key]
                 if value || hash.include?(key)
                   lookup_invocation.report_found(key, post_process(value, lookup_invocation))
@@ -227,10 +259,11 @@ module Puppet::Plugins::DataProviders
     #
     # @param name [String] the name of the created provider (for logging and debugging)
     # @param paths [Array<String>] array of resolved paths
+    # @param parent_data_provider [DataProvider] The data provider that is the container of this data provider
     # @return [DataProvider] The created data provider
     #
     # @api public
-    def create(name, paths)
+    def create(name, paths, parent_data_provider)
       raise NotImplementedError, "Subclass of PathBasedDataProviderFactory must implement 'create' method"
     end
 
@@ -250,6 +283,14 @@ module Puppet::Plugins::DataProviders
     def resolve_paths(datadir, declared_paths, paths, lookup_invocation)
       []
     end
+
+    # Returns the data provider factory version.
+    #
+    # return [Integer] the version of this data provider factory
+    # @api public
+    def version
+      2
+    end
   end
 
   # Factory for creating file based data providers. This is an extension of the path based