puppet 4.3.1 → 4.3.2

data/lib/puppet/functions/match.rb

@@ -30,6 +30,7 @@
 # ~~~ ruby
 # $matches = ["abc123","def456"].match(/([a-z]+)([1-9]+)/)
 # # $matches contains [[abc123, abc, 123], [def456, def, 456]]
+# ~~~
 #
 # @since 4.0.0
 #

data/lib/puppet/indirector/hiera.rb

@@ -23,7 +23,9 @@ class Puppet::Indirector::Hiera < Puppet::Indirector::Terminus
     throw :no_such_key if value.equal?(not_found)
     value
   rescue *DataBindingExceptions => detail
-    raise Puppet::DataBinding::LookupError.new(detail.message, detail)
+    error = Puppet::DataBinding::LookupError.new("DataBinding 'hiera': #{detail.message}")
+    error.set_backtrace(detail.backtrace)
+    raise error
   end
 
   private

data/lib/puppet/indirector/indirection.rb

@@ -207,10 +207,14 @@ class Puppet::Indirector::Indirection
         filtered = result
         if terminus.respond_to?(:filter)
           Puppet::Util::Profiler.profile("Filtered result for #{self.name} #{request.key}", [:indirector, :filter, self.name, request.key]) do
-            filtered = terminus.filter(result)
+            begin
+              filtered = terminus.filter(result)
+            rescue Puppet::Error => detail
+              Puppet.log_exception(detail)
+              raise detail
+            end
           end
         end
-
         filtered
       end
     end

data/lib/puppet/indirector/json.rb

@@ -15,7 +15,7 @@ class Puppet::Indirector::JSON < Puppet::Indirector::Terminus
     filename = path(request.key)
     FileUtils.mkdir_p(File.dirname(filename))
 
-    Puppet::Util.replace_file(filename, 0660) {|f| f.print to_json(request.instance) }
+    Puppet::Util.replace_file(filename, 0660) {|f| f.print to_json(request.instance).force_encoding(Encoding::ASCII_8BIT) }
   rescue TypeError => detail
     Puppet.log_exception(detail, "Could not save #{self.name} #{request.key}: #{detail}")
   end
@@ -52,7 +52,7 @@ class Puppet::Indirector::JSON < Puppet::Indirector::Terminus
     json = nil
 
     begin
-      json = File.read(file)
+      json = Puppet::FileSystem.read(file).force_encoding(Encoding::ASCII_8BIT)
     rescue Errno::ENOENT
       return nil
     rescue => detail

data/lib/puppet/module.rb

@@ -155,6 +155,8 @@ class Puppet::Module
 
       if attr == :dependencies
         value.each do |dep|
+          name = dep['name']
+          dep['name'] = name.tr('-', '/') unless name.nil?
           dep['version_requirement'] ||= '>= 0.0.0'
         end
       end
@@ -269,11 +271,10 @@ class Puppet::Module
 
     dependencies.each do |dependency|
       name = dependency['name']
-      forge_name = name.tr('-', '/')
       version_string = dependency['version_requirement'] || '>= 0.0.0'
 
       dep_mod = begin
-        environment.module_by_forge_name(forge_name)
+        environment.module_by_forge_name(name)
       rescue
         nil
       end

data/lib/puppet/node.rb

@@ -21,6 +21,8 @@ class Puppet::Node
 
   attr_reader :server_facts
 
+  ENVIRONMENT = 'environment'.freeze
+
   def initialize_from_hash(data)
     @name       = data['name']       || (raise ArgumentError, "No name provided in serialized data")
     @classes    = data['classes']    || []
@@ -48,7 +50,7 @@ class Puppet::Node
     if @environment
       @environment
     else
-      if env = parameters["environment"]
+      if env = parameters[ENVIRONMENT]
         self.environment = env
       elsif environment_name
         self.environment = environment_name
@@ -70,6 +72,13 @@ class Puppet::Node
     else
       @environment = env
     end
+
+    # Keep environment_name attribute and parameter in sync if they have been set
+    unless @environment.nil?
+      @parameters[ENVIRONMENT] = @environment.name if @parameters.include?(ENVIRONMENT)
+      self.environment_name = @environment.name if instance_variable_defined?(:@environment_name)
+    end
+    @environment
   end
 
   def has_environment_instance?
@@ -125,7 +134,7 @@ class Puppet::Node
       end
     end
 
-    @parameters["environment"] ||= self.environment.name.to_s
+    @parameters[ENVIRONMENT] ||= self.environment.name.to_s
   end
 
   def add_server_facts(facts)

data/lib/puppet/parser/compiler.rb

@@ -803,14 +803,7 @@ class Puppet::Parser::Compiler
     if trusted_param
       # Blows up if it is a parameter as it will be set as $trusted by the compiler as if it was a variable
       node.parameters.delete('trusted')
-      if trusted_param.is_a?(Hash) && %w{authenticated certname extensions}.all? {|key| trusted_param.has_key?(key) }
-        # looks like a hash of trusted data - resurrect it
-        # Allow root to trust the authenticated information if option --trusted is given
-        if !Puppet.features.root?
-          # Set as not trusted - but keep the information
-          trusted_param['authenticated'] = false
-        end
-      else
+      unless trusted_param.is_a?(Hash) && %w{authenticated certname extensions}.all? {|key| trusted_param.has_key?(key) }
         # trusted is some kind of garbage, do not resurrect
         trusted_param = nil
       end

data/lib/puppet/parser/functions/lookup.rb

@@ -1,154 +1,133 @@
 Puppet::Parser::Functions.newfunction(:lookup, :type => :rvalue, :arity => -2, :doc => <<-'ENDHEREDOC') do |args|
-Looks up data defined using Data Binding, and Data Providers using different strategies. The lookup searches in
-Data Bindings first (if configured; typically Hiera), then in the environments data provider (if any), and last in
-the module's data provider (if any) of the module the call to lookup originates from. Thus, the global Data Binding
-has higher priority than data provided in the environment, which has higher priority than data provided in a module,
-
-The lookup function can be called in one of these ways:
-
-    lookup(name)
-    lookup(name, value_type)
-    lookup(name, value_type, merge)
-    lookup(name, value_type, merge, default_value)
-    lookup(options_hash)
-    lookup(name, options_hash)
-
-The function may optionally be called with a code block / lambda with the following signature:
-
-    lookup(...) |$name| { ... }
-
-The block, if present, is mutually exclusive to the `default_value` and will be called with the `name` used in the
-lookup when no value is found. The value produced by the block then becomes the value produced by the lookup.
-
-The meaning of the parameters or content of the options hash is:
-
-* `name` - The name or array of names to lookup (first found is returned)
-* `value_type` - The type to assert. Defaults to 'Data' See 'Type Specification' below.
-* `default_value` - The default value if there was no value found (must comply with the data type)
-* `override` - a hash with map from names to values that are used instead of the underlying bindings. If the name
-  is found here it wins. Defaults to an empty hash.
-* `default_values_hash` - a hash with map from names to values that are used as a last resort to obtain a value.
-  Defaults to an empty hash.
-* `merge` - A string of type Enum[unique, hash, merge] or a hash with the key 'strategy' set to that string. See
-  'Merge Strategies' below.
-
-It is not permitted to pass the `name` as both a parameter and in the options hash.
-
-The search will proceed as follows:
-1. For each name given in the `name` array (or once, if it's just one name):
-  - If a matching key is found in the `override` hash, it's value is immediately type checked and returned
-  - Search and optionally merge Data Binding, environment data providers, and module data providers
-  - Type check and return the value if a matching key is found
-2. For each name given in the `name` array (or once, if it's just one name):
-  - Type check and return the value if a matching key is found in the `default_values_hash`
-3. Type check and return either the given `default_value` or the result of calling the code block if either exist
-4. Raise an error indicating that no matching value was found
-
-*Merge Strategies*
-
-The default behavior of the lookup is to return the first value that is found for the given `name`. The optional
-`merge` parameter will change this so that a lookup makes an attempt to find values in all three sources (the Data
-Binder, the environment, and the module scope) and then merge these values according to the given strategy. This
-does not apply to values found in the 'override' hash. Such values are returned immediately without merging.
-Note that `merge` is passed on to allow the underlying provider to return a merged result
-
-The valid strategies are:
-- 'hash' Performs a simple hash-merge by overwriting keys of lower lookup priority. Merged values must be of Hash type
-- 'unique' Appends everything to an array containing no nested arrays and where all duplicates have been removed. Can
-   append values of Scalar or Array[Scalar] type
-- 'deep' Performs a deep merge on values of Array and Hash type. See documentation for the DeepMerge gem's deep_merge
-   operation for details and options.
-
-The 'deep' strategy can use additional options to control its behavior. Options can be passed as top level
-keys in the `merge` parameter when it is a given as a hash. Recognized options are:
-- 'knockout_prefix' Set to string value to signify prefix which deletes elements from existing element. Defaults is _undef_
-- 'sort_merged_arrays' Set to _true_ to sort all arrays that are merged together. Default is _false_
-- 'unpack_arrays' Set to string value used as a deliminator to join all array values and then split them again. Default is _undef_
-- 'merge_hash_arrays' Set to _true_ to merge hashes within arrays. Default is _false_
-
-*Type Specification*
-
-The type specification is a type in the Puppet Type System, e.g.:
-  * `Integer`, an integral value with optional range e.g.:
-    * `Integer[0, default]` - 0 or positive
-    * `Integer[default, -1]` - negative,
-    * `Integer[1,100]` - value between 1 and 100 inclusive
-  * `String`- any string
-  * `Float` - floating point number (same signature as for Integer for `Integer` ranges)
-  * `Boolean` - true of false (strict)
-  * `Array` - an array (of Data by default), or parameterized as `Array[<element_type>]`, where
-    `<element_type>` is the expected type of elements
-  * `Hash`,  - a hash (of default `Literal` keys and `Data` values), or parameterized as
-    `Hash[<value_type>]`, `Hash[<key_type>, <value_type>]`, where `<key_type>`, and
-    `<value_type>` are the types of the keys and values respectively
-    (key is `Literal` by default).
-  * `Data` - abstract type representing any `Literal` (including _undef_), `Array[Data]`, or `Hash[Literal, Data]`
-  * `Pattern[<p1>, <p2>, ..., <pn>]` - an enumeration of valid patterns (one or more) where
-     a pattern is a regular expression string or regular expression,
-     e.g. `Pattern['.com$', '.net$']`, `Pattern[/[a-z]+[0-9]+/]`
-  * `Enum[<s1>, <s2>, ..., <sn>]`, - an enumeration of exact string values (one or more)
-     e.g. `Enum[blue, red, green]`.
-  * `Variant[<t1>, <t2>,...<tn>]` - matches one of the listed types (at least one must be given)
-    e.g. `Variant[Integer[8000,8999], Integer[20000, 99999]]` to accept a value in either range
-  * `Regexp`- a regular expression (i.e. the result is a regular expression, not a string
-     matching a regular expression).
-
-For more options and details about types, see the Puppet Language Reference
-
-*Handling of undef*
-
-When no match is found for the given `name` when searching all sources, (including the `override`and
-`default_values_hash`), then the value used is either the `default_value` or the value produced by the given block.
-If neither is provided, then the lookup will always raise an error. Note that this only applies when there's no
-match for the given `name`. It does not happen when a value is found and that value happens to be _undef_.
-
-*Validation of returned value*
+Uses the Puppet lookup system to retrieve a value for a given key. By default,
+this returns the first value found (and fails compilation if no values are
+available), but you can configure it to merge multiple values into one, fail
+gracefully, and more.
+
+When looking up a key, Puppet will search up to three tiers of data, in the
+following order:
+
+1. Hiera.
+2. The current environment's data provider.
+3. The indicated module's data provider, if the key is of the form
+   `<MODULE NAME>::<SOMETHING>`.
+
+#### Arguments
+
+You must provide the name of a key to look up, and can optionally provide other
+arguments. You can combine these arguments in the following ways:
+
+* `lookup( <NAME>, [<VALUE TYPE>], [<MERGE BEHAVIOR>], [<DEFAULT VALUE>] )`
+* `lookup( [<NAME>], <OPTIONS HASH> )`
+* `lookup( as above ) |$key| { # lambda returns a default value }`
+
+Arguments in `[square brackets]` are optional.
+
+The arguments accepted by `lookup` are as follows:
+
+1. `<NAME>` (string or array) --- The name of the key to look up.
+    * This can also be an array of keys. If Puppet doesn't find anything for the
+    first key, it will try again with the subsequent ones, only resorting to a
+    default value if none of them succeed.
+2. `<VALUE TYPE>` (data type) --- A
+[data type](https://docs.puppetlabs.com/puppet/latest/reference/lang_data_type.html)
+that must match the retrieved value; if not, the lookup (and catalog
+compilation) will fail. Defaults to `Data` (accepts any normal value).
+3. `<MERGE BEHAVIOR>` (string or hash; see **"Merge Behaviors"** below) ---
+Whether (and how) to combine multiple values. If present, this overrides any
+merge behavior specified in the data sources. Defaults to no value; Puppet will
+use merge behavior from the data sources if present, and will otherwise do a
+first-found lookup.
+4. `<DEFAULT VALUE>` (any normal value) --- If present, `lookup` returns this
+when it can't find a normal value. Default values are never merged with found
+values. Like a normal value, the default must match the value type. Defaults to
+no value; if Puppet can't find a normal value, the lookup (and compilation) will
+fail.
+5. `<OPTIONS HASH>` (hash) --- Alternate way to set the arguments above, plus
+some less-common extra options. If you pass an options hash, you can't combine
+it with any regular arguments (except `<NAME>`). An options hash can have the
+following keys:
+    * `'name'` --- Same as `<NAME>` (argument 1). You can pass this as an
+    argument or in the hash, but not both.
+    * `'value_type'` --- Same as `<VALUE TYPE>` (argument 2).
+    * `'merge'` --- Same as `<MERGE BEHAVIOR>` (argument 3).
+    * `'default_value'` --- Same as `<DEFAULT VALUE>` (argument 4).
+    * `'default_values_hash'` (hash) --- A hash of lookup keys and default
+    values. If Puppet can't find a normal value, it will check this hash for the
+    requested key before giving up. You can combine this with `default_value` or
+    a lambda, which will be used if the key isn't present in this hash. Defaults
+    to an empty hash.
+    * `'override'` (hash) --- A hash of lookup keys and override values. Puppet
+    will check for the requested key in the overrides hash _first;_ if found, it
+    returns that value as the _final_ value, ignoring merge behavior. Defaults
+    to an empty hash.
+
+Finally, `lookup` can take a lambda, which must accept a single parameter.
+This is yet another way to set a default value for the lookup; if no results are
+found, Puppet will pass the requested key to the lambda and use its result as
+the default value.
+
+#### Merge Behaviors
+
+Puppet lookup uses a hierarchy of data sources, and a given key might have
+values in multiple sources. By default, Puppet returns the first value it finds,
+but it can also continue searching and merge all the values together.
+
+> **Note:** Data sources can use the special `lookup_options` metadata key to
+request a specific merge behavior for a key. The `lookup` function will use that
+requested behavior unless you explicitly specify one.
+
+The valid merge behaviors are:
+
+* `'first'` --- Returns the first value found, with no merging. Puppet lookup's
+default behavior.
+* `'unique'` (called "array merge" in classic Hiera) --- Combines any number of
+arrays and scalar values to return a merged, flattened array with all duplicate
+values removed. The lookup will fail if any hash values are found.
+* `'hash'` --- Combines the keys and values of any number of hashes to return a
+merged hash. If the same key exists in multiple source hashes, Puppet will use
+the value from the highest-priority data source; it won't recursively merge the
+values.
+* `'deep'` --- Combines the keys and values of any number of hashes to return a
+merged hash. If the same key exists in multiple source hashes, Puppet will
+recursively merge hash or array values (with duplicate values removed from
+arrays). For conflicting scalar values, the highest-priority value will win.
+* `{'strategy' => 'first|unique|hash'}` --- Same as the string versions of these
+merge behaviors.
+* `{'strategy' => 'deep', <DEEP OPTION> => <VALUE>, ...}` --- Same as `'deep'`,
+but can adjust the merge with additional options. The available options are:
+    * `'knockout_prefix'` (string or undef) --- A string prefix to indicate a
+    value should be _removed_ from the final result. Defaults to `undef`, which
+    disables this feature.
+    * `'sort_merged_arrays'` (boolean) --- Whether to sort all arrays that are
+    merged together. Defaults to `false`.
+    * `'unpack_arrays'` (string or undef) --- A delimiter string; Puppet will
+    join merged arrays with this delimiter, then split them again. Defaults to
+    `undef`, which disables this feature.
+    * `'merge_hash_arrays'` (boolean) --- Whether to merge hashes within arrays.
+    Defaults to `false`.
+
+#### Examples
+
+Look up a key and return the first value found:
+
+    lookup('ntp::service_name')
+
+Do a unique merge lookup of class names, then add all of those classes to the
+catalog (like `hiera_include`):
+
+    lookup('classes', Array[String], 'unique').include
+
+Do a deep hash merge lookup of user data, but let higher priority sources
+remove values by prefixing them with `--`:
+
+    lookup( { 'name'  => 'users',
+              'merge' => {
+                'strategy'        => 'deep',
+                'knockout_prefix' => '--',
+              },
+    })
 
-The produced value is subject to type validation using the `value_type` (if given) and an error is raised unless
-the resulting value is of correct type.
-
-*Examples*
-
-When called with one argument; **the name**, it
-returns the bound value with the given name after having  asserted it has the default datatype `Data`:
-
-    lookup('the_name')
-
-When called with two arguments; **the name**, and **the expected type**, it
-returns the bound value with the given name after having asserted it has the given data
-type ('String' in the example):
-
-    lookup('the_name', String)
-
-When called with four arguments, **the name**, the **expected type**, the **merge** strategy, and a
-**default value**, it returns the bound value with the given name, or the default after having asserted the value
-has the given data type:
-
-    lookup('the_name', String, undef, 'Fred')
-    lookup('the_name', Array[String], 'unique', [Fred])
-
-Using a lambda to provide a default value by calling a function:
-
-    lookup('the_size', Integer[1,100]) |$name| {
-      obtain_size_default()
-    }
-
-There are two ways to make lookup return undef when no matching key was found instead of raising an error.
-Either call it with four arguments (the `merge` argument must be present even when using the default strategy
-to ensure that the four argument variant is used):
-
-     $are_you_there = lookup('peekaboo', Optional[String], undef, undef)
-
-or call it using an options hash:
-
-     $are_you_there = lookup('peekaboo', { 'default_value' => undef })
-     $are_you_there = lookup({ 'name' => 'peekaboo', 'default_value' => undef })
-
-or with a block that produces an undef value:
-
-     $are_you_there = lookup('peekaboo', Optional[String]) |$name| { undef }
-
-- Since 4.0.0
 ENDHEREDOC
   function_fail(["lookup() has been converted to 4x API"])
 end

data/lib/puppet/parser/functions/match.rb

@@ -34,6 +34,7 @@ $matches = "abc123".match(/([a-z]+)([1-9]+)/)
 ~~~ ruby
 $matches = ["abc123","def456"].match(/([a-z]+)([1-9]+)/)
 # $matches contains [[abc123, abc, 123], [def456, def, 456]]
+~~~
 
 - Since 4.0.0
 DOC

data/lib/puppet/plugins/data_providers/data_provider.rb

@@ -98,6 +98,7 @@ module Puppet::Plugins::DataProviders
   end
 
   class ModuleDataProvider
+    LOOKUP_OPTIONS = Puppet::Pops::Lookup::LOOKUP_OPTIONS
     include DataProvider
 
     # Retrieve the first segment of the qualified name _key_. This method will throw
@@ -106,7 +107,7 @@ module Puppet::Plugins::DataProviders
     # @param key [String] The key
     # @return [String] The first segment of the given key
     def data_key(key, lookup_invocation)
-      return lookup_invocation.module_name if key == 'lookup_options'
+      return lookup_invocation.module_name if key == LOOKUP_OPTIONS
       qual_index = key.index('::')
       throw :no_such_key if qual_index.nil?
       key[0..qual_index-1]
@@ -121,7 +122,7 @@ module Puppet::Plugins::DataProviders
     def validate_data(data, module_name)
       module_prefix = "#{module_name}::"
       data.each_key do |k|
-        unless k.is_a?(String) && (k == 'lookup_options' || k.start_with?(module_prefix))
+        unless k.is_a?(String) && (k == LOOKUP_OPTIONS || k.start_with?(module_prefix))
           raise Puppet::DataBinding::LookupError, "Module data for module '#{module_name}' must use keys qualified with the name of the module"
         end
       end