puppet 4.0.0 → 4.9.1

data/lib/puppet/parser/environment_compiler.rb

@@ -0,0 +1,201 @@
+require 'puppet/parser/compiler'
+
+class Puppet::Parser::EnvironmentCompiler < Puppet::Parser::Compiler
+  def self.compile(env, code_id=nil)
+    begin
+      env.check_for_reparse
+
+      node = Puppet::Node.new(env)
+      node.environment = env
+      new(node, :code_id => code_id).compile
+    rescue => detail
+      message = "#{detail} in environment #{env.name}"
+      Puppet.log_exception(detail, message)
+      raise Puppet::Error, message, detail.backtrace
+    end
+  end
+
+  def initialize(node, options = {})
+    super
+    @overridden_functions = {}
+  end
+
+  def add_function_overrides
+    add_function_override('hiera_include', proc { Puppet.debug "Ignoring hiera_include() during environment catalog compilation" })
+  end
+
+  def add_function_override(func_name, override)
+    typed_name = Puppet::Pops::Loader::TypedName.new(:function, func_name)
+    loader = loaders.puppet_system_loader
+
+    # Remove and preserve existing entry. A `nil` is also preserved to indicate
+    # an override that didn't replace a loaded function.
+    entry = loader.get_entry(typed_name)
+    existing = entry.nil? ? nil : entry.value
+    loader.remove_entry(typed_name) unless existing.nil?
+    @overridden_functions[typed_name] = existing
+
+    # Add the override to the loader
+    loader.set_entry(typed_name, override)
+  end
+
+  def remove_function_overrides
+    loader = loaders.puppet_system_loader
+    @overridden_functions.each_pair do |typed_name, overridden|
+      loader.remove_entry(typed_name)
+      loader.set_entry(typed_name, overridden) unless overridden.nil?
+    end
+  end
+
+  def add_catalog_validators
+    super
+    add_catalog_validator(CatalogValidator::SiteValidator)
+    add_catalog_validator(CatalogValidator::EnvironmentRelationshipValidator)
+  end
+
+  def compile
+    add_function_overrides
+    begin
+      Puppet.override(@context_overrides, "For compiling environment catalog #{environment.name}") do
+        @catalog.environment_instance = environment
+
+        Puppet::Util::Profiler.profile("Env Compile: Created settings scope", [:compiler, :create_settings_scope]) { create_settings_scope }
+
+        activate_binder
+
+        Puppet::Util::Profiler.profile("Env Compile: Evaluated main", [:compiler, :evaluate_main]) { evaluate_main }
+
+        Puppet::Util::Profiler.profile("Env Compile: Evaluated site", [:compiler, :evaluate_site]) { evaluate_site }
+
+        Puppet::Util::Profiler.profile("Env Compile: Evaluated application instances", [:compiler, :evaluate_applications]) { evaluate_applications }
+
+        Puppet::Util::Profiler.profile("Env Compile: Prune", [:compiler, :prune_catalog]) { prune_catalog }
+
+        Puppet::Util::Profiler.profile("Env Compile: Validate Catalog pre-finish", [:compiler, :validate_pre_finish]) do
+          validate_catalog(CatalogValidator::PRE_FINISH)
+        end
+
+        Puppet::Util::Profiler.profile("Env Compile: Finished catalog", [:compiler, :finish_catalog]) { finish }
+
+        fail_on_unevaluated
+
+        Puppet::Util::Profiler.profile("Env Compile: Validate Catalog final", [:compiler, :validate_final]) do
+          validate_catalog(CatalogValidator::FINAL)
+        end
+
+        if block_given?
+          yield @catalog
+        else
+          @catalog
+        end
+      end
+    ensure
+      remove_function_overrides
+    end
+  end
+
+  # @api private
+  def prune_catalog
+    prune_env_catalog
+  end
+
+  # Prunes the catalog by dropping all resources that are not contained under the Site (if a site expression is used).
+  # As a consequence all edges to/from dropped resources are also dropped.
+  # Once the pruning is performed, this compiler returns the pruned list when calling the #resources method.
+  # The pruning does not alter the order of resources in the resources list.
+  #
+  def prune_env_catalog
+    # Everything under Class[main], that is not under (inclusive of) Site[site] should be pruned as those resources
+    # are intended for nodes in a node catalog.
+    #
+    the_main_class_resource = @catalog.resource('Class', '')
+    the_site_resource = @catalog.resource('Site', 'site')
+
+    # Get downstream vertexes returns a hash where the keys are the resources and values nesting level
+    rooted_in_main = @catalog.downstream_from_vertex(the_main_class_resource).keys
+
+    to_be_removed =
+    if the_site_resource
+      keep_from_site = @catalog.downstream_from_vertex(the_site_resource).keys
+      keep_from_site << the_site_resource
+      rooted_in_main - keep_from_site
+    else
+      rooted_in_main
+    end
+
+    @catalog.remove_resource(*to_be_removed)
+    # The compiler keeps a list of added resources, this shadows that list with the now pruned result
+    @pruned_resources = @catalog.resources
+  end
+
+  def add_resource(scope, resource)
+    @resources << resource
+    @catalog.add_resource(resource)
+
+    if !resource.class? && resource[:stage]
+      raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage"
+    end
+
+    # Stages should not be inside of classes.  They are always a
+    # top-level container, regardless of where they appear in the
+    # manifest.
+    return if resource.stage?
+
+    # This adds a resource to the class it lexically appears in in the
+    # manifest.
+    unless resource.class?
+      @catalog.add_edge(scope.resource, resource)
+    end
+    resource.mark_unevaluated_consumer if is_capability_consumer?(resource)
+    assert_app_in_site(scope, resource)
+  end
+
+  def evaluate_ast_node()
+    # Do nothing, the environment catalog is not built for a particular node.
+  end
+
+  def on_empty_site
+    Puppet.warning("Environment Compiler: Could not find a site definition to evaluate")
+  end
+
+  def evaluate_applications
+    exceptwrap do
+      resources.select { |resource| type = resource.resource_type; type.is_a?(Puppet::Resource::Type) && type.application? }.each do |resource|
+        Puppet::Util::Profiler.profile("Evaluated application #{resource}", [:compiler, :evaluate_resource, resource]) do
+          resource.evaluate
+        end
+      end
+    end
+  end
+
+  def evaluate_classes(titles, scope, lazy)
+    # Always lazy in an Environment compilation
+    super(titles, scope, true)
+  end
+
+  # Overrides the regular compiler to be able to return the list of resources after a prune
+  # has taken place in the graph representation. Before a prune, the list is the same as in the regular
+  # compiler
+  #
+  def resources
+    @pruned_resources || super
+  end
+
+  def is_capability?(value)
+    if value.is_a?(Array)
+      value.find { |ev| is_capability?(ev) }
+    elsif value.is_a?(Puppet::Resource)
+      rstype = value.resource_type
+      rstype.nil? ? false : rstype.is_capability?
+    else
+      false
+    end
+  end
+  private :is_capability?
+
+  def is_capability_consumer?(resource)
+    resource.eachparam { |param| return true if (param.name == :consume || param.name == :require) && is_capability?(param.value) }
+    false
+  end
+  private :is_capability_consumer?
+end

data/lib/puppet/parser/functions/assert_type.rb

@@ -2,29 +2,59 @@ Puppet::Parser::Functions::newfunction(
   :assert_type,
   :type => :rvalue,
   :arity => -3,
-  :doc => "Returns the given value if it is an instance of the given type, and raises an error otherwise.
-Optionally, if a block is given (accepting two parameters), it will be called instead of raising
-an error. This to enable giving the user richer feedback, or to supply a default value.
+  :doc => <<DOC
+Returns the given value if it is of the given
+[data type](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html), or
+otherwise either raises an error or executes an optional two-parameter
+[lambda](https://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html).
 
-Example: assert that `$b` is a non empty `String` and assign to `$a`:
+The function takes two mandatory arguments, in this order:
 
-  $a = assert_type(String[1], $b)
+1. The expected data type.
+2. A value to compare against the expected data type.
 
-Example using custom error message:
+**Example**: Using `assert_type`
 
-  $a = assert_type(String[1], $b) |$expected, $actual| {
-    fail('The name cannot be empty')
-  }
+~~~ puppet
+$raw_username = 'Amy Berry'
 
-Example, using a warning and a default:
+# Assert that $raw_username is a non-empty string and assign it to $valid_username.
+$valid_username = assert_type(String[1], $raw_username)
 
-  $a = assert_type(String[1], $b) |$expected, $actual| {
-    warning('Name is empty, using default')
-    'anonymous'
-  }
+# $valid_username contains "Amy Berry".
+# If $raw_username was an empty string or a different data type, the Puppet run would
+# fail with an "Expected type does not match actual" error.
+~~~
+
+You can use an optional lambda to provide enhanced feedback. The lambda takes two
+mandatory parameters, in this order:
+
+1. The expected data type as described in the function's first argument.
+2. The actual data type of the value.
+
+**Example**: Using `assert_type` with a warning and default value
+
+~~~ puppet
+$raw_username = 'Amy Berry'
+
+# Assert that $raw_username is a non-empty string and assign it to $valid_username.
+# If it isn't, output a warning describing the problem and use a default value.
+$valid_username = assert_type(String[1], $raw_username) |$expected, $actual| {
+  warning( "The username should be \'${expected}\', not \'${actual}\'. Using 'anonymous'." )
+  'anonymous'
+}
+
+# $valid_username contains "Amy Berry".
+# If $raw_username was an empty string, the Puppet run would set $valid_username to
+# "anonymous" and output a warning: "The username should be 'String[1, default]', not
+# 'String[0, 0]'. Using 'anonymous'."
+~~~
+
+For more information about data types, see the
+[documentation](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html).
 
-See the documentation for 'The Puppet Type System' for more information about types.
 - Since 4.0.0
-") do |args|
-  function_fail(["assert_type() is only available when parser/evaluator future is in effect"])
+DOC
+) do |args|
+  Error.is4x('assert_type')
 end

data/lib/puppet/parser/functions/binary_file.rb

@@ -0,0 +1,24 @@
+Puppet::Parser::Functions::newfunction(
+  :binary_file,
+  :type => :rvalue,
+  :arity => 1,
+:doc => <<-DOC
+Loads a binary file from a module or file system and returns its contents as a Binary.
+
+The argument to this function should be a `<MODULE NAME>/<FILE>`
+reference, which will load `<FILE>` from a module's `files`
+directory. (For example, the reference `mysql/mysqltuner.pl` will load the
+file `<MODULES DIRECTORY>/mysql/files/mysqltuner.pl`.)
+
+This function also accepts an absolute file path that allows reading
+binary file content from anywhere on disk.
+
+An error is raised if the given file does not exists.
+
+To search for the existence of files, use the `find_file()` function.
+
+- since 4.8.0
+DOC
+) do |args|
+  Error.is4x('binary_file')
+end

data/lib/puppet/parser/functions/break.rb

@@ -0,0 +1,39 @@
+Puppet::Parser::Functions::newfunction(
+  :break,
+  :arity => 0,
+  :doc => <<-DOC
+Breaks the innermost iteration as if it encountered an end of input.
+This function does not return to the caller.
+
+The signal produced to stop the iteration bubbles up through
+the call stack until either terminating the innermost iteration or
+raising an error if the end of the call stack is reached.
+
+The break() function does not accept an argument.
+
+**Example:** Using `break`
+
+```puppet
+$data = [1,2,3]
+notice $data.map |$x| { if $x == 3 { break() } $x*10 }
+```
+
+Would notice the value `[10, 20]`
+
+**Example:** Using a nested `break`
+
+```puppet
+function break_if_even($x) {
+  if $x % 2 == 0 { break() }
+}
+$data = [1,2,3]
+notice $data.map |$x| { break_if_even($x); $x*10 }
+```
+Would notice the value `[10]`
+
+* Also see functions `next` and `return`
+* Since 4.8.0
+DOC
+) do |args|
+  Error.is4x('break')
+end

data/lib/puppet/parser/functions/contain.rb

@@ -17,22 +17,14 @@ relative names are not allowed. In addition to names in string form,
 you may also directly use Class and Resource Type values that are produced by
 evaluating resource and relationship expressions.
 
+The function returns an array of references to the classes that were contained thus
+allowing the function call to `contain` to directly continue.
+
 - Since 4.0.0 support for Class and Resource Type values, absolute names
+- Since 4.7.0 an Array[Type[Class[n]]] is returned with all the contained classes
 "
 ) do |classes|
-  scope = self
-
-  # Make call patterns uniform and protected against nested arrays, also make
-  # names absolute if so desired.
-  classes = transform_and_assert_classnames(classes.is_a?(Array) ? classes.flatten : [classes])
-
-  containing_resource = scope.resource
-
-  # This is the same as calling the include function but faster and does not rely on the include
-  # function (which is a statement) to return something (it should not).
-  (compiler.evaluate_classes(classes, self, false) || []).each do |resource|
-    if ! scope.catalog.edge?(containing_resource, resource)
-      scope.catalog.add_edge(containing_resource, resource)
-    end
-  end
+  # Call the 4.x version of this function in case 3.x ruby code uses this function
+  Puppet.warn_once(:deprecation, '3xfunction#contain', "Calling function_contain via the Scope class is deprecated. Use Scope#call_function instead")
+  call_function('contain', classes)
 end

data/lib/puppet/parser/functions/create_resources.rb

@@ -42,6 +42,10 @@ Puppet::Parser::Functions::newfunction(:create_resources, :arity => -3, :doc =>
 
         create_resources("@user", $myusers)
 
+    Note that `create_resources` will filter out parameter values that are `undef` so that normal
+    data binding and puppet default value expressions are considered (in that order) for the
+    final value of a parameter (just as when setting a parameter to `undef` in a puppet language
+    resource declaration).
   ENDHEREDOC
   raise ArgumentError, ("create_resources(): wrong number of arguments (#{args.length}; must be 2 or 3)") if args.length > 3
   raise ArgumentError, ('create_resources(): second argument must be a hash') unless args[1].is_a?(Hash)
@@ -49,34 +53,46 @@ Puppet::Parser::Functions::newfunction(:create_resources, :arity => -3, :doc =>
     raise ArgumentError, ('create_resources(): third argument, if provided, must be a hash') unless args[2].is_a?(Hash)
   end
 
-
   type, instances, defaults = args
   defaults ||= {}
-
-  resource = Puppet::Parser::AST::Resource.new(:type => type.sub(/^@{1,2}/, '').downcase, :instances =>
-    instances.collect do |title, params|
-      Puppet::Parser::AST::ResourceInstance.new(
-        :title => Puppet::Parser::AST::Leaf.new(:value => title),
-        :parameters => defaults.merge(params).collect do |name, value|
-          Puppet::Parser::AST::ResourceParam.new(
-            :param => name,
-            :value => Puppet::Parser::AST::Leaf.new(:value => value))
-        end)
-    end)
+  type_name = type.sub(/^@{1,2}/, '').downcase
+
+  # Get file/line information from the puppet stack (where call comes from in puppet source)
+  # If relayed via other puppet functions in ruby that do not nest their calls, the source position
+  # will be in the original puppet source.
+  #
+  stacktrace = Puppet::Pops::PuppetStack.stacktrace()
+  if stacktrace.size > 0
+    file, line = stacktrace[0]
+  else
+    file = nil
+    line = nil
+  end
 
   if type.start_with? '@@'
-    resource.exported = true
+    exported = true
   elsif type.start_with? '@'
-    resource.virtual = true
+    virtual = true
   end
 
-  begin
-    resource.safeevaluate(self)
-  rescue Puppet::ParseError => internal_error
-    if internal_error.original.nil?
-      raise internal_error
-    else
-      raise internal_error.original
-    end
-  end
+  instances.map do |title, params|
+    # Add support for iteration if title is an array
+    resource_titles = title.is_a?(Array) ? title  : [title]
+    Puppet::Pops::Evaluator::Runtime3ResourceSupport.create_resources(
+      file, line,
+      self,
+      virtual, exported,
+      type_name,
+      resource_titles,
+      defaults.merge(params).map do |name, value|
+        value = nil if value == :undef
+        Puppet::Parser::Resource::Param.new(
+          :name   => name,
+          :value  => value, # wide open to various data types, must be correct
+          :source => self.source, # TODO: support :line => line, :file => file,
+          :add    => false
+        )
+      end.compact
+      )
+  end.flatten.compact
 end

data/lib/puppet/parser/functions/defined.rb

@@ -1,70 +1,107 @@
-# Test whether a given class or definition is defined
-Puppet::Parser::Functions::newfunction(:defined, :type => :rvalue, :arity => -2,
-  :doc => "Determine whether
-  a given class or resource type is defined. This function can also determine whether a
-  specific resource has been defined, or whether a variable has been assigned a value
-  (including `undef` - as opposed to never having been assigned anything). Returns `true`
-  or `false`. Accepts class names, type names, resource references, and variable
-  reference strings of the form `'$name'`. When more than one argument is
-  supplied, `defined()` returns `true` if any are defined.
-
-  The `defined` function checks both native and defined types, including types
-  provided as plugins via modules. Types and classes are both checked using their names:
-
-      defined(\"file\")
-      defined(\"customtype\")
-      defined(\"foo\")
-      defined(\"foo::bar\")
-      defined(\'$name\')
-
-  Resource declarations are checked using resource references, e.g.
-  `defined( File['/tmp/myfile'] )`. Checking whether a given resource
-  has been declared is, unfortunately, dependent on the evaluation order of
-  the configuration, and the following code will not work:
-
-      if defined(File['/tmp/foo']) {
-          notify { \"This configuration includes the /tmp/foo file.\":}
-      }
-      file { \"/tmp/foo\":
-          ensure => present,
-      }
-
-  However, this order requirement refers to evaluation order only, and ordering of
-  resources in the configuration graph (e.g. with `before` or `require`) does not
-  affect the behavior of `defined`.
-
-  You may also search using types:
-
-      defined(Resource[\'file\',\'/some/file\'])
-      defined(File[\'/some/file\'])
-      defined(Class[\'foo\'])
-
- - Since 2.7.0
- - Since 3.6.0 variable reference and future parser types
- - Since 4.0.0") do |vals|
-  vals = [vals] unless vals.is_a?(Array)
-  vals.any? do |val|
-    case val
-    when String
-      if m = /^\$(.+)$/.match(val)
-        exist?(m[1])
-      else
-        find_resource_type(val) || find_definition(val) || find_hostclass(val)
-      end
-    when Puppet::Resource
-      compiler.findresource(val.type, val.title)
-    when Puppet::Pops::Types::PResourceType
-      raise ArgumentError, "The given resource type is a reference to all kind of types" if val.type_name.nil?
-      if val.title.nil?
-        find_builtin_resource_type(val.type_name) || find_definition(val.type_name)
-      else
-        compiler.findresource(val.type_name, val.title)
-      end
-    when Puppet::Pops::Types::PHostClassType
-      raise  ArgumentError, "The given class type is a reference to all classes" if val.class_name.nil?
-      find_hostclass(val.class_name)
-    else
-      raise ArgumentError, "Invalid argument of type '#{val.class}' to 'defined'"
-    end
-  end
+Puppet::Parser::Functions::newfunction(
+  :defined,
+  :type => :rvalue,
+  :arity => -2,
+  :doc => <<DOC
+Determines whether a given class or resource type is defined and returns a Boolean
+value. You can also use `defined` to determine whether a specific resource is defined,
+or whether a variable has a value (including `undef`, as opposed to the variable never
+being declared or assigned).
+
+This function takes at least one string argument, which can be a class name, type name,
+resource reference, or variable reference of the form `'$name'`.
+
+The `defined` function checks both native and defined types, including types
+provided by modules. Types and classes are matched by their names. The function matches
+resource declarations by using resource references.
+
+**Examples**: Different types of `defined` function matches
+
+~~~ puppet
+# Matching resource types
+defined("file")
+defined("customtype")
+
+# Matching defines and classes
+defined("foo")
+defined("foo::bar")
+
+# Matching variables
+defined('$name')
+
+# Matching declared resources
+defined(File['/tmp/file'])
+~~~
+
+Puppet depends on the configuration's evaluation order when checking whether a resource
+is declared.
+
+**Example**: Importance of evaluation order when using `defined`
+
+~~~ puppet
+# Assign values to $is_defined_before and $is_defined_after using identical `defined`
+# functions.
+
+$is_defined_before = defined(File['/tmp/file'])
+
+file { "/tmp/file":
+  ensure => present,
+}
+
+$is_defined_after = defined(File['/tmp/file'])
+
+# $is_defined_before returns false, but $is_defined_after returns true.
+~~~
+
+This order requirement only refers to evaluation order. The order of resources in the
+configuration graph (e.g. with `before` or `require`) does not affect the `defined`
+function's behavior.
+
+> **Warning:** Avoid relying on the result of the `defined` function in modules, as you
+> might not be able to guarantee the evaluation order well enough to produce consistent
+> results. This can cause other code that relies on the function's result to behave
+> inconsistently or fail.
+
+If you pass more than one argument to `defined`, the function returns `true` if _any_
+of the arguments are defined. You can also match resources by type, allowing you to
+match conditions of different levels of specificity, such as whether a specific resource
+is of a specific data type.
+
+**Example**: Matching multiple resources and resources by different types with `defined`
+
+~~~ puppet
+file { "/tmp/file1":
+  ensure => file,
+}
+
+$tmp_file = file { "/tmp/file2":
+  ensure => file,
+}
+
+# Each of these statements return `true` ...
+defined(File['/tmp/file1'])
+defined(File['/tmp/file1'],File['/tmp/file2'])
+defined(File['/tmp/file1'],File['/tmp/file2'],File['/tmp/file3'])
+# ... but this returns `false`.
+defined(File['/tmp/file3'])
+
+# Each of these statements returns `true` ...
+defined(Type[Resource['file','/tmp/file2']])
+defined(Resource['file','/tmp/file2'])
+defined(File['/tmp/file2'])
+defined('$tmp_file')
+# ... but each of these returns `false`.
+defined(Type[Resource['exec','/tmp/file2']])
+defined(Resource['exec','/tmp/file2'])
+defined(File['/tmp/file3'])
+defined('$tmp_file2')
+~~~
+
+- Since 2.7.0
+- Since 3.6.0 variable reference and future parser types
+- Since 3.8.1 type specific requests with future parser
+- Since 4.0.0 includes all future parser features
+DOC
+) do |vals|
+  Error.is4x('defined')
 end

data/lib/puppet/parser/functions/dig.rb

@@ -0,0 +1,29 @@
+Puppet::Parser::Functions::newfunction(
+  :dig,
+  :type => :rvalue,
+  :arity => -1,
+  :doc => <<-DOC
+Returns a value for a sequence of given keys/indexes into a structure.
+This function is used to "dig into" a complex data structure by
+using a sequence of keys / indexes to access a value from which
+the next key/index is accessed recursively.
+
+The first encountered `undef` value or key stops the "dig" and `undef` is returned.
+
+An error is raised if an attempt is made to "dig" into
+something other than an `undef` (which immediately returns `undef`), an `Array` or a `Hash`.
+
+**Example:** Using `dig`
+
+```puppet
+$data = {a => { b => [{x => 10, y => 20}, {x => 100, y => 200}]}}
+notice $data.dig(a, b, 1, x)
+```
+
+Would notice the value 100.
+
+* Since 4.5.0
+DOC
+) do |args|
+  Error.is4x('dig')
+end