puppet 4.0.0 → 4.5.0

data/lib/puppet/parser/environment_compiler.rb

@@ -0,0 +1,164 @@
+require 'puppet/parser/compiler'
+
+class Puppet::Parser::EnvironmentCompiler < Puppet::Parser::Compiler
+  def self.compile(env, code_id=nil)
+    begin
+      env.check_for_reparse
+
+      node = Puppet::Node.new(env)
+      node.environment = env
+      new(node, :code_id => code_id).compile
+    rescue => detail
+      message = "#{detail} in environment #{env.name}"
+      Puppet.log_exception(detail, message)
+      raise Puppet::Error, message, detail.backtrace
+    end
+  end
+
+  def add_catalog_validators
+    super
+    add_catalog_validator(CatalogValidator::SiteValidator)
+    add_catalog_validator(CatalogValidator::EnvironmentRelationshipValidator)
+  end
+
+  def compile
+    Puppet.override(@context_overrides, "For compiling environment catalog #{environment.name}") do
+      @catalog.environment_instance = environment
+
+      Puppet::Util::Profiler.profile("Env Compile: Created settings scope", [:compiler, :create_settings_scope]) { create_settings_scope }
+
+      activate_binder
+
+      Puppet::Util::Profiler.profile("Env Compile: Evaluated main", [:compiler, :evaluate_main]) { evaluate_main }
+
+      Puppet::Util::Profiler.profile("Env Compile: Evaluated site", [:compiler, :evaluate_site]) { evaluate_site }
+
+      Puppet::Util::Profiler.profile("Env Compile: Evaluated application instances", [:compiler, :evaluate_applications]) { evaluate_applications }
+
+      Puppet::Util::Profiler.profile("Env Compile: Prune", [:compiler, :prune_catalog]) { prune_catalog }
+
+      Puppet::Util::Profiler.profile("Env Compile: Validate Catalog pre-finish", [:compiler, :validate_pre_finish]) do
+        validate_catalog(CatalogValidator::PRE_FINISH)
+      end
+
+      Puppet::Util::Profiler.profile("Env Compile: Finished catalog", [:compiler, :finish_catalog]) { finish }
+
+      fail_on_unevaluated
+
+      Puppet::Util::Profiler.profile("Env Compile: Validate Catalog final", [:compiler, :validate_final]) do
+        validate_catalog(CatalogValidator::FINAL)
+      end
+
+      if block_given?
+        yield @catalog
+      else
+        @catalog
+      end
+    end
+  end
+
+  # @api private
+  def prune_catalog
+    prune_env_catalog
+  end
+
+  # Prunes the catalog by dropping all resources that are not contained under the Site (if a site expression is used).
+  # As a consequence all edges to/from dropped resources are also dropped.
+  # Once the pruning is performed, this compiler returns the pruned list when calling the #resources method.
+  # The pruning does not alter the order of resources in the resources list.
+  #
+  def prune_env_catalog
+    # Everything under Class[main], that is not under (inclusive of) Site[site] should be pruned as those resources
+    # are intended for nodes in a node catalog.
+    #
+    the_main_class_resource = @catalog.resource('Class', '')
+    the_site_resource = @catalog.resource('Site', 'site')
+
+    # Get downstream vertexes returns a hash where the keys are the resources and values nesting level
+    rooted_in_main = @catalog.downstream_from_vertex(the_main_class_resource).keys
+
+    to_be_removed =
+    if the_site_resource
+      keep_from_site = @catalog.downstream_from_vertex(the_site_resource).keys
+      keep_from_site << the_site_resource
+      rooted_in_main - keep_from_site
+    else
+      rooted_in_main
+    end
+
+    @catalog.remove_resource(*to_be_removed)
+    # The compiler keeps a list of added resources, this shadows that list with the now pruned result
+    @pruned_resources = @catalog.resources
+  end
+
+  def add_resource(scope, resource)
+    @resources << resource
+    @catalog.add_resource(resource)
+
+    if !resource.class? && resource[:stage]
+      raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage"
+    end
+
+    # Stages should not be inside of classes.  They are always a
+    # top-level container, regardless of where they appear in the
+    # manifest.
+    return if resource.stage?
+
+    # This adds a resource to the class it lexically appears in in the
+    # manifest.
+    unless resource.class?
+      @catalog.add_edge(scope.resource, resource)
+    end
+    resource.mark_unevaluated_consumer if is_capability_consumer?(resource)
+    assert_app_in_site(scope, resource)
+  end
+
+  def evaluate_ast_node()
+    # Do nothing, the environment catalog is not built for a particular node.
+  end
+
+  def on_empty_site
+    Puppet.warning("Environment Compiler: Could not find a site definition to evaluate")
+  end
+
+  def evaluate_applications
+    exceptwrap do
+      resources.select { |resource| type = resource.resource_type; type.is_a?(Puppet::Resource::Type) && type.application? }.each do |resource|
+        Puppet::Util::Profiler.profile("Evaluated application #{resource}", [:compiler, :evaluate_resource, resource]) do
+          resource.evaluate
+        end
+      end
+    end
+  end
+
+  def evaluate_classes(titles, scope, lazy)
+    # Always lazy in an Environment compilation
+    super(titles, scope, true)
+  end
+
+  # Overrides the regular compiler to be able to return the list of resources after a prune
+  # has taken place in the graph representation. Before a prune, the list is the same as in the regular
+  # compiler
+  #
+  def resources
+    @pruned_resources || super
+  end
+
+  def is_capability?(value)
+    if value.is_a?(Array)
+      value.find { |ev| is_capability?(ev) }
+    elsif value.is_a?(Puppet::Resource)
+      rstype = value.resource_type
+      rstype.nil? ? false : rstype.is_capability?
+    else
+      false
+    end
+  end
+  private :is_capability?
+
+  def is_capability_consumer?(resource)
+    resource.eachparam { |param| return true if (param.name == :consume || param.name == :require) && is_capability?(param.value) }
+    false
+  end
+  private :is_capability_consumer?
+end

data/lib/puppet/parser/functions/assert_type.rb

@@ -2,29 +2,59 @@ Puppet::Parser::Functions::newfunction(
   :assert_type,
   :type => :rvalue,
   :arity => -3,
-  :doc => "Returns the given value if it is an instance of the given type, and raises an error otherwise.
-Optionally, if a block is given (accepting two parameters), it will be called instead of raising
-an error. This to enable giving the user richer feedback, or to supply a default value.
+  :doc => <<DOC
+Returns the given value if it is of the given
+[data type](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html), or
+otherwise either raises an error or executes an optional two-parameter
+[lambda](https://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html).
 
-Example: assert that `$b` is a non empty `String` and assign to `$a`:
+The function takes two mandatory arguments, in this order:
 
-  $a = assert_type(String[1], $b)
+1. The expected data type.
+2. A value to compare against the expected data type.
 
-Example using custom error message:
+**Example**: Using `assert_type`
 
-  $a = assert_type(String[1], $b) |$expected, $actual| {
-    fail('The name cannot be empty')
-  }
+~~~ puppet
+$raw_username = 'Amy Berry'
 
-Example, using a warning and a default:
+# Assert that $raw_username is a non-empty string and assign it to $valid_username.
+$valid_username = assert_type(String[1], $raw_username)
 
-  $a = assert_type(String[1], $b) |$expected, $actual| {
-    warning('Name is empty, using default')
-    'anonymous'
-  }
+# $valid_username contains "Amy Berry".
+# If $raw_username was an empty string or a different data type, the Puppet run would
+# fail with an "Expected type does not match actual" error.
+~~~
+
+You can use an optional lambda to provide enhanced feedback. The lambda takes two
+mandatory parameters, in this order:
+
+1. The expected data type as described in the function's first argument.
+2. The actual data type of the value.
+
+**Example**: Using `assert_type` with a warning and default value
+
+~~~ puppet
+$raw_username = 'Amy Berry'
+
+# Assert that $raw_username is a non-empty string and assign it to $valid_username.
+# If it isn't, output a warning describing the problem and use a default value.
+$valid_username = assert_type(String[1], $raw_username) |$expected, $actual| {
+  warning( "The username should be \'${expected}\', not \'${actual}\'. Using 'anonymous'." )
+  'anonymous'
+}
+
+# $valid_username contains "Amy Berry".
+# If $raw_username was an empty string, the Puppet run would set $valid_username to
+# "anonymous" and output a warning: "The username should be 'String[1, default]', not
+# 'String[0, 0]'. Using 'anonymous'."
+~~~
+
+For more information about data types, see the
+[documentation](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html).
 
-See the documentation for 'The Puppet Type System' for more information about types.
 - Since 4.0.0
-") do |args|
+DOC
+) do |args|
   function_fail(["assert_type() is only available when parser/evaluator future is in effect"])
 end

data/lib/puppet/parser/functions/defined.rb

@@ -1,47 +1,108 @@
-# Test whether a given class or definition is defined
-Puppet::Parser::Functions::newfunction(:defined, :type => :rvalue, :arity => -2,
-  :doc => "Determine whether
-  a given class or resource type is defined. This function can also determine whether a
-  specific resource has been defined, or whether a variable has been assigned a value
-  (including `undef` - as opposed to never having been assigned anything). Returns `true`
-  or `false`. Accepts class names, type names, resource references, and variable
-  reference strings of the form `'$name'`. When more than one argument is
-  supplied, `defined()` returns `true` if any are defined.
-
-  The `defined` function checks both native and defined types, including types
-  provided as plugins via modules. Types and classes are both checked using their names:
-
-      defined(\"file\")
-      defined(\"customtype\")
-      defined(\"foo\")
-      defined(\"foo::bar\")
-      defined(\'$name\')
-
-  Resource declarations are checked using resource references, e.g.
-  `defined( File['/tmp/myfile'] )`. Checking whether a given resource
-  has been declared is, unfortunately, dependent on the evaluation order of
-  the configuration, and the following code will not work:
-
-      if defined(File['/tmp/foo']) {
-          notify { \"This configuration includes the /tmp/foo file.\":}
-      }
-      file { \"/tmp/foo\":
-          ensure => present,
-      }
-
-  However, this order requirement refers to evaluation order only, and ordering of
-  resources in the configuration graph (e.g. with `before` or `require`) does not
-  affect the behavior of `defined`.
-
-  You may also search using types:
-
-      defined(Resource[\'file\',\'/some/file\'])
-      defined(File[\'/some/file\'])
-      defined(Class[\'foo\'])
-
- - Since 2.7.0
- - Since 3.6.0 variable reference and future parser types
- - Since 4.0.0") do |vals|
+Puppet::Parser::Functions::newfunction(
+  :defined,
+  :type => :rvalue,
+  :arity => -2,
+  :doc => <<DOC
+Determines whether a given class or resource type is defined and returns a Boolean
+value. You can also use `defined` to determine whether a specific resource is defined,
+or whether a variable has a value (including `undef`, as opposed to the variable never
+being declared or assigned).
+
+This function takes at least one string argument, which can be a class name, type name,
+resource reference, or variable reference of the form `'$name'`.
+
+The `defined` function checks both native and defined types, including types
+provided by modules. Types and classes are matched by their names. The function matches 
+resource declarations by using resource references.
+
+**Examples**: Different types of `defined` function matches
+
+~~~ puppet
+# Matching resource types
+defined("file")
+defined("customtype")
+
+# Matching defines and classes
+defined("foo")
+defined("foo::bar")
+
+# Matching variables
+defined('$name')
+
+# Matching declared resources
+defined(File['/tmp/file'])
+~~~
+
+Puppet depends on the configuration's evaluation order when checking whether a resource
+is declared.
+
+**Example**: Importance of evaluation order when using `defined`
+
+~~~ puppet
+# Assign values to $is_defined_before and $is_defined_after using identical `defined`
+# functions.
+
+$is_defined_before = defined(File['/tmp/file'])
+
+file { "/tmp/file":
+  ensure => present,
+}
+
+$is_defined_after = defined(File['/tmp/file'])
+
+# $is_defined_before returns false, but $is_defined_after returns true.
+~~~
+
+This order requirement only refers to evaluation order. The order of resources in the
+configuration graph (e.g. with `before` or `require`) does not affect the `defined`
+function's behavior.
+
+> **Warning:** Avoid relying on the result of the `defined` function in modules, as you
+> might not be able to guarantee the evaluation order well enough to produce consistent
+> results. This can cause other code that relies on the function's result to behave
+> inconsistently or fail.
+
+If you pass more than one argument to `defined`, the function returns `true` if _any_
+of the arguments are defined. You can also match resources by type, allowing you to
+match conditions of different levels of specificity, such as whether a specific resource
+is of a specific data type.
+
+**Example**: Matching multiple resources and resources by different types with `defined`
+
+~~~ puppet
+file { "/tmp/file1":
+  ensure => file,
+}
+
+$tmp_file = file { "/tmp/file2":
+  ensure => file,
+}
+
+# Each of these statements return `true` ...
+defined(File['/tmp/file1'])
+defined(File['/tmp/file1'],File['/tmp/file2'])
+defined(File['/tmp/file1'],File['/tmp/file2'],File['/tmp/file3'])
+# ... but this returns `false`.
+defined(File['/tmp/file3'])
+
+# Each of these statements returns `true` ...
+defined(Type[Resource['file','/tmp/file2']])
+defined(Resource['file','/tmp/file2'])
+defined(File['/tmp/file2'])
+defined('$tmp_file')
+# ... but each of these returns `false`.
+defined(Type[Resource['exec','/tmp/file2']])
+defined(Resource['exec','/tmp/file2'])
+defined(File['/tmp/file3'])
+defined('$tmp_file2')
+~~~
+
+- Since 2.7.0
+- Since 3.6.0 variable reference and future parser types
+- Since 3.8.1 type specific requests with future parser
+- Since 4.0.0
+DOC
+) do |vals|
   vals = [vals] unless vals.is_a?(Array)
   vals.any? do |val|
     case val

data/lib/puppet/parser/functions/each.rb

@@ -3,41 +3,99 @@ Puppet::Parser::Functions::newfunction(
   :type => :rvalue,
   :arity => -3,
   :doc => <<-DOC
-Applies a parameterized block to each element in a sequence of selected entries from the first
-argument and returns the first argument.
+Runs a [lambda](http://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html)
+repeatedly using each value in a data structure, then returns the values unchanged.
 
-This function takes two mandatory arguments: the first should be an Array or a Hash or something that is
-of enumerable type (integer, Integer range, or String), and the second
-a parameterized block as produced by the puppet syntax:
+This function takes two mandatory arguments, in this order:
 
-      $a.each |$x| { ... }
-      each($a) |$x| { ... }
+1. An array or hash the function will iterate over.
+2. A lambda, which the function calls for each element in the first argument. It can
+request one or two parameters.
 
-When the first argument is an Array (or of enumerable type other than Hash), the parameterized block
-should define one or two block parameters.
-For each application of the block, the next element from the array is selected, and it is passed to
-the block if the block has one parameter. If the block has two parameters, the first is the elements
-index, and the second the value. The index starts from 0.
+**Example**: Using the `each` function
 
-      $a.each |$index, $value| { ... }
-      each($a) |$index, $value| { ... }
+`$data.each |$parameter| { <PUPPET CODE BLOCK> }`
 
-When the first argument is a Hash, the parameterized block should define one or two parameters.
-When one parameter is defined, the iteration is performed with each entry as an array of `[key, value]`,
-and when two parameters are defined the iteration is performed with key and value.
+or
 
-      $a.each |$entry|       { ..."key ${$entry[0]}, value ${$entry[1]}" }
-      $a.each |$key, $value| { ..."key ${key}, value ${value}" }
+`each($data) |$parameter| { <PUPPET CODE BLOCK> }`
 
-Example using each:
+When the first argument (`$data` in the above example) is an array, Puppet passes each
+value in turn to the lambda, then returns the original values.
 
-      [1,2,3].each |$val| { ... }                       # 1, 2, 3
-      [5,6,7].each |$index, $val| { ... }               # (0, 5), (1, 6), (2, 7)
-      {a=>1, b=>2, c=>3}].each |$val| { ... }           # ['a', 1], ['b', 2], ['c', 3]
-      {a=>1, b=>2, c=>3}.each |$key, $val| { ... }      # ('a', 1), ('b', 2), ('c', 3)
-      Integer[ 10, 20 ].each |$index, $value| { ... }   # (0, 10), (1, 11) ...
-      "hello".each |$char| { ... }                      # 'h', 'e', 'l', 'l', 'o'
-      3.each |$number| { ... }                          # 0, 1, 2
+**Example**: Using the `each` function with an array and a one-parameter lambda
+
+~~~ puppet
+# For the array $data, run a lambda that creates a resource for each item.
+$data = ["routers", "servers", "workstations"]
+$data.each |$item| {
+ notify { $item:
+   message => $item
+ }
+}
+# Puppet creates one resource for each of the three items in $data. Each resource is
+# named after the item's value and uses the item's value in a parameter.
+~~~
+
+When the first argument is a hash, Puppet passes each key and value pair to the lambda
+as an array in the form `[key, value]` and returns the original hash.
+
+**Example**: Using the `each` function with a hash and a one-parameter lambda
+
+~~~ puppet
+# For the hash $data, run a lambda using each item as a key-value array that creates a
+# resource for each item.
+$data = {"rtr" => "Router", "svr" => "Server", "wks" => "Workstation"}
+$data.each |$items| {
+ notify { $items[0]:
+   message => $items[1]
+ }
+}
+# Puppet creates one resource for each of the three items in $data, each named after the
+# item's key and containing a parameter using the item's value.
+~~~
+
+When the first argument is an array and the lambda has two parameters, Puppet passes the
+array's indexes (enumerated from 0) in the first parameter and its values in the second
+parameter.
+
+**Example**: Using the `each` function with an array and a two-parameter lambda
+
+~~~ puppet
+# For the array $data, run a lambda using each item's index and value that creates a
+# resource for each item.
+$data = ["routers", "servers", "workstations"]
+$data.each |$index, $value| {
+ notify { $value:
+   message => $index
+ }
+}
+# Puppet creates one resource for each of the three items in $data, each named after the
+# item's value and containing a parameter using the item's index.
+~~~
+
+When the first argument is a hash, Puppet passes its keys to the first parameter and its
+values to the second parameter.
+
+**Example**: Using the `each` function with a hash and a two-parameter lambda
+
+~~~ puppet
+# For the hash $data, run a lambda using each item's key and value to create a resource
+# for each item.
+$data = {"rtr" => "Router", "svr" => "Server", "wks" => "Workstation"}
+$data.each |$key, $value| {
+ notify { $key:
+   message => $value
+ }
+}
+# Puppet creates one resource for each of the three items in $data, each named after the
+# item's key and containing a parameter using the item's value.
+~~~
+
+For an example that demonstrates how to create multiple `file` resources using `each`,
+see the Puppet
+[iteration](https://docs.puppetlabs.com/puppet/latest/reference/lang_iteration.html)
+documentation.
 
 - Since 4.0.0
 DOC

data/lib/puppet/parser/functions/epp.rb

@@ -1,43 +1,35 @@
 Puppet::Parser::Functions::newfunction(:epp, :type => :rvalue, :arity => -2, :doc =>
-"Evaluates an Embedded Puppet Template (EPP) file and returns the rendered text result as a String.
+"Evaluates an Embedded Puppet (EPP) template file and returns the rendered text
+result as a String.
+
+`epp('<MODULE NAME>/<TEMPLATE FILE>', <PARAMETER HASH>)`
 
 The first argument to this function should be a `<MODULE NAME>/<TEMPLATE FILE>`
-reference, which will load `<TEMPLATE FILE>` from a module's `templates`
-directory. (For example, the reference `apache/vhost.conf.epp` will load the
-file `<MODULES DIRECTORY>/apache/templates/vhost.conf.epp`.)
-
-The second argument is optional; if present, it should be a hash containing parameters for the
-template. (See below.)
-
-EPP supports the following tags:
-
-* `<%= puppet expression %>` - This tag renders the value of the expression it contains.
-* `<% puppet expression(s) %>` - This tag will execute the expression(s) it contains, but renders nothing.
-* `<%# comment %>` - The tag and its content renders nothing.
-* `<%%` or `%%>` - Renders a literal `<%` or `%>` respectively.
-* `<%-` - Same as `<%` but suppresses any leading whitespace.
-* `-%>` - Same as `%>` but suppresses any trailing whitespace on the same line (including line break).
-* `<%- |parameters| -%>` - When placed as the first tag declares the template's parameters.
-
-File based EPP supports the following visibilities of variables in scope:
-
-* Global scope (i.e. top + node scopes) - global scope is always visible
-* Global + all given arguments - if the EPP template does not declare parameters, and arguments are given
-* Global + declared parameters - if the EPP declares parameters, given argument names must match
-
-EPP supports parameters by placing an optional parameter list as the very first element in the EPP. As an example,
-`<%- |$x, $y, $z = 'unicorn'| -%>` when placed first in the EPP text declares that the parameters `x` and `y` must be
-given as template arguments when calling `inline_epp`, and that `z` if not given as a template argument
-defaults to `'unicorn'`. Template parameters are available as variables, e.g.arguments `$x`, `$y` and `$z` in the example.
-Note that `<%-` must be used or any leading whitespace will be interpreted as text
-
-Arguments are passed to the template by calling `epp` with a Hash as the last argument, where parameters
-are bound to values, e.g. `epp('...', {'x'=>10, 'y'=>20})`. Excess arguments may be given
-(i.e. undeclared parameters) only if the EPP templates does not declare any parameters at all.
-Template parameters shadow variables in outer scopes. File based epp does never have access to variables in the
-scope where the `epp` function is called from.
-
-- See function inline_epp for examples of EPP
+reference, which loads `<TEMPLATE FILE>` from `<MODULE NAME>`'s `templates`
+directory. In most cases, the last argument is optional; if used, it should be a
+[hash](/puppet/latest/reference/lang_data_hash.html) that contains parameters to
+pass to the template.
+
+- See the [template](/puppet/latest/reference/lang_template.html) documentation
+for general template usage information.
+- See the [EPP syntax](/puppet/latest/reference/lang_template_epp.html)
+documentation for examples of EPP.
+
+For example, to call the apache module's `templates/vhost/_docroot.epp`
+template and pass the `docroot` and `virtual_docroot` parameters, call the `epp`
+function like this:
+
+`epp('apache/templates/vhost/_docroot.epp', { 'docroot' => '/var/www/html',
+'virtual_docroot' => '/var/www/example' })`
+
+Puppet produces a syntax error if you pass more parameters than are declared in
+the template's parameter tag. When passing parameters to a template that
+contains a parameter tag, use the same names as the tag's declared parameters.
+
+Parameters are required only if they are declared in the called template's
+parameter tag without default values. Puppet produces an error if the `epp`
+function fails to pass any required parameter.
+
 - Since 4.0.0") do |args|
 
   function_fail(["epp() is only available when parser/evaluator future is in effect"])

data/lib/puppet/parser/functions/file.rb

@@ -1,3 +1,5 @@
+require 'puppet/file_system'
+
 Puppet::Parser::Functions::newfunction(
   :file, :arity => -2, :type => :rvalue,
   :doc => "Loads a file from a module and returns its contents as a string.
@@ -24,7 +26,7 @@ Puppet::Parser::Functions::newfunction(
     end
 
     if path
-      File.read(path)
+      Puppet::FileSystem.read_preserve_line_endings(path)
     else
       raise Puppet::ParseError, "Could not find any files from #{vals.join(", ")}"
     end