puppet 4.2.3 → 4.3.0

data/lib/puppet/parser/compiler/catalog_validator.rb

@@ -0,0 +1,33 @@
+# Abstract class for a catalog validator that can be registered with the compiler to run at
+# a certain stage.
+class Puppet::Parser::Compiler
+  class CatalogValidator
+    PRE_FINISH = :pre_finish
+    FINAL = :final
+
+    # Returns true if the validator should run at the given stage. The default
+    # implementation will only run at stage `FINAL`
+    #
+    # @param stage [Symbol] One of the stage constants defined in this class
+    # @return [Boolean] true if the validator should run at the given stage
+    #
+    def self.validation_stage?(stage)
+      FINAL.equal?(stage)
+    end
+
+    attr_reader :catalog
+
+    # @param catalog [Puppet::Resource::Catalog] The catalog to validate
+    def initialize(catalog)
+      @catalog = catalog
+    end
+
+    # Validate some aspect of the catalog and raise a `CatalogValidationError` on failure
+    def validate
+    end
+  end
+
+  class CatalogValidationError < Puppet::Error
+    include Puppet::ExternalFileError
+  end
+end

data/lib/puppet/parser/compiler/catalog_validator/env_relationship_validator.rb

@@ -0,0 +1,64 @@
+class Puppet::Parser::Compiler
+  # Validator that asserts that all capability resources that are referenced by 'consume' or 'require' has
+  # been exported by some other resource in the environment
+ class CatalogValidator::EnvironmentRelationshipValidator < CatalogValidator
+
+    def validate
+      assumed_exports = {}
+      exported = {}
+      catalog.resources.each do |resource|
+        next unless resource.is_a?(Puppet::Parser::Resource)
+        resource.eachparam do |param|
+          pclass = Puppet::Type.metaparamclass(param.name)
+          validate_relationship(resource, param, assumed_exports, exported) if !pclass.nil? && pclass < Puppet::Type::RelationshipMetaparam
+        end
+      end
+      assumed_exports.each_pair do |key, (param, cap)|
+        raise CatalogValidationError.new("Capability '#{cap}' referenced by '#{param.name}' is never exported", param.file, param.line) unless exported.include?(key)
+      end
+      nil
+    end
+
+    private
+
+    def validate_relationship(resource, param, assumed_exports, exported)
+      case param.name
+      when :require, :consume
+        add_capability_ref(param, param.value, assumed_exports)
+      when :export
+        add_exported(resource, param, param.value, exported)
+      end
+    end
+
+    def add_capability_ref(param, value, assumed_exports)
+      case value
+      when Array
+        value.each { |v| add_capability_ref(param, v, assumed_exports) }
+      when Puppet::Resource
+        rt = value.resource_type
+        unless rt.nil? || !rt.is_capability?
+          title_key = catalog.title_key_for_ref(value.ref)
+          assumed_exports[title_key] = [param, value]
+        end
+        nil
+      end
+    end
+
+    def add_exported(resource, param, value, hash)
+      case value
+      when Array
+        value.each { |v| add_exported(resource, param, v, hash) }
+      when Puppet::Resource
+        rt = value.resource_type
+        unless rt.nil? || !rt.is_capability?
+          title_key = catalog.title_key_for_ref(value.ref)
+          if hash.include?(title_key)
+            raise CatalogValidationError.new("'#{value}' is exported by both '#{hash[title_key]}' and '#{resource}'", param.file, param.line)
+          else
+            hash[title_key] = resource
+          end
+        end
+      end
+    end
+  end
+end

data/lib/puppet/parser/compiler/catalog_validator/relationship_validator.rb

@@ -0,0 +1,38 @@
+class Puppet::Parser::Compiler
+  # Validator that asserts that only 'require', 'consume', and 'export' is used when declaring relationships
+  # to capability resources.
+  class CatalogValidator::RelationshipValidator < CatalogValidator
+    def validate
+      catalog.resources.each do |resource|
+        next unless resource.is_a?(Puppet::Parser::Resource)
+        resource.eachparam do |param|
+          pclass = Puppet::Type.metaparamclass(param.name)
+          validate_relationship(param) if !pclass.nil? && pclass < Puppet::Type::RelationshipMetaparam
+        end
+      end
+      nil
+    end
+
+    private
+
+    def validate_relationship(param)
+      unless [:require, :consume, :export].find {|pname| pname == param.name }
+        raise CatalogValidationError.new("'#{param.name}' is not a valid relationship to a capability", param.file, param.line) if has_capability?(param.value)
+      end
+    end
+
+    def has_capability?(value)
+      case value
+      when Array
+        value.find { |v| has_capability?(v) }
+      when Puppet::Resource
+        rt = value.resource_type
+        !rt.nil? && rt.is_capability?
+      else
+        false
+      end
+    end
+  end
+end
+
+

data/lib/puppet/parser/compiler/catalog_validator/site_validator.rb

@@ -0,0 +1,20 @@
+class Puppet::Parser::Compiler
+  # Validator that asserts that only application components can appear inside a site.
+  class CatalogValidator::SiteValidator < CatalogValidator
+    def self.validation_stage?(stage)
+      PRE_FINISH.equal?(stage)
+    end
+
+    def validate
+      the_site_resource = catalog.resource('Site', 'site')
+      return unless the_site_resource
+
+      catalog.downstream_from_vertex(the_site_resource).keys.each do |r|
+        unless r.is_application_component? || r.resource_type.application?
+          raise CatalogValidationError.new("Only application components can appear inside a site - #{r} is not allowed", r.file, r.line)
+        end
+      end
+    end
+  end
+end
+

data/lib/puppet/parser/environment_compiler.rb

@@ -0,0 +1,165 @@
+require 'puppet/parser/compiler'
+
+class Puppet::Parser::EnvironmentCompiler < Puppet::Parser::Compiler
+  def self.compile(env)
+    begin
+      $env_module_directories = nil
+      env.check_for_reparse
+
+      node = Puppet::Node.new(env)
+      node.environment = env
+      new(node).compile
+    rescue => detail
+      message = "#{detail} in environment #{env.name}"
+      Puppet.log_exception(detail, message)
+      raise Puppet::Error, message, detail.backtrace
+    end
+  end
+
+  def add_catalog_validators
+    super
+    add_catalog_validator(CatalogValidator::SiteValidator)
+    add_catalog_validator(CatalogValidator::EnvironmentRelationshipValidator)
+  end
+
+  def compile
+    Puppet.override(@context_overrides, "For compiling environment catalog #{environment.name}") do
+      @catalog.environment_instance = environment
+
+      Puppet::Util::Profiler.profile("Env Compile: Created settings scope", [:compiler, :create_settings_scope]) { create_settings_scope }
+
+      activate_binder
+
+      Puppet::Util::Profiler.profile("Env Compile: Evaluated main", [:compiler, :evaluate_main]) { evaluate_main }
+
+      Puppet::Util::Profiler.profile("Env Compile: Evaluated site", [:compiler, :evaluate_site]) { evaluate_site }
+
+      Puppet::Util::Profiler.profile("Env Compile: Evaluated application instances", [:compiler, :evaluate_applications]) { evaluate_applications }
+
+      Puppet::Util::Profiler.profile("Env Compile: Prune", [:compiler, :prune_catalog]) { prune_catalog }
+
+      Puppet::Util::Profiler.profile("Env Compile: Validate Catalog pre-finish", [:compiler, :validate_pre_finish]) do
+        validate_catalog(CatalogValidator::PRE_FINISH)
+      end
+
+      Puppet::Util::Profiler.profile("Env Compile: Finished catalog", [:compiler, :finish_catalog]) { finish }
+
+      fail_on_unevaluated
+
+      Puppet::Util::Profiler.profile("Env Compile: Validate Catalog final", [:compiler, :validate_final]) do
+        validate_catalog(CatalogValidator::FINAL)
+      end
+
+      if block_given?
+        yield @catalog
+      else
+        @catalog
+      end
+    end
+  end
+
+  # @api private
+  def prune_catalog
+    prune_env_catalog
+  end
+
+  # Prunes the catalog by dropping all resources that are not contained under the Site (if a site expression is used).
+  # As a consequence all edges to/from dropped resources are also dropped.
+  # Once the pruning is performed, this compiler returns the pruned list when calling the #resources method.
+  # The pruning does not alter the order of resources in the resources list.
+  #
+  def prune_env_catalog
+    # Everything under Class[main], that is not under (inclusive of) Site[site] should be pruned as those resources
+    # are intended for nodes in a node catalog.
+    #
+    the_main_class_resource = @catalog.resource('Class', '')
+    the_site_resource = @catalog.resource('Site', 'site')
+
+    # Get downstream vertexes returns a hash where the keys are the resources and values nesting level
+    rooted_in_main = @catalog.downstream_from_vertex(the_main_class_resource).keys
+
+    to_be_removed =
+    if the_site_resource
+      keep_from_site = @catalog.downstream_from_vertex(the_site_resource).keys
+      keep_from_site << the_site_resource
+      rooted_in_main - keep_from_site
+    else
+      rooted_in_main
+    end
+
+    @catalog.remove_resource(*to_be_removed)
+    # The compiler keeps a list of added resources, this shadows that list with the now pruned result
+    @pruned_resources = @catalog.resources
+  end
+
+  def add_resource(scope, resource)
+    @resources << resource
+    @catalog.add_resource(resource)
+
+    if !resource.class? && resource[:stage]
+      raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage"
+    end
+
+    # Stages should not be inside of classes.  They are always a
+    # top-level container, regardless of where they appear in the
+    # manifest.
+    return if resource.stage?
+
+    # This adds a resource to the class it lexically appears in in the
+    # manifest.
+    unless resource.class?
+      @catalog.add_edge(scope.resource, resource)
+    end
+    resource.mark_unevaluated_consumer if is_capability_consumer?(resource)
+    assert_app_in_site(scope, resource)
+  end
+
+  def evaluate_ast_node()
+    # Do nothing, the environment catalog is not built for a particular node.
+  end
+
+  def on_empty_site
+    Puppet.warning("Environment Compiler: Could not find a site definition to evaluate")
+  end
+
+  def evaluate_applications
+    exceptwrap do
+      resources.select { |resource| type = resource.resource_type; type.is_a?(Puppet::Resource::Type) && type.application? }.each do |resource|
+        Puppet::Util::Profiler.profile("Evaluated application #{resource}", [:compiler, :evaluate_resource, resource]) do
+          resource.evaluate
+        end
+      end
+    end
+  end
+
+  def evaluate_classes(titles, scope, lazy)
+    # Always lazy in an Environment compilation
+    super(titles, scope, true)
+  end
+
+  # Overrides the regular compiler to be able to return the list of resources after a prune
+  # has taken place in the graph representation. Before a prune, the list is the same as in the regular
+  # compiler
+  #
+  def resources
+    @pruned_resources || super
+  end
+
+  def is_capability?(value)
+    if value.is_a?(Array)
+      value.find { |ev| is_capability?(ev) }
+    elsif value.is_a?(Puppet::Resource)
+      rstype = value.resource_type
+      rstype.nil? ? false : rstype.is_capability?
+    else
+      false
+    end
+  end
+  private :is_capability?
+
+  def is_capability_consumer?(resource)
+    resource.eachparam { |param| return true if (param.name == :consume || param.name == :require) && is_capability?(param.value) }
+    false
+  end
+  private :is_capability_consumer?
+end

data/lib/puppet/parser/functions/assert_type.rb

@@ -2,29 +2,59 @@ Puppet::Parser::Functions::newfunction(
   :assert_type,
   :type => :rvalue,
   :arity => -3,
-  :doc => "Returns the given value if it is an instance of the given type, and raises an error otherwise.
-Optionally, if a block is given (accepting two parameters), it will be called instead of raising
-an error. This to enable giving the user richer feedback, or to supply a default value.
+  :doc => <<DOC
+Returns the given value if it is of the given
+[data type](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html), or
+otherwise either raises an error or executes an optional two-parameter
+[lambda](https://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html).
 
-Example: assert that `$b` is a non empty `String` and assign to `$a`:
+The function takes two mandatory arguments, in this order:
 
-  $a = assert_type(String[1], $b)
+1. The expected data type.
+2. A value to compare against the expected data type.
 
-Example using custom error message:
+**Example**: Using `assert_type`
 
-  $a = assert_type(String[1], $b) |$expected, $actual| {
-    fail('The name cannot be empty')
-  }
+~~~ puppet
+$raw_username = 'Amy Berry'
 
-Example, using a warning and a default:
+# Assert that $raw_username is a non-empty string and assign it to $valid_username.
+$valid_username = assert_type(String[1], $raw_username)
 
-  $a = assert_type(String[1], $b) |$expected, $actual| {
-    warning('Name is empty, using default')
-    'anonymous'
-  }
+# $valid_username contains "Amy Berry".
+# If $raw_username was an empty string or a different data type, the Puppet run would
+# fail with an "Expected type does not match actual" error.
+~~~
+
+You can use an optional lambda to provide enhanced feedback. The lambda takes two
+mandatory parameters, in this order:
+
+1. The expected data type as described in the function's first argument.
+2. The actual data type of the value.
+
+**Example**: Using `assert_type` with a warning and default value
+
+~~~ puppet
+$raw_username = 'Amy Berry'
+
+# Assert that $raw_username is a non-empty string and assign it to $valid_username.
+# If it isn't, output a warning describing the problem and use a default value.
+$valid_username = assert_type(String[1], $raw_username) |$expected, $actual| {
+  warning( "The username should be \'${expected}\', not \'${actual}\'. Using 'anonymous'." )
+  'anonymous'
+}
+
+# $valid_username contains "Amy Berry".
+# If $raw_username was an empty string, the Puppet run would set $valid_username to
+# "anonymous" and output a warning: "The username should be 'String[1, default]', not
+# 'String[0, 0]'. Using 'anonymous'."
+~~~
+
+For more information about data types, see the
+[documentation](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html).
 
-See the documentation for 'The Puppet Type System' for more information about types.
 - Since 4.0.0
-") do |args|
+DOC
+) do |args|
   function_fail(["assert_type() is only available when parser/evaluator future is in effect"])
 end

data/lib/puppet/parser/functions/defined.rb

@@ -1,71 +1,108 @@
-# Test whether a given class or definition is defined
-Puppet::Parser::Functions::newfunction(:defined, :type => :rvalue, :arity => -2,
-  :doc => "Determine whether
-  a given class or resource type is defined. This function can also determine whether a
-  specific resource has been defined, or whether a variable has been assigned a value
-  (including `undef` - as opposed to never having been assigned anything). Returns `true`
-  or `false`. Accepts class names, type names, resource references, and variable
-  reference strings of the form `'$name'`. When more than one argument is
-  supplied, `defined()` returns `true` if any are defined.
-
-  The `defined` function checks both native and defined types, including types
-  provided as plugins via modules. Types and classes are both checked using their names:
-
-      defined(\"file\")
-      defined(\"customtype\")
-      defined(\"foo\")
-      defined(\"foo::bar\")
-      defined(\'$name\')
-
-  Resource declarations are checked using resource references, e.g.
-  `defined( File['/tmp/myfile'] )`. Checking whether a given resource
-  has been declared is, unfortunately, dependent on the evaluation order of
-  the configuration, and the following code will not work:
-
-      if defined(File['/tmp/foo']) {
-          notify { \"This configuration includes the /tmp/foo file.\":}
-      }
-      file { \"/tmp/foo\":
-          ensure => present,
-      }
-
-  However, this order requirement refers to evaluation order only, and ordering of
-  resources in the configuration graph (e.g. with `before` or `require`) does not
-  affect the behavior of `defined`.
-
-  You may also search using types:
-
-      defined(Resource[\'file\',\'/some/file\'])
-      defined(File[\'/some/file\'])
-      defined(Class[\'foo\'])
-
-  The `defined` function does not answer if 4.x data types (e.g. `Integer`) are defined. If
-  given the string 'integer' the result is false, and if given a non CatalogEntry type,
-  an error is raised.
-
-  The rules for asking for undef, empty strings, and the main class are different from 3.x
-  (non future parser) and 4.x (with future parser or in Puppet 4.0.0 and later):
-
-      defined('')     # 3.x => true, 4.x => false
-      defined(undef)  # 3.x => true, 4.x => error
-      defined('main') # 3.x => false, 4.x => true
-
-  With the future parser, it is also possible to ask specifically if a name is
-  a resource type (built in or defined), or a class, by giving its type:
-
-      defined(Type[Class['foo']])
-      defined(Type[Resource['foo']])
-
-  Which is different from asking:
-
-      defined('foo')
-
-  Since the later returns true if 'foo' is either a class, a built-in resource type, or a user defined
-  resource type, and a specific request like `Type[Class['foo']]` only returns true if `'foo'` is a class.
-
- - Since 2.7.0
- - Since 3.6.0 variable reference and future parser types
- - Since 3.8.1 type specific requests with future parser") do |vals|
+Puppet::Parser::Functions::newfunction(
+  :defined,
+  :type => :rvalue,
+  :arity => -2,
+  :doc => <<DOC
+Determines whether a given class or resource type is defined and returns a Boolean
+value. You can also use `defined` to determine whether a specific resource is defined,
+or whether a variable has a value (including `undef`, as opposed to the variable never
+being declared or assigned).
+
+This function takes at least one string argument, which can be a class name, type name,
+resource reference, or variable reference of the form `'$name'`.
+
+The `defined` function checks both native and defined types, including types
+provided by modules. Types and classes are matched by their names. The function matches 
+resource declarations by using resource references.
+
+**Examples**: Different types of `defined` function matches
+
+~~~ puppet
+# Matching resource types
+defined("file")
+defined("customtype")
+
+# Matching defines and classes
+defined("foo")
+defined("foo::bar")
+
+# Matching variables
+defined('$name')
+
+# Matching declared resources
+defined(File['/tmp/file'])
+~~~
+
+Puppet depends on the configuration's evaluation order when checking whether a resource
+is declared.
+
+**Example**: Importance of evaluation order when using `defined`
+
+~~~ puppet
+# Assign values to $is_defined_before and $is_defined_after using identical `defined`
+# functions.
+
+$is_defined_before = defined(File['/tmp/file'])
+
+file { "/tmp/file":
+  ensure => present,
+}
+
+$is_defined_after = defined(File['/tmp/file'])
+
+# $is_defined_before returns false, but $is_defined_after returns true.
+~~~
+
+This order requirement only refers to evaluation order. The order of resources in the
+configuration graph (e.g. with `before` or `require`) does not affect the `defined`
+function's behavior.
+
+> **Warning:** Avoid relying on the result of the `defined` function in modules, as you
+> might not be able to guarantee the evaluation order well enough to produce consistent
+> results. This can cause other code that relies on the function's result to behave
+> inconsistently or fail.
+
+If you pass more than one argument to `defined`, the function returns `true` if _any_
+of the arguments are defined. You can also match resources by type, allowing you to
+match conditions of different levels of specificity, such as whether a specific resource
+is of a specific data type.
+
+**Example**: Matching multiple resources and resources by different types with `defined`
+
+~~~ puppet
+file { "/tmp/file1":
+  ensure => file,
+}
+
+$tmp_file = file { "/tmp/file2":
+  ensure => file,
+}
+
+# Each of these statements return `true` ...
+defined(File['/tmp/file1'])
+defined(File['/tmp/file1'],File['/tmp/file2'])
+defined(File['/tmp/file1'],File['/tmp/file2'],File['/tmp/file3'])
+# ... but this returns `false`.
+defined(File['/tmp/file3'])
+
+# Each of these statements returns `true` ...
+defined(Type[Resource['file','/tmp/file2']])
+defined(Resource['file','/tmp/file2'])
+defined(File['/tmp/file2'])
+defined('$tmp_file')
+# ... but each of these returns `false`.
+defined(Type[Resource['exec','/tmp/file2']])
+defined(Resource['exec','/tmp/file2'])
+defined(File['/tmp/file3'])
+defined('$tmp_file2')
+~~~
+
+- Since 2.7.0
+- Since 3.6.0 variable reference and future parser types
+- Since 3.8.1 type specific requests with future parser
+- Since 4.0.0
+DOC
+) do |vals|
   vals = [vals] unless vals.is_a?(Array)
   vals.any? do |val|
     case val