puppet 4.0.0 → 4.5.0

data/lib/puppet/parser/compiler/catalog_validator/relationship_validator.rb

@@ -0,0 +1,60 @@
+class Puppet::Parser::Compiler
+  # Validator that asserts that only 'require', 'consume', and 'export' is used when declaring relationships
+  # to capability resources.
+  class CatalogValidator::RelationshipValidator < CatalogValidator
+    def validate
+      catalog.resources.each do |resource|
+        next unless resource.is_a?(Puppet::Parser::Resource)
+        resource.eachparam do |param|
+          pclass = Puppet::Type.metaparamclass(param.name)
+          validate_relationship(param) if !pclass.nil? && pclass < Puppet::Type::RelationshipMetaparam
+        end
+      end
+      nil
+    end
+
+    private
+
+    # A hash lookup is 6x avg times faster than find among 3 values.
+    CAPABILITY_ACCEPTED_METAPARAMS = {:require => true, :consume => true, :export => true}.freeze
+
+    def validate_relationship(param)
+      # when relationship is to a capability
+      if has_capability?(param.value)
+        unless CAPABILITY_ACCEPTED_METAPARAMS[param.name]
+          raise CatalogValidationError.new(
+            "'#{param.name}' is not a valid relationship to a capability", 
+              param.file, param.line)
+        end
+      elsif Puppet[:strict] != :off
+        # all other relationships requires the referenced resource to exist when mode is strict
+        refs = param.value.is_a?(Array) ? param.value.flatten : [param.value]
+        refs.each do |r|
+          next if r.nil?
+          unless catalog.resource(r.to_s)
+            msg = "Could not find resource '#{r.to_s}' in parameter '#{param.name.to_s}'"
+            if Puppet[:strict] == :error
+              raise CatalogValidationError.new(msg, param.file, param.line)
+            else
+              Puppet.warn_once(:undefined_resources, r.to_s, msg, param.file, param.line)
+            end
+          end
+        end
+      end
+    end
+
+    def has_capability?(value)
+      case value
+      when Array
+        value.find { |v| has_capability?(v) }
+      when Puppet::Resource
+        rt = value.resource_type
+        !rt.nil? && rt.is_capability?
+      else
+        false
+      end
+    end
+  end
+end
+
+

data/lib/puppet/parser/compiler/catalog_validator/site_validator.rb

@@ -0,0 +1,20 @@
+class Puppet::Parser::Compiler
+  # Validator that asserts that only application components can appear inside a site.
+  class CatalogValidator::SiteValidator < CatalogValidator
+    def self.validation_stage?(stage)
+      PRE_FINISH.equal?(stage)
+    end
+
+    def validate
+      the_site_resource = catalog.resource('Site', 'site')
+      return unless the_site_resource
+
+      catalog.downstream_from_vertex(the_site_resource).keys.each do |r|
+        unless r.is_application_component? || r.resource_type.application?
+          raise CatalogValidationError.new("Only application components can appear inside a site - #{r} is not allowed", r.file, r.line)
+        end
+      end
+    end
+  end
+end
+

data/lib/puppet/parser/compiler/catalog_validator.rb

@@ -0,0 +1,33 @@
+# Abstract class for a catalog validator that can be registered with the compiler to run at
+# a certain stage.
+class Puppet::Parser::Compiler
+  class CatalogValidator
+    PRE_FINISH = :pre_finish
+    FINAL = :final
+
+    # Returns true if the validator should run at the given stage. The default
+    # implementation will only run at stage `FINAL`
+    #
+    # @param stage [Symbol] One of the stage constants defined in this class
+    # @return [Boolean] true if the validator should run at the given stage
+    #
+    def self.validation_stage?(stage)
+      FINAL.equal?(stage)
+    end
+
+    attr_reader :catalog
+
+    # @param catalog [Puppet::Resource::Catalog] The catalog to validate
+    def initialize(catalog)
+      @catalog = catalog
+    end
+
+    # Validate some aspect of the catalog and raise a `CatalogValidationError` on failure
+    def validate
+    end
+  end
+
+  class CatalogValidationError < Puppet::Error
+    include Puppet::ExternalFileError
+  end
+end

data/lib/puppet/parser/compiler.rb

@@ -6,6 +6,7 @@ require 'puppet/util/errors'
 
 require 'puppet/resource/type_collection_helper'
 require 'puppet/loaders'
+require 'puppet/pops'
 
 # Maintain a graph of scopes, along with a bunch of data
 # about the individual catalog we're compiling.
@@ -16,9 +17,9 @@ class Puppet::Parser::Compiler
   include Puppet::Util::Errors
   include Puppet::Util::MethodHelper
   include Puppet::Resource::TypeCollectionHelper
+  include Puppet::Pops::Evaluator::Runtime3Support
 
-  def self.compile(node)
-    $env_module_directories = nil
+  def self.compile(node, code_id = nil)
     node.environment.check_for_reparse
 
     errors = node.environment.validation_errors
@@ -26,12 +27,16 @@ class Puppet::Parser::Compiler
       errors.each { |e| Puppet.err(e) } if errors.size > 1
       errmsg = [
         "Compilation has been halted because: #{errors.first}",
-        "For more information, see http://docs.puppetlabs.com/puppet/latest/reference/environments.html",
+        "For more information, see https://docs.puppetlabs.com/puppet/latest/reference/environments.html",
       ]
       raise(Puppet::Error, errmsg.join(' '))
     end
 
-    new(node).compile {|resulting_catalog| resulting_catalog.to_resource }
+    new(node, :code_id => code_id).compile {|resulting_catalog| resulting_catalog.to_resource }
+  rescue Puppet::ParseErrorWithIssue => detail
+    detail.node = node.name
+    Puppet.log_exception(detail)
+    raise
   rescue => detail
     message = "#{detail} on node #{node.name}"
     Puppet.log_exception(detail, message)
@@ -61,6 +66,10 @@ class Puppet::Parser::Compiler
   #
   attr_accessor :boot_injector
 
+  # The id of code input to the compiler.
+  # @api private
+  attr_accessor :code_id
+
   # Add a collection to the global list.
   def_delegator :@collections,   :<<, :add_collection
   def_delegator :@relationships, :<<, :add_relationship
@@ -78,6 +87,22 @@ class Puppet::Parser::Compiler
   end
 
   def add_resource(scope, resource)
+    type = resource.resource_type
+    if type.is_a?(Puppet::Resource::Type) && type.application?
+      @applications << resource
+      assert_app_in_site(scope, resource)
+      return
+    end
+
+    if @current_app
+      # We are in the process of pulling application components out that
+      # apply to this node
+      Puppet.notice "Check #{resource}"
+      return unless @current_components.any? do |comp|
+        comp.type == resource.type && comp.title == resource.title
+      end
+    end
+
     @resources << resource
 
     # Note that this will fail if the resource is not unique.
@@ -95,7 +120,22 @@ class Puppet::Parser::Compiler
     # This adds a resource to the class it lexically appears in in the
     # manifest.
     unless resource.class?
-      return @catalog.add_edge(scope.resource, resource)
+      @catalog.add_edge(scope.resource, resource)
+    end
+  end
+
+  def assert_app_in_site(scope, resource)
+    if resource.type == 'App'
+      if scope.resource
+        # directly contained in a Site
+        return if scope.resource.type == 'Site'
+        # contained in something that may be contained in Site
+        upstream = @catalog.upstream_from_vertex(scope.resource)
+        if upstream
+          return if upstream.keys.map(&:type).include?('Site')
+        end
+      end
+      raise ArgumentError, "Application instances like '#{resource}' can only be contained within a Site"
     end
   end
 
@@ -107,6 +147,16 @@ class Puppet::Parser::Compiler
     @catalog.add_class(name) unless name == ""
   end
 
+  # Add a catalog validator that will run at some stage to this compiler
+  # @param catalog_validators [Class<CatalogValidator>] The catalog validator class to add
+  def add_catalog_validator(catalog_validators)
+    @catalog_validators << catalog_validators
+    nil
+  end
+
+  def add_catalog_validators
+    add_catalog_validator(CatalogValidator::RelationshipValidator)
+  end
 
   # Return a list of all of the defined classes.
   def_delegator :@catalog, :classes, :classlist
@@ -124,18 +174,37 @@ class Puppet::Parser::Compiler
 
       activate_binder
 
+      Puppet::Util::Profiler.profile("Compile: Evaluated capability mappings", [:compiler, :evaluate_capability_mappings]) { evaluate_capability_mappings }
+
       Puppet::Util::Profiler.profile("Compile: Evaluated main", [:compiler, :evaluate_main]) { evaluate_main }
 
+      Puppet::Util::Profiler.profile("Compile: Evaluated site", [:compiler, :evaluate_site]) { evaluate_site }
+
       Puppet::Util::Profiler.profile("Compile: Evaluated AST node", [:compiler, :evaluate_ast_node]) { evaluate_ast_node }
 
       Puppet::Util::Profiler.profile("Compile: Evaluated node classes", [:compiler, :evaluate_node_classes]) { evaluate_node_classes }
 
+      Puppet::Util::Profiler.profile("Compile: Evaluated application instances", [:compiler, :evaluate_applications]) { evaluate_applications }
+
+      # New capability mappings may have been defined when the site was evaluated
+      Puppet::Util::Profiler.profile("Compile: Evaluated site capability mappings", [:compiler, :evaluate_capability_mappings]) { evaluate_capability_mappings }
+
       Puppet::Util::Profiler.profile("Compile: Evaluated generators", [:compiler, :evaluate_generators]) { evaluate_generators }
 
+      Puppet::Util::Profiler.profile("Compile: Validate Catalog pre-finish", [:compiler, :validate_pre_finish]) do
+        validate_catalog(CatalogValidator::PRE_FINISH)
+      end
+
       Puppet::Util::Profiler.profile("Compile: Finished catalog", [:compiler, :finish_catalog]) { finish }
 
+      Puppet::Util::Profiler.profile("Compile: Prune", [:compiler, :prune_catalog]) { prune_catalog }
+
       fail_on_unevaluated
 
+      Puppet::Util::Profiler.profile("Compile: Validate Catalog final", [:compiler, :validate_final]) do
+        validate_catalog(CatalogValidator::FINAL)
+      end
+
       if block_given?
         yield @catalog
       else
@@ -144,6 +213,10 @@ class Puppet::Parser::Compiler
     end
   end
 
+  def validate_catalog(validation_stage)
+    @catalog_validators.select { |vclass| vclass.validation_stage?(validation_stage) }.each { |vclass| vclass.new(@catalog).validate }
+  end
+
   # Constructs the overrides for the context
   def context_overrides()
     {
@@ -184,6 +257,111 @@ class Puppet::Parser::Compiler
     evaluate_classes(classes_without_params, @node_scope || topscope)
   end
 
+  # Evaluates the site - the top container for an environment catalog
+  # The site contain behaves analogous to a node - for the environment catalog, node expressions are ignored
+  # as the result is cross node. The site expression serves as a container for everything that is across
+  # all nodes.
+  #
+  # @api private
+  #
+  def evaluate_site
+    # Has a site been defined? If not, do nothing but issue a warning.
+    #
+    site = known_resource_types.find_site()
+    unless site
+      on_empty_site()
+      return
+    end
+
+    # Create a resource to model this site and add it to catalog
+    resource = site.ensure_in_catalog(topscope)
+
+    # The site sets node scope to be able to shadow what is in top scope
+    @node_scope = topscope.class_scope(site)
+
+    # Evaluates the logic contain in the site expression
+    resource.evaluate
+  end
+
+  # @api private
+  def on_empty_site
+    # do nothing
+  end
+
+  # Prunes the catalog by dropping all resources are contained under the Site (if a site expression is used).
+  # As a consequence all edges to/from dropped resources are also dropped.
+  # Once the pruning is performed, this compiler returns the pruned list when calling the #resources method.
+  # The pruning does not alter the order of resources in the resources list.
+  #
+  # @api private
+  def prune_catalog
+    prune_node_catalog
+  end
+
+  def prune_node_catalog
+    # Everything under Site[site] should be pruned as that is for the environment catalog, not a node
+    #
+    the_site_resource = @catalog.resource('Site', 'site')
+
+    if the_site_resource
+      # Get downstream vertexes returns a hash where the keys are the resources and values nesting level
+      to_be_removed = @catalog.downstream_from_vertex(the_site_resource).keys
+
+      # Drop the Site[site] resource if it has no content
+      if to_be_removed.empty?
+        to_be_removed << the_site_resource
+      end
+    else
+      to_be_removed = []
+    end
+
+    # keep_from_site is populated with any App resources.
+    application_resources = @resources.select {|r| r.type == 'App' }
+    # keep all applications plus what is directly referenced from applications
+    keep_from_site = application_resources
+    keep_from_site += application_resources.map {|app| @catalog.direct_dependents_of(app) }.flatten
+
+    to_be_removed -= keep_from_site
+    @catalog.remove_resource(*to_be_removed)
+    # set the pruned result
+    @resources = @catalog.resources
+  end
+
+  # @api private
+  def evaluate_applications
+    @applications.each do |app|
+      components = []
+      mapping = app.parameters[:nodes] ? app.parameters[:nodes].value : {}
+      raise Puppet::Error, "Invalid node mapping in #{app.ref}: Mapping must be a hash" unless mapping.is_a?(Hash)
+      all_mapped = Set.new
+      mapping.each do |k,v|
+        raise Puppet::Error, "Invalid node mapping in #{app.ref}: Key #{k} is not a Node" unless k.is_a?(Puppet::Resource) && k.type == 'Node'
+        v = [v] unless v.is_a?(Array)
+        v.each do |res|
+          raise Puppet::Error, "Invalid node mapping in #{app.ref}: Value #{res} is not a resource" unless res.is_a?(Puppet::Resource)
+          raise Puppet::Error, "Application #{app.ref} maps component #{res} to multiple nodes" if all_mapped.add?(res.ref).nil?
+          components << res if k.title == node.name
+        end
+      end
+      begin
+        @current_app = app
+        @current_components = components
+        unless @current_components.empty?
+          Puppet.notice "EVAL APP #{app} #{components.inspect}"
+          # Add the app itself since components mapped to the current node
+          # will have a containment edge for it
+          # @todo lutter 2015-01-28: the node mapping winds up in the
+          # catalog, but probably shouldn't
+          @catalog.add_resource(@current_app)
+          @current_app.evaluate
+        end
+      ensure
+        @current_app = nil
+        @current_components = nil
+      end
+    end
+  end
+
   # Evaluates each specified class in turn. If there are any classes that 
   # can't be found, an error is raised. This method really just creates resource objects
   # that point back to the classes, and then the resources are themselves
@@ -199,6 +377,12 @@ class Puppet::Parser::Compiler
       classes = classes.keys
     end
 
+    unless @current_components.nil?
+      classes = classes.select do |title|
+        @current_components.any? { |comp| comp.class? && comp.title == title }
+      end
+    end
+
     hostclasses = classes.collect do |name|
       scope.find_hostclass(name) or raise Puppet::Error, "Could not find class #{name} for #{node.name}"
     end
@@ -228,9 +412,20 @@ class Puppet::Parser::Compiler
   def_delegator :@catalog, :resource, :findresource
 
   def initialize(node, options = {})
-    @node = node
+    @node = sanitize_node(node)
+    # Array of resources representing all application instances we've found
+    @applications = []
+    # We use @current_app and @current_components to signal to the
+    # evaluator that we are in the middle of evaluating an
+    # application. They are set in evaluate_applications to the application
+    # instance, resp. to an array of the components of that application
+    # that is mapped to the current node. They are only non-nil when we are
+    # in the middle of executing evaluate_applications
+    @current_app = nil
+    @current_components = nil
     set_options(options)
     initvars
+    add_catalog_validators
   end
 
   # Create a new scope, with either a specified parent scope or
@@ -311,6 +506,38 @@ class Puppet::Parser::Compiler
     [already_included, newly_included]
   end
 
+  def evaluate_capability_mappings
+    krt = known_resource_types
+    krt.capability_mappings.each_value do |capability_mapping|
+      args = capability_mapping.arguments
+      component_ref = args['component']
+      kind = args['kind']
+
+      # That component_ref is either a QNAME or a Class['literal'|QREF] is asserted during validation so no
+      # need to check that here
+      if component_ref.is_a?(Puppet::Pops::Model::QualifiedName)
+        component_name = component_ref.value
+        component_type = 'type'
+        component = krt.find_definition(component_name)
+      else
+        component_name = component_ref.keys[0].value
+        component_type = 'class'
+        component = krt.find_hostclass(component_name)
+      end
+      if component.nil?
+        raise Puppet::ParseError, "Capability mapping error: #{kind} clause references nonexistent #{component_type} #{component_name}"
+      end
+
+      blueprint = args['blueprint']
+      if kind == 'produces'
+        component.add_produces(blueprint)
+      else
+        component.add_consumes(blueprint)
+      end
+    end
+    krt.capability_mappings.clear # No longer needed
+  end
+
   # If ast nodes are enabled, then see if we can find and evaluate one.
   def evaluate_ast_node
     return unless ast_nodes?
@@ -322,7 +549,7 @@ class Puppet::Parser::Compiler
     end
 
     unless (astnode ||= known_resource_types.node("default"))
-      raise Puppet::ParseError, "Could not find default node or by name with '#{node.names.join(", ")}'"
+      raise Puppet::ParseError, "Could not find node statement with name 'default' or '#{node.names.join(", ")}'"
     end
 
     # Create a resource to model this node, and then add it to the list
@@ -359,11 +586,16 @@ class Puppet::Parser::Compiler
   def evaluate_definitions
     exceptwrap do
       Puppet::Util::Profiler.profile("Evaluated definitions", [:compiler, :evaluate_definitions]) do
-        !unevaluated_resources.each do |resource|
-          Puppet::Util::Profiler.profile("Evaluated resource #{resource}", [:compiler, :evaluate_resource, resource]) do
+        urs = unevaluated_resources.each do |resource|
+         begin
             resource.evaluate
+          rescue Puppet::Error => e
+            # PuppetError has the ability to wrap an exception, if so, use the wrapped exception's
+            # call stack instead
+            fail(Puppet::Pops::Issues::RUNTIME_ERROR, resource, {:detail => e.message}, e.original || e)
           end
-        end.empty?
+        end
+        !urs.empty?
       end
     end
   end
@@ -418,8 +650,7 @@ class Puppet::Parser::Compiler
     remaining = @resource_overrides.values.flatten.collect(&:ref)
 
     if !remaining.empty?
-      fail Puppet::ParseError,
-        "Could not find resource(s) #{remaining.join(', ')} for overriding"
+      raise Puppet::ParseError, "Could not find resource(s) #{remaining.join(', ')} for overriding"
     end
   end
 
@@ -513,7 +744,7 @@ class Puppet::Parser::Compiler
     @relationships = []
 
     # For maintaining the relationship between scopes and their resources.
-    @catalog = Puppet::Resource::Catalog.new(@node.name, @node.environment)
+    @catalog = Puppet::Resource::Catalog.new(@node.name, @node.environment, @code_id)
 
     # MOVED HERE - SCOPE IS NEEDED (MOVE-SCOPE)
     # Create the initial scope, it is needed early
@@ -545,17 +776,76 @@ class Puppet::Parser::Compiler
     else
       @catalog.add_class(*@node.classes)
     end
+
+    @catalog_validators = []
+  end
+
+  def sanitize_node(node)
+    # Resurrect "trusted information" that comes from node/fact terminus.
+    # The current way this is done in puppet db (currently the only one)
+    # is to store the node parameter 'trusted' as a hash of the trusted information.
+    #
+    # Thus here there are two main cases:
+    # 1. This terminus was used in a real agent call (only meaningful if someone curls the request as it would
+    #  fail since the result is a hash of two catalogs).
+    # 2  It is a command line call with a given node that use a terminus that:
+    # 2.1 does not include a 'trusted' fact - use local from node trusted information
+    # 2.2 has a 'trusted' fact - this in turn could be
+    # 2.2.1 puppet db having stored trusted node data as a fact (not a great design)
+    # 2.2.2 some other terminus having stored a fact called "trusted" (most likely that would have failed earlier, but could
+    #       be spoofed).
+    #
+    # For the reasons above, the resurection of trusted node data with authenticated => true is only performed
+    # if user is running as root, else it is resurrected as unauthenticated.
+    #
+    trusted_param = node.parameters['trusted']
+    if trusted_param
+      # Blows up if it is a parameter as it will be set as $trusted by the compiler as if it was a variable
+      node.parameters.delete('trusted')
+      unless trusted_param.is_a?(Hash) && %w{authenticated certname extensions}.all? {|key| trusted_param.has_key?(key) }
+        # trusted is some kind of garbage, do not resurrect
+        trusted_param = nil
+      end
+    else
+      # trusted may be boolean false if set as a fact by someone
+      trusted_param = nil
+    end
+
+    # The options for node.trusted_data in priority order are:
+    # 1) node came with trusted_data so use that
+    # 2) else if there is :trusted_information in the puppet context
+    # 3) else if the node provided a 'trusted' parameter (parsed out above)
+    # 4) last, fallback to local node trusted information
+    #
+    # Note that trusted_data should be a hash, but (2) and (4) are not
+    # hashes, so we to_h at the end
+    if !node.trusted_data
+      trusted = Puppet.lookup(:trusted_information) do
+        trusted_param || Puppet::Context::TrustedInformation.local(node)
+      end
+
+      # Ruby 1.9.3 can't apply to_h to a hash, so check first
+      node.trusted_data = trusted.is_a?(Hash) ? trusted : trusted.to_h
+    end
+
+    node
   end
 
   # Set the node's parameters into the top-scope as variables.
   def set_node_parameters
     node.parameters.each do |param, value|
-      @topscope[param.to_s] = value
+      # Ensure node does not leak Symbol instances in general
+      @topscope[param.to_s] = value.is_a?(Symbol) ? value.to_s : value
     end
     # These might be nil.
     catalog.client_version = node.parameters["clientversion"]
     catalog.server_version = node.parameters["serverversion"]
     @topscope.set_trusted(node.trusted_data)
+
+    if Puppet[:trusted_server_facts]
+      @topscope.set_server_facts(node.server_facts)
+    end
+
     facts_hash = node.facts.nil? ? {} : node.facts.values
     @topscope.set_facts(facts_hash)
   end
@@ -601,5 +891,5 @@ class Puppet::Parser::Compiler
     unless activate_binder()
       raise Puppet::DevError, "The Puppet Binder was not activated"
     end
-  end
+  end  # Creates a diagnostic producer
 end

data/lib/puppet/parser/e4_parser_adapter.rb

@@ -1,16 +1,16 @@
 require 'puppet/pops'
 
-module Puppet; module Parser; end; end;
+module Puppet
+module Parser
 # Adapts an egrammar/eparser to respond to the public API of the classic parser
 # and makes use of the new evaluator.
 #
-class Puppet::Parser::E4ParserAdapter
+class E4ParserAdapter
 
   def initialize
     @file = ''
     @string = ''
     @use = :unspecified
-    @@evaluating_parser ||= Puppet::Pops::Parser::EvaluatingParser.new()
   end
 
   def file=(file)
@@ -20,15 +20,15 @@ class Puppet::Parser::E4ParserAdapter
 
   def parse(string = nil)
     self.string= string if string
-
+    parser = Pops::Parser::EvaluatingParser.singleton
     parse_result =
     if @use == :string
       # Parse with a source_file to set in created AST objects (it was either given, or it may be unknown
       # if caller did not set a file and the present a string.
       #
-      @@evaluating_parser.parse_string(@string, @file || "unknown-source-location")
+      parser.parse_string(@string, @file || "unknown-source-location")
     else
-      @@evaluating_parser.parse_file(@file)
+      parser.parse_file(@file)
     end
 
     # the parse_result may be
@@ -38,19 +38,18 @@ class Puppet::Parser::E4ParserAdapter
     #
     model = parse_result.nil? ? nil : parse_result.current
     args = {}
-    Puppet::Pops::Model::AstTransformer.new(@file).merge_location(args, model)
+    Pops::Model::AstTransformer.new(@file).merge_location(args, model)
 
     ast_code =
-    if model.is_a? Puppet::Pops::Model::Program
-      Puppet::Parser::AST::PopsBridge::Program.new(model, args)
+    if model.is_a? Pops::Model::Program
+      AST::PopsBridge::Program.new(model, args)
     else
       args[:value] = model
-      Puppet::Parser::AST::PopsBridge::Expression.new(args)
+      AST::PopsBridge::Expression.new(args)
     end
 
     # Create the "main" class for the content - this content will get merged with all other "main" content
-    Puppet::Parser::AST::Hostclass.new('', :code => ast_code)
-
+    AST::Hostclass.new('', :code => ast_code)
   end
 
   def string=(string)
@@ -58,3 +57,5 @@ class Puppet::Parser::E4ParserAdapter
     @use = :string
   end
 end
+end
+end