puppet 4.0.0 → 4.9.1

data/lib/puppet/parser/scope.rb

@@ -6,7 +6,6 @@ require 'forwardable'
 require 'puppet/parser'
 require 'puppet/parser/templatewrapper'
 
-require 'puppet/resource/type_collection_helper'
 require 'puppet/util/methodhelper'
 
 # This class is part of the internal parser/evaluator/compiler functionality of Puppet.
@@ -18,13 +17,13 @@ class Puppet::Parser::Scope
   extend Forwardable
   include Puppet::Util::MethodHelper
 
-  include Puppet::Resource::TypeCollectionHelper
   require 'puppet/parser/resource'
 
   AST = Puppet::Parser::AST
 
   # Variables that always exist with nil value even if not set
   BUILT_IN_VARS = ['module_name'.freeze, 'caller_module_name'.freeze].freeze
+  EMPTY_HASH = {}.freeze
 
   Puppet::Util.logmethods(self)
 
@@ -85,11 +84,8 @@ class Puppet::Parser::Scope
     end
 
     def [](name)
-      if @symbols.include?(name)
-        @symbols[name]
-      else
-        super
-      end
+      val = @symbols[name]
+      val.nil? && !@symbols.include?(val) ? super : val
     end
 
     def is_local_scope?
@@ -174,13 +170,113 @@ class Puppet::Parser::Scope
 
   end
 
+  # @api private
+  class ParameterScope < Ephemeral
+    class Access
+      attr_accessor :value
+
+      def assigned?
+        instance_variable_defined?(:@value)
+      end
+    end
+
+    # A parameter default must be evaluated using a special scope. The scope that is given to this method must
+    # have a `ParameterScope` as its last ephemeral scope. This method will then push a `MatchScope` while the
+    # given `expression` is evaluated. The method will catch any throw of `:unevaluated_parameter` and produce
+    # an error saying that the evaluated parameter X tries to access the unevaluated parameter Y.
+    #
+    # @param name [String] the name of the currently evaluated parameter
+    # @param expression [Puppet::Parser::AST] the expression to evaluate
+    # @param scope [Puppet::Parser::Scope] a scope where a `ParameterScope` has been pushed
+    # @return [Object] the result of the evaluation
+    #
+    # @api private
+    def evaluate3x(name, expression, scope)
+      scope.with_guarded_scope do
+        bad = catch(:unevaluated_parameter) do
+          scope.new_match_scope(nil)
+          return as_read_only { expression.safeevaluate(scope) }
+        end
+        parameter_reference_failure(name, bad)
+      end
+    end
+
+    def evaluate(name, expression, scope, evaluator)
+      scope.with_guarded_scope do
+        bad = catch(:unevaluated_parameter) do
+          scope.new_match_scope(nil)
+          return as_read_only { evaluator.evaluate(expression, scope) }
+        end
+        parameter_reference_failure(name, bad)
+      end
+    end
+
+    def parameter_reference_failure(from, to)
+      # Parameters are evaluated in the order they have in the @params hash.
+      keys = @params.keys
+      raise Puppet::Error, "#{@callee_name}: expects a value for parameter $#{to}" if keys.index(to) < keys.index(from)
+      raise Puppet::Error, "#{@callee_name}: default expression for $#{from} tries to illegally access not yet evaluated $#{to}"
+    end
+    private :parameter_reference_failure
+
+    def initialize(parent, callee_name, param_names)
+      super(parent)
+      @callee_name = callee_name
+      @params = {}
+      param_names.each { |name| @params[name] = Access.new }
+    end
+
+    def [](name)
+      access = @params[name]
+      return super if access.nil?
+      throw(:unevaluated_parameter, name) unless access.assigned?
+      access.value
+    end
+
+    def []=(name, value)
+      raise Puppet::Error, "Attempt to assign variable #{name} when evaluating parameters" if @read_only
+      @params[name] ||= Access.new
+      @params[name].value = value
+    end
+
+    def bound?(name)
+      @params.include?(name)
+    end
+
+    def include?(name)
+      @params.include?(name) || super
+    end
+
+    def is_local_scope?
+      true
+    end
+
+    def as_read_only
+      read_only = @read_only
+      @read_only = true
+      begin
+        yield
+      ensure
+        @read_only = read_only
+      end
+    end
+
+    def to_hash
+      Hash[@params.select {|_, access| access.assigned? }.map { |name, access| [name, access.value] }]
+    end
+  end
+
+
   # Returns true if the variable of the given name has a non nil value.
   # TODO: This has vague semantics - does the variable exist or not?
   #       use ['name'] to get nil or value, and if nil check with exist?('name')
   #       this include? is only useful because of checking against the boolean value false.
   #
   def include?(name)
-    ! self[name].nil?
+    catch(:undefined_variable) {
+      return ! self[name].nil?
+    }
+    false
   end
 
   # Returns true if the variable of the given name is set to any value (including nil)
@@ -249,11 +345,11 @@ class Puppet::Parser::Scope
   end
 
   def find_hostclass(name)
-    known_resource_types.find_hostclass(name)
+    environment.known_resource_types.find_hostclass(name)
   end
 
   def find_definition(name)
-    known_resource_types.find_definition(name)
+    environment.known_resource_types.find_definition(name)
   end
 
   def find_global_scope()
@@ -367,7 +463,8 @@ class Puppet::Parser::Scope
   # Look up a defined type.
   def lookuptype(name)
     # This happens a lot, avoid making a call to make a call
-    known_resource_types.find_definition(name) || known_resource_types.find_hostclass(name)
+    krt = environment.known_resource_types
+    krt.find_definition(name) || krt.find_hostclass(name)
   end
 
   def undef_as(x,v)
@@ -387,41 +484,54 @@ class Puppet::Parser::Scope
   # @return Object the value of the variable, or nil if it's not found
   #
   # @api public
-  def lookupvar(name, options = {})
+  def lookupvar(name, options = EMPTY_HASH)
     unless name.is_a? String
       raise Puppet::ParseError, "Scope variable name #{name.inspect} is a #{name.class}, not a string"
     end
 
-    table = @ephemeral.last
-
     if name =~ /^(.*)::(.+)$/
-      class_name = $1
-      variable_name = $2
-      lookup_qualified_variable(class_name, variable_name, options)
+      return lookup_qualified_variable($1, $2, options)
+    end
+
+    table = @ephemeral.last
+    val = table[name]
+    return val unless val.nil? && !table.include?(name)
 
-    # TODO: optimize with an assoc instead, this searches through scopes twice for a hit
-    elsif table.include?(name)
-      table[name]
+    next_scope = inherited_scope || enclosing_scope
+    if next_scope
+      next_scope.lookupvar(name, options)
     else
-      next_scope = inherited_scope || enclosing_scope
-      if next_scope
-        next_scope.lookupvar(name, options)
-      else
-        variable_not_found(name)
-      end
+      variable_not_found(name)
     end
   end
 
+  UNDEFINED_VARIABLES_KIND = 'undefined_variables'.freeze
+  DEPRECATION_KIND = 'deprecation'.freeze
+
+  # The exception raised when a throw is uncaught is different in different versions
+  # of ruby. In >=2.2.0 it is UncaughtThrowError (which did not exist prior to this)
+  #
+  UNCAUGHT_THROW_EXCEPTION = defined?(UncaughtThrowError) ? UncaughtThrowError : ArgumentError
+
   def variable_not_found(name, reason=nil)
-    # Built in variables always exist
-    if BUILT_IN_VARS.include?(name)
+    # Built in variables and numeric variables always exist
+    if BUILT_IN_VARS.include?(name) || name =~ Puppet::Pops::Patterns::NUMERIC_VAR_NAME
       return nil
     end
-    if Puppet[:strict_variables]
-      throw :undefined_variable
-    else
-      nil
+    begin
+      throw(:undefined_variable, reason)
+    rescue  UNCAUGHT_THROW_EXCEPTION
+      case Puppet[:strict]
+      when :off
+        # do nothing
+      when :warning
+        Puppet.warn_once(UNDEFINED_VARIABLES_KIND, "Variable: #{name}",
+        "Undefined variable '#{name}'; #{reason}" )
+      when :error
+        raise ArgumentError, "Undefined variable '#{name}'; #{reason}"
+      end
     end
+    nil
   end
 
   # Retrieves the variable value assigned to the name given as an argument. The name must be a String,
@@ -484,24 +594,37 @@ class Puppet::Parser::Scope
   def lookup_qualified_variable(class_name, variable_name, position)
     begin
       if lookup_as_local_name?(class_name, variable_name)
-        self[variable_name]
+        if is_topscope?
+          # This is the case where $::x is looked up from within the topscope itself, or from a local scope
+          # parented at the top scope. In this case, the lookup must ignore local and ephemeral scopes.
+          #
+          handle_not_found(class_name, variable_name, position) unless @symtable.include?(variable_name)
+          @symtable[variable_name]
+        else
+          self[variable_name]
+        end
       else
         qualified_scope(class_name).lookupvar(variable_name, position)
       end
     rescue RuntimeError => e
-      unless Puppet[:strict_variables]
-        # Do not issue warning if strict variables are on, as an error will be raised by variable_not_found
-        location = if position[:lineproc]
-                     " at #{position[:lineproc].call}"
-                   elsif position[:file] && position[:line]
-                     " at #{position[:file]}:#{position[:line]}"
-                   else
-                     ""
-                   end
-        warning "Could not look up qualified variable '#{class_name}::#{variable_name}'; #{e.message}#{location}"
-      end
-      variable_not_found("#{class_name}::#{variable_name}", e.message)
+      handle_not_found(class_name, variable_name, position, e.message)
+    end
+  end
+
+  def handle_not_found(class_name, variable_name, position, reason = nil)
+    unless Puppet[:strict_variables]
+      # Do not issue warning if strict variables are on, as an error will be raised by variable_not_found
+      location = if position[:lineproc]
+                   " at #{position[:lineproc].call}"
+                 elsif position[:file] && position[:line]
+                   " at #{position[:file]}:#{position[:line]}"
+                 else
+                   ""
+                 end
+      variable_not_found("#{class_name}::#{variable_name}", "#{reason}#{location}")
+      return nil
     end
+    variable_not_found("#{class_name}::#{variable_name}", reason)
   end
 
   # Handles the special case of looking up fully qualified variable in not yet evaluated top scope
@@ -581,7 +704,7 @@ class Puppet::Parser::Scope
 
     params.each { |param|
       if table.include?(param.name)
-        raise Puppet::ParseError.new("Default already defined for #{type} { #{param.name} }; cannot redefine", param.line, param.file)
+        raise Puppet::ParseError.new("Default already defined for #{type} { #{param.name} }; cannot redefine", param.file, param.line)
       end
       table[param.name] = param
     }
@@ -607,12 +730,17 @@ class Puppet::Parser::Scope
       raise Puppet::ParseError, "Attempt to assign to a reserved variable name: '#{name}'"
     end
 
+    # Check for server_facts reserved variable name if the trusted_sever_facts setting is true
+    if Puppet[:trusted_server_facts] && name == 'server_facts' && !options[:privileged]
+      raise Puppet::ParseError, "Attempt to assign to a reserved variable name: '#{name}'"
+    end
+
     table = effective_symtable(options[:ephemeral])
     if table.bound?(name)
       if options[:append]
-        error = Puppet::ParseError.new("Cannot append, variable #{name} is defined in this scope")
+        error = Puppet::ParseError.new("Cannot append, variable '$#{name}' is defined in this scope")
       else
-        error = Puppet::ParseError.new("Cannot reassign variable #{name}")
+        error = Puppet::ParseError.new("Cannot reassign variable '$#{name}'")
       end
       error.file = options[:file] if options[:file]
       error.line = options[:line] if options[:line]
@@ -636,6 +764,10 @@ class Puppet::Parser::Scope
     setvar('facts', deep_freeze(hash), :privileged => true)
   end
 
+  def set_server_facts(hash)
+    setvar('server_facts', deep_freeze(hash), :privileged => true)
+  end
+
   # Deeply freezes the given object. The object and its content must be of the types:
   # Array, Hash, Numeric, Boolean, Symbol, Regexp, NilClass, or String. All other types raises an Error.
   # (i.e. if they are assignable to Puppet::Pops::Types::Data type).
@@ -715,10 +847,14 @@ class Puppet::Parser::Scope
     "Scope(#{@resource})"
   end
 
-  # remove ephemeral scope up to level
-  # TODO: Who uses :all ? Remove ??
+  alias_method :inspect, :to_s
+
+  # Pop ephemeral scopes up to level and return them
   #
+  # @deprecated use #pop_epehemeral
+  # @api private
   def unset_ephemeral_var(level=:all)
+    Puppet.deprecation_warning('Method Parser::Scope#unset_ephemeral_var() is deprecated')
     if level == :all
       @ephemeral = [ MatchScope.new(@symtable, nil)]
     else
@@ -726,6 +862,22 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Pop ephemeral scopes up to level and return them
+  #
+  # @param level [Fixnum] a positive integer
+  # @return [Array] the removed ephemeral scopes
+  # @api private
+  def pop_ephemerals(level)
+    @ephemeral.pop(@ephemeral.size - level)
+  end
+
+  # Push ephemeral scopes onto the ephemeral scope stack
+  # @param ephemeral_scopes [Array]
+  # @api private
+  def push_ephemerals(ephemeral_scopes)
+    ephemeral_scopes.each { |ephemeral_scope| @ephemeral.push(ephemeral_scope) } unless ephemeral_scopes.nil?
+  end
+
   def ephemeral_level
     @ephemeral.size
   end
@@ -739,6 +891,56 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Execute given block in global scope with no ephemerals present
+  #
+  # @yieldparam [Scope] global_scope the global and ephemeral less scope
+  # @return [Object] the return of the block
+  #
+  # @api private
+  def with_global_scope(&block)
+    find_global_scope.without_ephemeral_scopes(&block)
+  end
+
+  # Execute given block with all ephemeral popped from the ephemeral stack
+  #
+  # @api private
+  def without_ephemeral_scopes
+    save_ephemeral = @ephemeral
+    begin
+      @ephemeral = [ @symtable ]
+      yield(self)
+    ensure
+      @ephemeral = save_ephemeral
+    end
+  end
+
+  # Nests a parameter scope
+  # @param [String] callee_name the name of the function, template, or resource that defines the parameters
+  # @param [Array<String>] param_names list of parameter names
+  # @yieldparam [ParameterScope] param_scope the nested scope
+  # @api private
+  def with_parameter_scope(callee_name, param_names)
+    param_scope = ParameterScope.new(@ephemeral.last, callee_name, param_names)
+    with_guarded_scope do
+      @ephemeral.push(param_scope)
+      yield(param_scope)
+    end
+  end
+
+  # Execute given block and ensure that ephemeral level is restored
+  #
+  # @return [Object] the return of the block
+  #
+  # @api private
+  def with_guarded_scope
+    elevel = ephemeral_level
+    begin
+      yield
+    ensure
+      pop_ephemerals(elevel)
+    end
+  end
+
   # Sets match data in the most nested scope (which always is a MatchScope), it clobbers match data already set there
   #
   def set_match_data(match_data)
@@ -767,18 +969,19 @@ class Puppet::Parser::Scope
     end
   end
 
+  # @api private
   def find_resource_type(type)
-    # It still works fine without the type == 'class' short-cut, but it is a lot slower.
-    return nil if ["class", "node"].include? type.to_s.downcase
-    find_builtin_resource_type(type) || find_defined_resource_type(type)
+    raise Puppet::DevError, "Scope#find_resource_type() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
   end
 
+  # @api private
   def find_builtin_resource_type(type)
-    Puppet::Type.type(type.to_s.downcase.to_sym)
+    raise Puppet::DevError, "Scope#find_builtin_resource_type() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
   end
 
+  # @api private
   def find_defined_resource_type(type)
-    known_resource_types.find_definition(type.to_s.downcase)
+    raise Puppet::DevError, "Scope#find_defined_resource_type() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
   end
 
 
@@ -796,25 +999,33 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Called from two places:
+  # runtime3support when creating resources
+  # ast::resource - used by create resources ?
+  # @api private
   def resolve_type_and_titles(type, titles)
-    raise ArgumentError, "titles must be an array" unless titles.is_a?(Array)
-
-    case type.downcase
-    when "class"
-      # resolve the titles
-      titles = titles.collect do |a_title|
-        hostclass = find_hostclass(a_title)
-        hostclass ?  hostclass.name : a_title
-      end
-    when "node"
-      # no-op
-    else
-      # resolve the type
-      resource_type = find_resource_type(type)
-      type = resource_type.name if resource_type
-    end
+    raise Puppet::DevError, "Scope#resolve_type_and_title() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
 
-    return [type, titles]
+#    Puppet.deprecation_warning('Scope#resolve_type_and_titles is deprecated.')
+#
+#    raise ArgumentError, "titles must be an array" unless titles.is_a?(Array)
+#
+#    case type.downcase
+#    when "class"
+#      # resolve the titles
+#      titles = titles.collect do |a_title|
+#        hostclass = find_hostclass(a_title)
+#        hostclass ?  hostclass.name : a_title
+#      end
+#    when "node"
+#      # no-op
+#    else
+#      # resolve the type
+#      resource_type = find_resource_type(type)
+#      type = resource_type.name if resource_type
+#    end
+#
+#    return [type, titles]
   end
 
   # Transforms references to classes to the form suitable for
@@ -847,10 +1058,22 @@ class Puppet::Parser::Scope
       when Puppet::Pops::Types::PResourceType
         assert_class_and_title(name.type_name, name.title)
         name.title.sub(/^([^:]{1,2})/, '::\1')
-      end
+      end.downcase
     end
   end
 
+  # Calls a 3.x or 4.x function by name with arguments given in an array using the 4.x calling convention
+  # and returns the result.
+  # Note that it is the caller's responsibility to rescue the given ArgumentError and provide location information
+  # to aid the user find the problem. The problem is otherwise reported against the source location that
+  # invoked the function that ultimately called this method.
+  #
+  # @return [Object] the result of the called function
+  # @raise ArgumentError if the function does not exist
+  def call_function(func_name, args, &block)
+    Puppet::Pops::Parser::EvaluatingParser.new.evaluator.external_call_function(func_name, args, self, &block)
+  end
+
   private
 
   def assert_class_and_title(type_name, title)

data/lib/puppet/parser/templatewrapper.rb

@@ -1,5 +1,6 @@
 require 'puppet/parser/files'
 require 'erb'
+require 'puppet/file_system'
 
 # A simple wrapper for templates, so they don't have full access to
 # the scope objects.
@@ -70,7 +71,7 @@ class Puppet::Parser::TemplateWrapper
     if string
       template_source = "inline template"
     else
-      string = File.read(@__file__)
+      string = Puppet::FileSystem.read_preserve_line_endings(@__file__)
       template_source = @__file__
     end
 

data/lib/puppet/parser/type_loader.rb

@@ -71,7 +71,7 @@ class Puppet::Parser::TypeLoader
           return result
         end
       rescue TypeLoaderError => detail
-        # I'm not convienced we should just drop these errors, but this
+        # I'm not convinced we should just drop these errors, but this
         # preserves existing behaviours.
       end
     end

data/lib/puppet/parser.rb

@@ -3,15 +3,17 @@ module Puppet::Parser; end
 
 require 'puppet/parser/ast'
 require 'puppet/parser/compiler'
+require 'puppet/parser/compiler/catalog_validator'
 require 'puppet/resource/type_collection'
 
 require 'puppet/parser/functions'
 require 'puppet/parser/files'
 require 'puppet/parser/relationship'
 
-require 'puppet/resource/type_collection_helper'
 require 'puppet/resource/type'
 require 'monitor'
 
+Dir[File.dirname(__FILE__) + '/parser/compiler/catalog_validator/*.rb'].each { |f| require f }
+
 # PUP-3274 This should probably go someplace else
 class Puppet::LexError < RuntimeError; end

data/lib/puppet/plugins/configuration.rb

@@ -11,6 +11,7 @@ module Puppet::Plugins::Configuration
     require 'puppet/plugins/binding_schemes'
     require 'puppet/plugins/syntax_checkers'
     require 'puppet/plugins/data_providers'
+    require 'puppet/plugins/data_providers/registry'
 
     # Extension-points are registered here:
     #
@@ -33,7 +34,7 @@ module Puppet::Plugins::Configuration
 
     extensions.multibind(checkers_name).name(checkers_name).hash_of(checkers_type)
     extensions.multibind(schemes_name).name(schemes_name).hash_of(schemes_type)
-    Puppet::Plugins::DataProviders::register_extensions(extensions)
+    Puppet::Plugins::DataProviders::Registry.register_extensions(extensions)
 
     # Register injector boot bindings
     # -------------------------------
@@ -64,5 +65,13 @@ module Puppet::Plugins::Configuration
       in_multibind(checkers_name)
       to_instance('Puppet::SyntaxCheckers::Json')
     end
-    Puppet::Plugins::DataProviders::register_defaults(bindings)
+
+    bindings.bind do
+      name('base64')
+      instance_of(checkers_type)
+      in_multibind(checkers_name)
+      to_instance('Puppet::SyntaxCheckers::Base64')
+    end
+
+    Puppet::Plugins::DataProviders::Registry.register_defaults(bindings)
 end