puppet 4.1.0 → 4.10.0

data/lib/puppet/parser/scope.rb

@@ -6,7 +6,6 @@ require 'forwardable'
 require 'puppet/parser'
 require 'puppet/parser/templatewrapper'
 
-require 'puppet/resource/type_collection_helper'
 require 'puppet/util/methodhelper'
 
 # This class is part of the internal parser/evaluator/compiler functionality of Puppet.
@@ -18,13 +17,13 @@ class Puppet::Parser::Scope
   extend Forwardable
   include Puppet::Util::MethodHelper
 
-  include Puppet::Resource::TypeCollectionHelper
   require 'puppet/parser/resource'
 
   AST = Puppet::Parser::AST
 
   # Variables that always exist with nil value even if not set
   BUILT_IN_VARS = ['module_name'.freeze, 'caller_module_name'.freeze].freeze
+  EMPTY_HASH = {}.freeze
 
   Puppet::Util.logmethods(self)
 
@@ -85,11 +84,8 @@ class Puppet::Parser::Scope
     end
 
     def [](name)
-      if @symbols.include?(name)
-        @symbols[name]
-      else
-        super
-      end
+      val = @symbols[name]
+      val.nil? && !@symbols.include?(val) ? super : val
     end
 
     def is_local_scope?
@@ -174,13 +170,113 @@ class Puppet::Parser::Scope
 
   end
 
+  # @api private
+  class ParameterScope < Ephemeral
+    class Access
+      attr_accessor :value
+
+      def assigned?
+        instance_variable_defined?(:@value)
+      end
+    end
+
+    # A parameter default must be evaluated using a special scope. The scope that is given to this method must
+    # have a `ParameterScope` as its last ephemeral scope. This method will then push a `MatchScope` while the
+    # given `expression` is evaluated. The method will catch any throw of `:unevaluated_parameter` and produce
+    # an error saying that the evaluated parameter X tries to access the unevaluated parameter Y.
+    #
+    # @param name [String] the name of the currently evaluated parameter
+    # @param expression [Puppet::Parser::AST] the expression to evaluate
+    # @param scope [Puppet::Parser::Scope] a scope where a `ParameterScope` has been pushed
+    # @return [Object] the result of the evaluation
+    #
+    # @api private
+    def evaluate3x(name, expression, scope)
+      scope.with_guarded_scope do
+        bad = catch(:unevaluated_parameter) do
+          scope.new_match_scope(nil)
+          return as_read_only { expression.safeevaluate(scope) }
+        end
+        parameter_reference_failure(name, bad)
+      end
+    end
+
+    def evaluate(name, expression, scope, evaluator)
+      scope.with_guarded_scope do
+        bad = catch(:unevaluated_parameter) do
+          scope.new_match_scope(nil)
+          return as_read_only { evaluator.evaluate(expression, scope) }
+        end
+        parameter_reference_failure(name, bad)
+      end
+    end
+
+    def parameter_reference_failure(from, to)
+      # Parameters are evaluated in the order they have in the @params hash.
+      keys = @params.keys
+      raise Puppet::Error, "#{@callee_name}: expects a value for parameter $#{to}" if keys.index(to) < keys.index(from)
+      raise Puppet::Error, "#{@callee_name}: default expression for $#{from} tries to illegally access not yet evaluated $#{to}"
+    end
+    private :parameter_reference_failure
+
+    def initialize(parent, callee_name, param_names)
+      super(parent)
+      @callee_name = callee_name
+      @params = {}
+      param_names.each { |name| @params[name] = Access.new }
+    end
+
+    def [](name)
+      access = @params[name]
+      return super if access.nil?
+      throw(:unevaluated_parameter, name) unless access.assigned?
+      access.value
+    end
+
+    def []=(name, value)
+      raise Puppet::Error, "Attempt to assign variable #{name} when evaluating parameters" if @read_only
+      @params[name] ||= Access.new
+      @params[name].value = value
+    end
+
+    def bound?(name)
+      @params.include?(name)
+    end
+
+    def include?(name)
+      @params.include?(name) || super
+    end
+
+    def is_local_scope?
+      true
+    end
+
+    def as_read_only
+      read_only = @read_only
+      @read_only = true
+      begin
+        yield
+      ensure
+        @read_only = read_only
+      end
+    end
+
+    def to_hash
+      Hash[@params.select {|_, access| access.assigned? }.map { |name, access| [name, access.value] }]
+    end
+  end
+
+
   # Returns true if the variable of the given name has a non nil value.
   # TODO: This has vague semantics - does the variable exist or not?
   #       use ['name'] to get nil or value, and if nil check with exist?('name')
   #       this include? is only useful because of checking against the boolean value false.
   #
   def include?(name)
-    ! self[name].nil?
+    catch(:undefined_variable) {
+      return ! self[name].nil?
+    }
+    false
   end
 
   # Returns true if the variable of the given name is set to any value (including nil)
@@ -249,11 +345,11 @@ class Puppet::Parser::Scope
   end
 
   def find_hostclass(name)
-    known_resource_types.find_hostclass(name)
+    environment.known_resource_types.find_hostclass(name)
   end
 
   def find_definition(name)
-    known_resource_types.find_definition(name)
+    environment.known_resource_types.find_definition(name)
   end
 
   def find_global_scope()
@@ -367,7 +463,8 @@ class Puppet::Parser::Scope
   # Look up a defined type.
   def lookuptype(name)
     # This happens a lot, avoid making a call to make a call
-    known_resource_types.find_definition(name) || known_resource_types.find_hostclass(name)
+    krt = environment.known_resource_types
+    krt.find_definition(name) || krt.find_hostclass(name)
   end
 
   def undef_as(x,v)
@@ -387,41 +484,54 @@ class Puppet::Parser::Scope
   # @return Object the value of the variable, or nil if it's not found
   #
   # @api public
-  def lookupvar(name, options = {})
+  def lookupvar(name, options = EMPTY_HASH)
     unless name.is_a? String
       raise Puppet::ParseError, "Scope variable name #{name.inspect} is a #{name.class}, not a string"
     end
 
-    table = @ephemeral.last
-
     if name =~ /^(.*)::(.+)$/
-      class_name = $1
-      variable_name = $2
-      lookup_qualified_variable(class_name, variable_name, options)
+      return lookup_qualified_variable($1, $2, options)
+    end
 
-    # TODO: optimize with an assoc instead, this searches through scopes twice for a hit
-    elsif table.include?(name)
-      table[name]
+    table = @ephemeral.last
+    val = table[name]
+    return val unless val.nil? && !table.include?(name)
+
+    next_scope = inherited_scope || enclosing_scope
+    if next_scope
+      next_scope.lookupvar(name, options)
     else
-      next_scope = inherited_scope || enclosing_scope
-      if next_scope
-        next_scope.lookupvar(name, options)
-      else
-        variable_not_found(name)
-      end
+      variable_not_found(name)
     end
   end
 
+  UNDEFINED_VARIABLES_KIND = 'undefined_variables'.freeze
+  DEPRECATION_KIND = 'deprecation'.freeze
+
+  # The exception raised when a throw is uncaught is different in different versions
+  # of ruby. In >=2.2.0 it is UncaughtThrowError (which did not exist prior to this)
+  #
+  UNCAUGHT_THROW_EXCEPTION = defined?(UncaughtThrowError) ? UncaughtThrowError : ArgumentError
+
   def variable_not_found(name, reason=nil)
-    # Built in variables always exist
-    if BUILT_IN_VARS.include?(name)
+    # Built in variables and numeric variables always exist
+    if BUILT_IN_VARS.include?(name) || name =~ Puppet::Pops::Patterns::NUMERIC_VAR_NAME
       return nil
     end
-    if Puppet[:strict_variables]
-      throw :undefined_variable
-    else
-      nil
+    begin
+      throw(:undefined_variable, reason)
+    rescue  UNCAUGHT_THROW_EXCEPTION
+      case Puppet[:strict]
+      when :off
+        # do nothing
+      when :warning
+        Puppet.warn_once(UNDEFINED_VARIABLES_KIND, "Variable: #{name}",
+        "Undefined variable '#{name}'; #{reason}" )
+      when :error
+        raise ArgumentError, "Undefined variable '#{name}'; #{reason}"
+      end
     end
+    nil
   end
 
   # Retrieves the variable value assigned to the name given as an argument. The name must be a String,
@@ -478,32 +588,45 @@ class Puppet::Parser::Scope
   end
 
   def is_topscope?
-    compiler and self == compiler.topscope
+    @compiler && equal?(@compiler.topscope)
   end
 
   def lookup_qualified_variable(class_name, variable_name, position)
     begin
       if lookup_as_local_name?(class_name, variable_name)
-        self[variable_name]
+        if is_topscope?
+          # This is the case where $::x is looked up from within the topscope itself, or from a local scope
+          # parented at the top scope. In this case, the lookup must ignore local and ephemeral scopes.
+          #
+          handle_not_found(class_name, variable_name, position) unless @symtable.include?(variable_name)
+          @symtable[variable_name]
+        else
+          self[variable_name]
+        end
       else
         qualified_scope(class_name).lookupvar(variable_name, position)
       end
     rescue RuntimeError => e
-      unless Puppet[:strict_variables]
-        # Do not issue warning if strict variables are on, as an error will be raised by variable_not_found
-        location = if position[:lineproc]
-                     " at #{position[:lineproc].call}"
-                   elsif position[:file] && position[:line]
-                     " at #{position[:file]}:#{position[:line]}"
-                   else
-                     ""
-                   end
-        warning "Could not look up qualified variable '#{class_name}::#{variable_name}'; #{e.message}#{location}"
-      end
-      variable_not_found("#{class_name}::#{variable_name}", e.message)
+      handle_not_found(class_name, variable_name, position, e.message)
     end
   end
 
+  def handle_not_found(class_name, variable_name, position, reason = nil)
+    unless Puppet[:strict_variables]
+      # Do not issue warning if strict variables are on, as an error will be raised by variable_not_found
+      location = if position[:lineproc]
+                   " at #{position[:lineproc].call}"
+                 elsif position[:file] && position[:line]
+                   " at #{position[:file]}:#{position[:line]}"
+                 else
+                   ""
+                 end
+      variable_not_found("#{class_name}::#{variable_name}", "#{reason}#{location}")
+      return nil
+    end
+    variable_not_found("#{class_name}::#{variable_name}", reason)
+  end
+
   # Handles the special case of looking up fully qualified variable in not yet evaluated top scope
   # This is ok if the lookup request originated in topscope (this happens when evaluating
   # bindings; using the top scope to provide the values for facts.
@@ -581,20 +704,51 @@ class Puppet::Parser::Scope
 
     params.each { |param|
       if table.include?(param.name)
-        raise Puppet::ParseError.new("Default already defined for #{type} { #{param.name} }; cannot redefine", param.line, param.file)
+        raise Puppet::ParseError.new("Default already defined for #{type} { #{param.name} }; cannot redefine", param.file, param.line)
       end
       table[param.name] = param
     }
   end
 
-  RESERVED_VARIABLE_NAMES = ['trusted', 'facts'].freeze
+  # Merge all settings for the given _env_name_ into this scope
+  # @param env_name [Symbol] the name of the environment
+  def merge_settings(env_name)
+    settings = Puppet.settings
+    table = effective_symtable(false)
+    settings.each_key do |name|
+      next if :name == name
+      table[name.to_s] = transform_setting(settings.value_sym(name, env_name))
+    end
+    nil
+  end
+
+  def transform_setting(val)
+    if val.is_a?(String) || val.is_a?(Numeric) || true == val || false == val || nil == val
+      val
+    elsif val.is_a?(Array)
+      val.map {|entry| transform_setting(entry) }
+    elsif val.is_a?(Hash)
+      result = {}
+      val.each {|k,v| result[transform_setting(k)] = transform_setting(v) }
+      result
+    else
+      # not ideal, but required as there are settings values that are special
+      :undef == val ? nil : val.to_s
+    end
+  end
+  private :transform_setting
+
+  VARNAME_TRUSTED = 'trusted'.freeze
+  VARNAME_FACTS = 'facts'.freeze
+  VARNAME_SERVER_FACTS = 'server_facts'.freeze
+  RESERVED_VARIABLE_NAMES = [VARNAME_TRUSTED, VARNAME_FACTS].freeze
 
   # Set a variable in the current scope.  This will override settings
   # in scopes above, but will not allow variables in the current scope
   # to be reassigned.
   #   It's preferred that you use self[]= instead of this; only use this
   # when you need to set options.
-  def setvar(name, value, options = {})
+  def setvar(name, value, options = EMPTY_HASH)
     if name =~ /^[0-9]+$/
       raise Puppet::ParseError.new("Cannot assign to a numeric match result variable '$#{name}'") # unless options[:ephemeral]
     end
@@ -603,12 +757,12 @@ class Puppet::Parser::Scope
     end
 
     # Check for reserved variable names
-    if !options[:privileged] && RESERVED_VARIABLE_NAMES.include?(name)
+    if (name == VARNAME_TRUSTED || name == VARNAME_FACTS) && !options[:privileged]
       raise Puppet::ParseError, "Attempt to assign to a reserved variable name: '#{name}'"
     end
 
     # Check for server_facts reserved variable name if the trusted_sever_facts setting is true
-    if Puppet[:trusted_server_facts] && name == 'server_facts' && !options[:privileged]
+    if name == VARNAME_SERVER_FACTS && !options[:privileged] && Puppet[:trusted_server_facts]
       raise Puppet::ParseError, "Attempt to assign to a reserved variable name: '#{name}'"
     end
 
@@ -675,15 +829,13 @@ class Puppet::Parser::Scope
   #
   # @param use_ephemeral [Boolean] whether the top most ephemeral (of any kind) should be used or not
   def effective_symtable(use_ephemeral)
-    s = @ephemeral.last
-    if use_ephemeral
-      return s || @symtable
-    else
-      while s && !s.is_local_scope?()
-        s = s.parent
-      end
-      s || @symtable
+    s = @ephemeral[-1]
+    return s || @symtable if use_ephemeral
+
+    while s && !s.is_local_scope?()
+      s = s.parent
     end
+    s || @symtable
   end
 
   # Sets the variable value of the name given as an argument to the given value. The value is
@@ -693,12 +845,12 @@ class Puppet::Parser::Scope
   #
   # @param varname [String] The variable name to which the value is assigned. Must not contain `::`
   # @param value [String] The value to assign to the given variable name.
-  # @param options [Hash] Additional options, not part of api.
+  # @param options [Hash] Additional options, not part of api and no longer used.
   #
   # @api public
   #
-  def []=(varname, value, options = {})
-    setvar(varname, value, options = {})
+  def []=(varname, value, _ = nil)
+    setvar(varname, value)
   end
 
   def append_value(bound_value, new_value)
@@ -724,10 +876,14 @@ class Puppet::Parser::Scope
     "Scope(#{@resource})"
   end
 
-  # remove ephemeral scope up to level
-  # TODO: Who uses :all ? Remove ??
+  alias_method :inspect, :to_s
+
+  # Pop ephemeral scopes up to level and return them
   #
+  # @deprecated use #pop_epehemeral
+  # @api private
   def unset_ephemeral_var(level=:all)
+    Puppet.deprecation_warning('Method Parser::Scope#unset_ephemeral_var() is deprecated')
     if level == :all
       @ephemeral = [ MatchScope.new(@symtable, nil)]
     else
@@ -735,6 +891,22 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Pop ephemeral scopes up to level and return them
+  #
+  # @param level [Fixnum] a positive integer
+  # @return [Array] the removed ephemeral scopes
+  # @api private
+  def pop_ephemerals(level)
+    @ephemeral.pop(@ephemeral.size - level)
+  end
+
+  # Push ephemeral scopes onto the ephemeral scope stack
+  # @param ephemeral_scopes [Array]
+  # @api private
+  def push_ephemerals(ephemeral_scopes)
+    ephemeral_scopes.each { |ephemeral_scope| @ephemeral.push(ephemeral_scope) } unless ephemeral_scopes.nil?
+  end
+
   def ephemeral_level
     @ephemeral.size
   end
@@ -748,6 +920,69 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Execute given block in global scope with no ephemerals present
+  #
+  # @yieldparam [Scope] global_scope the global and ephemeral less scope
+  # @return [Object] the return of the block
+  #
+  # @api private
+  def with_global_scope(&block)
+    find_global_scope.without_ephemeral_scopes(&block)
+  end
+
+  # Execute given block with a ephemeral scope containing the given variables
+  # @api private
+  def with_local_scope(scope_variables)
+    local = LocalScope.new(@ephemeral.last)
+    scope_variables.each_pair { |k, v| local[k] = v }
+    @ephemeral.push(local)
+    begin
+      yield(self)
+    ensure
+      @ephemeral.pop
+    end
+  end
+
+  # Execute given block with all ephemeral popped from the ephemeral stack
+  #
+  # @api private
+  def without_ephemeral_scopes
+    save_ephemeral = @ephemeral
+    begin
+      @ephemeral = [ @symtable ]
+      yield(self)
+    ensure
+      @ephemeral = save_ephemeral
+    end
+  end
+
+  # Nests a parameter scope
+  # @param [String] callee_name the name of the function, template, or resource that defines the parameters
+  # @param [Array<String>] param_names list of parameter names
+  # @yieldparam [ParameterScope] param_scope the nested scope
+  # @api private
+  def with_parameter_scope(callee_name, param_names)
+    param_scope = ParameterScope.new(@ephemeral.last, callee_name, param_names)
+    with_guarded_scope do
+      @ephemeral.push(param_scope)
+      yield(param_scope)
+    end
+  end
+
+  # Execute given block and ensure that ephemeral level is restored
+  #
+  # @return [Object] the return of the block
+  #
+  # @api private
+  def with_guarded_scope
+    elevel = ephemeral_level
+    begin
+      yield
+    ensure
+      pop_ephemerals(elevel)
+    end
+  end
+
   # Sets match data in the most nested scope (which always is a MatchScope), it clobbers match data already set there
   #
   def set_match_data(match_data)
@@ -766,7 +1001,7 @@ class Puppet::Parser::Scope
       new_ephemeral(true)
       match.each {|k,v| setvar(k, v, :file => file, :line => line, :ephemeral => true) }
       # Must always have an inner match data scope (that starts out as transparent)
-      # In 3x slightly wasteful, since a new nested scope is created for a match 
+      # In 3x slightly wasteful, since a new nested scope is created for a match
       # (TODO: Fix that problem)
       new_ephemeral(false)
     else
@@ -776,18 +1011,19 @@ class Puppet::Parser::Scope
     end
   end
 
+  # @api private
   def find_resource_type(type)
-    # It still works fine without the type == 'class' short-cut, but it is a lot slower.
-    return nil if ["class", "node"].include? type.to_s.downcase
-    find_builtin_resource_type(type) || find_defined_resource_type(type)
+    raise Puppet::DevError, "Scope#find_resource_type() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
   end
 
+  # @api private
   def find_builtin_resource_type(type)
-    Puppet::Type.type(type.to_s.downcase.to_sym)
+    raise Puppet::DevError, "Scope#find_builtin_resource_type() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
   end
 
+  # @api private
   def find_defined_resource_type(type)
-    known_resource_types.find_definition(type.to_s.downcase)
+    raise Puppet::DevError, "Scope#find_defined_resource_type() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
   end
 
 
@@ -805,25 +1041,33 @@ class Puppet::Parser::Scope
     end
   end
 
+  # Called from two places:
+  # runtime3support when creating resources
+  # ast::resource - used by create resources ?
+  # @api private
   def resolve_type_and_titles(type, titles)
-    raise ArgumentError, "titles must be an array" unless titles.is_a?(Array)
-
-    case type.downcase
-    when "class"
-      # resolve the titles
-      titles = titles.collect do |a_title|
-        hostclass = find_hostclass(a_title)
-        hostclass ?  hostclass.name : a_title
-      end
-    when "node"
-      # no-op
-    else
-      # resolve the type
-      resource_type = find_resource_type(type)
-      type = resource_type.name if resource_type
-    end
+    raise Puppet::DevError, "Scope#resolve_type_and_title() is no longer supported, use Puppet::Pops::Evaluator::Runtime3ResourceSupport instead"
 
-    return [type, titles]
+#    Puppet.deprecation_warning('Scope#resolve_type_and_titles is deprecated.')
+#
+#    raise ArgumentError, "titles must be an array" unless titles.is_a?(Array)
+#
+#    case type.downcase
+#    when "class"
+#      # resolve the titles
+#      titles = titles.collect do |a_title|
+#        hostclass = find_hostclass(a_title)
+#        hostclass ?  hostclass.name : a_title
+#      end
+#    when "node"
+#      # no-op
+#    else
+#      # resolve the type
+#      resource_type = find_resource_type(type)
+#      type = resource_type.name if resource_type
+#    end
+#
+#    return [type, titles]
   end
 
   # Transforms references to classes to the form suitable for
@@ -856,10 +1100,22 @@ class Puppet::Parser::Scope
       when Puppet::Pops::Types::PResourceType
         assert_class_and_title(name.type_name, name.title)
         name.title.sub(/^([^:]{1,2})/, '::\1')
-      end
+      end.downcase
     end
   end
 
+  # Calls a 3.x or 4.x function by name with arguments given in an array using the 4.x calling convention
+  # and returns the result.
+  # Note that it is the caller's responsibility to rescue the given ArgumentError and provide location information
+  # to aid the user find the problem. The problem is otherwise reported against the source location that
+  # invoked the function that ultimately called this method.
+  #
+  # @return [Object] the result of the called function
+  # @raise ArgumentError if the function does not exist
+  def call_function(func_name, args, &block)
+    Puppet::Pops::Parser::EvaluatingParser.new.evaluator.external_call_function(func_name, args, self, &block)
+  end
+
   private
 
   def assert_class_and_title(type_name, title)

data/lib/puppet/parser/type_loader.rb

@@ -71,7 +71,7 @@ class Puppet::Parser::TypeLoader
           return result
         end
       rescue TypeLoaderError => detail
-        # I'm not convienced we should just drop these errors, but this
+        # I'm not convinced we should just drop these errors, but this
         # preserves existing behaviours.
       end
     end

data/lib/puppet/parser.rb

@@ -3,15 +3,17 @@ module Puppet::Parser; end
 
 require 'puppet/parser/ast'
 require 'puppet/parser/compiler'
+require 'puppet/parser/compiler/catalog_validator'
 require 'puppet/resource/type_collection'
 
 require 'puppet/parser/functions'
 require 'puppet/parser/files'
 require 'puppet/parser/relationship'
 
-require 'puppet/resource/type_collection_helper'
 require 'puppet/resource/type'
 require 'monitor'
 
+Dir[File.dirname(__FILE__) + '/parser/compiler/catalog_validator/*.rb'].each { |f| require f }
+
 # PUP-3274 This should probably go someplace else
 class Puppet::LexError < RuntimeError; end

data/lib/puppet/plugins/configuration.rb

@@ -11,6 +11,7 @@ module Puppet::Plugins::Configuration
     require 'puppet/plugins/binding_schemes'
     require 'puppet/plugins/syntax_checkers'
     require 'puppet/plugins/data_providers'
+    require 'puppet/plugins/data_providers/registry'
 
     # Extension-points are registered here:
     #
@@ -33,7 +34,7 @@ module Puppet::Plugins::Configuration
 
     extensions.multibind(checkers_name).name(checkers_name).hash_of(checkers_type)
     extensions.multibind(schemes_name).name(schemes_name).hash_of(schemes_type)
-    Puppet::Plugins::DataProviders::register_extensions(extensions)
+    Puppet::Plugins::DataProviders::Registry.register_extensions(extensions)
 
     # Register injector boot bindings
     # -------------------------------
@@ -64,5 +65,13 @@ module Puppet::Plugins::Configuration
       in_multibind(checkers_name)
       to_instance('Puppet::SyntaxCheckers::Json')
     end
-    Puppet::Plugins::DataProviders::register_defaults(bindings)
+
+    bindings.bind do
+      name('base64')
+      instance_of(checkers_type)
+      in_multibind(checkers_name)
+      to_instance('Puppet::SyntaxCheckers::Base64')
+    end
+
+    Puppet::Plugins::DataProviders::Registry.register_defaults(bindings)
 end