puppet 3.4.3 → 3.5.0.rc1

data/lib/puppet/parser/templatewrapper.rb

@@ -84,7 +84,7 @@ class Puppet::Parser::TemplateWrapper
 
   # @api private
   def file=(filename)
-    unless @__file__ = Puppet::Parser::Files.find_template(filename, scope.compiler.environment.to_s)
+    unless @__file__ = Puppet::Parser::Files.find_template(filename, scope.compiler.environment)
       raise Puppet::ParseError, "Could not find template '#{filename}'"
     end
 

data/lib/puppet/parser/type_loader.rb

@@ -5,7 +5,6 @@ require 'puppet/parser/parser_factory'
 
 class Puppet::Parser::TypeLoader
   extend  Forwardable
-  include Puppet::Node::Environment::Helper
 
   # Import manifest files that match a given file glob pattern.
   #
@@ -50,6 +49,18 @@ class Puppet::Parser::TypeLoader
     self.environment = env
   end
 
+  def environment
+    @environment
+  end
+
+  def environment=(env)
+    if env.is_a?(String) or env.is_a?(Symbol)
+      @environment = Puppet.lookup(:environments).get(env)
+    else
+      @environment = env
+    end
+  end
+
   # Try to load the object with the given fully qualified name.
   def try_load_fqname(type, fqname)
     return nil if fqname == "" # special-case main.
@@ -101,7 +112,7 @@ class Puppet::Parser::TypeLoader
       # The use case is that parsing for the purpose of searching for information
       # should not abort. There is currently one such use case in indirector/resourcetype/parser
       #
-      if $squelsh_parse_errors
+    if Puppet.lookup(:squelch_parse_errors) {|| false }
         begin
           loaded_asts << parse_file(file)
         rescue => e

data/lib/puppet/pops.rb

@@ -25,6 +25,7 @@ module Puppet
       require 'puppet/pops/types/type_factory'
       require 'puppet/pops/types/type_parser'
       require 'puppet/pops/types/class_loader'
+      require 'puppet/pops/types/enumeration'
     end
 
     module Model
@@ -54,11 +55,11 @@ module Puppet
       require 'puppet/pops/binder/injector_entry'
       require 'puppet/pops/binder/key_factory'
       require 'puppet/pops/binder/injector'
-      require 'puppet/pops/binder/hiera2'
       require 'puppet/pops/binder/bindings_composer'
       require 'puppet/pops/binder/bindings_model_dumper'
       require 'puppet/pops/binder/system_bindings'
       require 'puppet/pops/binder/bindings_loader'
+      require 'puppet/pops/binder/lookup'
 
       module Config
         require 'puppet/pops/binder/config/binder_config'
@@ -71,15 +72,28 @@ module Puppet
     module Parser
       require 'puppet/pops/parser/eparser'
       require 'puppet/pops/parser/parser_support'
+      require 'puppet/pops/parser/locator'
+      require 'puppet/pops/parser/locatable'
       require 'puppet/pops/parser/lexer'
+      require 'puppet/pops/parser/lexer2'
       require 'puppet/pops/parser/evaluating_parser'
+      require 'puppet/pops/parser/epp_parser'
     end
 
     module Validation
       require 'puppet/pops/validation/checker3_1'
       require 'puppet/pops/validation/validator_factory_3_1'
+      require 'puppet/pops/validation/checker4_0'
+      require 'puppet/pops/validation/validator_factory_4_0'
+    end
+
+    module Evaluator
+      require 'puppet/pops/evaluator/runtime3_support'
+      require 'puppet/pops/evaluator/evaluator_impl'
+      require 'puppet/pops/evaluator/epp_evaluator'
     end
   end
 
+  require 'puppet/parser/ast/pops_bridge'
   require 'puppet/bindings'
 end

data/lib/puppet/pops/adapters.rb

@@ -10,46 +10,78 @@ module Puppet::Pops::Adapters
     attr_accessor :documentation
   end
 
-  # An origin adapter adapts an object with where it came from. This origin
-  # describes the resource (a file, etc.) where source text originates.
-  # Instances of SourcePosAdapter is then used on other objects in a model to
-  # describe their relative position versus the origin.
-  #
-  # @see Puppet::Pops::Utils#find_adapter
-  #
-  class OriginAdapter < Puppet::Pops::Adaptable::Adapter
-    # @return [String] the origin of the adapted (usually a filename)
-    attr_accessor :origin
-  end
-
-  # A SourcePosAdapter describes a position relative to an origin. (Typically an {OriginAdapter} is
-  # associated with the root of a model. This origin has a URI to the resource, and a line number.
-  # The offset in the SourcePosAdapter is then relative to this origin.
-  # (This somewhat complex structure makes it possible to correctly refer to a source position
+  # A SourcePosAdapter holds a reference to  a *Positioned* object (object that has offset and length).
+  # This somewhat complex structure makes it possible to correctly refer to a source position
   # in source that is embedded in some resource; a parser only sees the embedded snippet of source text
-  # and does not know where it was embedded).
+  # and does not know where it was embedded. It also enables lazy evaluation of source positions (they are
+  # rarely needed - typically just when there is an error to report.
   #
-  # @see Puppet::Pops::Utils#find_adapter
+  # @note It is relatively expensive to compute line and position on line - it is not something that
+  #   should be done for every token or model object.
+  #
+  # @see Puppet::Pops::Utils#find_adapter, Puppet::Pops::Utils#find_closest_positioned
   #
   class SourcePosAdapter < Puppet::Pops::Adaptable::Adapter
-    # @return [Fixnum] The start line in source starting from 1
-    attr_accessor :line
+    attr_accessor :locator
+
+    def self.create_adapter(o)
+      new(o)
+    end
+
+    def initialize(o)
+      @adapted = o
+    end
+
+    def locator
+      # The locator is always the parent locator, all positioned objects are positioned within their
+      # parent. If a positioned object also has a locator that locator is for its children!
+      #
+      @locator ||= find_locator(@adapted.eContainer)
+    end
+
+    def find_locator(o)
+      if o.nil?
+        raise ArgumentError, "InternalError: SourcePosAdapter for something that has no locator among parents"
+      end
+      case
+      when o.is_a?(Puppet::Pops::Model::Program)
+        return o.locator
+      # TODO_HEREDOC use case of SubLocator instead
+      when o.is_a?(Puppet::Pops::Model::SubLocatedExpression) && !(found_locator = o.locator).nil?
+        return found_locator
+      when adapter = self.class.get(o)
+        return adapter.locator
+      else
+        find_locator(o.eContainer)
+      end
+    end
+    private :find_locator
 
-    # @return [Fixnum] The position on the start_line (in characters) starting from 0
-    attr_accessor :pos
+    def offset
+      @adapted.offset
+    end
 
-    # @return [Fixnum] The (start) offset of source text characters
-    #   (starting from 0) representing the adapted object.
-    #   Value may be nil
-    attr_accessor :offset
+    def length
+      @adapted.length
+    end
 
-    # @return [Fixnum] The length (count) of characters of source text
-    #   representing the adapted object from the origin. Not including any
-    #   trailing whitespace.
-    attr_accessor :length
+    # Produces the line number for the given offset.
+    # @note This is an expensive operation
+    #
+    def line
+      locator.line_for_offset(offset)
+    end
+
+    # Produces the position on the line of the given offset.
+    # @note This is an expensive operation
+    #
+    def pos
+      locator.pos_on_line(offset)
+    end
 
-    def extract_text_from_string(string)
-      string.slice(offset, length)
+    # Extracts the text represented by this source position (the string is obtained from the locator)
+    def extract_text
+      locator.string.slice(offset, length)
     end
   end
 

data/lib/puppet/pops/binder/binder.rb

@@ -1,107 +1,64 @@
 # The Binder is responsible for processing layered bindings that can be used to setup an Injector.
 #
-# An instance should be created, and calls should then be made to {#define_categories} to define the available categories, and
-# their precedence. This should be followed by a call to {#define_layers} which will match the layered bindings against the
-# effective categories (filtering out everything that does not apply, handle overrides, abstract entries etc.).
+# An instance should be created and a call to {#define_layers} should be made which will process the layered bindings
+# (handle overrides, abstract entries etc.).
 # The constructed hash with `key => InjectorEntry` mappings is obtained as {#injector_entries}, and is used to initialize an
 # {Puppet::Pops::Binder::Injector Injector}.
 #
 # @api public
 #
 class Puppet::Pops::Binder::Binder
-  # This limits the number of available categorizations, including "common".
+
   # @api private
-  PRECEDENCE_MAX = 1000
+  attr_reader :injector_entries
 
   # @api private
-  attr_reader :category_precedences
+  attr :id_index
 
   # @api private
-  attr_reader :category_values
+  attr_reader :key_factory
 
+  # A parent Binder or nil
   # @api private
-  attr_reader :injector_entries
+  attr_reader :parent
 
+  # The next anonymous key to use
   # @api private
-  attr_reader :key_factory
+  attr_reader :anonymous_key
 
-  # Whether the binder is fully configured or not
-  # @api public
-  #
-  attr_reader :configured
+  # This binder's precedence
+  # @api private
+  attr_reader :binder_precedence
 
   # @api public
-  def initialize
-    @category_precedences = {}
-    @category_values = {}
+  def initialize(layered_bindings, parent_binder=nil)
+    @parent = parent_binder
+    @id_index = Hash.new() { |k, v| [] }
+
     @key_factory = Puppet::Pops::Binder::KeyFactory.new()
 
     # Resulting hash of all key -> binding
     @injector_entries = {}
 
-    # Not configured until the fat lady sings
-    @configured = false
-
-    @next_anonymous_key = 0
-  end
-
-  # Answers the question 'is this binder configured?' to the point it can be used to instantiate an Injector
-  # @api public
-  def configured?()
-    configured()
-  end
-
-  # Defines the effective categories in precedence order (highest precedence first).
-  # The 'common' (lowest precedence) category should not be included in the list.
-  # A sanity check is made that there are no more than 1000 categorizations (which is pretty wild).
-  #
-  # The term 'effective categories' refers to the evaluated list of tuples (categorization, category-value) represented with
-  # an instance of Puppet::Pops::Binder::Bindings::EffectiveCategories.
-  #
-  # @param effective_categories [Puppet::Pops::Binder::Bindings::EffectiveCategories] effective categories (i.e. with evaluated values)
-  # @raise ArgumentError if this binder is already configured
-  # @raise ArgumentError if the argument is not an EffectiveCategories
-  # @raise ArgumentError if there is an attempt to redefine a category (non unique, or 'common').
-  # @return [Puppet::Pops::Binder::Binder] self
-  # @api public
-  #
-  def define_categories(effective_categories)
-    raise ArgumentError, "This categories are already defined. Cannot redefine." unless @category_precedences.empty?
-
-    # Note: a model instance is used since a Hash does not have a defined order in all Rubies.
-    unless effective_categories.is_a?(Puppet::Pops::Binder::Bindings::EffectiveCategories)
-      raise ArgumentError, "Expected Puppet::Pops::Binder::Bindings::EffectiveCategories, but got a: #{effective_categories.class}"
-    end
-    categories = effective_categories.categories
-    raise ArgumentError, "Category limit (#{PRECEDENCE_MAX}) exceeded" unless categories.size <= PRECEDENCE_MAX
-
-    # Automatically add the 'common' category with lowest precedence
-    @category_precedences['common'] = 0
-
-    # if categories contains "common", it should be last - simply drop it if present
-    if last = categories[-1]
-      if last.categorization == 'common'
-        categories.delete_at(-1)
-      end
-    end
-    # Process the given categories (highest precedence is first in the list)
-    categories.each_with_index do |c, index|
-      cname = c.categorization
-      raise ArgumentError, "Attempt to redefine categorization: #{cname}" if @category_precedences[cname]
-      @category_precedences[cname] = PRECEDENCE_MAX - index
-      @category_values[cname] = c.value
+    if @parent.nil?
+      @anonymous_key = 0
+      @binder_precedence = 0
+    else
+      # First anonymous key is the parent's next (non incremented key). (The parent can not change, it is
+      # the final, free key).
+      @anonymous_key = @parent.anonymous_key
+      @binder_precedence = @parent.binder_precedence + 1
     end
-    self
+    define_layers(layered_bindings)
   end
 
   # Binds layers from highest to lowest as defined by the given LayeredBindings.
   # @note
-  #   Categories must be set with #define_categories before calling this method. The model should have been
+  #   The model should have been
   #   validated to get better error messages if the model is invalid. This implementation expects the model
   #   to be valid, and any errors raised will be more technical runtime errors.
   #
   # @param layered_bindings [Puppet::Pops::Binder::Bindings::LayeredBindings] the named and ordered layers
-  # @raise ArgumentError if categories have not been defined
   # @raise ArgumentError if this binder is already configured
   # @raise ArgumentError if bindings with unresolved 'override' surfaces as an effective binding
   # @raise ArgumentError if the given argument has the wrong type, or if model is invalid in some way
@@ -109,27 +66,76 @@ class Puppet::Pops::Binder::Binder
   # @api public
   #
   def define_layers(layered_bindings)
-    raise ArgumentError, "This binder is already configured. Cannot redefine its content." if configured?()
 
-    raise ArgumentError, "Categories must be defined first" if @category_precedences.empty?
     LayerProcessor.new(self, key_factory).bind(layered_bindings)
-    injector_entries.each  do |k,v|
-      unless key_factory.is_contributions_key?(k) || v.is_resolved?()
+    contribution_keys = []
+    # make one pass over entries to collect contributions, and check overrides
+    injector_entries.each do |k,v|
+      if key_factory.is_contributions_key?(k)
+        contribution_keys << [k,v]
+      elsif !v.is_resolved?()
         raise ArgumentError, "Binding with unresolved 'override' detected: #{self.class.format_binding(v.binding)}}"
+      else
+        # if binding has an id, add it to the index
+        add_id_to_index(v.binding)
       end
     end
-    # and the fat lady has sung
-    @configured = true
-    self
+
+    # If a lower level binder has contributions for a key also contributed to in this binder
+    # they must included in the higher shadowing contribution.
+    # If a contribution is made to an id that is defined in a parent
+    # contribute to an id that is defined in a lower binder, it must be promoted to this binder (copied) or
+    # there is risk of making the lower level injector dirty.
+    #
+    contribution_keys.each do |kv|
+      parent_contribution = lookup_in_parent(kv[0])
+      next unless parent_contribution
+      injector_entries[kv[0]] = kv[1] + parent_contributions
+
+      # key the multibind_id from the contribution key
+      multibind_id = key_factory.multibind_contribution_key_to_id(kv[0])
+      promote_matching_bindings(self, @parent, multibind_id)
+    end
   end
+  private :define_layers
 
   # @api private
   def next_anonymous_key
-    tmp = @next_anonymous_key
-    @next_anonymous_key += 1
+    tmp = @anonymous_key
+    @anonymous_key += 1
     tmp
   end
 
+  def add_id_to_index(binding)
+    return unless binding.is_a?(Puppet::Pops::Binder::Bindings::Multibinding) && !(id = binding.id).nil?
+    @id_index[id] = @id_index[id] << binding
+  end
+
+  def promote_matching_bindings(to_binder, from_binder, multibind_id)
+    return if from_binder.nil?
+    from_binder.id_index[ multibind_id ].each do |binding|
+      key = key_factory.binding_key(binding)
+      entry = lookup(key)
+      unless entry.precedence == @binder_precedence
+        # it is from a lower layer it must be promoted
+        injector_entries[ key ] = Puppet::Pops::Binder::InjectorEntry.new(binding, binder_precedence)
+      end
+    end
+    # recursive "up the parent chain" to promote all
+    promote_matching_bindings(to_binder, from_binder.parent, multibind_id)
+  end
+
+  def lookup_in_parent(key)
+    @parent.nil? ? nil : @parent.lookup(key)
+  end
+
+  def lookup(key)
+    if x = injector_entries[key]
+      return x
+    end
+    @parent ?  @parent.lookup(key) :  nil
+  end
+
   # @api private
   def self.format_binding(b)
     type_name = Puppet::Pops::Types::TypeCalculator.new().string(b.type)
@@ -163,18 +169,16 @@ class Puppet::Pops::Binder::Binder
   # @api private
   #
   class LayerProcessor
-    attr :effective_prec
-    attr :prec_stack
     attr :bindings
     attr :binder
     attr :key_factory
     attr :contributions
+    attr :binder_precedence
 
     def initialize(binder, key_factory)
       @binder = binder
+      @binder_precedence = binder.binder_precedence
       @key_factory = key_factory
-      @prec_stack = []
-      @effective_prec = nil
       @bindings = []
       @contributions = []
       @@bind_visitor ||= Puppet::Pops::Visitor.new(nil,"bind",0,0)
@@ -184,14 +188,14 @@ class Puppet::Pops::Binder::Binder
     # @api private
     #
     def add(b)
-      bindings << Puppet::Pops::Binder::InjectorEntry.new(effective_prec, b)
+      bindings << Puppet::Pops::Binder::InjectorEntry.new(b, binder_precedence)
     end
 
     # Add a multibind contribution
     # @api private
     #
     def add_contribution(b)
-      contributions << Puppet::Pops::Binder::InjectorEntry.new(effective_prec, b)
+      contributions << Puppet::Pops::Binder::InjectorEntry.new(b, binder_precedence)
     end
 
     # Bind given abstract binding
@@ -201,35 +205,41 @@ class Puppet::Pops::Binder::Binder
       @@bind_visitor.visit_this(self, binding)
     end
 
-    # @return [Puppet::Pops::Binder::InjectorEntry] the entry with the highest (category) precedence
+    # @return [Puppet::Pops::Binder::InjectorEntry] the entry with the highest precedence
     # @api private
     def highest(b1, b2)
-      case b1.precedence <=> b2.precedence
-      when 1
-        b1
-      when -1
-        b2
-      when 0
-        raise_conflicting_binding(b1, b2)
+      if b1.is_abstract? != b2.is_abstract?
+        # if one is abstract and the other is not, the non abstract wins
+        b1.is_abstract? ? b2 : b1
+      else
+        case b1.precedence <=> b2.precedence
+        when 1
+          b1
+        when -1
+          b2
+        when 0
+          raise_conflicting_binding(b1, b2)
+        end
       end
     end
 
     # Raises a conflicting bindings error given two InjectorEntry's with same precedence in the same layer
     # (if they are in different layers, something is seriously wrong)
     def raise_conflicting_binding(b1, b2)
-      b1_layer_name, b1_bindings_name = Puppet::Pops::Binder::Binder.get_named_binding_layer_and_name(b1.binding)
-      b2_layer_name, b2_bindings_name = Puppet::Pops::Binder::Binder.get_named_binding_layer_and_name(b2.binding)
+      b1_layer_name, b1_bindings_name = binder.class.get_named_binding_layer_and_name(b1.binding)
+      b2_layer_name, b2_bindings_name = binder.class.get_named_binding_layer_and_name(b2.binding)
 
-      # The resolution is per layer, and if they differ something is serious wrong as a higher layer
-      # overrides a lower; so no such conflict should be possible:
+      finality_msg = (b1.is_final? || b2.is_final?) ? ". Override of final binding not allowed" : ''
+
+      # TODO: Use of layer_name is not very good, it is not guaranteed to be unique
       unless b1_layer_name == b2_layer_name
         raise ArgumentError, [
-          'Internal Error: Conflicting binding for',
+          'Conflicting binding for',
           "'#{b1.binding.name}'",
           'being resolved across layers',
           "'#{b1_layer_name}' and",
           "'#{b2_layer_name}'"
-          ].join(' ')
+          ].join(' ')+finality_msg
       end
 
       # Conflicting bindings made from the same source
@@ -241,7 +251,7 @@ class Puppet::Pops::Binder::Binder
           "'#{b1_layer_name}', ",
           'both from:',
           "'#{b1_bindings_name}'"
-          ].join(' ')
+          ].join(' ')+finality_msg
       end
 
       # Conflicting bindings from different sources
@@ -253,7 +263,7 @@ class Puppet::Pops::Binder::Binder
         'from:',
         "'#{b1_bindings_name}', and",
         "'#{b2_bindings_name}'"
-        ].join(' ')
+        ].join(' ')+finality_msg
     end
 
 
@@ -276,33 +286,6 @@ class Puppet::Pops::Binder::Binder
       ! binding.multibind_id.nil?
     end
 
-    # @api private
-    def push_precedences(precedences)
-      prec_stack.push(precedences)
-      @effective_prec = nil # clear cache
-    end
-
-    # @api private
-    def pop_precedences()
-      prec_stack.pop()
-      @effective_prec = nil # clear cache
-    end
-
-    # Returns the effective precedence as an array with highest precedence first.
-    # Internally the precedence is an array with the highest precedence first.
-    #
-    # @api private
-    #
-    def effective_prec()
-      unless @effective_prec
-        @effective_prec = prec_stack.flatten.uniq.sort.reverse
-        if @effective_prec.size == 0
-          @effective_prec = [ 0 ] # i.e. "common"
-        end
-      end
-      @effective_prec
-    end
-
     # @api private
     def bind_Binding(o)
       if is_contribution?(o)
@@ -323,27 +306,6 @@ class Puppet::Pops::Binder::Binder
       o.bindings.each {|b| bind(b) }
     end
 
-    # Process CategorizedBindings by calculating precedence, and then if satisfying the predicates, process the contained
-    # bindings.
-    # @api private
-    #
-    def bind_CategorizedBindings(o)
-      precedences = o.predicates.collect do |p|
-        prec = binder.category_precedences[p.categorization]
-
-        # Skip bindings if the categorization is not present, or
-        # if the category value is not the effective value for the categorization
-        # Ignore the value for the common category (it is not possible to state common 'false' etc.)
-        #
-        return unless prec
-        return unless binder.category_values[p.categorization] == p.value.downcase || p.categorization == 'common'
-        prec
-      end
-      push_precedences(precedences)
-      o.bindings.each {|b| bind(b) }
-      pop_precedences()
-    end
-
     # Process layered bindings from highest to lowest layer
     # @api private
     #
@@ -380,12 +342,22 @@ class Puppet::Pops::Binder::Binder
       bindings.each do |b|
         bkey = key(b.binding)
 
-        # ignore if a higher layer defined it, but ensure override gets resolved
+        # ignore if a higher layer defined it (unless the lower is final), but ensure override gets resolved
+        # (override is not resolved across binders)
         if x = binder.injector_entries[bkey]
+          if b.is_final?
+            raise_conflicting_binding(x, b)
+          end
           x.mark_override_resolved()
           next
         end
 
+        # If a lower (parent) binder exposes a final binding it may not be overridden
+        #
+        if (x = binder.lookup_in_parent(bkey)) && x.is_final?
+          raise_conflicting_binding(x, b)
+        end
+
         # if already found in this layer, one wins (and resolves override), or it is an error
         existing = this_layer[bkey]
         winner = existing ? highest(existing, b) : b