puppet 3.8.7 → 4.0.0.rc1

data/lib/puppet/pops/loader/dependency_loader.rb

@@ -26,9 +26,9 @@ class Puppet::Pops::Loader::DependencyLoader < Puppet::Pops::Loader::BaseLoader
   # Finds name in a loader this loader depends on / can see
   #
   def find(typed_name)
-    if typed_name.qualified
-      if loader = index()[typed_name.name_parts[0]]
-        loader.load_typed(typed_name)
+    if typed_name.qualified?
+      if l = index()[typed_name.name_parts[0]]
+        l.load_typed(typed_name)
       else
         # no module entered as dependency with name matching first segment of wanted name
         nil

data/lib/puppet/pops/loader/loader.rb

@@ -137,43 +137,44 @@ class Puppet::Pops::Loader::Loader
   # A name/type combination that can be used as a compound hash key
   #
   class TypedName
+    DOUBLE_COLON = '::'
+
+    attr_reader :hash
     attr_reader :type
     attr_reader :name
     attr_reader :name_parts
-
-    # True if name is qualified (more than a single segment)
-    attr_reader :qualified
+    attr_reader :compound_name
 
     def initialize(type, name)
       @type = type
       # relativize the name (get rid of leading ::), and make the split string available
-      @name_parts = name.to_s.split(/::/)
-      @name_parts.shift if name_parts[0].empty?
-      @name = name_parts.join('::')
-      @qualified = name_parts.size > 1
-      # precompute hash - the name is frozen, so this is safe to do
-      @hash = [self.class, type, @name].hash
-
-      # Not allowed to have numeric names - 0, 010, 0x10, 1.2 etc
-      if Puppet::Pops::Utils.is_numeric?(@name)
-        raise ArgumentError, "Illegal attempt to use a numeric name '#{name}' at #{origin_label(origin)}."
+      parts = name.to_s.split(DOUBLE_COLON)
+      if parts[0].empty?
+        parts.shift
+        @name = name[2..-1]
+      else
+        @name = name
       end
+      @name_parts = parts
 
-      freeze()
-    end
-
-    def hash
-      @hash
+      # Use a frozen compound key for the hash and comparison
+      @compound_name = "#{type}/#{name}".freeze
+      @hash = @compound_name.hash
+      freeze
     end
 
     def ==(o)
-      o.class == self.class && type == o.type && name == o.name
+      o.class == self.class && o.compound_name == @compound_name
     end
 
     alias eql? ==
 
+    def qualified?
+      @name_parts.size > 1
+    end
+
     def to_s
-      "#{type}/#{name}"
+      @compound_name
     end
   end
 end

data/lib/puppet/pops/loaders.rb

@@ -12,11 +12,9 @@ class Puppet::Pops::Loaders
 
     # Create the set of loaders
     # 1. Puppet, loads from the "running" puppet - i.e. bundled functions, types, extension points and extensions
-    #    These cannot be cached since a  loaded instance will be bound to its closure scope which holds on to
-    #    a compiler and all loaded types. Subsequent request would find remains of the environment that loaded
-    #    the content. PUP-4461.
+    #    Does not change without rebooting the service running puppet.
     #
-    @puppet_system_loader = create_puppet_system_loader()
+    @@puppet_system_loader ||= create_puppet_system_loader()
 
     # 2. Environment loader - i.e. what is bound across the environment, may change for each setup
     #    TODO: loaders need to work when also running in an agent doing catalog application. There is no
@@ -32,7 +30,7 @@ class Puppet::Pops::Loaders
   #
   def self.clear
     @@static_loader = nil
-    @puppet_system_loader = nil
+    @@puppet_system_loader = nil
   end
 
   def static_loader
@@ -40,7 +38,7 @@ class Puppet::Pops::Loaders
   end
 
   def puppet_system_loader
-    @puppet_system_loader
+    @@puppet_system_loader
   end
 
   def public_loader_for_module(module_name)
@@ -88,7 +86,14 @@ class Puppet::Pops::Loaders
     # The environment is not a namespace, so give it a nil "module_name"
     module_name = nil
     loader_name = "environment:#{environment.name}"
-    loader = Puppet::Pops::Loader::SimpleEnvironmentLoader.new(puppet_system_loader, loader_name)
+    env_conf = Puppet.lookup(:environments).get_conf(environment.name)
+    if env_conf.nil? || !env_conf.is_a?(Puppet::Settings::EnvironmentConf)
+      # Not a real directory environment, cannot work as a module TODO: Drop when legacy env are dropped?
+      loader = Puppet::Pops::Loader::SimpleEnvironmentLoader.new(puppet_system_loader, loader_name)
+    else
+      # View the environment as a module to allow loading from it - this module is always called 'environment'
+      loader = Puppet::Pops::Loader::ModuleLoaders.module_loader_from(puppet_system_loader, self, 'environment', env_conf.path_to_env)
+    end
 
     # An environment has a module path even if it has a null loader
     configure_loaders_for_modules(loader, environment)
@@ -215,7 +220,7 @@ class Puppet::Pops::Loaders
     private
 
     def create_loader_with_all_modules_visible(from_module_data)
-      Puppet.debug("ModuleLoader: module '#{from_module_data.name}' has unknown dependencies - it will have all other modules visible")
+      Puppet.debug{"ModuleLoader: module '#{from_module_data.name}' has unknown dependencies - it will have all other modules visible"}
 
       Puppet::Pops::Loader::DependencyLoader.new(from_module_data.public_loader, from_module_data.name, all_module_loaders())
     end

data/lib/puppet/pops/lookup.rb

@@ -0,0 +1,96 @@
+# This class is the backing implementation of the Puppet function 'lookup'.
+# See puppet/functions/lookup.rb for documentation.
+#
+class Puppet::Pops::Lookup
+  # Performs a lookup in the configured scopes and optionally merges the default.
+  #
+  # This is a backing function and all parameters are assumed to have been type checked.
+  # See puppet/functions/lookup.rb for full documentation and all parameter combinations.
+  #
+  # @param scope [Puppet::Parser::Scope] The scope to use for the lookup
+  # @param name [String|Array<String>] The name or names to lookup
+  # @param type [Puppet::Pops::Types::PAnyType|nil] The expected type of the found value
+  # @param default_value [Object] The value to use as default when no value is found
+  # @param has_default [Boolean] Set to _true_ if _default_value_ is included (_nil_ is a valid _default_value_)
+  # @param override [Hash<String,Object>|nil] A map to use as override. Values found here are returned immediately (no merge)
+  # @param default_values_hash [Hash<String,Object>] A map to use as the last resort (but before default)
+  # @param merge [String|Hash<String,Object>|nil] Merge strategy or hash with strategy and options
+  # @return [Object] The found value
+  #
+  def self.lookup(scope, name, value_type, default_value, has_default, override, default_values_hash, merge)
+    value_type = Puppet::Pops::Types::PDataType.new if value_type.nil?
+    names = name.is_a?(Array) ? name : [name]
+
+    # find first name that yields a non-nil result and wrap it in a two element array
+    # with name and value.
+    not_found = Object.new
+    result_with_name = names.reduce([nil,not_found]) do |memo, key|
+      value = override.include?(key) ? assert_type('override', value_type, override[key]) : not_found
+      value = search_and_merge(key, scope, merge, not_found) if value.equal?(not_found)
+      break [key, assert_type('found', value_type, value)] unless value.equal?(not_found)
+      memo
+    end
+
+    # Use the 'default_values_hash' map as a last resort if nothing is found
+    if result_with_name[1].equal?(not_found) && !default_values_hash.empty?
+      result_with_name = names.reduce(result_with_name) do |memo, key|
+        value = default_values_hash.include?(key) ? assert_type('default_values_hash', value_type, default_values_hash[key]) : not_found
+        memo = [ key, value ]
+        break memo unless value.equal?(not_found)
+        memo
+      end
+    end
+
+    answer = result_with_name[1]
+    if answer.equal?(not_found)
+      if block_given?
+        answer = assert_type('default_block', value_type, yield(name))
+      elsif has_default
+        answer = assert_type('default_value', value_type, default_value)
+      else
+        fail_lookup(names)
+      end
+    end
+    answer
+  end
+
+  def self.search_and_merge(name, scope, merge, not_found)
+    in_global = lambda { lookup_with_databinding(name, scope, merge) }
+    in_env = lambda { Puppet::DataProviders.lookup_in_environment(name, scope, merge) }
+    in_module = lambda { Puppet::DataProviders.lookup_in_module(name, scope, merge) }
+
+    [in_global, in_env, in_module].reduce(not_found) do |memo, f|
+      found = false # can't trust catch return value since nil is valid
+      value = catch (:no_such_key) do
+        answer = f.call
+        found = true
+        answer
+      end
+      next memo unless found
+      break value if merge.nil? # value found and no merge
+      strategy = Puppet::Pops::MergeStrategy.strategy(merge)
+      memo.equal?(not_found) ? strategy.convert_value(value) : strategy.merge(memo, value)
+    end
+  end
+  private_class_method :search_and_merge
+
+  def self.lookup_with_databinding(key, scope, merge)
+    begin
+      Puppet::DataBinding.indirection.find(key, { :environment => scope.environment.to_s, :variables => scope, :merge => merge })
+    rescue Puppet::DataBinding::LookupError => e
+      raise Puppet::Error, "Error from DataBinding '#{Puppet[:data_binding_terminus]}' while looking up '#{name}': #{e.message}", e
+    end
+  end
+  private_class_method :lookup_with_databinding
+
+  def self.assert_type(subject, type, value)
+    Puppet::Pops::Types::TypeAsserter.assert_instance_of(subject, type, value)
+  end
+  private_class_method :assert_type
+
+  def self.fail_lookup(names)
+    name_part = names.size == 1 ? "the name '#{names[0]}'" : 'any of the names [' + names.map {|n| "'#{n}'"} .join(', ') + ']'
+    raise Puppet::Error, "Function lookup() did not find a value for #{name_part}"
+  end
+  private_class_method :fail_lookup
+end

data/lib/puppet/pops/merge_strategy.rb

@@ -0,0 +1,283 @@
+require 'deep_merge/core'
+
+module Puppet::Pops
+  # Merges to objects into one based on an implemented strategy.
+  #
+  class MergeStrategy
+    TypeAsserter = Puppet::Pops::Types::TypeAsserter
+    TypeParser = Puppet::Pops::Types::TypeParser
+
+    # The type used for validation of the _merge_ argument
+    def self.merge_t
+      @@merge_t ||=  TypeParser.new.parse("Variant[String[1],Runtime[ruby,'Symbol'],Hash[Variant[String[1],Runtime[ruby,'Symbol']],Scalar,1]]")
+    end
+    private_class_method :merge_t
+
+    def self.strategies
+      @@strategies ||= {}
+    end
+    private_class_method :strategies
+
+    # Finds the merge strategy for the given _merge_, creates an instance of it and returns that instance.
+    #
+    # @param merge [String|Symbol|Hash<String,Object>] The merge strategy. Can be a string or symbol denoting the key
+    #   identifier or a hash with options where the key 'strategy' denotes the key
+    # @return [MergeStrategy] The matching merge strategy
+    #
+    def self.strategy(merge)
+      TypeAsserter.assert_instance_of("MergeStrategies.merge 'merge' parameter", merge_t, merge)
+      if merge.is_a?(Hash)
+        merge_strategy = merge['strategy']
+        if merge_strategy.nil?
+          raise ArgumentError, "The hash given as 'merge' must contain the name of a strategy in string form for the key 'strategy'"
+        end
+        merge_options  = merge
+      else
+        merge_strategy = merge
+        merge_options  = {}
+      end
+      strategy = strategies[merge_strategy.to_sym]
+      raise ArgumentError, "Unknown merge strategy: '#{merge_strategy}'" if strategy.nil?
+      strategy.new(merge_options)
+    end
+
+    # Returns the list of merge strategy keys known to this class
+    #
+    # @return [Array<Symbol>] List of strategy keys
+    #
+    def self.strategy_keys
+      strategies.keys
+    end
+
+    # Adds a new merge strategy to the map of strategies known to this class
+    #
+    # @param strategy_class [Class<MergeStrategy>] The class of the added strategy
+    #
+    def self.add_strategy(strategy_class)
+      unless MergeStrategy > strategy_class
+        raise ArgumentError, "MergeStrategies.add_strategy 'strategy_class' must be a 'MergeStrategy' class. Got #{strategy_class}"
+      end
+      strategies[strategy_class.key] = strategy_class
+      nil
+    end
+
+    # Finds a merge strategy that corresponds to the given _merge_ argument and delegates the task of merging the elements of _e1_ and _e2_ to it.
+    #
+    # @param e1 [Object] The first element
+    # @param e2 [Object] The second element
+    # @return [Object] The result of the merge
+    #
+    def self.merge(e1, e2, merge)
+      strategy(merge).merge(e1, e2)
+    end
+
+    def self.key
+      raise NotImplementedError, "Subclass must implement 'key'"
+    end
+
+    # Create a new instance of this strategy configured with the given _options_
+    # @param merge_options [Hash<String,Object>] Merge options
+    def initialize(options)
+      assert_type('merge_options', options_t, options)
+      @options = options
+    end
+
+    # Merges the elements of _e1_ and _e2_ accoring to the implemented strategy and options given when this
+    # instance was created
+    #
+    # @param e1 [Object] The first element
+    # @param e2 [Object] The second element
+    # @return [Object] The result of the merge
+    #
+    def merge(e1, e2)
+      checked_merge(
+        assert_type('e1', value_t, e1),
+        assert_type('e2', value_t, e2))
+    end
+
+    # Converts a single value to the type expeced when peforming a merge of two elements
+    # @param value [Object] the value to convert
+    # @return [Object] the converted value
+    def convert_value(value)
+      value
+    end
+
+    protected
+
+    def options
+      @options
+    end
+
+    # Returns the type used to validate the options hash
+    #
+    # @return [Puppet::Pops::Types::PStructType] the puppet type
+    #
+    def options_t
+      @options_t ||=TypeParser.new.parse("Struct[{strategy=>Optional[Pattern[#{self.class.key}]]}]")
+    end
+
+    # Returns the type used to validate the options hash
+    #
+    # @return [Puppet::Pops::Types::PAnyType] the puppet type
+    #
+    def value_t
+      raise NotImplementedError, "Subclass must implement 'value_t'"
+    end
+
+    def checked_merge(e1, e2)
+      raise NotImplementedError, "Subclass must implement 'checked_merge(e1,e2)'"
+    end
+
+    def assert_type(param, type, value)
+      TypeAsserter.assert_instance_of(param, type, value)
+    end
+  end
+
+  # Produces a new hash by merging hash e1 with hash e2 in such a way that the values of duplicate keys
+  # will be those of e1
+  #
+  class HashMergeStrategy < MergeStrategy
+    def self.key
+      :hash
+    end
+
+    # @param e1 [Hash<String,Object>] The hash that will act as the source of the merge
+    # @param e2 [Hash<String,Object>] The hash that will act as the receiver for the merge
+    # @return [Hash<String,Object]] The merged hash
+    # @see Hash#merge
+    def checked_merge(e1, e2)
+      e2.merge(e1)
+    end
+
+    protected
+
+    def value_t
+      @value_t ||= Puppet::Pops::Types::TypeParser.new.parse('Hash[String,Data]')
+    end
+
+    MergeStrategy.add_strategy(self)
+  end
+
+  # Merges two values that must be either scalar or arrays into a unique set of values.
+  #
+  # Scalar values will be converted into a one element arrays and array values will be flattened
+  # prior to forming the unique set. The order of the elements is preserved with e1 being the
+  # first contributor of elements and e2 the second.
+  #
+  class UniqueMergeStrategy < MergeStrategy
+    def self.key
+      :unique
+    end
+
+    # @param e1 [Array<Object>] The first array
+    # @param e2 [Array<Object>] The second array
+    # @return [Array<Object>] The unique set of elements
+    #
+    def checked_merge(e1, e2)
+      convert_value(e1) | convert_value(e2)
+    end
+
+    def convert_value(e)
+      e.is_a?(Array) ? e.flatten : [e]
+    end
+
+    protected
+
+    def value_t
+      @value_t ||= Puppet::Pops::Types::TypeParser.new.parse('Variant[Scalar,Array[Data]]')
+    end
+
+    MergeStrategy.add_strategy(self)
+  end
+
+  # Documentation copied from https://github.com/danielsdeleo/deep_merge/blob/master/lib/deep_merge/core.rb
+  # altered with respect to _preserve_unmergeables_ since this implementation always disables that option.
+  #
+  # The destination is dup'ed before the deep_merge is called to allow frozen objects as values.
+  #
+  # deep_merge method permits merging of arbitrary child elements. The two top level
+  # elements must be hashes. These hashes can contain unlimited (to stack limit) levels
+  # of child elements. These child elements to not have to be of the same types.
+  # Where child elements are of the same type, deep_merge will attempt to merge them together.
+  # Where child elements are not of the same type, deep_merge will skip or optionally overwrite
+  # the destination element with the contents of the source element at that level.
+  # So if you have two hashes like this:
+  #   source = {:x => [1,2,3], :y => 2}
+  #   dest =   {:x => [4,5,'6'], :y => [7,8,9]}
+  #   dest.deep_merge!(source)
+  #   Results: {:x => [1,2,3,4,5,'6'], :y => 2}
+  #
+  # "deep_merge" will unconditionally overwrite any unmergeables and merge everything else.
+  #
+  # Options:
+  #   Options are specified in the last parameter passed, which should be in hash format:
+  #   hash.deep_merge!({:x => [1,2]}, {:knockout_prefix => '--'})
+  #   - 'knockout_prefix' Set to string value to signify prefix which deletes elements from existing element. Defaults is _undef_
+  #   - 'sort_merged_arrays' Set to _true_ to sort all arrays that are merged together. Default is _false_
+  #   - 'unpack_arrays' Set to string value used as a deliminator to join all array values and then split them again. Default is _undef_
+  #   - 'merge_hash_arrays' Set to _true_ to merge hashes within arrays. Default is _false_
+  #
+  # Selected Options Details:
+  # :knockout_prefix => The purpose of this is to provide a way to remove elements
+  #   from existing Hash by specifying them in a special way in incoming hash
+  #    source = {:x => ['--1', '2']}
+  #    dest   = {:x => ['1', '3']}
+  #    dest.ko_deep_merge!(source)
+  #    Results: {:x => ['2','3']}
+  #   Additionally, if the knockout_prefix is passed alone as a string, it will cause
+  #   the entire element to be removed:
+  #    source = {:x => '--'}
+  #    dest   = {:x => [1,2,3]}
+  #    dest.ko_deep_merge!(source)
+  #    Results: {:x => ""}
+  # :unpack_arrays => The purpose of this is to permit compound elements to be passed
+  #   in as strings and to be converted into discrete array elements
+  #   irsource = {:x => ['1,2,3', '4']}
+  #   dest   = {:x => ['5','6','7,8']}
+  #   dest.deep_merge!(source, {:unpack_arrays => ','})
+  #   Results: {:x => ['1','2','3','4','5','6','7','8']}
+  #   Why: If receiving data from an HTML form, this makes it easy for a checkbox
+  #    to pass multiple values from within a single HTML element
+  #
+  # :merge_hash_arrays => merge hashes within arrays
+  #   source = {:x => [{:y => 1}]}
+  #   dest   = {:x => [{:z => 2}]}
+  #   dest.deep_merge!(source, {:merge_hash_arrays => true})
+  #   Results: {:x => [{:y => 1, :z => 2}]}
+  #
+  class DeepMergeStrategy < MergeStrategy
+    def self.key
+      :deep
+    end
+
+    def checked_merge(e1, e2)
+      dm_options = { :preserve_unmergeables => false }
+      options.each_pair { |k,v| dm_options[k.to_sym] = v unless k == 'strategy' }
+      # e2 (the destination) is dup'ed to avoid that the passed in object mutates
+      DeepMerge.deep_merge!(e1, e2.dup, dm_options)
+    end
+
+    protected
+
+    # Returns a type that allows all deep_merge options except 'preserve_unmergeables' since we force
+    # the setting of that option to false
+    #
+    # @return [Puppet::Pops::Types::PAnyType] the puppet type used when validating the options hash
+    def options_t
+      @options_t ||= Puppet::Pops::Types::TypeParser.new.parse('Struct[{'\
+          "strategy=>Optional[Pattern[#{self.class.key}]],"\
+          'knockout_prefix=>Optional[String],'\
+          'merge_debug=>Optional[Boolean],'\
+          'merge_hash_arrays=>Optional[Boolean],'\
+          'sort_merge_arrays=>Optional[Boolean],'\
+          'unpack_arrays=>Optional[String]'\
+          '}]')
+    end
+
+    def value_t
+      @value_t ||= Puppet::Pops::Types::TypeParser.new.parse('Variant[Array[Data],Hash[String,Data]]')
+    end
+
+    MergeStrategy.add_strategy(self)
+  end
+end