chef 13.6.4 → 13.7.16

data/lib/chef/node/attribute_collections.rb

@@ -63,13 +63,13 @@ class Chef
       MUTATOR_METHODS.each do |mutator|
         define_method(mutator) do |*args, &block|
           ret = super(*args, &block)
-          send_reset_cache
+          send_reset_cache(__path__)
           ret
         end
       end
 
       def delete(key, &block)
-        send_reset_cache(__path__, key)
+        send_reset_cache(__path__)
         super
       end
 
@@ -147,13 +147,13 @@ class Chef
       # object.
 
       def delete(key, &block)
-        send_reset_cache(__path__, key)
+        send_reset_cache(__path__)
         super
       end
 
       MUTATOR_METHODS.each do |mutator|
         define_method(mutator) do |*args, &block|
-          send_reset_cache
+          send_reset_cache(__path__)
           super(*args, &block)
         end
       end
@@ -174,7 +174,7 @@ class Chef
 
       def []=(key, value)
         ret = super
-        send_reset_cache(__path__, key)
+        send_reset_cache(__path__)
         ret
       end
 

data/lib/chef/node/common_api.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright:: Copyright 2016, Chef Software, Inc.
+# Copyright:: Copyright 2016-2017, Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/chef/node/immutable_collections.rb

@@ -30,22 +30,16 @@ class Chef
         e
       end
 
-      def convert_value(value)
+      def convert_value(value, path = nil)
         case value
         when Hash
-          ImmutableMash.new(value, __root__, __node__, __precedence__)
+          ImmutableMash.new({}, __root__, __node__, __precedence__, path)
         when Array
-          ImmutableArray.new(value, __root__, __node__, __precedence__)
-        when ImmutableMash, ImmutableArray
-          value
+          ImmutableArray.new([], __root__, __node__, __precedence__, path)
         else
           safe_dup(value).freeze
         end
       end
-
-      def immutablize(value)
-        convert_value(value)
-      end
     end
 
     # == ImmutableArray
@@ -59,17 +53,51 @@ class Chef
     #   Chef::Node::Attribute's values, it overrides all reader methods to
     #   detect staleness and raise an error if accessed when stale.
     class ImmutableArray < Array
+      alias_method :internal_clear, :clear
+      alias_method :internal_replace, :replace
+      alias_method :internal_push, :<<
+      alias_method :internal_to_a, :to_a
+      alias_method :internal_each, :each
+      private :internal_push, :internal_replace, :internal_clear, :internal_each
+      protected :internal_to_a
+
       include Immutablize
 
-      alias :internal_push :<<
-      private :internal_push
+      methods = Array.instance_methods - Object.instance_methods +
+        [ :!, :!=, :<=>, :==, :===, :eql?, :to_s, :hash, :key, :has_key?, :inspect, :pretty_print, :pretty_print_inspect, :pretty_print_cycle, :pretty_print_instance_variables ]
 
-      def initialize(array_data = [])
-        array_data.each do |value|
-          internal_push(immutablize(value))
+      methods.each do |method|
+        define_method method do |*args, &block|
+          ensure_generated_cache!
+          super(*args, &block)
         end
       end
 
+      def each
+        ensure_generated_cache!
+        # aggressively pre generate the cache, works around ruby being too smart and fiddling with internals
+        internal_each { |i| i.ensure_generated_cache! if i.respond_to?(:ensure_generated_cache!) }
+        super
+      end
+
+      # because sometimes ruby gives us back Arrays or ImmutableArrays out of objects from things like #uniq or array slices
+      def return_normal_array(array)
+        if array.respond_to?(:internal_to_a, true)
+          array.internal_to_a
+        else
+          array.to_a
+        end
+      end
+
+      def uniq
+        ensure_generated_cache!
+        return_normal_array(super)
+      end
+
+      def initialize(array_data = [])
+        # Immutable collections no longer have initialized state
+      end
+
       # For elements like Fixnums, true, nil...
       def safe_dup(e)
         e.dup
@@ -96,8 +124,81 @@ class Chef
 
       alias_method :to_array, :to_a
 
+      def [](*args)
+        ensure_generated_cache!
+        args.length > 1 ? return_normal_array(super) : super # correctly handle array slices
+      end
+
+      def reset
+        @generated_cache = false
+        @short_circuit_attr_level = nil
+        internal_clear # redundant?
+      end
+
+      # @api private
+      def ensure_generated_cache!
+        generate_cache unless @generated_cache
+        @generated_cache = true
+      end
+
+      # This can be set to e.g. [ :@default ] by the parent container to cause this container
+      # to only use the default level and to bypass deep merging (the common case is either
+      # default-level or automatic-level and we aren't doing any deep merging).  Right now it
+      # "optimized" for the case where we're no longer merging anything and only tracking a
+      # single level, and setting this to anything other than a size=1 array would behave
+      # in a broken fashion.  That could be fixed, but the perf boost would likely not be
+      # that large in the typical case.
+      #
+      # @api private
+      attr_accessor :short_circuit_attr_levels
+
       private
 
+      # deep merging of array attribute within normal and override where they are merged together
+      def combined_components(components)
+        combined_values = nil
+        components.each do |component|
+          values = __node__.attributes.instance_variable_get(component).read(*__path__)
+          next unless values.is_a?(Array)
+          @tracked_components << component
+          combined_values ||= []
+          combined_values += values
+        end
+        combined_values
+      end
+
+      def get_array(component)
+        array = __node__.attributes.instance_variable_get(component).read(*__path__)
+        if array.is_a?(Array)
+          @tracked_components << component
+          array
+        end # else nil
+      end
+
+      def generate_cache
+        internal_clear
+        components = []
+        @tracked_components = []
+        if short_circuit_attr_levels
+          components << get_array(short_circuit_attr_levels.first)
+        else
+          components << combined_components(Attribute::DEFAULT_COMPONENTS)
+          components << get_array(:@normal)
+          components << combined_components(Attribute::OVERRIDE_COMPONENTS)
+          components << get_array(:@automatic)
+        end
+        highest = components.compact.last
+        if highest.is_a?(Array)
+          internal_replace( highest.each_with_index.map { |x, i| convert_value(x, __path__ + [ i ] ) } )
+        end
+        if @tracked_components.size == 1
+          # tracked_components is accurate enough to tell us if we're not really merging
+          internal_each do |key, value|
+            value.short_circuit_attr_levels = @tracked_components if value.respond_to?(:short_circuit_attr_levels)
+          end
+        end
+      end
+
       # needed for __path__
       def convert_key(key)
         key
@@ -120,19 +221,31 @@ class Chef
     #   it is stale.
     # * Values can be accessed in attr_reader-like fashion via method_missing.
     class ImmutableMash < Mash
+      alias_method :internal_clear, :clear
+      alias_method :internal_key?, :key? # FIXME: could bypass convert_key in Mash for perf
+      alias_method :internal_each, :each
+
       include Immutablize
       include CommonAPI
 
+      methods = Hash.instance_methods - Object.instance_methods +
+        [ :!, :!=, :<=>, :==, :===, :eql?, :to_s, :hash, :key, :has_key?, :inspect, :pretty_print, :pretty_print_inspect, :pretty_print_cycle, :pretty_print_instance_variables ]
+
+      methods.each do |method|
+        define_method method do |*args, &block|
+          ensure_generated_cache!
+          super(*args, &block)
+        end
+      end
+
       # this is for deep_merge usage, chef users must never touch this API
       # @api private
       def internal_set(key, value)
-        regular_writer(key, convert_value(value))
+        regular_writer(key, convert_value(value, __path__ + [ key ]))
       end
 
       def initialize(mash_data = {})
-        mash_data.each do |key, value|
-          internal_set(key, value)
-        end
+        # Immutable collections no longer have initialized state
       end
 
       alias :attribute? :has_key?
@@ -168,11 +281,55 @@ class Chef
 
       alias_method :to_hash, :to_h
 
-      # For elements like Fixnums, true, nil...
-      def safe_dup(e)
-        e.dup
-      rescue TypeError
-        e
+      def [](key)
+        ensure_generated_cache!
+        super
+      end
+
+      alias_method :to_hash, :to_h
+
+      def reset
+        @generated_cache = false
+        @short_circuit_attr_level = nil
+        internal_clear # redundant?
+      end
+
+      # @api private
+      def ensure_generated_cache!
+        generate_cache unless @generated_cache
+        @generated_cache = true
+      end
+
+      # @api private
+      attr_accessor :short_circuit_attr_levels
+
+      private
+
+      def generate_cache
+        internal_clear
+        components = short_circuit_attr_levels ? short_circuit_attr_levels : Attribute::COMPONENTS.reverse
+        # tracked_components is not entirely accurate due to the short-circuit
+        tracked_components = []
+        components.each do |component|
+          subhash = __node__.attributes.instance_variable_get(component).read(*__path__)
+          unless subhash.nil? # FIXME: nil is used for not present
+            tracked_components << component
+            if subhash.kind_of?(Hash)
+              subhash.keys.each do |key|
+                next if internal_key?(key)
+                internal_set(key, subhash[key])
+              end
+            else
+              break
+            end
+          end
+        end
+        if tracked_components.size == 1
+          # tracked_components is accurate enough to tell us if we're not really merging
+          internal_each do |key, value|
+            value.short_circuit_attr_levels = tracked_components if value.respond_to?(:short_circuit_attr_levels)
+          end
+        end
       end
 
       prepend Chef::Node::Mixin::StateTracking

data/lib/chef/node/mixin/state_tracking.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright:: Copyright 2016, Chef Software, Inc.
+# Copyright:: Copyright 2016-2017, Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,11 +24,12 @@ class Chef
         attr_reader :__node__
         attr_reader :__precedence__
 
-        def initialize(data = nil, root = self, node = nil, precedence = nil)
+        def initialize(data = nil, root = self, node = nil, precedence = nil, path = nil)
           # __path__ and __root__ must be nil when we call super so it knows
           # to avoid resetting the cache on construction
           data.nil? ? super() : super(data)
-          @__path__ = []
+          @__path__ = path
+          @__path__ ||= []
           @__root__ = root
           @__node__ = node
           @__precedence__ = precedence
@@ -76,9 +77,8 @@ class Chef
           end
         end
 
-        def send_reset_cache(path = nil, key = nil)
-          next_path = [ path, key ].flatten.compact
-          __root__.reset_cache(next_path.first) if !__root__.nil? && __root__.respond_to?(:reset_cache) && !next_path.nil?
+        def send_reset_cache(path)
+          __root__.reset_cache(*path) if !__root__.nil? && __root__.respond_to?(:reset_cache) && !path.nil?
         end
 
         def copy_state_to(ret, next_path)

data/lib/chef/node_map.rb

@@ -16,6 +16,25 @@
 # limitations under the License.
 #
 
+#
+# example of a NodeMap entry for the user resource (as typed on the DSL):
+#
+#  :user=>
+#  [{:klass=>Chef::Resource::User::AixUser, :os=>"aix"},
+#   {:klass=>Chef::Resource::User::DsclUser, :os=>"darwin"},
+#   {:klass=>Chef::Resource::User::PwUser, :os=>"freebsd"},
+#   {:klass=>Chef::Resource::User::LinuxUser, :os=>"linux"},
+#   {:klass=>Chef::Resource::User::SolarisUser,
+#    :os=>["omnios", "solaris2"]},
+#   {:klass=>Chef::Resource::User::WindowsUser, :os=>"windows"}],
+#
+# the entries in the array are pre-sorted into priority order (blocks/platform_version/platform/platform_family/os/none) so that
+# the first entry's :klass that matches the filter is returned when doing a get.
+#
+# note that as this examples show filter values may be a scalar string or an array of scalar strings.
+#
+# XXX: confusingly, in the *_priority_map the :klass may be an array of Strings of class names
+#
 class Chef
   class NodeMap
 
@@ -31,13 +50,12 @@ class Chef
     #
     # @return [NodeMap] Returns self for possible chaining
     #
-    def set(key, value, platform: nil, platform_version: nil, platform_family: nil, os: nil, canonical: nil, override: nil, &block)
-      filters = {}
-      filters[:platform] = platform if platform
-      filters[:platform_version] = platform_version if platform_version
-      filters[:platform_family] = platform_family if platform_family
-      filters[:os] = os if os
-      new_matcher = { value: value, filters: filters }
+    def set(key, klass, platform: nil, platform_version: nil, platform_family: nil, os: nil, canonical: nil, override: nil, &block)
+      new_matcher = { klass: klass }
+      new_matcher[:platform] = platform if platform
+      new_matcher[:platform_version] = platform_version if platform_version
+      new_matcher[:platform_family] = platform_family if platform_family
+      new_matcher[:os] = os if os
       new_matcher[:block] = block if block
       new_matcher[:canonical] = canonical if canonical
       new_matcher[:override] = override if override
@@ -48,7 +66,10 @@ class Chef
       map[key] ||= []
       map[key].each_with_index do |matcher, index|
         cmp = compare_matchers(key, new_matcher, matcher)
-        insert_at ||= index if cmp && cmp <= 0
+        if cmp && cmp <= 0
+          insert_at = index
+          break
+        end
       end
       if insert_at
         map[key].insert(insert_at, new_matcher)
@@ -68,11 +89,14 @@ class Chef
     # @param canonical [Boolean] `true` or `false` to match canonical or
     #   non-canonical values only. `nil` to ignore canonicality.  Default: `nil`
     #
-    # @return [Object] Value
+    # @return [Object] Class
     #
     def get(node, key, canonical: nil)
-      raise ArgumentError, "first argument must be a Chef::Node" unless node.is_a?(Chef::Node) || node.nil?
-      list(node, key, canonical: canonical).first
+      return nil unless map.has_key?(key)
+      map[key].map do |matcher|
+        return matcher[:klass] if node_matches?(node, matcher) && canonical_matches?(canonical, matcher)
+      end
+      nil
     end
 
     #
@@ -85,23 +109,22 @@ class Chef
     # @param canonical [Boolean] `true` or `false` to match canonical or
     #   non-canonical values only. `nil` to ignore canonicality.  Default: `nil`
     #
-    # @return [Object] Value
+    # @return [Object] Class
     #
     def list(node, key, canonical: nil)
-      raise ArgumentError, "first argument must be a Chef::Node" unless node.is_a?(Chef::Node) || node.nil?
       return [] unless map.has_key?(key)
       map[key].select do |matcher|
         node_matches?(node, matcher) && canonical_matches?(canonical, matcher)
-      end.map { |matcher| matcher[:value] }
+      end.map { |matcher| matcher[:klass] }
     end
 
     # Seriously, don't use this, it's nearly certain to change on you
     # @return remaining
     # @api private
-    def delete_canonical(key, value)
+    def delete_canonical(key, klass)
       remaining = map[key]
       if remaining
-        remaining.delete_if { |matcher| matcher[:canonical] && Array(matcher[:value]) == Array(value) }
+        remaining.delete_if { |matcher| matcher[:canonical] && Array(matcher[:klass]) == Array(klass) }
         if remaining.empty?
           map.delete(key)
           remaining = nil
@@ -143,7 +166,7 @@ class Chef
 
       filter_values.empty? ||
         Array(filter_values).any? do |v|
-          Chef::VersionConstraint::Platform.new(v).include?(value)
+          Gem::Requirement.new(v).satisfied_by?(Gem::Version.new(value))
         end
     end
 
@@ -161,7 +184,7 @@ class Chef
 
     def node_matches?(node, matcher)
       return true if !node
-      filters_match?(node, matcher[:filters]) && block_matches?(node, matcher[:block])
+      filters_match?(node, matcher) && block_matches?(node, matcher[:block])
     end
 
     def canonical_matches?(canonical, matcher)
@@ -171,17 +194,17 @@ class Chef
 
     # @api private
     def dispatch_compare_matchers(key, new_matcher, matcher)
-      cmp = compare_matcher_properties(new_matcher, matcher) { |m| m[:block] }
+      cmp = compare_matcher_properties(new_matcher[:block], matcher[:block])
       return cmp if cmp != 0
-      cmp = compare_matcher_properties(new_matcher, matcher) { |m| m[:filters][:platform_version] }
+      cmp = compare_matcher_properties(new_matcher[:platform_version], matcher[:platform_version])
       return cmp if cmp != 0
-      cmp = compare_matcher_properties(new_matcher, matcher) { |m| m[:filters][:platform] }
+      cmp = compare_matcher_properties(new_matcher[:platform], matcher[:platform])
       return cmp if cmp != 0
-      cmp = compare_matcher_properties(new_matcher, matcher) { |m| m[:filters][:platform_family] }
+      cmp = compare_matcher_properties(new_matcher[:platform_family], matcher[:platform_family])
       return cmp if cmp != 0
-      cmp = compare_matcher_properties(new_matcher, matcher) { |m| m[:filters][:os] }
+      cmp = compare_matcher_properties(new_matcher[:os], matcher[:os])
       return cmp if cmp != 0
-      cmp = compare_matcher_properties(new_matcher, matcher) { |m| m[:override] }
+      cmp = compare_matcher_properties(new_matcher[:override], matcher[:override])
       return cmp if cmp != 0
       # If all things are identical, return 0
       0
@@ -195,37 +218,32 @@ class Chef
       if cmp == 0
         # Sort by class name (ascending) as well, if all other properties
         # are exactly equal
-        if new_matcher[:value].is_a?(Class) && !new_matcher[:override]
-          cmp = compare_matcher_properties(new_matcher, matcher) { |m| m[:value].name }
+        # XXX: remove this in Chef-14 and use last-writer-wins (prepend if they match)
+        if !new_matcher[:override]
+          # we only sort classes, which only sorts the handler array, this magically does not sort
+          # the priority array via the invisible else here.
+          if new_matcher[:klass].is_a?(Class)
+            cmp = compare_matcher_properties(new_matcher[:klass].name, matcher[:klass].name)
+          end
         end
       end
       cmp
     end
 
-    def compare_matcher_properties(new_matcher, matcher)
-      a = yield(new_matcher)
-      b = yield(matcher)
+    def compare_matcher_properties(a, b)
+      # falsity comparisons here handle both "nil" and "false"
+      return 1 if !a && b
+      return -1 if !b && a
+      return 0 if !a && !b
 
-      # Check for blcacklists ('!windows'). Those always come *after* positive
+      # Check for blacklists ('!windows'). Those always come *after* positive
       # whitelists.
       a_negated = Array(a).any? { |f| f.is_a?(String) && f.start_with?("!") }
       b_negated = Array(b).any? { |f| f.is_a?(String) && f.start_with?("!") }
-      if a_negated != b_negated
-        return 1 if a_negated
-        return -1 if b_negated
-      end
+      return 1 if a_negated && !b_negated
+      return -1 if b_negated && !a_negated
 
-      # We treat false / true and nil / not-nil with the same comparison
-      a = nil if a == false
-      b = nil if b == false
-      cmp = a <=> b
-      # This is the case where one is non-nil, and one is nil. The one that is
-      # nil is "greater" (i.e. it should come last).
-      if cmp.nil?
-        return 1 if a.nil?
-        return -1 if b.nil?
-      end
-      cmp
+      a <=> b
     end
 
     def map