inventory_refresh 0.1.2 → 0.2.3

data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb

@@ -0,0 +1,132 @@
+require_relative "../helpers"
+
+module InventoryRefresh
+  class InventoryCollection
+    module Helpers
+      module QuestionsHelper
+        # @return [Boolean] true means we want to call .changed? on every ActiveRecord object before saving it
+        def check_changed?
+          check_changed
+        end
+
+        # @return [Boolean] true means we want to use ActiveRecord object for writing attributes and we want to perform
+        #         casting on all columns
+        def use_ar_object?
+          use_ar_object
+        end
+
+        # @return [Boolean] true means the data is not complete, leading to only creating and updating data
+        def complete?
+          complete
+        end
+
+        # @return [Boolean] true means we want to only update data
+        def update_only?
+          update_only
+        end
+
+        # @return [Boolean] true means we will delete/soft-delete data
+        def delete_allowed?
+          complete? && !update_only?
+        end
+
+        # @return [Boolean] true means we will delete/soft-delete data
+        def create_allowed?
+          !update_only?
+        end
+
+        # @return [Boolean] true means that only create of new data is allowed
+        def create_only?
+          create_only
+        end
+
+        # @return [Boolean] true if the whole InventoryCollection object has all data persisted
+        def saved?
+          saved
+        end
+
+        # @return [Boolean] true if all dependencies have all data persisted
+        def saveable?
+          dependencies.all?(&:saved?)
+        end
+
+        # @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
+        def parallel_safe?
+          @parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
+        end
+
+        # @return [Boolean] true if the model_class supports STI
+        def supports_sti?
+          @supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
+          @supports_sti_cache
+        end
+
+        # @param column_name [Symbol, String]
+        # @return [Boolean] true if the model_class supports given column
+        def supports_column?(column_name)
+          @supported_cols_cache ||= {}
+          return @supported_cols_cache[column_name.to_sym] unless @supported_cols_cache[column_name.to_sym].nil?
+
+          include_col = model_class.column_names.include?(column_name.to_s)
+          if %w(created_on created_at updated_on updated_at).include?(column_name.to_s)
+            include_col &&= ActiveRecord::Base.record_timestamps
+          end
+
+          @supported_cols_cache[column_name.to_sym] = include_col
+        end
+
+        # @return [Boolean] true if no more data will be added to this InventoryCollection object, that usually happens
+        #         after the parsing step is finished
+        def data_collection_finalized?
+          data_collection_finalized
+        end
+
+        # @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
+        def targeted?
+          targeted
+        end
+
+        # True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
+        # stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
+        #
+        # @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
+        def noop?
+          # If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
+          saving_noop? && delete_complement_noop?
+        end
+
+        # @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
+        def saving_noop?
+          saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
+        end
+
+        # @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
+        def delete_complement_noop?
+          all_manager_uuids.nil?
+        end
+
+        private
+
+        # @return true if it's a noop parent targeted InventoryCollection
+        def saving_targeted_parent_collection_noop?
+          targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
+        end
+
+        # @return true if it's a noop child targeted InventoryCollection
+        def saving_targeted_child_collection_noop?
+          targeted_noop_condition && parent_inventory_collections.present? &&
+            parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
+        end
+
+        # @return true if it's a noop full InventoryCollection refresh
+        def saving_full_collection_noop?
+          !targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
+        end
+
+        def targeted_noop_condition
+          targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
+        end
+      end
+    end
+  end
+end

data/lib/inventory_refresh/inventory_collection/index/type/skeletal.rb

@@ -87,19 +87,19 @@ module InventoryRefresh
           #
           # @param attributes [Hash] Attributes we want to extend with version related attributes
           def fill_versions!(attributes)
-            if inventory_collection.supports_resource_timestamps_max? && attributes[:resource_timestamp]
+            if inventory_collection.supports_column?(:resource_timestamps_max) && attributes[:resource_timestamp]
               fill_specific_version_attr(:resource_timestamps, :resource_timestamp, attributes)
-            elsif inventory_collection.supports_resource_versions_max? && attributes[:resource_version]
-              fill_specific_version_attr(:resource_versions, :resource_version, attributes)
+            elsif inventory_collection.supports_column?(:resource_counters_max) && attributes[:resource_counter]
+              fill_specific_version_attr(:resource_counters, :resource_counter, attributes)
             end
           end
 
           # Add specific versions columns into the passed attributes
           #
           # @param partial_row_version_attr [Symbol] Attr name for partial rows, allowed values are
-          #        [:resource_timestamps, :resource_versions]
+          #        [:resource_timestamps, :resource_counters]
           # @param full_row_version_attr [Symbol] Attr name for full rows, allowed values are
-          #        [:resource_timestamp, :resource_version]
+          #        [:resource_timestamp, :resource_counter]
           # @param attributes [Hash] Attributes we want to extend with version related attributes
           def fill_specific_version_attr(partial_row_version_attr, full_row_version_attr, attributes)
             # We have to symbolize, since serializing persistor makes these strings

data/lib/inventory_refresh/inventory_collection/reference.rb

@@ -35,6 +35,10 @@ module InventoryRefresh
         nested_secondary_index
       end
 
+      def loadable?
+        keys.any? { |attribute| full_reference[attribute].present? }
+      end
+
       class << self
         # Builds string uuid from passed Hash and keys
         #

data/lib/inventory_refresh/inventory_collection/scanner.rb

@@ -13,7 +13,7 @@ module InventoryRefresh
           indexed_inventory_collections = inventory_collections.index_by(&:name)
 
           inventory_collections.each do |inventory_collection|
-            new(inventory_collection, indexed_inventory_collections).scan!
+            new(inventory_collection, indexed_inventory_collections, build_association_hash(inventory_collections)).scan!
           end
 
           inventory_collections.each do |inventory_collection|
@@ -22,9 +22,21 @@ module InventoryRefresh
             end
           end
         end
+
+        def build_association_hash(inventory_collections)
+          associations_hash = {}
+          parents = inventory_collections.map(&:parent).compact.uniq
+          parents.each do |parent|
+            parent.class.reflect_on_all_associations(:has_many).each do |association|
+              through_assoc = association.options.try(:[], :through)
+              associations_hash[association.name] = through_assoc if association.options.try(:[], :through)
+            end
+          end
+          associations_hash
+        end
       end
 
-      attr_reader :inventory_collection, :indexed_inventory_collections
+      attr_reader :associations_hash, :inventory_collection, :indexed_inventory_collections
 
       # Boolean helpers the scanner uses from the :inventory_collection
       delegate :inventory_object_lazy?,
@@ -40,19 +52,26 @@ module InventoryRefresh
                :to => :inventory_collection
 
       # The data scanner modifies inside of the :inventory_collection
-      delegate :attribute_references,
+      delegate :all_manager_uuids_scope,
+               :association,
+               :arel,
+               :attribute_references,
+               :custom_save_block,
                :data_collection_finalized=,
                :dependency_attributes,
+               :targeted?,
                :targeted_scope,
+               :parent,
                :parent_inventory_collections,
                :parent_inventory_collections=,
                :references,
                :transitive_dependency_attributes,
                :to => :inventory_collection
 
-      def initialize(inventory_collection, indexed_inventory_collections)
+      def initialize(inventory_collection, indexed_inventory_collections, associations_hash)
         @inventory_collection          = inventory_collection
         @indexed_inventory_collections = indexed_inventory_collections
+        @associations_hash             = associations_hash
       end
 
       def scan!
@@ -66,26 +85,85 @@ module InventoryRefresh
           end
         end
 
-        # Transform :parent_inventory_collections symbols to InventoryCollection objects
-        if parent_inventory_collections.present?
-          self.parent_inventory_collections = parent_inventory_collections.map do |inventory_collection_index|
-            ic = indexed_inventory_collections[inventory_collection_index]
-            if ic.nil?
-              raise "Can't find InventoryCollection #{inventory_collection_index} from #{inventory_collection}" if targeted?
-            else
-              # Add parent_inventory_collection as a dependency, so e.g. disconnect is done in a right order
-              (dependency_attributes[:__parent_inventory_collections] ||= Set.new) << ic
-              ic
-            end
-          end.compact
+        # Scan InventoryCollection skeletal data
+        inventory_collection.skeletal_primary_index.each_value do |inventory_object|
+          scan_inventory_object!(inventory_object)
         end
 
+        build_parent_inventory_collections!
+        scan_all_manager_uuids_scope!
+
         # Mark InventoryCollection as finalized aka. scanned
         self.data_collection_finalized = true
       end
 
       private
 
+      def scan_all_manager_uuids_scope!
+        return if all_manager_uuids_scope.nil?
+
+        all_manager_uuids_scope.each do |scope|
+          scope.each_value do |value|
+            scan_all_manager_uuids_scope_attribute!(value)
+          end
+        end
+      end
+
+      def build_parent_inventory_collections!
+        if parent_inventory_collections.nil?
+          build_parent_inventory_collection!
+        else
+          # We can't figure out what immediate parent is, we'll add parent_inventory_collections as dependencies
+          parent_inventory_collections.each { |ic_name| add_parent_inventory_collection_dependency!(ic_name) }
+        end
+
+        return if parent_inventory_collections.blank?
+
+        # Transform InventoryCollection object names to actual objects
+        self.parent_inventory_collections = parent_inventory_collections.map { |x| load_inventory_collection_by_name(x) }
+      end
+
+      def build_parent_inventory_collection!
+        return unless supports_building_inventory_collection?
+
+        if association.present? && parent.present? && associations_hash[association].present?
+          # Add immediate parent IC as dependency
+          add_parent_inventory_collection_dependency!(associations_hash[association])
+          # Add root IC in parent_inventory_collections
+          self.parent_inventory_collections = [find_parent_inventory_collection(associations_hash, inventory_collection.association)]
+        end
+      end
+
+      def supports_building_inventory_collection?
+        # Don't try to introspect ICs with custom query or saving code
+        return if arel.present? || custom_save_block.present?
+        # We support :parent_inventory_collections only for targeted mode, where all ICs are present
+        return unless targeted?
+
+        true
+      end
+
+      def add_parent_inventory_collection_dependency!(ic_name)
+        ic = load_inventory_collection_by_name(ic_name)
+        (dependency_attributes[:__parent_inventory_collections] ||= Set.new) << ic if ic
+      end
+
+      def find_parent_inventory_collection(hash, name)
+        if hash[name]
+          find_parent_inventory_collection(hash, hash[name])
+        else
+          name
+        end
+      end
+
+      def load_inventory_collection_by_name(name)
+        ic = indexed_inventory_collections[name]
+        if ic.nil?
+          raise "Can't find InventoryCollection :#{name} referenced from #{inventory_collection}" if targeted?
+        end
+        ic
+      end
+
       def scan_inventory_object!(inventory_object)
         inventory_object.data.each do |key, value|
           if value.kind_of?(Array)
@@ -96,8 +174,23 @@ module InventoryRefresh
         end
       end
 
+      def loadable?(value)
+        inventory_object_lazy?(value) || inventory_object?(value)
+      end
+
+      def add_reference(value_inventory_collection, value)
+        value_inventory_collection.add_reference(value.reference, :key => value.key)
+      end
+
+      def scan_all_manager_uuids_scope_attribute!(value)
+        return unless loadable?(value)
+
+        add_reference(value.inventory_collection, value)
+        (dependency_attributes[:__all_manager_uuids_scope] ||= Set.new) << value.inventory_collection
+      end
+
       def scan_inventory_object_attribute!(key, value)
-        return if !inventory_object_lazy?(value) && !inventory_object?(value)
+        return unless loadable?(value)
         value_inventory_collection = value.inventory_collection
 
         # Storing attributes and their dependencies
@@ -105,7 +198,7 @@ module InventoryRefresh
 
         # Storing a reference in the target inventory_collection, then each IC knows about all the references and can
         # e.g. load all the referenced uuids from a DB
-        value_inventory_collection.add_reference(value.reference, :key => value.key)
+        add_reference(value_inventory_collection, value)
 
         if inventory_object_lazy?(value)
           # Storing if attribute is a transitive dependency, so a lazy_find :key results in dependency

data/lib/inventory_refresh/inventory_collection/serialization.rb

@@ -4,6 +4,7 @@ module InventoryRefresh
   class InventoryCollection
     class Serialization
       delegate :all_manager_uuids,
+               :all_manager_uuids=,
                :build,
                :targeted_scope,
                :data,
@@ -27,18 +28,17 @@ module InventoryRefresh
       # @param available_inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of available
       #        InventoryCollection objects
       def from_hash(inventory_objects_data, available_inventory_collections)
-        targeted_scope.merge!(inventory_objects_data["manager_uuids"].map(&:symbolize_keys!))
+        targeted_scope.merge!(inventory_objects_data["manager_uuids"].map(&:symbolize_keys!)) if inventory_objects_data["manager_uuids"]
 
-        inventory_objects_data['data'].each do |inventory_object_data|
+        (inventory_objects_data['data'] || []).each do |inventory_object_data|
           build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
         end
 
-        inventory_objects_data['partial_data'].each do |inventory_object_data|
+        (inventory_objects_data['partial_data'] || []).each do |inventory_object_data|
           skeletal_primary_index.build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
         end
 
-        # TODO(lsmola) add support for all_manager_uuids serialization
-        # self.all_manager_uuids = inventory_objects_data['all_manager_uuids']
+        self.all_manager_uuids = inventory_objects_data['all_manager_uuids']
       end
 
       # Serializes InventoryCollection's data storage into Array of Hashes
@@ -105,9 +105,9 @@ module InventoryRefresh
         raise "Nested lazy_relation of #{inventory_collection} is too deep, left processing: #{hash}" if depth > 20
 
         hash.transform_values do |value|
-          if value.kind_of?(Hash) && value['type'] == "InventoryRefresh::InventoryObjectLazy"
+          if value.kind_of?(Hash) && value['inventory_collection_name']
             hash_to_lazy_relation(value, available_inventory_collections, depth)
-          elsif value.kind_of?(Array) && value.first.kind_of?(Hash) && value.first['type'] == "InventoryRefresh::InventoryObjectLazy"
+          elsif value.kind_of?(Array) && value.first.kind_of?(Hash) && value.first['inventory_collection_name']
             # TODO(lsmola) do we need to compact it sooner? What if first element is nil? On the other hand, we want to
             # deprecate this Vm HABTM assignment because it's not effective
             value.compact.map { |x| hash_to_lazy_relation(x, available_inventory_collections, depth) }

data/lib/inventory_refresh/inventory_collection/unconnected_edge.rb

@@ -0,0 +1,19 @@
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class InventoryCollection
+    class UnconnectedEdge
+      attr_reader :inventory_object, :inventory_object_key, :inventory_object_lazy
+
+      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject that couldn't connect the relation
+      # @param inventory_object_key [String] Relation name that couldn't be connected
+      # @param inventory_object_lazy [InventoryRefresh::InventoryObjectLazy] The lazy relation that failed to load
+      #                              this can happen only for relations with key, or pointing to DB only relations
+      def initialize(inventory_object, inventory_object_key, inventory_object_lazy)
+        @inventory_object      = inventory_object
+        @inventory_object_key  = inventory_object_key
+        @inventory_object_lazy = inventory_object_lazy
+      end
+    end
+  end
+end

data/lib/inventory_refresh/inventory_object.rb

@@ -25,8 +25,13 @@ module InventoryRefresh
       reference.stringified_reference
     end
 
+    # @return [Hash] hash reference having :manager_ref keys, which can uniquely identity entity under a manager
+    def uuid
+      reference.full_reference.slice(*reference.keys).stringify_keys!
+    end
+
     # @return [InventoryRefresh::InventoryObject] returns self
-    def load
+    def load(*_args)
       self
     end
 
@@ -92,8 +97,9 @@ module InventoryRefresh
     #
     # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
     # @param all_attribute_keys [Array<Symbol>] Attribute keys we will modify based on object's data
+    # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject object owning these attributes
     # @return [Hash] Data in DB format
-    def attributes_with_keys(inventory_collection_scope = nil, all_attribute_keys = [])
+    def attributes_with_keys(inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
       # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
       # different blacklist and whitelist. The generic code always passes a scope.
       inventory_collection_scope ||= inventory_collection
@@ -106,7 +112,7 @@ module InventoryRefresh
         elsif loadable?(value) || inventory_collection_scope.association_to_foreign_key_mapping[key]
           # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
           # result
-          data[key] = value.load if value.respond_to?(:load)
+          data[key] = value.load(inventory_object, key) if value.respond_to?(:load)
           if (foreign_key = inventory_collection_scope.association_to_foreign_key_mapping[key])
             # We have an association to fill, lets fill also the :key, cause some other InventoryObject can refer to it
             record_id = data[key].try(:id)
@@ -143,12 +149,12 @@ module InventoryRefresh
       attributes.each do |k, v|
         # We don't want timestamps or resource versions to be overwritten here, since those are driving the conditions
         next if %i(resource_timestamps resource_timestamps_max resource_timestamp).include?(k)
-        next if %i(resource_versions resource_versions_max resource_version).include?(k)
+        next if %i(resource_counters resource_counters_max resource_counter).include?(k)
 
         if data[:resource_timestamp] && attributes[:resource_timestamp]
           assign_only_newest(:resource_timestamp, :resource_timestamps, attributes, data, k, v)
-        elsif data[:resource_version] && attributes[:resource_version]
-          assign_only_newest(:resource_version, :resource_versions, attributes, data, k, v)
+        elsif data[:resource_counter] && attributes[:resource_counter]
+          assign_only_newest(:resource_counter, :resource_counters, attributes, data, k, v)
         else
           public_send("#{k}=", v)
         end
@@ -156,8 +162,8 @@ module InventoryRefresh
 
       if attributes[:resource_timestamp]
         assign_full_row_version_attr(:resource_timestamp, attributes, data)
-      elsif attributes[:resource_version]
-        assign_full_row_version_attr(:resource_version, attributes, data)
+      elsif attributes[:resource_counter]
+        assign_full_row_version_attr(:resource_counter, attributes, data)
       end
 
       self
@@ -206,9 +212,9 @@ module InventoryRefresh
     # newer than existing attribute.
     #
     # @param full_row_version_attr [Symbol] Attr name for full rows, allowed values are
-    #        [:resource_timestamp, :resource_version]
+    #        [:resource_timestamp, :resource_counter]
     # @param partial_row_version_attr [Symbol] Attr name for partial rows, allowed values are
-    #        [:resource_timestamps, :resource_versions]
+    #        [:resource_timestamps, :resource_counters]
     # @param attributes [Hash] New attributes we are assigning
     # @param data [Hash] Existing attributes of the InventoryObject
     # @param k [Symbol] Name of the attribute we are assigning
@@ -247,7 +253,7 @@ module InventoryRefresh
     # Assigns attribute representing version of the whole row
     #
     # @param full_row_version_attr [Symbol] Attr name for full rows, allowed values are
-    #        [:resource_timestamp, :resource_version]
+    #        [:resource_timestamp, :resource_counter]
     # @param attributes [Hash] New attributes we are assigning
     # @param data [Hash] Existing attributes of the InventoryObject
     def assign_full_row_version_attr(full_row_version_attr, attributes, data)