inventory_refresh 0.3.3 → 1.0.0

data/lib/inventory_refresh/inventory_object.rb

@@ -42,12 +42,64 @@ module InventoryRefresh
     # Transforms InventoryObject object data into hash format with keys that are column names and resolves correct
     # values of the foreign keys (even the polymorphic ones)
     #
-    # @param data [Array<Object>] Array of objects that we want to map to DB table columns
+    # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
+    # @return [Hash] Data in DB format
+    def attributes(inventory_collection_scope = nil)
+      # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
+      # different blacklist and whitelist. The generic code always passes a scope.
+      inventory_collection_scope ||= inventory_collection
+
+      attributes_for_saving = {}
+      # First transform the values
+      data.each do |key, value|
+        if !allowed?(inventory_collection_scope, key)
+          next
+        elsif value.kind_of?(Array) && value.any? { |x| loadable?(x) }
+          # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
+          # result
+          data[key] = value.compact.map(&:load).compact
+          # We can use built in _ids methods to assign array of ids into has_many relations. So e.g. the :key_pairs=
+          # relation setter will become :key_pair_ids=
+          attributes_for_saving["#{key.to_s.singularize}_ids".to_sym] = data[key].map(&:id).compact.uniq
+        elsif loadable?(value) || inventory_collection_scope.association_to_foreign_key_mapping[key]
+          # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
+          # result
+          data[key] = value.load if value.respond_to?(:load)
+          if (foreign_key = inventory_collection_scope.association_to_foreign_key_mapping[key])
+            # We have an association to fill, lets fill also the :key, cause some other InventoryObject can refer to it
+            record_id                                 = data[key].try(:id)
+            attributes_for_saving[foreign_key.to_sym] = record_id
+
+            if (foreign_type = inventory_collection_scope.association_to_foreign_type_mapping[key])
+              # If we have a polymorphic association, we need to also fill a base class name, but we want to nullify it
+              # if record_id is missing
+              base_class = data[key].try(:base_class_name) || data[key].class.try(:base_class).try(:name)
+              attributes_for_saving[foreign_type.to_sym] = record_id ? base_class : nil
+            end
+          elsif data[key].kind_of?(::InventoryRefresh::InventoryObject)
+            # We have an association to fill but not an Activerecord association, so e.g. Ancestry, lets just load
+            # it here. This way of storing ancestry is ineffective in DB call count, but RAM friendly
+            attributes_for_saving[key.to_sym] = data[key].base_class_name.constantize.find_by(:id => data[key].id)
+          else
+            # We have a normal attribute to fill
+            attributes_for_saving[key.to_sym] = data[key]
+          end
+        else
+          attributes_for_saving[key.to_sym] = value
+        end
+      end
+
+      attributes_for_saving
+    end
+
+    # Transforms InventoryObject object data into hash format with keys that are column names and resolves correct
+    # values of the foreign keys (even the polymorphic ones)
+    #
     # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
     # @param all_attribute_keys [Array<Symbol>] Attribute keys we will modify based on object's data
     # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject object owning these attributes
     # @return [Hash] Data in DB format
-    def self.attributes_with_keys(data, inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
+    def attributes_with_keys(inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
       # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
       # different blacklist and whitelist. The generic code always passes a scope.
       inventory_collection_scope ||= inventory_collection
@@ -96,8 +148,8 @@ module InventoryRefresh
     def assign_attributes(attributes)
       attributes.each do |k, v|
         # We don't want timestamps or resource versions to be overwritten here, since those are driving the conditions
-        next if %i(resource_timestamps resource_timestamps_max resource_timestamp).include?(k)
-        next if %i(resource_counters resource_counters_max resource_counter).include?(k)
+        next if %i[resource_timestamps resource_timestamps_max resource_timestamp].include?(k)
+        next if %i[resource_counters resource_counters_max resource_counter].include?(k)
 
         if data[:resource_timestamp] && attributes[:resource_timestamp]
           assign_only_newest(:resource_timestamp, :resource_timestamps, attributes, data, k, v)
@@ -146,44 +198,14 @@ module InventoryRefresh
           end
         end
 
-        unless defined_methods.include?(attr.to_sym)
-          define_method(attr) do
-            data[attr]
-          end
+        next if defined_methods.include?(attr.to_sym)
+
+        define_method(attr) do
+          data[attr]
         end
       end
     end
 
-    # Return true if the attribute is allowed to be saved into the DB
-    #
-    # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] InventoryCollection object owning the
-    #        attribute
-    # @param key [Symbol] attribute name
-    # @return true if the attribute is allowed to be saved into the DB
-    def self.allowed?(inventory_collection_scope, key)
-      foreign_to_association = (inventory_collection_scope.foreign_key_to_association_mapping[key] ||
-        inventory_collection_scope.foreign_type_to_association_mapping[key])
-
-      return false if inventory_collection_scope.attributes_blacklist.present? &&
-        (inventory_collection_scope.attributes_blacklist.include?(key) ||
-          (foreign_to_association && inventory_collection_scope.attributes_blacklist.include?(foreign_to_association)))
-
-      return false if inventory_collection_scope.attributes_whitelist.present? &&
-        (!inventory_collection_scope.attributes_whitelist.include?(key) &&
-          (!foreign_to_association || (foreign_to_association && inventory_collection_scope.attributes_whitelist.include?(foreign_to_association))))
-
-      true
-    end
-
-    # Return true if the object is loadable, which we determine by a list of loadable classes.
-    #
-    # @param value [Object] object we test
-    # @return true if the object is loadable
-    def self.loadable?(value)
-      value.kind_of?(::InventoryRefresh::InventoryObjectLazy) || value.kind_of?(::InventoryRefresh::InventoryObject) ||
-        value.kind_of?(::InventoryRefresh::ApplicationRecordReference)
-    end
-
     private
 
     # Assigns value based on the version attributes. If versions are specified, it asigns attribute only if it's
@@ -261,7 +283,18 @@ module InventoryRefresh
     # @param key [Symbol] attribute name
     # @return true if the attribute is allowed to be saved into the DB
     def allowed?(inventory_collection_scope, key)
-      self.class.allowed?(inventory_collection_scope, key)
+      foreign_to_association = inventory_collection_scope.foreign_key_to_association_mapping[key] ||
+                               inventory_collection_scope.foreign_type_to_association_mapping[key]
+
+      return false if inventory_collection_scope.attributes_blacklist.present? &&
+                      (inventory_collection_scope.attributes_blacklist.include?(key) ||
+                        (foreign_to_association && inventory_collection_scope.attributes_blacklist.include?(foreign_to_association)))
+
+      return false if inventory_collection_scope.attributes_whitelist.present? &&
+                      (!inventory_collection_scope.attributes_whitelist.include?(key) &&
+                        (!foreign_to_association || (foreign_to_association && inventory_collection_scope.attributes_whitelist.include?(foreign_to_association))))
+
+      true
     end
 
     # Return true if the object is loadable, which we determine by a list of loadable classes.
@@ -269,7 +302,8 @@ module InventoryRefresh
     # @param value [Object] object we test
     # @return true if the object is loadable
     def loadable?(value)
-      self.class.loadable?(value)
+      value.kind_of?(::InventoryRefresh::InventoryObjectLazy) || value.kind_of?(::InventoryRefresh::InventoryObject) ||
+        value.kind_of?(::InventoryRefresh::ApplicationRecordReference)
     end
   end
 end

data/lib/inventory_refresh/inventory_object_lazy.rb

@@ -30,6 +30,7 @@ module InventoryRefresh
 
     # @return [String] stringified reference
     def to_s
+      # TODO(lsmola) do we need this method?
       stringified_reference
     end
 
@@ -70,6 +71,10 @@ module InventoryRefresh
 
     # @return [Boolean] true if the key is an association on inventory_collection_scope model class
     def association?(key)
+      # TODO(lsmola) remove this if there will be better dependency scan, probably with transitive dependencies filled
+      # in a second pass, then we can get rid of this hardcoded symbols. Right now we are not able to introspect these.
+      return true if [:parent, :genealogy_parent].include?(key)
+
       inventory_collection.dependency_attributes.key?(key) ||
         !inventory_collection.association_to_foreign_key_mapping[key].nil?
     end
@@ -95,7 +100,7 @@ module InventoryRefresh
 
     private
 
-    delegate :saved?, :saver_strategy, :skeletal_primary_index, :to => :inventory_collection
+    delegate :parallel_safe?, :saved?, :saver_strategy, :skeletal_primary_index, :targeted?, :to => :inventory_collection
     delegate :nested_secondary_index?, :primary?, :full_reference, :keys, :primary?, :to => :reference
 
     attr_writer :reference
@@ -107,18 +112,20 @@ module InventoryRefresh
     #
     # @return [InventoryRefresh::InventoryObject, NilClass] Returns pre-created InventoryObject or nil
     def skeletal_precreate!
+      # We can do skeletal pre-create only for strategies using unique indexes. Since this can build records out of
+      # the given :arel scope, we will always attempt to create the recod, so we need unique index to avoid duplication
+      # of records.
+      return unless parallel_safe?
       # Pre-create only for strategies that will be persisting data, i.e. are not saved already
       return if saved?
       # We can only do skeletal pre-create for primary index reference, since that is needed to create DB unique index
-      # and full reference must be present
-      return if !primary? || full_reference.blank?
-
-      # To avoid pre-creating invalid records all fields of a primary key must have non null value
-      # TODO(lsmola) for composite keys, it's still valid to have one of the keys nil, figure out how to allow this. We
-      # will need to scan the DB for NOT NULL constraint and allow it based on that. So we would move this check to
-      # saving code, but this will require bigger change, since having the column nil means we will have to batch it
-      # smartly, since having nil means we will need to use different unique index for the upsert/update query.
-      return if keys.any? { |x| full_reference[x].nil? }
+      return unless primary?
+      # Full reference must be present
+      return if full_reference.blank?
+
+      # To avoid pre-creating invalid records all fields of a primary key must have value
+      # TODO(lsmola) for composite keys, it's still valid to have one of the keys nil, figure out how to allow this
+      return if keys.any? { |x| full_reference[x].blank? }
 
       skeletal_primary_index.build(full_reference)
     end

data/lib/inventory_refresh/null_logger.rb

@@ -3,10 +3,10 @@ require "logger"
 module InventoryRefresh
   class NullLogger < Logger
     def initialize(*_args)
-    end 
+    end
 
     def add(*_args, &_block)
-    end 
+    end
 
     def debug?
       false

data/lib/inventory_refresh/persister.rb

@@ -3,13 +3,15 @@ module InventoryRefresh
     require 'json'
     require 'yaml'
 
-    attr_reader :manager, :collections
+    attr_reader :manager, :target, :collections
 
     attr_accessor :refresh_state_uuid, :refresh_state_part_uuid, :total_parts, :sweep_scope, :retry_count, :retry_max
 
     # @param manager [ManageIQ::Providers::BaseManager] A manager object
-    def initialize(manager)
+    # @param target [Object] A refresh Target object
+    def initialize(manager, target = nil)
       @manager = manager
+      @target  = target
 
       @collections = {}
 
@@ -43,6 +45,9 @@ module InventoryRefresh
                                            &block)
 
       builder.add_properties(extra_properties) if extra_properties.present?
+
+      builder.add_properties({:manager_uuids => target.try(:references, collection_name) || []}, :if_missing) if targeted?
+
       builder.evaluate_lambdas!(self)
 
       collections[collection_name] = builder.to_inventory_collection
@@ -61,7 +66,7 @@ module InventoryRefresh
     # @return [InventoryRefresh::InventoryCollection] returns a defined InventoryCollection or undefined method
     def method_missing(method_name, *arguments, &block)
       if inventory_collections_names.include?(method_name)
-        self.define_collections_reader(method_name)
+        define_collections_reader(method_name)
         send(method_name)
       else
         super
@@ -91,7 +96,7 @@ module InventoryRefresh
 
     # Returns serialized Persisted object to JSON
     # @return [String] serialized Persisted object to JSON
-    def to_json
+    def to_json(*_args)
       JSON.dump(to_hash)
     end
 
@@ -99,6 +104,8 @@ module InventoryRefresh
     def to_hash
       collections_data = collections.map do |_, collection|
         next if collection.data.blank? &&
+                collection.targeted_scope.primary_references.blank? &&
+                collection.all_manager_uuids.nil? &&
                 collection.skeletal_primary_index.index_data.blank?
 
         collection.to_hash
@@ -110,7 +117,7 @@ module InventoryRefresh
         :retry_count             => retry_count,
         :retry_max               => retry_max,
         :total_parts             => total_parts,
-        :sweep_scope             => sweep_scope_to_hash(sweep_scope),
+        :sweep_scope             => sweep_scope,
         :collections             => collections_data,
       }
     end
@@ -120,19 +127,22 @@ module InventoryRefresh
       #
       # @param json_data [String] input JSON data
       # @return [ManageIQ::Providers::Inventory::Persister] Persister object loaded from a passed JSON
-      def from_json(json_data, manager)
-        from_hash(JSON.parse(json_data), manager)
+      def from_json(json_data, manager, target = nil)
+        from_hash(JSON.parse(json_data), manager, target)
       end
 
       # Returns Persister object built from serialized data
       #
       # @param persister_data [Hash] serialized Persister object in hash
       # @return [ManageIQ::Providers::Inventory::Persister] Persister object built from serialized data
-      def from_hash(persister_data, manager)
-        new(manager).tap do |persister|
+      def from_hash(persister_data, manager, target = nil)
+        # TODO(lsmola) we need to pass serialized targeted scope here
+        target ||= InventoryRefresh::TargetCollection.new(:manager => manager)
+
+        new(manager, target).tap do |persister|
           persister_data['collections'].each do |collection|
             inventory_collection = persister.collections[collection['name'].try(:to_sym)]
-            raise "Unrecognized InventoryCollection name: #{collection['name']}" if inventory_collection.blank?
+            raise "Unrecognized InventoryCollection name: #{inventory_collection}" if inventory_collection.blank?
 
             inventory_collection.from_hash(collection, persister.collections)
           end
@@ -142,44 +152,9 @@ module InventoryRefresh
           persister.retry_count             = persister_data['retry_count']
           persister.retry_max               = persister_data['retry_max']
           persister.total_parts             = persister_data['total_parts']
-          persister.sweep_scope             = sweep_scope_from_hash(persister_data['sweep_scope'], persister.collections)
+          persister.sweep_scope             = persister_data['sweep_scope']
         end
       end
-
-      private
-
-      def assert_sweep_scope!(sweep_scope)
-        return unless sweep_scope
-
-        allowed_format_message = "Allowed format of sweep scope is Array<String> or Hash{String => Hash}, got #{sweep_scope}"
-
-        if sweep_scope.kind_of?(Array)
-          return if sweep_scope.all? { |x| x.kind_of?(String) || x.kind_of?(Symbol) }
-
-          raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
-        elsif sweep_scope.kind_of?(Hash)
-          return if sweep_scope.values.all? { |x| x.kind_of?(Array) }
-
-          raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
-        end
-
-        raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
-      end
-
-      def sweep_scope_from_hash(sweep_scope, available_inventory_collections)
-        assert_sweep_scope!(sweep_scope)
-
-        return sweep_scope unless sweep_scope.kind_of?(Hash)
-
-        sweep_scope.each_with_object({}) do |(k, v), obj|
-          inventory_collection = available_inventory_collections[k.try(:to_sym)]
-          raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
-
-          serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
-
-          obj[k] = serializer.sweep_scope_from_hash(v, available_inventory_collections)
-        end.symbolize_keys!
-      end
     end
 
     protected
@@ -210,37 +185,28 @@ module InventoryRefresh
       nil
     end
 
-    def assert_graph_integrity?
+    def saver_strategy
+      :default
+    end
+
+    # Persisters for targeted refresh can override to true
+    def targeted?
       false
     end
 
-    def use_ar_object?
-      true
+    def assert_graph_integrity?
+      false
     end
 
     # @return [Hash] kwargs shared for all InventoryCollection objects
     def shared_options
       {
+        :saver_strategy         => saver_strategy,
         :strategy               => strategy,
+        :targeted               => targeted?,
         :parent                 => manager.presence,
         :assert_graph_integrity => assert_graph_integrity?,
-        :use_ar_object          => use_ar_object?,
       }
     end
-
-    private
-
-    def sweep_scope_to_hash(sweep_scope)
-      return sweep_scope unless sweep_scope.kind_of?(Hash)
-
-      sweep_scope.each_with_object({}) do |(k, v), obj|
-        inventory_collection = collections[k.try(:to_sym)]
-        raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
-
-        serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
-
-        obj[k] = serializer.sweep_scope_to_hash(v)
-      end
-    end
   end
 end

data/lib/inventory_refresh/save_collection/base.rb

@@ -1,5 +1,7 @@
 require "inventory_refresh/logging"
+require "inventory_refresh/save_collection/saver/batch"
 require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
+require "inventory_refresh/save_collection/saver/default"
 
 module InventoryRefresh::SaveCollection
   class Base
@@ -14,7 +16,7 @@ module InventoryRefresh::SaveCollection
         return if skip?(inventory_collection)
 
         logger.debug("----- BEGIN ----- Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
-                     " the database, for the manager: '#{ems.id}'...")
+                     " the database, for the manager: '#{ems.name}'...")
 
         if inventory_collection.custom_save_block.present?
           logger.debug("Saving collection #{inventory_collection} using a custom save block")
@@ -22,7 +24,7 @@ module InventoryRefresh::SaveCollection
         else
           save_inventory(inventory_collection)
         end
-        logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.id}'...Complete")
+        logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.name}'...Complete")
         inventory_collection.saved = true
       end
 

data/lib/inventory_refresh/save_collection/saver/base.rb

@@ -12,6 +12,8 @@ module InventoryRefresh::SaveCollection
       # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we will be saving
       def initialize(inventory_collection)
         @inventory_collection = inventory_collection
+        # TODO(lsmola) do I need to reload every time? Also it should be enough to clear the associations.
+        inventory_collection.parent&.reload
         @association = inventory_collection.db_collection_for_comparison
 
         # Private attrs
@@ -19,17 +21,21 @@ module InventoryRefresh::SaveCollection
         @table_name             = @model_class.table_name
         @q_table_name           = get_connection.quote_table_name(@table_name)
         @primary_key            = @model_class.primary_key
-        @arel_primary_key       = @model_class.arel_attribute(@primary_key)
+        @arel_primary_key       = @model_class.arel_table[@primary_key]
         @unique_index_keys      = inventory_collection.unique_index_keys
         @unique_index_keys_to_s = inventory_collection.manager_ref_to_cols.map(&:to_s)
         @select_keys            = [@primary_key] + @unique_index_keys_to_s + internal_columns.map(&:to_s)
         @unique_db_primary_keys = Set.new
         @unique_db_indexes      = Set.new
 
+        # Right now ApplicationRecordIterator in association is used for targeted refresh. Given the small amount of
+        # records flowing through there, we probably don't need to optimize that association to fetch a pure SQL.
+        @pure_sql_records_fetching = !inventory_collection.use_ar_object? && !@association.kind_of?(InventoryRefresh::ApplicationRecordIterator)
+
         @batch_size_for_persisting = inventory_collection.batch_size_pure_sql
-        @batch_size                = inventory_collection.use_ar_object? ? @batch_size_for_persisting : inventory_collection.batch_size
 
-        @record_key_method   = inventory_collection.pure_sql_record_fetching? ? :pure_sql_record_key : :ar_record_key
+        @batch_size          = @pure_sql_records_fetching ? @batch_size_for_persisting : inventory_collection.batch_size
+        @record_key_method   = @pure_sql_records_fetching ? :pure_sql_record_key : :ar_record_key
         @select_keys_indexes = @select_keys.each_with_object({}).with_index { |(key, obj), index| obj[key.to_s] = index }
         @pg_types            = @model_class.attribute_names.each_with_object({}) do |key, obj|
           obj[key.to_sym] = inventory_collection.model_class.columns_hash[key]
@@ -69,8 +75,14 @@ module InventoryRefresh::SaveCollection
 
       # Saves the InventoryCollection
       def save_inventory_collection!
+        # If we have a targeted InventoryCollection that wouldn't do anything, quickly skip it
+        return if inventory_collection.noop?
+
+        # Delete_complement strategy using :all_manager_uuids attribute
+        delete_complement unless inventory_collection.delete_complement_noop?
+
         # Create/Update/Archive/Delete records based on InventoryCollection data and scope
-        save!(association)
+        save!(association) unless inventory_collection.saving_noop?
       end
 
       protected
@@ -89,8 +101,6 @@ module InventoryRefresh::SaveCollection
       # @param attributes [Hash] attributes hash
       # @return [Hash] modified hash from parameter attributes with casted values
       def values_for_database!(all_attribute_keys, attributes)
-        # TODO(lsmola) we'll need to fill default value from the DB to the NOT_NULL columns here, since sending NULL
-        # to column with NOT_NULL constraint always fails, even if there is a default value
         all_attribute_keys.each do |key|
           next unless attributes.key?(key)
 
@@ -102,7 +112,11 @@ module InventoryRefresh::SaveCollection
       end
 
       def transform_to_hash!(all_attribute_keys, hash)
-        if serializable_keys?
+        if inventory_collection.use_ar_object?
+          record = inventory_collection.model_class.new(hash)
+          values_for_database!(all_attribute_keys,
+                               record.attributes.slice(*record.changed_attributes.keys).symbolize_keys)
+        elsif serializable_keys?
           values_for_database!(all_attribute_keys,
                                hash)
         else
@@ -113,15 +127,99 @@ module InventoryRefresh::SaveCollection
       private
 
       attr_reader :unique_index_keys, :unique_index_keys_to_s, :select_keys, :unique_db_primary_keys, :unique_db_indexes,
-                  :primary_key, :arel_primary_key, :record_key_method, :select_keys_indexes,
+                  :primary_key, :arel_primary_key, :record_key_method, :pure_sql_records_fetching, :select_keys_indexes,
                   :batch_size, :batch_size_for_persisting, :model_class, :serializable_keys, :deserializable_keys, :pg_types, :table_name,
                   :q_table_name
 
       delegate :supports_column?, :to => :inventory_collection
 
+      # Saves the InventoryCollection
+      #
+      # @param association [Symbol] An existing association on manager
+      def save!(association)
+        attributes_index        = {}
+        inventory_objects_index = {}
+        inventory_collection.each do |inventory_object|
+          attributes = inventory_object.attributes(inventory_collection)
+          index      = build_stringified_reference(attributes, unique_index_keys)
+
+          attributes_index[index]        = attributes
+          inventory_objects_index[index] = inventory_object
+        end
+
+        logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
+        # Records that are in the DB, we will be updating or deleting them.
+        ActiveRecord::Base.transaction do
+          association.find_each do |record|
+            index = build_stringified_reference_for_record(record, unique_index_keys)
+
+            next unless assert_distinct_relation(record.id)
+            next unless assert_unique_record(record, index)
+
+            inventory_object = inventory_objects_index.delete(index)
+            hash             = attributes_index.delete(index)
+
+            if inventory_object.nil?
+              # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
+              # delete it from the DB.
+              delete_record!(record) if inventory_collection.delete_allowed?
+            elsif assert_referential_integrity(hash)
+              # Record was found in the DB and sent for saving, we will be updating the DB.
+              update_record!(record, hash, inventory_object)
+            end
+          end
+        end
+
+        inventory_collection.custom_reconnect_block&.call(inventory_collection, inventory_objects_index, attributes_index)
+
+        # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
+        if inventory_collection.create_allowed?
+          ActiveRecord::Base.transaction do
+            inventory_objects_index.each do |index, inventory_object|
+              hash = attributes_index.delete(index)
+
+              create_record!(hash, inventory_object) if assert_referential_integrity(hash)
+            end
+          end
+        end
+        logger.debug("Processing #{inventory_collection}, "\
+                     "created=#{inventory_collection.created_records.count}, "\
+                     "updated=#{inventory_collection.updated_records.count}, "\
+                     "deleted=#{inventory_collection.deleted_records.count}...Complete")
+      rescue => e
+        logger.error("Error when saving #{inventory_collection} with #{inventory_collection_details}. Message: #{e.message}")
+        raise e
+      end
+
       # @return [String] a string for logging purposes
       def inventory_collection_details
-        "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}"
+        "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}, targeted: #{inventory_collection.targeted?}"
+      end
+
+      # @param record [ApplicationRecord] ApplicationRecord object
+      # @param key [Symbol] A key that is an attribute of the AR object
+      # @return [Object] Value of attribute name :key on the :record
+      def record_key(record, key)
+        record.public_send(key)
+      end
+
+      # Deletes a complement of referenced data
+      def delete_complement
+        raise(":delete_complement method is supported only for :saver_strategy => [:batch, :concurrent_safe_batch]")
+      end
+
+      # Deletes/soft-deletes a given record
+      #
+      # @param [ApplicationRecord] record we want to delete
+      def delete_record!(record)
+        record.public_send(inventory_collection.delete_method)
+        inventory_collection.store_deleted_records(record)
+      end
+
+      # @return [TrueClass] always return true, this method is redefined in default saver
+      def assert_unique_record(_record, _index)
+        # TODO(lsmola) can go away once we indexed our DB with unique indexes
+        true
       end
 
       # Check if relation provided is distinct, i.e. the relation should not return the same primary key value twice.
@@ -134,12 +232,12 @@ module InventoryRefresh::SaveCollection
           # Change the InventoryCollection's :association or :arel parameter to return distinct results. The :through
           # relations can return the same record multiple times. We don't want to do SELECT DISTINCT by default, since
           # it can be very slow.
-          unless inventory_collection.assert_graph_integrity
+          if inventory_collection.assert_graph_integrity
+            raise("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. ")
+          else
             logger.warn("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. "\
                         " The duplicate value is being ignored.")
             return false
-          else
-            raise("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. ")
           end
         else
           unique_db_primary_keys << primary_key_value
@@ -157,14 +255,15 @@ module InventoryRefresh::SaveCollection
       def assert_referential_integrity(hash)
         inventory_collection.fixed_foreign_keys.each do |x|
           next unless hash[x].nil?
+
           subject = "#{hash} of #{inventory_collection} because of missing foreign key #{x} for "\
                     "#{inventory_collection.parent.class.name}:"\
                     "#{inventory_collection.parent.try(:id)}"
-          unless inventory_collection.assert_graph_integrity
+          if inventory_collection.assert_graph_integrity
+            raise("Referential integrity check violated for #{subject}")
+          else
             logger.warn("Referential integrity check violated, ignoring #{subject}")
             return false
-          else
-            raise("Referential integrity check violated for #{subject}")
           end
         end
         true

data/lib/inventory_refresh/save_collection/saver/batch.rb

@@ -0,0 +1,17 @@
+require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
+
+module InventoryRefresh::SaveCollection
+  module Saver
+    class Batch < InventoryRefresh::SaveCollection::Saver::ConcurrentSafeBatch
+      private
+
+      # Just returning manager ref transformed to column names, for strategies that do not expect to have unique DB
+      # indexes.
+      #
+      # @return [Array<Symbol>] manager ref transformed to column names
+      def unique_index_columns
+        inventory_collection.manager_ref_to_cols.map(&:to_sym)
+      end
+    end
+  end
+end