inventory_refresh 0.3.5 → 1.1.0

data/lib/inventory_refresh/save_collection/saver/base.rb

@@ -12,6 +12,8 @@ module InventoryRefresh::SaveCollection
       # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we will be saving
       def initialize(inventory_collection)
         @inventory_collection = inventory_collection
+        # TODO(lsmola) do I need to reload every time? Also it should be enough to clear the associations.
+        inventory_collection.parent&.reload
         @association = inventory_collection.db_collection_for_comparison
 
         # Private attrs
@@ -19,17 +21,21 @@ module InventoryRefresh::SaveCollection
         @table_name             = @model_class.table_name
         @q_table_name           = get_connection.quote_table_name(@table_name)
         @primary_key            = @model_class.primary_key
-        @arel_primary_key       = @model_class.arel_attribute(@primary_key)
+        @arel_primary_key       = @model_class.arel_table[@primary_key]
         @unique_index_keys      = inventory_collection.unique_index_keys
         @unique_index_keys_to_s = inventory_collection.manager_ref_to_cols.map(&:to_s)
         @select_keys            = [@primary_key] + @unique_index_keys_to_s + internal_columns.map(&:to_s)
         @unique_db_primary_keys = Set.new
         @unique_db_indexes      = Set.new
 
+        # Right now ApplicationRecordIterator in association is used for targeted refresh. Given the small amount of
+        # records flowing through there, we probably don't need to optimize that association to fetch a pure SQL.
+        @pure_sql_records_fetching = !inventory_collection.use_ar_object? && !@association.kind_of?(InventoryRefresh::ApplicationRecordIterator)
+
         @batch_size_for_persisting = inventory_collection.batch_size_pure_sql
-        @batch_size                = inventory_collection.use_ar_object? ? @batch_size_for_persisting : inventory_collection.batch_size
 
-        @record_key_method   = inventory_collection.pure_sql_record_fetching? ? :pure_sql_record_key : :ar_record_key
+        @batch_size          = @pure_sql_records_fetching ? @batch_size_for_persisting : inventory_collection.batch_size
+        @record_key_method   = @pure_sql_records_fetching ? :pure_sql_record_key : :ar_record_key
         @select_keys_indexes = @select_keys.each_with_object({}).with_index { |(key, obj), index| obj[key.to_s] = index }
         @pg_types            = @model_class.attribute_names.each_with_object({}) do |key, obj|
           obj[key.to_sym] = inventory_collection.model_class.columns_hash[key]
@@ -69,8 +75,14 @@ module InventoryRefresh::SaveCollection
 
       # Saves the InventoryCollection
       def save_inventory_collection!
+        # If we have a targeted InventoryCollection that wouldn't do anything, quickly skip it
+        return if inventory_collection.noop?
+
+        # Delete_complement strategy using :all_manager_uuids attribute
+        delete_complement unless inventory_collection.delete_complement_noop?
+
         # Create/Update/Archive/Delete records based on InventoryCollection data and scope
-        save!(association)
+        save!(association) unless inventory_collection.saving_noop?
       end
 
       protected
@@ -89,8 +101,6 @@ module InventoryRefresh::SaveCollection
       # @param attributes [Hash] attributes hash
       # @return [Hash] modified hash from parameter attributes with casted values
       def values_for_database!(all_attribute_keys, attributes)
-        # TODO(lsmola) we'll need to fill default value from the DB to the NOT_NULL columns here, since sending NULL
-        # to column with NOT_NULL constraint always fails, even if there is a default value
         all_attribute_keys.each do |key|
           next unless attributes.key?(key)
 
@@ -102,7 +112,11 @@ module InventoryRefresh::SaveCollection
       end
 
       def transform_to_hash!(all_attribute_keys, hash)
-        if serializable_keys?
+        if inventory_collection.use_ar_object?
+          record = inventory_collection.model_class.new(hash)
+          values_for_database!(all_attribute_keys,
+                               record.attributes.slice(*record.changed_attributes.keys).symbolize_keys)
+        elsif serializable_keys?
           values_for_database!(all_attribute_keys,
                                hash)
         else
@@ -113,15 +127,99 @@ module InventoryRefresh::SaveCollection
       private
 
       attr_reader :unique_index_keys, :unique_index_keys_to_s, :select_keys, :unique_db_primary_keys, :unique_db_indexes,
-                  :primary_key, :arel_primary_key, :record_key_method, :select_keys_indexes,
+                  :primary_key, :arel_primary_key, :record_key_method, :pure_sql_records_fetching, :select_keys_indexes,
                   :batch_size, :batch_size_for_persisting, :model_class, :serializable_keys, :deserializable_keys, :pg_types, :table_name,
                   :q_table_name
 
       delegate :supports_column?, :to => :inventory_collection
 
+      # Saves the InventoryCollection
+      #
+      # @param association [Symbol] An existing association on manager
+      def save!(association)
+        attributes_index        = {}
+        inventory_objects_index = {}
+        inventory_collection.each do |inventory_object|
+          attributes = inventory_object.attributes(inventory_collection)
+          index      = build_stringified_reference(attributes, unique_index_keys)
+
+          attributes_index[index]        = attributes
+          inventory_objects_index[index] = inventory_object
+        end
+
+        logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
+        # Records that are in the DB, we will be updating or deleting them.
+        ActiveRecord::Base.transaction do
+          association.find_each do |record|
+            index = build_stringified_reference_for_record(record, unique_index_keys)
+
+            next unless assert_distinct_relation(record.id)
+            next unless assert_unique_record(record, index)
+
+            inventory_object = inventory_objects_index.delete(index)
+            hash             = attributes_index.delete(index)
+
+            if inventory_object.nil?
+              # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
+              # delete it from the DB.
+              delete_record!(record) if inventory_collection.delete_allowed?
+            elsif assert_referential_integrity(hash)
+              # Record was found in the DB and sent for saving, we will be updating the DB.
+              update_record!(record, hash, inventory_object)
+            end
+          end
+        end
+
+        inventory_collection.custom_reconnect_block&.call(inventory_collection, inventory_objects_index, attributes_index)
+
+        # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
+        if inventory_collection.create_allowed?
+          ActiveRecord::Base.transaction do
+            inventory_objects_index.each do |index, inventory_object|
+              hash = attributes_index.delete(index)
+
+              create_record!(hash, inventory_object) if assert_referential_integrity(hash)
+            end
+          end
+        end
+        logger.debug("Processing #{inventory_collection}, "\
+                     "created=#{inventory_collection.created_records.count}, "\
+                     "updated=#{inventory_collection.updated_records.count}, "\
+                     "deleted=#{inventory_collection.deleted_records.count}...Complete")
+      rescue => e
+        logger.error("Error when saving #{inventory_collection} with #{inventory_collection_details}. Message: #{e.message}")
+        raise e
+      end
+
       # @return [String] a string for logging purposes
       def inventory_collection_details
-        "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}"
+        "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}, targeted: #{inventory_collection.targeted?}"
+      end
+
+      # @param record [ApplicationRecord] ApplicationRecord object
+      # @param key [Symbol] A key that is an attribute of the AR object
+      # @return [Object] Value of attribute name :key on the :record
+      def record_key(record, key)
+        record.public_send(key)
+      end
+
+      # Deletes a complement of referenced data
+      def delete_complement
+        raise(":delete_complement method is supported only for :saver_strategy => [:batch, :concurrent_safe_batch]")
+      end
+
+      # Deletes/soft-deletes a given record
+      #
+      # @param [ApplicationRecord] record we want to delete
+      def delete_record!(record)
+        record.public_send(inventory_collection.delete_method)
+        inventory_collection.store_deleted_records(record)
+      end
+
+      # @return [TrueClass] always return true, this method is redefined in default saver
+      def assert_unique_record(_record, _index)
+        # TODO(lsmola) can go away once we indexed our DB with unique indexes
+        true
       end
 
       # Check if relation provided is distinct, i.e. the relation should not return the same primary key value twice.
@@ -134,12 +232,12 @@ module InventoryRefresh::SaveCollection
           # Change the InventoryCollection's :association or :arel parameter to return distinct results. The :through
           # relations can return the same record multiple times. We don't want to do SELECT DISTINCT by default, since
           # it can be very slow.
-          unless inventory_collection.assert_graph_integrity
+          if inventory_collection.assert_graph_integrity
+            raise("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. ")
+          else
             logger.warn("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. "\
                         " The duplicate value is being ignored.")
             return false
-          else
-            raise("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. ")
           end
         else
           unique_db_primary_keys << primary_key_value
@@ -157,14 +255,15 @@ module InventoryRefresh::SaveCollection
       def assert_referential_integrity(hash)
         inventory_collection.fixed_foreign_keys.each do |x|
           next unless hash[x].nil?
+
           subject = "#{hash} of #{inventory_collection} because of missing foreign key #{x} for "\
                     "#{inventory_collection.parent.class.name}:"\
                     "#{inventory_collection.parent.try(:id)}"
-          unless inventory_collection.assert_graph_integrity
+          if inventory_collection.assert_graph_integrity
+            raise("Referential integrity check violated for #{subject}")
+          else
             logger.warn("Referential integrity check violated, ignoring #{subject}")
             return false
-          else
-            raise("Referential integrity check violated for #{subject}")
           end
         end
         true

data/lib/inventory_refresh/save_collection/saver/batch.rb

@@ -0,0 +1,17 @@
+require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
+
+module InventoryRefresh::SaveCollection
+  module Saver
+    class Batch < InventoryRefresh::SaveCollection::Saver::ConcurrentSafeBatch
+      private
+
+      # Just returning manager ref transformed to column names, for strategies that do not expect to have unique DB
+      # indexes.
+      #
+      # @return [Array<Symbol>] manager ref transformed to column names
+      def unique_index_columns
+        inventory_collection.manager_ref_to_cols.map(&:to_sym)
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb

@@ -46,6 +46,41 @@ module InventoryRefresh::SaveCollection
         record[select_keys_indexes[key]]
       end
 
+      # Returns iterator or relation based on settings
+      #
+      # @param association [Symbol] An existing association on manager
+      # @return [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or relation based on settings
+      def batch_iterator(association)
+        if pure_sql_records_fetching
+          # Building fast iterator doing pure SQL query and therefore avoiding redundant creation of AR objects. The
+          # iterator responds to find_in_batches, so it acts like the AR relation. For targeted refresh, the association
+          # can already be ApplicationRecordIterator, so we will skip that.
+          pure_sql_iterator = lambda do |&block|
+            primary_key_offset = nil
+            loop do
+              relation    = association.select(*select_keys)
+                                       .reorder("#{primary_key} ASC")
+                                       .limit(batch_size)
+              # Using rails way of comparing primary key instead of offset
+              relation    = relation.where(arel_primary_key.gt(primary_key_offset)) if primary_key_offset
+              records     = get_connection.query(relation.to_sql)
+              last_record = records.last
+              block.call(records)
+
+              break if records.size < batch_size
+
+              primary_key_offset = record_key(last_record, primary_key)
+            end
+          end
+
+          InventoryRefresh::ApplicationRecordIterator.new(:iterator => pure_sql_iterator)
+        else
+          # Normal Rails ActiveRecord::Relation where we can call find_in_batches or
+          # InventoryRefresh::ApplicationRecordIterator passed from targeted refresh
+          association
+        end
+      end
+
       # Saves the InventoryCollection
       #
       # @param association [Symbol] An existing association on manager
@@ -55,7 +90,7 @@ module InventoryRefresh::SaveCollection
         all_attribute_keys      = Set.new + inventory_collection.batch_extra_attributes
 
         inventory_collection.each do |inventory_object|
-          attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
+          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
           index      = build_stringified_reference(attributes, unique_index_keys)
 
           # Interesting fact: not building attributes_index and using only inventory_objects_index doesn't do much
@@ -69,7 +104,7 @@ module InventoryRefresh::SaveCollection
         logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
 
         unless inventory_collection.create_only?
-          load_and_update_records!(association, inventory_objects_index, attributes_index, all_attribute_keys)
+          update_or_destroy_records!(batch_iterator(association), inventory_objects_index, attributes_index, all_attribute_keys)
         end
 
         unless inventory_collection.create_only?
@@ -78,11 +113,15 @@ module InventoryRefresh::SaveCollection
 
         # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
         if inventory_collection.create_allowed?
+          on_conflict = inventory_collection.parallel_safe? ? :do_update : nil
+
           inventory_objects_index.each_slice(batch_size_for_persisting) do |batch|
-            create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => :do_update)
+            create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => on_conflict)
           end
 
-          create_or_update_partial_records(all_attribute_keys)
+          if inventory_collection.parallel_safe?
+            create_or_update_partial_records(all_attribute_keys)
+          end
         end
 
         logger.debug("Marking :last_seen_at of #{inventory_collection} of size #{inventory_collection.size}...")
@@ -103,7 +142,7 @@ module InventoryRefresh::SaveCollection
       end
 
       def expand_all_attribute_keys!(all_attribute_keys)
-        %i(created_at updated_at created_on updated_on).each do |col|
+        %i[created_at updated_at created_on updated_on].each do |col|
           all_attribute_keys << col if supports_column?(col)
         end
         all_attribute_keys << :type if supports_sti?
@@ -111,7 +150,7 @@ module InventoryRefresh::SaveCollection
       end
 
       def mark_last_seen_at(attributes_index)
-        return unless supports_column?(:last_seen_at)
+        return unless supports_column?(:last_seen_at) && inventory_collection.parallel_safe?
         return if attributes_index.blank?
 
         all_attribute_keys = [:last_seen_at]
@@ -124,7 +163,8 @@ module InventoryRefresh::SaveCollection
         get_connection.execute(query)
       end
 
-      # Batch updates existing records that are in the DB using attributes_index.
+      # Batch updates existing records that are in the DB using attributes_index. And delete the ones that were not
+      # present in inventory_objects_index.
       #
       # @param records_batch_iterator [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or
       #        relation, both responding to :find_in_batches method
@@ -132,11 +172,12 @@ module InventoryRefresh::SaveCollection
       # @param attributes_index [Hash{String => Hash}] Hash of data hashes with only keys that are column names of the
       #        models's table
       # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
-      def load_and_update_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
-        hashes_for_update         = []
+      def update_or_destroy_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
+        hashes_for_update   = []
+        records_for_destroy = []
         indexed_inventory_objects = {}
 
-        records_batch_iterator.find_in_batches(:batch_size => batch_size, :attributes_index => attributes_index) do |batch|
+        records_batch_iterator.find_in_batches(:batch_size => batch_size) do |batch|
           update_time = time_now
 
           batch.each do |record|
@@ -149,14 +190,20 @@ module InventoryRefresh::SaveCollection
             inventory_object = inventory_objects_index.delete(index)
             hash             = attributes_index[index]
 
-            if inventory_object
+            if inventory_object.nil?
+              # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
+              # delete it from the DB.
+              if inventory_collection.delete_allowed?
+                records_for_destroy << record
+              end
+            else
               # Record was found in the DB and sent for saving, we will be updating the DB.
               inventory_object.id = primary_key_value
               next unless assert_referential_integrity(hash)
+              next unless changed?(record, hash, all_attribute_keys)
 
-              record_version = nil
-              record_version_max = nil
-              if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
+              if inventory_collection.parallel_safe? &&
+                 (supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys))
 
                 version_attr, max_version_attr = if supports_remote_data_timestamp?(all_attribute_keys)
                                                    [:resource_timestamp, :resource_timestamps_max]
@@ -164,16 +211,16 @@ module InventoryRefresh::SaveCollection
                                                    [:resource_counter, :resource_counters_max]
                                                  end
 
-                record_version = record_key(record, version_attr.to_s)
-                record_version_max = record_key(record, max_version_attr.to_s)
+                next if skeletonize_or_skip_record(record_key(record, version_attr),
+                                                   hash[version_attr],
+                                                   record_key(record, max_version_attr),
+                                                   inventory_object)
               end
 
               hash_for_update = if inventory_collection.use_ar_object?
                                   record.assign_attributes(hash.except(:id))
-                                  next unless changed?(record)
-
                                   values_for_database!(all_attribute_keys,
-                                                       hash)
+                                                       record.attributes.symbolize_keys)
                                 elsif serializable_keys?
                                   # TODO(lsmola) hash data with current DB data to allow subset of data being sent,
                                   # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
@@ -184,14 +231,6 @@ module InventoryRefresh::SaveCollection
                                   # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
                                   hash
                                 end
-
-              if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
-                next if skeletonize_or_skip_record(record_version,
-                                                   hash[version_attr],
-                                                   record_version_max,
-                                                   inventory_object)
-              end
-
               assign_attributes_for_update!(hash_for_update, update_time)
 
               hash_for_update[:id] = primary_key_value
@@ -207,22 +246,39 @@ module InventoryRefresh::SaveCollection
             hashes_for_update = []
             indexed_inventory_objects = {}
           end
+
+          # Destroy in batches
+          if records_for_destroy.size >= batch_size_for_persisting
+            destroy_records!(records_for_destroy)
+            records_for_destroy = []
+          end
         end
 
         # Update the last batch
         update_records!(all_attribute_keys, hashes_for_update, indexed_inventory_objects)
         hashes_for_update = [] # Cleanup so GC can release it sooner
+
+        # Destroy the last batch
+        destroy_records!(records_for_destroy)
+        records_for_destroy = [] # Cleanup so GC can release it sooner
       end
 
-      def changed?(record)
+      def changed?(_record, _hash, _all_attribute_keys)
         return true unless inventory_collection.check_changed?
 
-        # If object was archived before, pass it to update so it can be unarchived
-        return true if record.respond_to?(:archived_at) && record.archived_at
-        # Skip if nothing changed
-        return false if record.changed_attributes.empty?
-        # Skip if we only changed the resource_timestamp, but data stays the same
-        return false if record.changed_attributes.keys == ["resource_timestamp"]
+        # TODO(lsmola) this check needs to be disabled now, because it doesn't work with lazy_find having secondary
+        # indexes. Examples: we save a pod before we save a project, that means the project lazy_find won't evaluate,
+        # because we load it with secondary index and can't do skeletal precreate. Then when the object is being saved
+        # again, the lazy_find is evaluated, but the resource version is not changed, so the row is not saved.
+        #
+        # To keep this quick .changed? check, we might need to extend this, so the resource_version doesn't save until
+        # all lazy_links of the row are evaluated.
+        #
+        # if supports_resource_version?(all_attribute_keys) && supports_column?(resource_version_column)
+        #   record_resource_version = record_key(record, resource_version_column.to_s)
+        #
+        #   return record_resource_version != hash[resource_version_column]
+        # end
 
         true
       end
@@ -230,7 +286,10 @@ module InventoryRefresh::SaveCollection
       def db_columns_index(record, pure_sql: false)
         # Incoming values are in SQL string form.
         # TODO(lsmola) unify this behavior with object_index_with_keys method in InventoryCollection
+        # TODO(lsmola) maybe we can drop the whole pure sql fetching, since everything will be targeted refresh
         # with streaming refresh? Maybe just metrics and events will not be, but those should be upsert only
+        # TODO(lsmola) taking ^ in account, we can't drop pure sql, since that is returned by batch insert and
+        # update queries
         unique_index_keys_to_s.map do |attribute|
           value = if pure_sql
                     record[attribute]
@@ -261,13 +320,20 @@ module InventoryRefresh::SaveCollection
       def update_records!(all_attribute_keys, hashes, indexed_inventory_objects)
         return if hashes.blank?
 
+        unless inventory_collection.parallel_safe?
+          # We need to update the stored records before we save it, since hashes are modified
+          inventory_collection.store_updated_records(hashes)
+        end
+
         query = build_update_query(all_attribute_keys, hashes)
         result = get_connection.execute(query)
 
-        # We will check for timestamp clashes of full row update and we will fallback to skeletal update
-        inventory_collection.store_updated_records(result)
+        if inventory_collection.parallel_safe?
+          # We will check for timestamp clashes of full row update and we will fallback to skeletal update
+          inventory_collection.store_updated_records(result)
 
-        skeletonize_ignored_records!(indexed_inventory_objects, result)
+          skeletonize_ignored_records!(indexed_inventory_objects, result)
+        end
 
         result
       end
@@ -287,7 +353,11 @@ module InventoryRefresh::SaveCollection
         hashes                    = []
         create_time               = time_now
         batch.each do |index, inventory_object|
-          hash = if serializable_keys?
+          hash = if inventory_collection.use_ar_object?
+                   record = inventory_collection.model_class.new(attributes_index[index])
+                   values_for_database!(all_attribute_keys,
+                                        record.attributes.symbolize_keys)
+                 elsif serializable_keys?
                    values_for_database!(all_attribute_keys,
                                         attributes_index[index])
                  else
@@ -309,19 +379,24 @@ module InventoryRefresh::SaveCollection
           build_insert_query(all_attribute_keys, hashes, :on_conflict => on_conflict, :mode => :full)
         )
 
-        # We've done upsert, so records were either created or updated. We can recognize that by checking if
-        # created and updated timestamps are the same
-        created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
-        created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
-        updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
-        updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
-
-        if created_attr && updated_attr
-          created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
-          inventory_collection.store_created_records(created)
-          inventory_collection.store_updated_records(updated)
+        if inventory_collection.parallel_safe?
+          # We've done upsert, so records were either created or updated. We can recognize that by checking if
+          # created and updated timestamps are the same
+          created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
+          created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
+          updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
+          updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
+
+          if created_attr && updated_attr
+            created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
+            inventory_collection.store_created_records(created)
+            inventory_collection.store_updated_records(updated)
+          else
+            # The record doesn't have both created and updated attrs, so we'll take all as created
+            inventory_collection.store_created_records(result)
+          end
         else
-          # The record doesn't have both created and updated attrs, so we'll take all as created
+          # We've done just insert, so all records were created
           inventory_collection.store_created_records(result)
         end
 
@@ -334,7 +409,9 @@ module InventoryRefresh::SaveCollection
                                        :on_conflict => on_conflict)
         end
 
-        skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
+        if inventory_collection.parallel_safe?
+          skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
+        end
       end
 
       # Stores primary_key values of created records into associated InventoryObject objects.
@@ -365,6 +442,7 @@ module InventoryRefresh::SaveCollection
               inventory_object[ref] = attributes[ref]
 
               next unless (foreign_key = association_to_foreign_key_mapping[ref])
+
               base_class_name       = attributes[association_to_foreign_type_mapping[ref].try(:to_sym)] || association_to_base_class_mapping[ref]
               id                    = attributes[foreign_key.to_sym]
               inventory_object[ref] = InventoryRefresh::ApplicationRecordReference.new(base_class_name, id)

data/lib/inventory_refresh/save_collection/saver/default.rb

@@ -0,0 +1,57 @@
+require "inventory_refresh/save_collection/saver/base"
+
+module InventoryRefresh::SaveCollection
+  module Saver
+    class Default < InventoryRefresh::SaveCollection::Saver::Base
+      private
+
+      # Updates the passed record with hash data and stores primary key value into inventory_object.
+      #
+      # @param record [ApplicationRecord] record we want to update in DB
+      # @param hash [Hash] data we want to update the record with
+      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
+      #        key value
+      def update_record!(record, hash, inventory_object)
+        record.assign_attributes(hash.except(:id))
+        if !inventory_collection.check_changed? || record.changed?
+          record.save
+          inventory_collection.store_updated_records(record)
+        end
+
+        inventory_object.id = record.id
+      end
+
+      # Creates a new record in the DB using the passed hash data
+      #
+      # @param hash [Hash] hash with data we want to persist to DB
+      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
+      #        key value
+      def create_record!(hash, inventory_object)
+        record = inventory_collection.model_class.create!(hash.except(:id))
+        inventory_collection.store_created_records(record)
+
+        inventory_object.id = record.id
+      end
+
+      # Asserts we do not have duplicate records in the DB. If the record is duplicate we will destroy it.
+      #
+      # @param record [ApplicationRecord] record we want to update in DB
+      # @param index [String] manager_uuid of the record
+      # @return [Boolean] false if the record is duplicate
+      def assert_unique_record(record, index)
+        # TODO(lsmola) can go away once we indexed our DB with unique indexes
+        if unique_db_indexes.include?(index) # Include on Set is O(1)
+          # We have a duplicate in the DB, destroy it. A find_each method does automatically .order(:id => :asc)
+          # so we always keep the oldest record in the case of duplicates.
+          logger.warn("A duplicate record was detected and destroyed, inventory_collection: "\
+                      "'#{inventory_collection}', record: '#{record}', duplicate_index: '#{index}'")
+          record.destroy
+          return false
+        else
+          unique_db_indexes << index
+        end
+        true
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb

@@ -59,7 +59,7 @@ module InventoryRefresh::SaveCollection
         skeletal_inventory_objects_index = {}
 
         inventory_collection.skeletal_primary_index.each_value do |inventory_object|
-          attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
+          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
           index      = build_stringified_reference(attributes, unique_index_keys)
 
           skeletal_attributes_index[index]        = attributes
@@ -135,6 +135,7 @@ module InventoryRefresh::SaveCollection
         # We need to set correct timestamps_max for this particular attribute, based on what is in timestamps
         batch.each do |x|
           next unless x[:__non_serialized_versions][column_name]
+
           x[comparables_max_name] = x[:__non_serialized_versions][column_name]
         end
       end
@@ -200,25 +201,7 @@ module InventoryRefresh::SaveCollection
         )
       end
 
-      def comparable_timestamp(timestamp)
-        # Lets cast all timestamps to to_f, rounding the time comparing precision to miliseconds, that should be
-        # enough, since we are the ones setting the record version in collector. Otherwise we will have hard time with
-        # doing equality, since the value changes going through DB (DB. cuts it at 5 decimal places)
-
-        if timestamp.kind_of?(String)
-          Time.use_zone('UTC') { Time.zone.parse(timestamp) }.to_f.round(3)
-        elsif timestamp.kind_of?(Time)
-          timestamp.in_time_zone('UTC').to_f.round(3)
-        else
-          timestamp
-        end
-      end
-
       def skeletonize_or_skip_record(record_version, hash_version, record_versions_max, inventory_object)
-        record_version      = comparable_timestamp(record_version)
-        record_versions_max = comparable_timestamp(record_versions_max) if record_versions_max
-        hash_version        = comparable_timestamp(hash_version)
-
         # Skip updating this record, because it is old
         return true if record_version && hash_version && record_version >= hash_version