inventory_refresh 0.1.3 → 0.2.0

data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb

@@ -1,4 +1,6 @@
 require "inventory_refresh/save_collection/saver/base"
+require "inventory_refresh/save_collection/saver/partial_upsert_helper"
+require "inventory_refresh/save_collection/saver/retention_helper"
 require "active_support/core_ext/module/delegation"
 
 module InventoryRefresh::SaveCollection
@@ -6,16 +8,21 @@ module InventoryRefresh::SaveCollection
     class ConcurrentSafeBatch < InventoryRefresh::SaveCollection::Saver::Base
       private
 
+      # Methods for archiving or deleting non existent records
+      include InventoryRefresh::SaveCollection::Saver::PartialUpsertHelper
+      include InventoryRefresh::SaveCollection::Saver::RetentionHelper
+
       delegate :association_to_base_class_mapping,
                :association_to_foreign_key_mapping,
                :association_to_foreign_type_mapping,
                :attribute_references,
+               :resource_version_column,
                :to => :inventory_collection
 
       # Attribute accessor to ApplicationRecord object or Hash
       #
       # @param record [Hash, ApplicationRecord] record or hash
-      # @param key [Symbol] key pointing to attribute of the record
+      # @param key [String] key pointing to attribute of the record
       # @return [Object] value of the record on the key
       def record_key(record, key)
         send(record_key_method, record, key)
@@ -24,7 +31,7 @@ module InventoryRefresh::SaveCollection
       # Attribute accessor to ApplicationRecord object
       #
       # @param record [ApplicationRecord] record
-      # @param key [Symbol] key pointing to attribute of the record
+      # @param key [String] key pointing to attribute of the record
       # @return [Object] value of the record on the key
       def ar_record_key(record, key)
         record.public_send(key)
@@ -33,7 +40,7 @@ module InventoryRefresh::SaveCollection
       # Attribute accessor to Hash object
       #
       # @param record [Hash] hash
-      # @param key [Symbol] key pointing to attribute of the record
+      # @param key [String] key pointing to attribute of the record
       # @return [Object] value of the record on the key
       def pure_sql_record_key(record, key)
         record[select_keys_indexes[key]]
@@ -82,7 +89,7 @@ module InventoryRefresh::SaveCollection
         all_attribute_keys      = Set.new + inventory_collection.batch_extra_attributes
 
         inventory_collection.each do |inventory_object|
-          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys)
+          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
           index      = build_stringified_reference(attributes, unique_index_keys)
 
           # Interesting fact: not building attributes_index and using only inventory_objects_index doesn't do much
@@ -91,11 +98,7 @@ module InventoryRefresh::SaveCollection
           inventory_objects_index[index] = inventory_object
         end
 
-        all_attribute_keys << :created_at if supports_created_at?
-        all_attribute_keys << :updated_at if supports_updated_at?
-        all_attribute_keys << :created_on if supports_created_on?
-        all_attribute_keys << :updated_on if supports_updated_on?
-        all_attribute_keys << :type if supports_sti?
+        expand_all_attribute_keys!(all_attribute_keys)
 
         logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
 
@@ -115,14 +118,19 @@ module InventoryRefresh::SaveCollection
             create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => on_conflict)
           end
 
-          # Let the GC clean this up
-          inventory_objects_index = nil
-          attributes_index = nil
-
           if inventory_collection.parallel_safe?
             create_or_update_partial_records(all_attribute_keys)
           end
         end
+
+        logger.debug("Marking :last_seen_at of #{inventory_collection} of size #{inventory_collection.size}...")
+
+        mark_last_seen_at(attributes_index)
+
+        # Let the GC clean this up
+        inventory_objects_index = nil
+        attributes_index = nil
+
         logger.debug("Processing #{inventory_collection}, "\
                      "created=#{inventory_collection.created_records.count}, "\
                      "updated=#{inventory_collection.updated_records.count}, "\
@@ -132,6 +140,28 @@ module InventoryRefresh::SaveCollection
         raise e
       end
 
+      def expand_all_attribute_keys!(all_attribute_keys)
+        %i(created_at updated_at created_on updated_on).each do |col|
+          all_attribute_keys << col if supports_column?(col)
+        end
+        all_attribute_keys << :type if supports_sti?
+        all_attribute_keys << :archived_at if supports_column?(:archived_at)
+      end
+
+      def mark_last_seen_at(attributes_index)
+        return unless supports_column?(:last_seen_at) && inventory_collection.parallel_safe?
+        return if attributes_index.blank?
+
+        all_attribute_keys = [:last_seen_at]
+
+        last_seen_at = Time.now.utc
+        attributes_index.each_value { |v| v[:last_seen_at] = last_seen_at }
+
+        query = build_partial_update_query(all_attribute_keys, attributes_index.values)
+
+        get_connection.execute(query)
+      end
+
       # Batch updates existing records that are in the DB using attributes_index. And delete the ones that were not
       # present in inventory_objects_index.
       #
@@ -157,7 +187,7 @@ module InventoryRefresh::SaveCollection
             index = db_columns_index(record)
 
             inventory_object = inventory_objects_index.delete(index)
-            hash             = attributes_index.delete(index)
+            hash             = attributes_index[index]
 
             if inventory_object.nil?
               # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
@@ -167,8 +197,9 @@ module InventoryRefresh::SaveCollection
               end
             else
               # Record was found in the DB and sent for saving, we will be updating the DB.
-              next unless assert_referential_integrity(hash)
               inventory_object.id = primary_key_value
+              next unless assert_referential_integrity(hash)
+              next unless changed?(record, hash, all_attribute_keys)
 
               if inventory_collection.parallel_safe? &&
                  (supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys))
@@ -176,12 +207,12 @@ module InventoryRefresh::SaveCollection
                 version_attr, max_version_attr = if supports_remote_data_timestamp?(all_attribute_keys)
                                                    [:resource_timestamp, :resource_timestamps_max]
                                                  elsif supports_remote_data_version?(all_attribute_keys)
-                                                   [:resource_version, :resource_versions_max]
+                                                   [:resource_counter, :resource_counters_max]
                                                  end
 
-                next if skeletonize_or_skip_record(record.try(version_attr) || record.try(:[], version_attr),
+                next if skeletonize_or_skip_record(record_key(record, version_attr),
                                                    hash[version_attr],
-                                                   record.try(max_version_attr) || record.try(:[], max_version_attr),
+                                                   record_key(record, max_version_attr),
                                                    inventory_object)
               end
 
@@ -231,6 +262,26 @@ module InventoryRefresh::SaveCollection
         records_for_destroy = [] # Cleanup so GC can release it sooner
       end
 
+      def changed?(_record, _hash, _all_attribute_keys)
+        return true unless inventory_collection.check_changed?
+
+        # TODO(lsmola) this check needs to be disabled now, because it doesn't work with lazy_find having secondary
+        # indexes. Examples: we save a pod before we save a project, that means the project lazy_find won't evaluate,
+        # because we load it with secondary index and can't do skeletal precreate. Then when the object is being saved
+        # again, the lazy_find is evaluated, but the resource version is not changed, so the row is not saved.
+        #
+        # To keep this quick .changed? check, we might need to extend this, so the resource_version doesn't save until
+        # all lazy_links of the row are evaluated.
+        #
+        # if supports_resource_version?(all_attribute_keys) && supports_column?(resource_version_column)
+        #   record_resource_version = record_key(record, resource_version_column.to_s)
+        #
+        #   return record_resource_version != hash[resource_version_column]
+        # end
+
+        true
+      end
+
       def db_columns_index(record, pure_sql: false)
         # Incoming values are in SQL string form.
         # TODO(lsmola) unify this behavior with object_index_with_keys method in InventoryCollection
@@ -261,39 +312,6 @@ module InventoryRefresh::SaveCollection
         end
       end
 
-      # Deletes or sof-deletes records. If the model_class supports a custom class delete method, we will use it for
-      # batch soft-delete.
-      #
-      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
-      #        to fetch ApplicationRecord objects from the DB
-      def destroy_records!(records)
-        return false unless inventory_collection.delete_allowed?
-        return if records.blank?
-
-        # Is the delete_method rails standard deleting method?
-        rails_delete = %i(destroy delete).include?(inventory_collection.delete_method)
-        if !rails_delete && inventory_collection.model_class.respond_to?(inventory_collection.delete_method)
-          # We have custom delete method defined on a class, that means it supports batch destroy
-          inventory_collection.store_deleted_records(records.map { |x| {:id => record_key(x, primary_key)} })
-          inventory_collection.model_class.public_send(inventory_collection.delete_method, records.map { |x| record_key(x, primary_key) })
-        else
-          # We have either standard :destroy and :delete rails method, or custom instance level delete method
-          # Note: The standard :destroy and :delete rails method can't be batched because of the hooks and cascade destroy
-          ActiveRecord::Base.transaction do
-            if pure_sql_records_fetching
-              # For pure SQL fetching, we need to get the AR objects again, so we can call destroy
-              inventory_collection.model_class.where(:id => records.map { |x| record_key(x, primary_key) }).find_each do |record|
-                delete_record!(record)
-              end
-            else
-              records.each do |record|
-                delete_record!(record)
-              end
-            end
-          end
-        end
-      end
-
       # Batch updates existing records
       #
       # @param hashes [Array<Hash>] data used for building a batch update sql query
@@ -319,153 +337,6 @@ module InventoryRefresh::SaveCollection
         result
       end
 
-      # Taking result from update or upsert of the row. The records that were not saved will be turned into skeletal
-      # records and we will save them attribute by attribute.
-      #
-      # @param hash [Hash{String => InventoryObject}>] Hash with indexed data we want to save
-      # @param result [Array<Hash>] Result from the DB containing the data that were actually saved
-      # @param all_unique_columns [Boolean] True if index is consisted from all columns of the unique index. False if
-      #        index is just made from manager_ref turned in DB column names.
-      def skeletonize_ignored_records!(hash, result, all_unique_columns: false)
-        updated = if all_unique_columns
-                    result.map { |x| unique_index_columns_to_s.map { |key| x[key] } }
-                  else
-                    result.map { |x| db_columns_index(x, :pure_sql => true) }
-                  end
-
-        updated.each { |x| hash.delete(x) }
-
-        # Now lets skeletonize all inventory_objects that were not saved by update or upsert. Old rows that can't be
-        # saved are not being sent here. We have only rows that are new, but become old as we send the query (so other
-        # parallel process saved the data in the meantime). Or if some attributes are newer than the whole row
-        # being sent.
-        hash.each_key do |db_index|
-          inventory_collection.skeletal_primary_index.skeletonize_primary_index(hash[db_index].manager_uuid)
-        end
-      end
-
-      # Saves partial records using upsert, taking records from skeletal_primary_index. This is used both for
-      # skeletal precreate as well as for saving partial rows.
-      #
-      # @param all_attribute_keys [Set] Superset of all keys of all records being saved
-      def create_or_update_partial_records(all_attribute_keys)
-        skeletal_attributes_index        = {}
-        skeletal_inventory_objects_index = {}
-
-        inventory_collection.skeletal_primary_index.each_value do |inventory_object|
-          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys)
-          index      = build_stringified_reference(attributes, unique_index_keys)
-
-          skeletal_attributes_index[index]        = attributes
-          skeletal_inventory_objects_index[index] = inventory_object
-        end
-
-        if supports_remote_data_timestamp?(all_attribute_keys)
-          all_attribute_keys << :resource_timestamps
-          all_attribute_keys << :resource_timestamps_max
-        elsif supports_remote_data_version?(all_attribute_keys)
-          all_attribute_keys << :resource_versions
-          all_attribute_keys << :resource_versions_max
-        end
-
-        indexed_inventory_objects = {}
-        hashes                    = []
-        create_time               = time_now
-
-        skeletal_inventory_objects_index.each do |index, inventory_object|
-          hash = skeletal_attributes_index.delete(index)
-          # Partial create or update must never set a timestamp for the whole row
-          timestamps = if supports_remote_data_timestamp?(all_attribute_keys) && supports_resource_timestamps_max?
-                         assign_partial_row_version_attributes!(:resource_timestamp,
-                                                                :resource_timestamps,
-                                                                :resource_timestamps_max,
-                                                                hash,
-                                                                all_attribute_keys)
-                       elsif supports_remote_data_version?(all_attribute_keys) && supports_resource_versions_max?
-                         assign_partial_row_version_attributes!(:resource_version,
-                                                                :resource_versions,
-                                                                :resource_versions_max,
-                                                                hash,
-                                                                all_attribute_keys)
-                       end
-          # Transform hash to DB format
-          hash = transform_to_hash!(all_attribute_keys, hash)
-
-          assign_attributes_for_create!(hash, create_time)
-
-          next unless assert_referential_integrity(hash)
-
-          hash[:__non_serialized_versions] = timestamps # store non serialized timestamps for the partial updates
-          hashes << hash
-          # Index on Unique Columns values, so we can easily fill in the :id later
-          indexed_inventory_objects[unique_index_columns.map { |x| hash[x] }] = inventory_object
-        end
-
-        return if hashes.blank?
-
-        # First, lets try to create all partial records
-        hashes.each_slice(batch_size_for_persisting) do |batch|
-          result = create_partial!(all_attribute_keys,
-                                   batch,
-                                   :on_conflict => :do_nothing)
-          inventory_collection.store_created_records(result)
-        end
-
-        # We need only skeletal records with timestamp. We can't save the ones without timestamp, because e.g. skeletal
-        # precreate would be updating records with default values, that are not correct.
-        pre_filtered = hashes.select { |x| x[:resource_timestamps_max] || x[:resource_versions_max] }
-
-        results = {}
-        # TODO(lsmola) we don't need to process rows that were save by the create -> oncoflict do nothing
-        (all_attribute_keys - inventory_collection.base_columns).each do |column_name|
-          filtered = pre_filtered.select { |x| x.key?(column_name) }
-
-          filtered.each_slice(batch_size_for_persisting) do |batch|
-            # We need to set correct timestamps_max for this particular attribute, based on what is in timestamps
-            if supports_remote_data_timestamp?(all_attribute_keys)
-              batch.each { |x| x[:resource_timestamps_max] = x[:__non_serialized_versions][column_name] if x[:__non_serialized_versions][column_name] }
-            elsif supports_remote_data_version?(all_attribute_keys)
-              batch.each { |x| x[:resource_versions_max] = x[:__non_serialized_versions][column_name] if x[:__non_serialized_versions][column_name] }
-            end
-
-            result = create_partial!(inventory_collection.base_columns + [column_name],
-                                     batch,
-                                     :on_conflict => :do_update,
-                                     :column_name => column_name)
-            result.each do |res|
-              results[res["id"]] = res
-            end
-          end
-        end
-
-        inventory_collection.store_updated_records(results.values)
-
-        # TODO(lsmola) we need to move here the hash loading ar object etc. otherwise the lazy_find with key will not
-        # be correct
-        if inventory_collection.dependees.present?
-          # We need to get primary keys of the created objects, but only if there are dependees that would use them
-          map_ids_to_inventory_objects(indexed_inventory_objects,
-                                       all_attribute_keys,
-                                       hashes,
-                                       nil,
-                                       :on_conflict => :do_nothing)
-        end
-      end
-
-      # Batch upserts 1 data column of the row, plus the internal columns
-      #
-      # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
-      # @param hashes [Array<InventoryRefresh::InventoryObject>] Array of InventoryObject objects we will be inserting
-      #        into the DB
-      # @param on_conflict [Symbol, NilClass] defines behavior on conflict with unique index constraint, allowed values
-      #        are :do_update, :do_nothing, nil
-      # @param column_name [Symbol] Name of the data column we will be upserting
-      def create_partial!(all_attribute_keys, hashes, on_conflict: nil, column_name: nil)
-        get_connection.execute(
-          build_insert_query(all_attribute_keys, hashes, :on_conflict => on_conflict, :mode => :partial, :column_name => column_name)
-        )
-      end
-
       # Batch inserts records using attributes_index data. With on_conflict option using :do_update, this method
       # does atomic upsert.
       #
@@ -510,10 +381,10 @@ module InventoryRefresh::SaveCollection
         if inventory_collection.parallel_safe?
           # We've done upsert, so records were either created or updated. We can recognize that by checking if
           # created and updated timestamps are the same
-          created_attr = "created_on" if inventory_collection.supports_created_on?
-          created_attr ||= "created_at" if inventory_collection.supports_created_at?
-          updated_attr = "updated_on" if inventory_collection.supports_updated_on?
-          updated_attr ||= "updated_at" if inventory_collection.supports_updated_at?
+          created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
+          created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
+          updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
+          updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
 
           if created_attr && updated_attr
             created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
@@ -604,29 +475,6 @@ module InventoryRefresh::SaveCollection
           end
         end
       end
-
-      def skeletonize_or_skip_record(record_version, hash_version, record_versions_max, inventory_object)
-        # Skip updating this record, because it is old
-        return true if record_version && hash_version && record_version >= hash_version
-
-        # Some column has bigger version than the whole row, we need to store the row partially
-        if record_versions_max && hash_version && record_versions_max > hash_version
-          inventory_collection.skeletal_primary_index.skeletonize_primary_index(inventory_object.manager_uuid)
-          return true
-        end
-
-        false
-      end
-
-      def assign_partial_row_version_attributes!(full_row_version_attr, partial_row_version_attr,
-                                                 partial_row_version_attr_max, hash, all_attribute_keys)
-        hash[partial_row_version_attr_max] = hash.delete(full_row_version_attr)
-
-        if hash[partial_row_version_attr].present?
-          # Lets clean to only what we save, since when we build the skeletal object, we can set more
-          hash[partial_row_version_attr] = hash[partial_row_version_attr].slice(*all_attribute_keys)
-        end
-      end
     end
   end
 end

data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb

@@ -0,0 +1,226 @@
+module InventoryRefresh::SaveCollection
+  module Saver
+    module PartialUpsertHelper
+      private
+
+      # Taking result from update or upsert of the row. The records that were not saved will be turned into skeletal
+      # records and we will save them attribute by attribute.
+      #
+      # @param hash [Hash{String => InventoryObject}>] Hash with indexed data we want to save
+      # @param result [Array<Hash>] Result from the DB containing the data that were actually saved
+      # @param all_unique_columns [Boolean] True if index is consisted from all columns of the unique index. False if
+      #        index is just made from manager_ref turned in DB column names.
+      def skeletonize_ignored_records!(hash, result, all_unique_columns: false)
+        updated = if all_unique_columns
+                    result.map { |x| unique_index_columns_to_s.map { |key| x[key] } }
+                  else
+                    result.map { |x| db_columns_index(x, :pure_sql => true) }
+                  end
+
+        updated.each { |x| hash.delete(x) }
+
+        # Now lets skeletonize all inventory_objects that were not saved by update or upsert. Old rows that can't be
+        # saved are not being sent here. We have only rows that are new, but become old as we send the query (so other
+        # parallel process saved the data in the meantime). Or if some attributes are newer than the whole row
+        # being sent.
+        hash.each_key do |db_index|
+          inventory_collection.skeletal_primary_index.skeletonize_primary_index(hash[db_index].manager_uuid)
+        end
+      end
+
+      # Saves partial records using upsert, taking records from skeletal_primary_index. This is used both for
+      # skeletal precreate as well as for saving partial rows.
+      #
+      # @param all_attribute_keys [Set] Superset of all keys of all records being saved
+      def create_or_update_partial_records(all_attribute_keys)
+        skeletal_inventory_objects_index, skeletal_attributes_index = load_partial_attributes(all_attribute_keys)
+
+        indexed_inventory_objects, hashes = process_partial_data(skeletal_inventory_objects_index,
+                                                                 skeletal_attributes_index,
+                                                                 all_attribute_keys)
+        return if hashes.blank?
+
+        processed_record_refs = skeletal_precreate!(hashes, all_attribute_keys)
+        hashes_for_update = hashes_for_update(hashes, processed_record_refs)
+        partial_updates!(hashes_for_update, all_attribute_keys)
+
+        if inventory_collection.dependees.present?
+          # We need to get primary keys of the created objects, but only if there are dependees that would use them
+          map_ids_to_inventory_objects(indexed_inventory_objects,
+                                       all_attribute_keys,
+                                       hashes,
+                                       nil,
+                                       :on_conflict => :do_nothing)
+        end
+      end
+
+      def load_partial_attributes(all_attribute_keys)
+        skeletal_attributes_index        = {}
+        skeletal_inventory_objects_index = {}
+
+        inventory_collection.skeletal_primary_index.each_value do |inventory_object|
+          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
+          index      = build_stringified_reference(attributes, unique_index_keys)
+
+          skeletal_attributes_index[index]        = attributes
+          skeletal_inventory_objects_index[index] = inventory_object
+        end
+
+        if supports_remote_data_timestamp?(all_attribute_keys)
+          all_attribute_keys << :resource_timestamps
+          all_attribute_keys << :resource_timestamps_max
+        elsif supports_remote_data_version?(all_attribute_keys)
+          all_attribute_keys << :resource_counters
+          all_attribute_keys << :resource_counters_max
+        end
+
+        # We cannot set the resource_version doing partial update
+        all_attribute_keys.delete(resource_version_column)
+
+        return skeletal_inventory_objects_index, skeletal_attributes_index
+      end
+
+      def hashes_for_update(hashes, processed_record_refs)
+        indexed_hashes = hashes.each_with_object({}) { |hash, obj| obj[unique_index_columns.map { |x| hash[x] }] = hash }
+        indexed_hashes.except!(*processed_record_refs)
+        hashes_for_update = indexed_hashes.values
+
+        # We need only skeletal records with timestamp. We can't save the ones without timestamp, because e.g. skeletal
+        # precreate would be updating records with default values, that are not correct.
+        hashes_for_update.select { |x| x[:resource_timestamps_max] || x[:resource_counters_max] }
+      end
+
+      def skeletal_precreate!(hashes, all_attribute_keys)
+        processed_record_refs = []
+        # First, lets try to create all partial records
+        hashes.each_slice(batch_size_for_persisting) do |batch|
+          result = create_partial!(all_attribute_keys,
+                                   batch,
+                                   :on_conflict => :do_nothing)
+          inventory_collection.store_created_records(result)
+          # Store refs of created records, so we can ignore them for update
+          result.each { |hash| processed_record_refs << unique_index_columns.map { |x| hash[x.to_s] } }
+        end
+
+        processed_record_refs
+      end
+
+      def partial_updates!(hashes, all_attribute_keys)
+        results = {}
+        (all_attribute_keys - inventory_collection.base_columns).each do |column_name|
+          filtered = hashes.select { |x| x.key?(column_name) }
+
+          filtered.each_slice(batch_size_for_persisting) do |batch|
+            partial_update!(batch, all_attribute_keys, column_name, results)
+          end
+        end
+
+        inventory_collection.store_updated_records(results.values)
+      end
+
+      def partial_update!(batch, all_attribute_keys, column_name, results)
+        fill_comparables_max!(batch, all_attribute_keys, column_name)
+        result = create_partial!((inventory_collection.base_columns + [column_name]).to_set & all_attribute_keys,
+                                 batch,
+                                 :on_conflict => :do_update,
+                                 :column_name => column_name)
+        result.each do |res|
+          results[res["id"]] = res
+        end
+      end
+
+      def fill_comparables_max!(batch, all_attribute_keys, column_name)
+        comparables_max_name = comparable_max_column_name(all_attribute_keys)
+
+        # We need to set correct timestamps_max for this particular attribute, based on what is in timestamps
+        batch.each do |x|
+          next unless x[:__non_serialized_versions][column_name]
+          x[comparables_max_name] = x[:__non_serialized_versions][column_name]
+        end
+      end
+
+      def comparable_max_column_name(all_attribute_keys)
+        if supports_remote_data_timestamp?(all_attribute_keys)
+          :resource_timestamps_max
+        elsif supports_remote_data_version?(all_attribute_keys)
+          :resource_counters_max
+        end
+      end
+
+      def process_partial_data(skeletal_inventory_objects_index, skeletal_attributes_index, all_attribute_keys)
+        indexed_inventory_objects = {}
+        hashes                    = []
+        create_time               = time_now
+
+        skeletal_inventory_objects_index.each do |index, inventory_object|
+          hash = prepare_partial_hash(skeletal_attributes_index.delete(index), all_attribute_keys, create_time)
+          next unless assert_referential_integrity(hash)
+
+          hashes << hash
+          # Index on Unique Columns values, so we can easily fill in the :id later
+          indexed_inventory_objects[unique_index_columns.map { |x| hash[x] }] = inventory_object
+        end
+
+        return indexed_inventory_objects, hashes
+      end
+
+      def prepare_partial_hash(hash, all_attribute_keys, create_time)
+        # Partial create or update must never set a timestamp for the whole row
+        timestamps = if supports_remote_data_timestamp?(all_attribute_keys) && supports_column?(:resource_timestamps_max)
+                       assign_partial_row_version_attributes!(:resource_timestamp,
+                                                              :resource_timestamps,
+                                                              hash,
+                                                              all_attribute_keys)
+                     elsif supports_remote_data_version?(all_attribute_keys) && supports_column?(:resource_counters_max)
+                       assign_partial_row_version_attributes!(:resource_counter,
+                                                              :resource_counters,
+                                                              hash,
+                                                              all_attribute_keys)
+                     end
+        # Transform hash to DB format
+        hash = transform_to_hash!(all_attribute_keys, hash)
+
+        assign_attributes_for_create!(hash, create_time)
+
+        hash[:__non_serialized_versions] = timestamps # store non serialized timestamps for the partial updates
+        hash
+      end
+
+      # Batch upserts 1 data column of the row, plus the internal columns
+      #
+      # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
+      # @param hashes [Array<InventoryRefresh::InventoryObject>] Array of InventoryObject objects we will be inserting
+      #        into the DB
+      # @param on_conflict [Symbol, NilClass] defines behavior on conflict with unique index constraint, allowed values
+      #        are :do_update, :do_nothing, nil
+      # @param column_name [Symbol] Name of the data column we will be upserting
+      def create_partial!(all_attribute_keys, hashes, on_conflict: nil, column_name: nil)
+        get_connection.execute(
+          build_insert_query(all_attribute_keys, hashes, :on_conflict => on_conflict, :mode => :partial, :column_name => column_name)
+        )
+      end
+
+      def skeletonize_or_skip_record(record_version, hash_version, record_versions_max, inventory_object)
+        # Skip updating this record, because it is old
+        return true if record_version && hash_version && record_version >= hash_version
+
+        # Some column has bigger version than the whole row, we need to store the row partially
+        if record_versions_max && hash_version && record_versions_max > hash_version
+          inventory_collection.skeletal_primary_index.skeletonize_primary_index(inventory_object.manager_uuid)
+          return true
+        end
+
+        false
+      end
+
+      def assign_partial_row_version_attributes!(full_row_version_attr, partial_row_version_attr, hash, all_attribute_keys)
+        hash[comparable_max_column_name(all_attribute_keys)] = hash.delete(full_row_version_attr)
+
+        return if hash[partial_row_version_attr].blank?
+
+        # Lets clean to only what we save, since when we build the skeletal object, we can set more
+        hash[partial_row_version_attr] = hash[partial_row_version_attr].slice(*all_attribute_keys)
+      end
+    end
+  end
+end