inventory_refresh 0.3.3 → 1.0.0

data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb

@@ -46,6 +46,41 @@ module InventoryRefresh::SaveCollection
         record[select_keys_indexes[key]]
       end
 
+      # Returns iterator or relation based on settings
+      #
+      # @param association [Symbol] An existing association on manager
+      # @return [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or relation based on settings
+      def batch_iterator(association)
+        if pure_sql_records_fetching
+          # Building fast iterator doing pure SQL query and therefore avoiding redundant creation of AR objects. The
+          # iterator responds to find_in_batches, so it acts like the AR relation. For targeted refresh, the association
+          # can already be ApplicationRecordIterator, so we will skip that.
+          pure_sql_iterator = lambda do |&block|
+            primary_key_offset = nil
+            loop do
+              relation    = association.select(*select_keys)
+                                       .reorder("#{primary_key} ASC")
+                                       .limit(batch_size)
+              # Using rails way of comparing primary key instead of offset
+              relation    = relation.where(arel_primary_key.gt(primary_key_offset)) if primary_key_offset
+              records     = get_connection.query(relation.to_sql)
+              last_record = records.last
+              block.call(records)
+
+              break if records.size < batch_size
+
+              primary_key_offset = record_key(last_record, primary_key)
+            end
+          end
+
+          InventoryRefresh::ApplicationRecordIterator.new(:iterator => pure_sql_iterator)
+        else
+          # Normal Rails ActiveRecord::Relation where we can call find_in_batches or
+          # InventoryRefresh::ApplicationRecordIterator passed from targeted refresh
+          association
+        end
+      end
+
       # Saves the InventoryCollection
       #
       # @param association [Symbol] An existing association on manager
@@ -55,7 +90,7 @@ module InventoryRefresh::SaveCollection
         all_attribute_keys      = Set.new + inventory_collection.batch_extra_attributes
 
         inventory_collection.each do |inventory_object|
-          attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
+          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
           index      = build_stringified_reference(attributes, unique_index_keys)
 
           # Interesting fact: not building attributes_index and using only inventory_objects_index doesn't do much
@@ -69,7 +104,7 @@ module InventoryRefresh::SaveCollection
         logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
 
         unless inventory_collection.create_only?
-          load_and_update_records!(association, inventory_objects_index, attributes_index, all_attribute_keys)
+          update_or_destroy_records!(batch_iterator(association), inventory_objects_index, attributes_index, all_attribute_keys)
         end
 
         unless inventory_collection.create_only?
@@ -78,11 +113,15 @@ module InventoryRefresh::SaveCollection
 
         # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
         if inventory_collection.create_allowed?
+          on_conflict = inventory_collection.parallel_safe? ? :do_update : nil
+
           inventory_objects_index.each_slice(batch_size_for_persisting) do |batch|
-            create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => :do_update)
+            create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => on_conflict)
           end
 
-          create_or_update_partial_records(all_attribute_keys)
+          if inventory_collection.parallel_safe?
+            create_or_update_partial_records(all_attribute_keys)
+          end
         end
 
         logger.debug("Marking :last_seen_at of #{inventory_collection} of size #{inventory_collection.size}...")
@@ -103,7 +142,7 @@ module InventoryRefresh::SaveCollection
       end
 
       def expand_all_attribute_keys!(all_attribute_keys)
-        %i(created_at updated_at created_on updated_on).each do |col|
+        %i[created_at updated_at created_on updated_on].each do |col|
           all_attribute_keys << col if supports_column?(col)
         end
         all_attribute_keys << :type if supports_sti?
@@ -111,7 +150,7 @@ module InventoryRefresh::SaveCollection
       end
 
       def mark_last_seen_at(attributes_index)
-        return unless supports_column?(:last_seen_at)
+        return unless supports_column?(:last_seen_at) && inventory_collection.parallel_safe?
         return if attributes_index.blank?
 
         all_attribute_keys = [:last_seen_at]
@@ -124,7 +163,8 @@ module InventoryRefresh::SaveCollection
         get_connection.execute(query)
       end
 
-      # Batch updates existing records that are in the DB using attributes_index.
+      # Batch updates existing records that are in the DB using attributes_index. And delete the ones that were not
+      # present in inventory_objects_index.
       #
       # @param records_batch_iterator [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or
       #        relation, both responding to :find_in_batches method
@@ -132,11 +172,12 @@ module InventoryRefresh::SaveCollection
       # @param attributes_index [Hash{String => Hash}] Hash of data hashes with only keys that are column names of the
       #        models's table
       # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
-      def load_and_update_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
-        hashes_for_update         = []
+      def update_or_destroy_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
+        hashes_for_update   = []
+        records_for_destroy = []
         indexed_inventory_objects = {}
 
-        records_batch_iterator.find_in_batches(:batch_size => batch_size, :attributes_index => attributes_index) do |batch|
+        records_batch_iterator.find_in_batches(:batch_size => batch_size) do |batch|
           update_time = time_now
 
           batch.each do |record|
@@ -149,14 +190,20 @@ module InventoryRefresh::SaveCollection
             inventory_object = inventory_objects_index.delete(index)
             hash             = attributes_index[index]
 
-            if inventory_object
+            if inventory_object.nil?
+              # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
+              # delete it from the DB.
+              if inventory_collection.delete_allowed?
+                records_for_destroy << record
+              end
+            else
               # Record was found in the DB and sent for saving, we will be updating the DB.
               inventory_object.id = primary_key_value
               next unless assert_referential_integrity(hash)
+              next unless changed?(record, hash, all_attribute_keys)
 
-              record_version = nil
-              record_version_max = nil
-              if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
+              if inventory_collection.parallel_safe? &&
+                 (supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys))
 
                 version_attr, max_version_attr = if supports_remote_data_timestamp?(all_attribute_keys)
                                                    [:resource_timestamp, :resource_timestamps_max]
@@ -164,16 +211,16 @@ module InventoryRefresh::SaveCollection
                                                    [:resource_counter, :resource_counters_max]
                                                  end
 
-                record_version = record_key(record, version_attr.to_s)
-                record_version_max = record_key(record, max_version_attr.to_s)
+                next if skeletonize_or_skip_record(record_key(record, version_attr),
+                                                   hash[version_attr],
+                                                   record_key(record, max_version_attr),
+                                                   inventory_object)
               end
 
               hash_for_update = if inventory_collection.use_ar_object?
                                   record.assign_attributes(hash.except(:id))
-                                  next unless changed?(record)
-
                                   values_for_database!(all_attribute_keys,
-                                                       hash)
+                                                       record.attributes.symbolize_keys)
                                 elsif serializable_keys?
                                   # TODO(lsmola) hash data with current DB data to allow subset of data being sent,
                                   # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
@@ -184,14 +231,6 @@ module InventoryRefresh::SaveCollection
                                   # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
                                   hash
                                 end
-
-              if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
-                next if skeletonize_or_skip_record(record_version,
-                                                   hash[version_attr],
-                                                   record_version_max,
-                                                   inventory_object)
-              end
-
               assign_attributes_for_update!(hash_for_update, update_time)
 
               hash_for_update[:id] = primary_key_value
@@ -207,22 +246,39 @@ module InventoryRefresh::SaveCollection
             hashes_for_update = []
             indexed_inventory_objects = {}
           end
+
+          # Destroy in batches
+          if records_for_destroy.size >= batch_size_for_persisting
+            destroy_records!(records_for_destroy)
+            records_for_destroy = []
+          end
         end
 
         # Update the last batch
         update_records!(all_attribute_keys, hashes_for_update, indexed_inventory_objects)
         hashes_for_update = [] # Cleanup so GC can release it sooner
+
+        # Destroy the last batch
+        destroy_records!(records_for_destroy)
+        records_for_destroy = [] # Cleanup so GC can release it sooner
       end
 
-      def changed?(record)
+      def changed?(_record, _hash, _all_attribute_keys)
         return true unless inventory_collection.check_changed?
 
-        # If object was archived before, pass it to update so it can be unarchived
-        return true if record.respond_to?(:archived_at) && record.archived_at
-        # Skip if nothing changed
-        return false if record.changed_attributes.empty?
-        # Skip if we only changed the resource_timestamp, but data stays the same
-        return false if record.changed_attributes.keys == ["resource_timestamp"]
+        # TODO(lsmola) this check needs to be disabled now, because it doesn't work with lazy_find having secondary
+        # indexes. Examples: we save a pod before we save a project, that means the project lazy_find won't evaluate,
+        # because we load it with secondary index and can't do skeletal precreate. Then when the object is being saved
+        # again, the lazy_find is evaluated, but the resource version is not changed, so the row is not saved.
+        #
+        # To keep this quick .changed? check, we might need to extend this, so the resource_version doesn't save until
+        # all lazy_links of the row are evaluated.
+        #
+        # if supports_resource_version?(all_attribute_keys) && supports_column?(resource_version_column)
+        #   record_resource_version = record_key(record, resource_version_column.to_s)
+        #
+        #   return record_resource_version != hash[resource_version_column]
+        # end
 
         true
       end
@@ -230,7 +286,10 @@ module InventoryRefresh::SaveCollection
       def db_columns_index(record, pure_sql: false)
         # Incoming values are in SQL string form.
         # TODO(lsmola) unify this behavior with object_index_with_keys method in InventoryCollection
+        # TODO(lsmola) maybe we can drop the whole pure sql fetching, since everything will be targeted refresh
         # with streaming refresh? Maybe just metrics and events will not be, but those should be upsert only
+        # TODO(lsmola) taking ^ in account, we can't drop pure sql, since that is returned by batch insert and
+        # update queries
         unique_index_keys_to_s.map do |attribute|
           value = if pure_sql
                     record[attribute]
@@ -261,13 +320,20 @@ module InventoryRefresh::SaveCollection
       def update_records!(all_attribute_keys, hashes, indexed_inventory_objects)
         return if hashes.blank?
 
+        unless inventory_collection.parallel_safe?
+          # We need to update the stored records before we save it, since hashes are modified
+          inventory_collection.store_updated_records(hashes)
+        end
+
         query = build_update_query(all_attribute_keys, hashes)
         result = get_connection.execute(query)
 
-        # We will check for timestamp clashes of full row update and we will fallback to skeletal update
-        inventory_collection.store_updated_records(result)
+        if inventory_collection.parallel_safe?
+          # We will check for timestamp clashes of full row update and we will fallback to skeletal update
+          inventory_collection.store_updated_records(result)
 
-        skeletonize_ignored_records!(indexed_inventory_objects, result)
+          skeletonize_ignored_records!(indexed_inventory_objects, result)
+        end
 
         result
       end
@@ -287,7 +353,11 @@ module InventoryRefresh::SaveCollection
         hashes                    = []
         create_time               = time_now
         batch.each do |index, inventory_object|
-          hash = if serializable_keys?
+          hash = if inventory_collection.use_ar_object?
+                   record = inventory_collection.model_class.new(attributes_index[index])
+                   values_for_database!(all_attribute_keys,
+                                        record.attributes.symbolize_keys)
+                 elsif serializable_keys?
                    values_for_database!(all_attribute_keys,
                                         attributes_index[index])
                  else
@@ -309,19 +379,24 @@ module InventoryRefresh::SaveCollection
           build_insert_query(all_attribute_keys, hashes, :on_conflict => on_conflict, :mode => :full)
         )
 
-        # We've done upsert, so records were either created or updated. We can recognize that by checking if
-        # created and updated timestamps are the same
-        created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
-        created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
-        updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
-        updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
-
-        if created_attr && updated_attr
-          created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
-          inventory_collection.store_created_records(created)
-          inventory_collection.store_updated_records(updated)
+        if inventory_collection.parallel_safe?
+          # We've done upsert, so records were either created or updated. We can recognize that by checking if
+          # created and updated timestamps are the same
+          created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
+          created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
+          updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
+          updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
+
+          if created_attr && updated_attr
+            created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
+            inventory_collection.store_created_records(created)
+            inventory_collection.store_updated_records(updated)
+          else
+            # The record doesn't have both created and updated attrs, so we'll take all as created
+            inventory_collection.store_created_records(result)
+          end
         else
-          # The record doesn't have both created and updated attrs, so we'll take all as created
+          # We've done just insert, so all records were created
           inventory_collection.store_created_records(result)
         end
 
@@ -334,7 +409,9 @@ module InventoryRefresh::SaveCollection
                                        :on_conflict => on_conflict)
         end
 
-        skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
+        if inventory_collection.parallel_safe?
+          skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
+        end
       end
 
       # Stores primary_key values of created records into associated InventoryObject objects.
@@ -365,6 +442,7 @@ module InventoryRefresh::SaveCollection
               inventory_object[ref] = attributes[ref]
 
               next unless (foreign_key = association_to_foreign_key_mapping[ref])
+
               base_class_name       = attributes[association_to_foreign_type_mapping[ref].try(:to_sym)] || association_to_base_class_mapping[ref]
               id                    = attributes[foreign_key.to_sym]
               inventory_object[ref] = InventoryRefresh::ApplicationRecordReference.new(base_class_name, id)

data/lib/inventory_refresh/save_collection/saver/default.rb

@@ -0,0 +1,57 @@
+require "inventory_refresh/save_collection/saver/base"
+
+module InventoryRefresh::SaveCollection
+  module Saver
+    class Default < InventoryRefresh::SaveCollection::Saver::Base
+      private
+
+      # Updates the passed record with hash data and stores primary key value into inventory_object.
+      #
+      # @param record [ApplicationRecord] record we want to update in DB
+      # @param hash [Hash] data we want to update the record with
+      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
+      #        key value
+      def update_record!(record, hash, inventory_object)
+        record.assign_attributes(hash.except(:id))
+        if !inventory_collection.check_changed? || record.changed?
+          record.save
+          inventory_collection.store_updated_records(record)
+        end
+
+        inventory_object.id = record.id
+      end
+
+      # Creates a new record in the DB using the passed hash data
+      #
+      # @param hash [Hash] hash with data we want to persist to DB
+      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
+      #        key value
+      def create_record!(hash, inventory_object)
+        record = inventory_collection.model_class.create!(hash.except(:id))
+        inventory_collection.store_created_records(record)
+
+        inventory_object.id = record.id
+      end
+
+      # Asserts we do not have duplicate records in the DB. If the record is duplicate we will destroy it.
+      #
+      # @param record [ApplicationRecord] record we want to update in DB
+      # @param index [String] manager_uuid of the record
+      # @return [Boolean] false if the record is duplicate
+      def assert_unique_record(record, index)
+        # TODO(lsmola) can go away once we indexed our DB with unique indexes
+        if unique_db_indexes.include?(index) # Include on Set is O(1)
+          # We have a duplicate in the DB, destroy it. A find_each method does automatically .order(:id => :asc)
+          # so we always keep the oldest record in the case of duplicates.
+          logger.warn("A duplicate record was detected and destroyed, inventory_collection: "\
+                      "'#{inventory_collection}', record: '#{record}', duplicate_index: '#{index}'")
+          record.destroy
+          return false
+        else
+          unique_db_indexes << index
+        end
+        true
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb

@@ -59,7 +59,7 @@ module InventoryRefresh::SaveCollection
         skeletal_inventory_objects_index = {}
 
         inventory_collection.skeletal_primary_index.each_value do |inventory_object|
-          attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
+          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
           index      = build_stringified_reference(attributes, unique_index_keys)
 
           skeletal_attributes_index[index]        = attributes
@@ -135,6 +135,7 @@ module InventoryRefresh::SaveCollection
         # We need to set correct timestamps_max for this particular attribute, based on what is in timestamps
         batch.each do |x|
           next unless x[:__non_serialized_versions][column_name]
+
           x[comparables_max_name] = x[:__non_serialized_versions][column_name]
         end
       end
@@ -200,25 +201,7 @@ module InventoryRefresh::SaveCollection
         )
       end
 
-      def comparable_timestamp(timestamp)
-        # Lets cast all timestamps to to_f, rounding the time comparing precision to miliseconds, that should be
-        # enough, since we are the ones setting the record version in collector. Otherwise we will have hard time with
-        # doing equality, since the value changes going through DB (DB. cuts it at 5 decimal places)
-
-        if timestamp.kind_of?(String)
-          Time.use_zone('UTC') { Time.zone.parse(timestamp) }.to_f.round(3)
-        elsif timestamp.kind_of?(Time)
-          timestamp.in_time_zone('UTC').to_f.round(3)
-        else
-          timestamp
-        end
-      end
-
       def skeletonize_or_skip_record(record_version, hash_version, record_versions_max, inventory_object)
-        record_version      = comparable_timestamp(record_version)
-        record_versions_max = comparable_timestamp(record_versions_max) if record_versions_max
-        hash_version        = comparable_timestamp(hash_version)
-
         # Skip updating this record, because it is old
         return true if record_version && hash_version && record_version >= hash_version
 

data/lib/inventory_refresh/save_collection/saver/retention_helper.rb

@@ -3,6 +3,28 @@ module InventoryRefresh::SaveCollection
     module RetentionHelper
       private
 
+      # Deletes a complement of referenced data
+      def delete_complement
+        return unless inventory_collection.delete_allowed?
+
+        all_manager_uuids_size = inventory_collection.all_manager_uuids.size
+
+        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
+                     "#{all_manager_uuids_size}...")
+
+        query = complement_of!(inventory_collection.all_manager_uuids,
+                               inventory_collection.all_manager_uuids_scope,
+                               inventory_collection.all_manager_uuids_timestamp)
+
+        ids_of_non_active_entities = ActiveRecord::Base.connection.execute(query.to_sql).to_a
+        ids_of_non_active_entities.each_slice(10_000) do |batch|
+          destroy_records!(batch)
+        end
+
+        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
+                     "#{all_manager_uuids_size}, deleted=#{inventory_collection.deleted_records.size}...Complete")
+      end
+
       # Applies strategy based on :retention_strategy parameter, or fallbacks to legacy_destroy_records.
       #
       # @param records [Array<ApplicationRecord, Hash, Array>] Records we want to delete or archive
@@ -13,9 +35,13 @@ module InventoryRefresh::SaveCollection
         return false unless inventory_collection.delete_allowed?
         return if records.blank?
 
-        ids = ids_array(records)
-        inventory_collection.store_deleted_records(ids)
-        send("#{inventory_collection.retention_strategy}_all_records!", ids)
+        if inventory_collection.retention_strategy
+          ids = ids_array(records)
+          inventory_collection.store_deleted_records(ids)
+          send("#{inventory_collection.retention_strategy}_all_records!", ids)
+        else
+          legacy_destroy_records!(records)
+        end
       end
 
       # Convert records to list of ids in format [{:id => X}, {:id => Y}...]
@@ -45,6 +71,45 @@ module InventoryRefresh::SaveCollection
       def destroy_all_records!(records)
         inventory_collection.model_class.where(:id => records.map { |x| x[:id] }).delete_all
       end
+
+      # Deletes or sof-deletes records. If the model_class supports a custom class delete method, we will use it for
+      # batch soft-delete. This is the legacy method doing either ineffective deletion/archiving or requiring a method
+      # on a class.
+      #
+      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
+      #        to fetch ApplicationRecord objects from the DB
+      def legacy_destroy_records!(records)
+        # Is the delete_method rails standard deleting method?
+        rails_delete = %i[destroy delete].include?(inventory_collection.delete_method)
+        if !rails_delete && inventory_collection.model_class.respond_to?(inventory_collection.delete_method)
+          # We have custom delete method defined on a class, that means it supports batch destroy
+          inventory_collection.store_deleted_records(records.map { |x| {:id => record_key(x, primary_key)} })
+          inventory_collection.model_class.public_send(inventory_collection.delete_method, records.map { |x| record_key(x, primary_key) })
+        else
+          legacy_ineffective_destroy_records(records)
+        end
+      end
+
+      # Very ineffective way of deleting records, but is needed if we want to invoke hooks.
+      #
+      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
+      #        to fetch ApplicationRecord objects from the DB
+      def legacy_ineffective_destroy_records(records)
+        # We have either standard :destroy and :delete rails method, or custom instance level delete method
+        # Note: The standard :destroy and :delete rails method can't be batched because of the hooks and cascade destroy
+        ActiveRecord::Base.transaction do
+          if pure_sql_records_fetching
+            # For pure SQL fetching, we need to get the AR objects again, so we can call destroy
+            inventory_collection.model_class.where(:id => records.map { |x| record_key(x, primary_key) }).find_each do |record|
+              delete_record!(record)
+            end
+          else
+            records.each do |record|
+              delete_record!(record)
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/inventory_refresh/save_collection/saver/sql_helper.rb

@@ -8,6 +8,9 @@ module InventoryRefresh::SaveCollection
     module SqlHelper
       include InventoryRefresh::Logging
 
+      # TODO(lsmola) all below methods should be rewritten to arel, but we need to first extend arel to be able to do
+      # this
+
       extend ActiveSupport::Concern
 
       included do
@@ -80,6 +83,128 @@ module InventoryRefresh::SaveCollection
           "#{value}::#{sql_type}"
         end
       end
+
+      # Effective way of doing multiselect
+      #
+      # If we use "(col1, col2) IN [(a,e), (b,f), (b,e)]" it's not great, just with 10k batch, we see
+      # *** ActiveRecord::StatementInvalid Exception: PG::StatementTooComplex: ERROR:  stack depth limit exceeded
+      # HINT:  Increase the configuration parameter "max_stack_depth" (currently 2048kB), after ensuring the
+      # platform's stack depth limit is adequate.
+      #
+      # If we use "(col1 = a AND col2 = e) OR (col1 = b AND col2 = f) OR (col1 = b AND col2 = e)" with 10k batch, it
+      # takes about 6s and consumes 300MB, with 100k it takes ~1h and consume 3GB in Postgre process
+      #
+      # The best way seems to be using CTE, where the list of values we want to map is turned to 'table' and we just
+      # do RIGHT OUTER JOIN to get the complement of given identifiers. Tested on getting complement of 100k items,
+      # using 2 cols (:ems_ref and :uid_ems) from total 150k rows. It takes ~1s and 350MB in Postgre process
+      #
+      # @param manager_uuids [Array<String>, Array[Hash]] Array with manager_uuids of entities. The keys have to match
+      #        inventory_collection.manager_ref. We allow passing just array of strings, if manager_ref.size ==1, to
+      #        spare some memory
+      # @return [Arel::SelectManager] Arel for getting complement of uuids. This method modifies the passed
+      #         manager_uuids to spare some memory
+      def complement_of!(manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
+        all_attribute_keys       = inventory_collection.manager_ref
+        all_attribute_keys_array = inventory_collection.manager_ref.map(&:to_s)
+
+        active_entities     = Arel::Table.new(:active_entities)
+        active_entities_cte = Arel::Nodes::As.new(
+          active_entities,
+          Arel.sql("(#{active_entities_query(all_attribute_keys_array, manager_uuids)})")
+        )
+
+        all_entities     = Arel::Table.new(:all_entities)
+        all_entities_cte = Arel::Nodes::As.new(
+          all_entities,
+          Arel.sql("(#{all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp).select(:id, *all_attribute_keys_array).to_sql})")
+        )
+        join_condition   = all_attribute_keys.map { |key| active_entities[key].eq(all_entities[key]) }.inject(:and)
+        where_condition  = all_attribute_keys.map { |key| active_entities[key].eq(nil) }.inject(:and)
+
+        active_entities
+          .project(all_entities[:id])
+          .join(all_entities, Arel::Nodes::RightOuterJoin)
+          .on(join_condition)
+          .with(active_entities_cte, all_entities_cte)
+          .where(where_condition)
+      end
+
+      private
+
+      def all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp)
+        all_entities_query = inventory_collection.full_collection_for_comparison
+        all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive
+
+        if all_manager_uuids_scope
+          scope_keys = all_manager_uuids_scope.first.keys.map { |x| association_to_foreign_key_mapping[x.to_sym] }.map(&:to_s)
+          scope = load_scope(all_manager_uuids_scope)
+          condition = inventory_collection.build_multi_selection_condition(scope, scope_keys)
+          all_entities_query = all_entities_query.where(condition)
+        end
+
+        if all_manager_uuids_timestamp && supports_column?(:resource_timestamp)
+          all_manager_uuids_timestamp = Time.parse(all_manager_uuids_timestamp).utc
+
+          date_field = model_class.arel_table[:resource_timestamp]
+          all_entities_query = all_entities_query.where(date_field.lt(all_manager_uuids_timestamp))
+        end
+        all_entities_query
+      end
+
+      def load_scope(all_manager_uuids_scope)
+        scope_keys = all_manager_uuids_scope.first.keys.to_set
+
+        all_manager_uuids_scope.map do |cond|
+          assert_scope!(scope_keys, cond)
+
+          cond.map do |key, value|
+            foreign_key       = association_to_foreign_key_mapping[key.to_sym]
+            foreign_key_value = value.load&.id
+
+            assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
+
+            [foreign_key, foreign_key_value]
+          end.to_h
+        end
+      end
+
+      def assert_scope!(scope_keys, cond)
+        if cond.keys.to_set != scope_keys
+          raise "'#{inventory_collection}' expected keys for :all_manager_uuids_scope are #{scope_keys.to_a}, got"\
+                " #{cond.keys}. Keys must be the same for all scopes provided."
+        end
+      end
+
+      def assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
+        unless foreign_key
+          raise "'#{inventory_collection}' doesn't have relation :#{key} provided in :all_manager_uuids_scope."
+        end
+
+        unless foreign_key_value
+          raise "'#{inventory_collection}' couldn't load scope value :#{key} => #{value.inspect} provided in :all_manager_uuids_scope"
+        end
+      end
+
+      def active_entities_query(all_attribute_keys_array, manager_uuids)
+        connection = ActiveRecord::Base.connection
+
+        all_attribute_keys_array_q = all_attribute_keys_array.map { |x| quote_column_name(x) }
+        # For Postgre, only first set of values should contain the type casts
+        first_value = manager_uuids.shift.to_h
+        first_value = "(#{all_attribute_keys_array.map { |x| quote(connection, first_value[x], x, true) }.join(",")})"
+
+        # Rest of the values, without the type cast
+        values = manager_uuids.map! do |hash|
+          "(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, false) }.join(",")})"
+        end.join(",")
+
+        values = values.blank? ? first_value : [first_value, values].join(",")
+
+        <<-SQL
+          SELECT *
+          FROM   (VALUES #{values}) AS active_entities_table(#{all_attribute_keys_array_q.join(",")})
+        SQL
+      end
     end
   end
 end