inventory_refresh 0.3.5 → 1.1.0

data/lib/inventory_refresh/save_collection/saver/retention_helper.rb

@@ -3,6 +3,28 @@ module InventoryRefresh::SaveCollection
     module RetentionHelper
       private
 
+      # Deletes a complement of referenced data
+      def delete_complement
+        return unless inventory_collection.delete_allowed?
+
+        all_manager_uuids_size = inventory_collection.all_manager_uuids.size
+
+        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
+                     "#{all_manager_uuids_size}...")
+
+        query = complement_of!(inventory_collection.all_manager_uuids,
+                               inventory_collection.all_manager_uuids_scope,
+                               inventory_collection.all_manager_uuids_timestamp)
+
+        ids_of_non_active_entities = ActiveRecord::Base.connection.execute(query.to_sql).to_a
+        ids_of_non_active_entities.each_slice(10_000) do |batch|
+          destroy_records!(batch)
+        end
+
+        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
+                     "#{all_manager_uuids_size}, deleted=#{inventory_collection.deleted_records.size}...Complete")
+      end
+
       # Applies strategy based on :retention_strategy parameter, or fallbacks to legacy_destroy_records.
       #
       # @param records [Array<ApplicationRecord, Hash, Array>] Records we want to delete or archive
@@ -13,9 +35,13 @@ module InventoryRefresh::SaveCollection
         return false unless inventory_collection.delete_allowed?
         return if records.blank?
 
-        ids = ids_array(records)
-        inventory_collection.store_deleted_records(ids)
-        send("#{inventory_collection.retention_strategy}_all_records!", ids)
+        if inventory_collection.retention_strategy
+          ids = ids_array(records)
+          inventory_collection.store_deleted_records(ids)
+          send("#{inventory_collection.retention_strategy}_all_records!", ids)
+        else
+          legacy_destroy_records!(records)
+        end
       end
 
       # Convert records to list of ids in format [{:id => X}, {:id => Y}...]
@@ -45,6 +71,45 @@ module InventoryRefresh::SaveCollection
       def destroy_all_records!(records)
         inventory_collection.model_class.where(:id => records.map { |x| x[:id] }).delete_all
       end
+
+      # Deletes or sof-deletes records. If the model_class supports a custom class delete method, we will use it for
+      # batch soft-delete. This is the legacy method doing either ineffective deletion/archiving or requiring a method
+      # on a class.
+      #
+      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
+      #        to fetch ApplicationRecord objects from the DB
+      def legacy_destroy_records!(records)
+        # Is the delete_method rails standard deleting method?
+        rails_delete = %i[destroy delete].include?(inventory_collection.delete_method)
+        if !rails_delete && inventory_collection.model_class.respond_to?(inventory_collection.delete_method)
+          # We have custom delete method defined on a class, that means it supports batch destroy
+          inventory_collection.store_deleted_records(records.map { |x| {:id => record_key(x, primary_key)} })
+          inventory_collection.model_class.public_send(inventory_collection.delete_method, records.map { |x| record_key(x, primary_key) })
+        else
+          legacy_ineffective_destroy_records(records)
+        end
+      end
+
+      # Very ineffective way of deleting records, but is needed if we want to invoke hooks.
+      #
+      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
+      #        to fetch ApplicationRecord objects from the DB
+      def legacy_ineffective_destroy_records(records)
+        # We have either standard :destroy and :delete rails method, or custom instance level delete method
+        # Note: The standard :destroy and :delete rails method can't be batched because of the hooks and cascade destroy
+        ActiveRecord::Base.transaction do
+          if pure_sql_records_fetching
+            # For pure SQL fetching, we need to get the AR objects again, so we can call destroy
+            inventory_collection.model_class.where(:id => records.map { |x| record_key(x, primary_key) }).find_each do |record|
+              delete_record!(record)
+            end
+          else
+            records.each do |record|
+              delete_record!(record)
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/inventory_refresh/save_collection/saver/sql_helper.rb

@@ -8,6 +8,9 @@ module InventoryRefresh::SaveCollection
     module SqlHelper
       include InventoryRefresh::Logging
 
+      # TODO(lsmola) all below methods should be rewritten to arel, but we need to first extend arel to be able to do
+      # this
+
       extend ActiveSupport::Concern
 
       included do
@@ -80,6 +83,128 @@ module InventoryRefresh::SaveCollection
           "#{value}::#{sql_type}"
         end
       end
+
+      # Effective way of doing multiselect
+      #
+      # If we use "(col1, col2) IN [(a,e), (b,f), (b,e)]" it's not great, just with 10k batch, we see
+      # *** ActiveRecord::StatementInvalid Exception: PG::StatementTooComplex: ERROR:  stack depth limit exceeded
+      # HINT:  Increase the configuration parameter "max_stack_depth" (currently 2048kB), after ensuring the
+      # platform's stack depth limit is adequate.
+      #
+      # If we use "(col1 = a AND col2 = e) OR (col1 = b AND col2 = f) OR (col1 = b AND col2 = e)" with 10k batch, it
+      # takes about 6s and consumes 300MB, with 100k it takes ~1h and consume 3GB in Postgre process
+      #
+      # The best way seems to be using CTE, where the list of values we want to map is turned to 'table' and we just
+      # do RIGHT OUTER JOIN to get the complement of given identifiers. Tested on getting complement of 100k items,
+      # using 2 cols (:ems_ref and :uid_ems) from total 150k rows. It takes ~1s and 350MB in Postgre process
+      #
+      # @param manager_uuids [Array<String>, Array[Hash]] Array with manager_uuids of entities. The keys have to match
+      #        inventory_collection.manager_ref. We allow passing just array of strings, if manager_ref.size ==1, to
+      #        spare some memory
+      # @return [Arel::SelectManager] Arel for getting complement of uuids. This method modifies the passed
+      #         manager_uuids to spare some memory
+      def complement_of!(manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
+        all_attribute_keys       = inventory_collection.manager_ref
+        all_attribute_keys_array = inventory_collection.manager_ref.map(&:to_s)
+
+        active_entities     = Arel::Table.new(:active_entities)
+        active_entities_cte = Arel::Nodes::As.new(
+          active_entities,
+          Arel.sql("(#{active_entities_query(all_attribute_keys_array, manager_uuids)})")
+        )
+
+        all_entities     = Arel::Table.new(:all_entities)
+        all_entities_cte = Arel::Nodes::As.new(
+          all_entities,
+          Arel.sql("(#{all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp).select(:id, *all_attribute_keys_array).to_sql})")
+        )
+        join_condition   = all_attribute_keys.map { |key| active_entities[key].eq(all_entities[key]) }.inject(:and)
+        where_condition  = all_attribute_keys.map { |key| active_entities[key].eq(nil) }.inject(:and)
+
+        active_entities
+          .project(all_entities[:id])
+          .join(all_entities, Arel::Nodes::RightOuterJoin)
+          .on(join_condition)
+          .with(active_entities_cte, all_entities_cte)
+          .where(where_condition)
+      end
+
+      private
+
+      def all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp)
+        all_entities_query = inventory_collection.full_collection_for_comparison
+        all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive
+
+        if all_manager_uuids_scope
+          scope_keys = all_manager_uuids_scope.first.keys.map { |x| association_to_foreign_key_mapping[x.to_sym] }.map(&:to_s)
+          scope = load_scope(all_manager_uuids_scope)
+          condition = inventory_collection.build_multi_selection_condition(scope, scope_keys)
+          all_entities_query = all_entities_query.where(condition)
+        end
+
+        if all_manager_uuids_timestamp && supports_column?(:resource_timestamp)
+          all_manager_uuids_timestamp = Time.parse(all_manager_uuids_timestamp).utc
+
+          date_field = model_class.arel_table[:resource_timestamp]
+          all_entities_query = all_entities_query.where(date_field.lt(all_manager_uuids_timestamp))
+        end
+        all_entities_query
+      end
+
+      def load_scope(all_manager_uuids_scope)
+        scope_keys = all_manager_uuids_scope.first.keys.to_set
+
+        all_manager_uuids_scope.map do |cond|
+          assert_scope!(scope_keys, cond)
+
+          cond.map do |key, value|
+            foreign_key       = association_to_foreign_key_mapping[key.to_sym]
+            foreign_key_value = value.load&.id
+
+            assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
+
+            [foreign_key, foreign_key_value]
+          end.to_h
+        end
+      end
+
+      def assert_scope!(scope_keys, cond)
+        if cond.keys.to_set != scope_keys
+          raise "'#{inventory_collection}' expected keys for :all_manager_uuids_scope are #{scope_keys.to_a}, got"\
+                " #{cond.keys}. Keys must be the same for all scopes provided."
+        end
+      end
+
+      def assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
+        unless foreign_key
+          raise "'#{inventory_collection}' doesn't have relation :#{key} provided in :all_manager_uuids_scope."
+        end
+
+        unless foreign_key_value
+          raise "'#{inventory_collection}' couldn't load scope value :#{key} => #{value.inspect} provided in :all_manager_uuids_scope"
+        end
+      end
+
+      def active_entities_query(all_attribute_keys_array, manager_uuids)
+        connection = ActiveRecord::Base.connection
+
+        all_attribute_keys_array_q = all_attribute_keys_array.map { |x| quote_column_name(x) }
+        # For Postgre, only first set of values should contain the type casts
+        first_value = manager_uuids.shift.to_h
+        first_value = "(#{all_attribute_keys_array.map { |x| quote(connection, first_value[x], x, true) }.join(",")})"
+
+        # Rest of the values, without the type cast
+        values = manager_uuids.map! do |hash|
+          "(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, false) }.join(",")})"
+        end.join(",")
+
+        values = values.blank? ? first_value : [first_value, values].join(",")
+
+        <<-SQL
+          SELECT *
+          FROM   (VALUES #{values}) AS active_entities_table(#{all_attribute_keys_array_q.join(",")})
+        SQL
+      end
     end
   end
 end

data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb

@@ -23,13 +23,13 @@ module InventoryRefresh::SaveCollection
         connection = get_connection
 
         # We want to ignore create timestamps when updating
-        all_attribute_keys_array = all_attribute_keys.to_a.delete_if { |x| %i(created_at created_on).include?(x) }
+        all_attribute_keys_array = all_attribute_keys.to_a.delete_if { |x| %i[created_at created_on].include?(x) }
         all_attribute_keys_array << :id
 
         # If there is not version attribute, the version conditions will be ignored
-        version_attribute = if supports_remote_data_timestamp?(all_attribute_keys)
+        version_attribute = if inventory_collection.parallel_safe? && supports_remote_data_timestamp?(all_attribute_keys)
                               :resource_timestamp
-                            elsif supports_remote_data_version?(all_attribute_keys)
+                            elsif inventory_collection.parallel_safe? && supports_remote_data_version?(all_attribute_keys)
                               :resource_counter
                             end
 
@@ -130,9 +130,13 @@ module InventoryRefresh::SaveCollection
       end
 
       def update_query_returning
-        <<-SQL
-          RETURNING updated_values.#{quote_column_name("id")}, #{unique_index_columns.map { |x| "updated_values.#{quote_column_name(x)}" }.join(",")}
-        SQL
+        if inventory_collection.parallel_safe?
+          <<-SQL
+            RETURNING updated_values.#{quote_column_name("id")}, #{unique_index_columns.map { |x| "updated_values.#{quote_column_name(x)}" }.join(",")}
+          SQL
+        else
+          ""
+        end
       end
     end
   end

data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb

@@ -19,7 +19,7 @@ module InventoryRefresh::SaveCollection
       #        columns of a row, :partial is when we save only few columns, so a partial row.
       # @param on_conflict [Symbol, NilClass] defines behavior on conflict with unique index constraint, allowed values
       #        are :do_update, :do_nothing, nil
-      def build_insert_query(all_attribute_keys, hashes, on_conflict: nil, mode:, column_name: nil)
+      def build_insert_query(all_attribute_keys, hashes, mode:, on_conflict: nil, column_name: nil)
         logger.debug("Building insert query for #{inventory_collection} of size #{inventory_collection.size}...")
 
         # Cache the connection for the batch
@@ -55,6 +55,8 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_on_conflict_behavior(all_attribute_keys, on_conflict, mode, ignore_cols, column_name)
+        return "" unless inventory_collection.parallel_safe?
+
         insert_query_on_conflict = insert_query_on_conflict_do(on_conflict)
         if on_conflict == :do_update
           insert_query_on_conflict += insert_query_on_conflict_update(all_attribute_keys, mode, ignore_cols, column_name)
@@ -63,11 +65,12 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_on_conflict_do(on_conflict)
-        if on_conflict == :do_nothing
+        case on_conflict
+        when :do_nothing
           <<-SQL
             ON CONFLICT DO NOTHING
           SQL
-        elsif on_conflict == :do_update
+        when :do_update
           index_where_condition = unique_index_for(unique_index_keys).where
           where_to_sql          = index_where_condition ? "WHERE #{index_where_condition}" : ""
 
@@ -92,6 +95,8 @@ module InventoryRefresh::SaveCollection
                               :resource_counter
                             end
 
+        # TODO(lsmola) should we add :deleted => false to the update clause? That should handle a reconnect, without a
+        # a need to list :deleted anywhere in the parser. We just need to check that a model has the :deleted attribute
         query = <<-SQL
           SET #{(all_attribute_keys - ignore_cols).map { |key| build_insert_set_cols(key) }.join(", ")}
         SQL
@@ -103,10 +108,12 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_on_conflict_update_mode(mode, version_attribute, column_name)
-        if mode == :full
+        case mode
+        when :full
           full_update_condition(version_attribute)
-        elsif mode == :partial
+        when :partial
           raise "Column name must be provided" unless column_name
+
           partial_update_condition(version_attribute, column_name)
         end
       end
@@ -124,7 +131,7 @@ module InventoryRefresh::SaveCollection
           , #{attr_partial} = '{}', #{attr_partial_max} = NULL
 
           WHERE EXCLUDED.#{attr_full} IS NULL OR (
-            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_full} >= #{q_table_name}.#{attr_full}) AND
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_full} > #{q_table_name}.#{attr_full}) AND
             (#{q_table_name}.#{attr_partial_max} IS NULL OR EXCLUDED.#{attr_full} >= #{q_table_name}.#{attr_partial_max})
           )
         SQL
@@ -133,9 +140,10 @@ module InventoryRefresh::SaveCollection
       def partial_update_condition(attr_full, column_name)
         attr_partial     = attr_full.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
         attr_partial_max = "#{attr_partial}_max"
-        cast             = if attr_full == :resource_timestamp
+        cast             = case attr_full
+                           when :resource_timestamp
                              "timestamp"
-                           elsif attr_full == :resource_counter
+                           when :resource_counter
                              "integer"
                            end
 
@@ -150,9 +158,9 @@ module InventoryRefresh::SaveCollection
           #{insert_query_set_jsonb_version(cast, attr_partial, attr_partial_max, column_name)}
           , #{attr_partial_max} = greatest(#{q_table_name}.#{attr_partial_max}::#{cast}, EXCLUDED.#{attr_partial_max}::#{cast})
           WHERE EXCLUDED.#{attr_partial_max} IS NULL OR (
-            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_partial_max} >= #{q_table_name}.#{attr_full}) AND (
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_partial_max} > #{q_table_name}.#{attr_full}) AND (
               (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast} IS NULL OR
-              EXCLUDED.#{attr_partial_max}::#{cast} >= (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast}
+              EXCLUDED.#{attr_partial_max}::#{cast} > (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast}
             )
           )
         SQL
@@ -179,12 +187,16 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_returning_timestamps
-        # For upsert, we'll return also created and updated timestamps, so we can recognize what was created and what
-        # updated
-        if inventory_collection.internal_timestamp_columns.present?
-          <<-SQL
-            , #{inventory_collection.internal_timestamp_columns.map { |x| quote_column_name(x) }.join(",")}
-          SQL
+        if inventory_collection.parallel_safe?
+          # For upsert, we'll return also created and updated timestamps, so we can recognize what was created and what
+          # updated
+          if inventory_collection.internal_timestamp_columns.present?
+            <<-SQL
+              , #{inventory_collection.internal_timestamp_columns.map { |x| quote_column_name(x) }.join(",")}
+            SQL
+          end
+        else
+          ""
         end
       end
     end

data/lib/inventory_refresh/save_collection/sweeper.rb

@@ -1,7 +1,5 @@
-require "inventory_refresh/exception"
 require "inventory_refresh/logging"
 require "inventory_refresh/save_collection/saver/retention_helper"
-require "inventory_refresh/inventory_collection/index/type/local_db"
 
 module InventoryRefresh::SaveCollection
   class Sweeper < InventoryRefresh::SaveCollection::Base
@@ -12,111 +10,39 @@ module InventoryRefresh::SaveCollection
       # @param _ems [ActiveRecord] Manager owning the inventory_collections
       # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] Array of InventoryCollection objects
       #        for sweeping
-      # @param sweep_scope [Array<String, Symbol, Hash>] Array of inventory collection names marking sweep. Or for
-      #        targeted sweeping it's array of hashes, where key is inventory collection name pointing to an array of
-      #        identifiers of inventory objects we want to target for sweeping.
       # @param refresh_state [ActiveRecord] Record of :refresh_states
-      def sweep(_ems, inventory_collections, sweep_scope, refresh_state)
-        scope_set = build_scope_set(sweep_scope)
-
+      def sweep(_ems, inventory_collections, refresh_state)
         inventory_collections.each do |inventory_collection|
-          next unless sweep_possible?(inventory_collection, scope_set)
+          next unless sweep_possible?(inventory_collection, refresh_state)
 
-          new(inventory_collection, refresh_state, sweep_scope).sweep
+          new(inventory_collection, refresh_state).sweep
         end
       end
 
-      def sweep_possible?(inventory_collection, scope_set)
-        inventory_collection.supports_column?(:last_seen_at) && in_scope?(inventory_collection, scope_set)
-      end
-
-      def in_scope?(inventory_collection, scope_set)
-        scope_set.include?(inventory_collection&.name)
+      def sweep_possible?(inventory_collection, refresh_state)
+        inventory_collection.supports_column?(:last_seen_at) && inventory_collection.parallel_safe? &&
+          inventory_collection.strategy == :local_db_find_missing_references &&
+          in_scope?(inventory_collection, refresh_state.sweep_scope)
       end
 
-      def build_scope_set(sweep_scope)
-        return [] unless sweep_scope
+      def in_scope?(inventory_collection, sweep_scope)
+        return true unless sweep_scope
+        return true if sweep_scope.kind_of?(Array) && sweep_scope.include?(inventory_collection&.name&.to_s)
 
-        if sweep_scope.kind_of?(Array)
-          sweep_scope.map(&:to_sym).to_set
-        elsif sweep_scope.kind_of?(Hash)
-          sweep_scope.keys.map(&:to_sym).to_set
-        end
+        false
       end
     end
 
     include InventoryRefresh::SaveCollection::Saver::RetentionHelper
 
-    attr_reader :inventory_collection, :refresh_state, :sweep_scope, :model_class, :primary_key
+    attr_reader :inventory_collection, :refresh_state, :model_class, :primary_key
 
-    delegate :inventory_object_lazy?,
-             :inventory_object?,
-             :to => :inventory_collection
-
-    def initialize(inventory_collection, refresh_state, sweep_scope)
+    def initialize(inventory_collection, refresh_state)
       @inventory_collection = inventory_collection
+      @refresh_state        = refresh_state
 
-      @refresh_state = refresh_state
-      @sweep_scope   = sweep_scope
-
-      @model_class = inventory_collection.model_class
-      @primary_key = @model_class.primary_key
-    end
-
-    def apply_targeted_sweep_scope(all_entities_query)
-      if sweep_scope.kind_of?(Hash)
-        scope = sweep_scope[inventory_collection.name]
-        return all_entities_query if scope.nil? || scope.empty?
-
-        # Scan the scope to find all references, so we can load them from DB in batches
-        scan_sweep_scope!(scope)
-
-        scope_keys = Set.new
-        conditions = scope.map { |x| InventoryRefresh::InventoryObject.attributes_with_keys(x, inventory_collection, scope_keys) }
-        assert_conditions!(conditions, scope_keys)
-
-        all_entities_query.where(inventory_collection.build_multi_selection_condition(conditions, scope_keys))
-      else
-        all_entities_query
-      end
-    end
-
-    def loadable?(value)
-      inventory_object_lazy?(value) || inventory_object?(value)
-    end
-
-    def scan_sweep_scope!(scope)
-      scope.each do |sc|
-        sc.each_value do |value|
-          next unless loadable?(value)
-
-          value_inventory_collection = value.inventory_collection
-          value_inventory_collection.add_reference(value.reference, :key => value.key)
-        end
-      end
-    end
-
-    def assert_conditions!(conditions, scope_keys)
-      conditions.each do |cond|
-        assert_uniform_keys!(cond, scope_keys)
-        assert_non_existent_keys!(cond)
-      end
-    end
-
-    def assert_uniform_keys!(cond, scope_keys)
-      return if (diff = (scope_keys - cond.keys.to_set)).empty?
-
-      raise(InventoryRefresh::Exception::SweeperNonUniformScopeKeyFoundError,
-            "Sweeping scope for #{inventory_collection} contained non uniform keys. All keys for the"\
-            "scope must be the same, it's possible to send multiple sweeps with different key set. Missing keys"\
-            " for a scope were: #{diff.to_a}")
-    end
-
-    def assert_non_existent_keys!(cond)
-      return if (diff = (cond.keys.to_set - inventory_collection.all_column_names)).empty?
-
-      raise(InventoryRefresh::Exception::SweeperNonExistentScopeKeyFoundError,
-            "Sweeping scope for #{inventory_collection} contained keys that are not columns: #{diff.to_a}")
+      @model_class            = inventory_collection.model_class
+      @primary_key            = @model_class.primary_key
     end
 
     def sweep
@@ -126,9 +52,7 @@ module InventoryRefresh::SaveCollection
       table       = model_class.arel_table
       date_field  = table[:last_seen_at]
       all_entities_query = inventory_collection.full_collection_for_comparison
-      all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive && inventory_collection.supports_column?(:archived_at)
-
-      all_entities_query = apply_targeted_sweep_scope(all_entities_query)
+      all_entities_query.active if inventory_collection.retention_strategy == :archive
 
       query = all_entities_query
               .where(date_field.lt(refresh_start)).or(all_entities_query.where(:last_seen_at => nil))

data/lib/inventory_refresh/save_collection/topological_sort.rb

@@ -17,21 +17,21 @@ module InventoryRefresh::SaveCollection
 
         layers = InventoryRefresh::Graph::TopologicalSort.new(graph).topological_sort
 
-        logger.debug("Saving manager #{ems.id}...")
+        logger.debug("Saving manager #{ems.name}...")
 
-        sorted_graph_log = "Topological sorting of manager #{ems.id} resulted in these layers processable in parallel:\n"
+        sorted_graph_log = "Topological sorting of manager #{ems.name} resulted in these layers processable in parallel:\n"
         sorted_graph_log += graph.to_graphviz(:layers => layers)
         logger.debug(sorted_graph_log)
 
         layers.each_with_index do |layer, index|
-          logger.debug("Saving manager #{ems.id} | Layer #{index}")
+          logger.debug("Saving manager #{ems.name} | Layer #{index}")
           layer.each do |inventory_collection|
             save_inventory_object_inventory(ems, inventory_collection) unless inventory_collection.saved?
           end
-          logger.debug("Saved manager #{ems.id} | Layer #{index}")
+          logger.debug("Saved manager #{ems.name} | Layer #{index}")
         end
 
-        logger.debug("Saving manager #{ems.id}...Complete")
+        logger.debug("Saving manager #{ems.name}...Complete")
       end
     end
   end

data/lib/inventory_refresh/save_inventory.rb

@@ -29,18 +29,11 @@ module InventoryRefresh
       # @param ems [ExtManagementSystem] manager owning the inventory_collections
       # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
       #        for sweeping
-      # @param sweep_scope [Array<String, Symbol, Hash>] Array of inventory collection names marking sweep. Or for
-      #        targeted sweeping it's array of hashes, where key is inventory collection name pointing to an array of
-      #        identifiers of inventory objects we want to target for sweeping.
       # @param refresh_state [ActiveRecord] Record of :refresh_states
-      def sweep_inactive_records(ems, inventory_collections, sweep_scope, refresh_state)
-        inventory_collections.each do |inventory_collection|
-          inventory_collection.strategy = :local_db_find_references
-        end
-
-        logger.info("#{log_header(ems)} Sweeping EMS Inventory with scope #{sweep_scope} and date #{refresh_state.created_at} ...")
-        InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, sweep_scope, refresh_state)
-        logger.info("#{log_header(ems)} Sweeping EMS Inventory with scope #{sweep_scope} and date #{refresh_state.created_at}...Complete")
+      def sweep_inactive_records(ems, inventory_collections, refresh_state)
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory...")
+        InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, refresh_state)
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory...Complete")
 
         ems
       end
@@ -50,7 +43,7 @@ module InventoryRefresh
       # @param ems [ExtManagementSystem] manager owning the inventory_collections
       # @return [String] helper string for logging
       def log_header(ems)
-        "EMS: [#{ems.id}]"
+        "EMS: [#{ems.name}], id: [#{ems.id}]"
       end
     end
   end

data/lib/inventory_refresh/target.rb

@@ -0,0 +1,73 @@
+module InventoryRefresh
+  class Target
+    attr_reader :association, :manager_ref, :event_id, :options
+
+    # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
+    #                             should be the same of association of a counterpart InventoryCollection object
+    # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
+    #                           The keys should be the same as the keys of the counterpart InventoryObject
+    # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    # @param manager_id [Integer] A primary key of the Manager owning the Target
+    # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
+    # @param options [Hash] A free form options hash
+    def initialize(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
+      raise "Provide either :manager or :manager_id argument" if manager.nil? && manager_id.nil?
+
+      @manager     = manager
+      @manager_id  = manager_id
+      @association = association
+      @manager_ref = manager_ref
+      @event_id    = event_id
+      @options     = options
+    end
+
+    # A Rails recommended interface for deserializing an object
+    # @return [InventoryRefresh::Target] InventoryRefresh::Target instance
+    def self.load(**args)
+      new(**args)
+    end
+
+    # A Rails recommended interface for serializing an object
+    #
+    # @param obj [InventoryRefresh::Target] InventoryRefresh::Target instance we want to serialize
+    # @return [Hash] serialized object
+    def self.dump(obj)
+      obj.dump
+    end
+
+    # Returns a serialized InventoryRefresh::Target object. This can be used to initialize a new object, then the object
+    # target acts the same as the object InventoryRefresh::Target.new(target.serialize)
+    #
+    # @return [Hash] serialized object
+    def dump
+      {
+        :manager_id  => manager_id,
+        :association => association,
+        :manager_ref => manager_ref,
+        :event_id    => event_id,
+        :options     => options
+      }
+    end
+
+    alias id dump
+    alias name manager_ref
+
+    # @return [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    def manager
+      @manager || ManageIQ::Providers::BaseManager.find(@manager_id)
+    end
+
+    # @return [Integer] A primary key of the Manager owning the Target
+    def manager_id
+      @manager_id || manager.try(:id)
+    end
+
+    # Loads InventoryRefresh::Target ApplicationRecord representation from our DB, this requires that InventoryRefresh::Target
+    # has been refreshed, otherwise the AR object can be missing.
+    #
+    # @return [ApplicationRecord] A InventoryRefresh::Target loaded from the database as AR object
+    def load_from_db
+      manager.public_send(association).find_by(manager_ref)
+    end
+  end
+end