inventory_refresh 0.1.1 → 0.2.2

data/lib/inventory_refresh/save_collection/saver/retention_helper.rb

@@ -0,0 +1,115 @@
+module InventoryRefresh::SaveCollection
+  module Saver
+    module RetentionHelper
+      private
+
+      # Deletes a complement of referenced data
+      def delete_complement
+        return unless inventory_collection.delete_allowed?
+
+        all_manager_uuids_size = inventory_collection.all_manager_uuids.size
+
+        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
+                     "#{all_manager_uuids_size}...")
+
+        query = complement_of!(inventory_collection.all_manager_uuids,
+                               inventory_collection.all_manager_uuids_scope,
+                               inventory_collection.all_manager_uuids_timestamp)
+
+        ids_of_non_active_entities = ActiveRecord::Base.connection.execute(query.to_sql).to_a
+        ids_of_non_active_entities.each_slice(10_000) do |batch|
+          destroy_records!(batch)
+        end
+
+        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
+                     "#{all_manager_uuids_size}, deleted=#{inventory_collection.deleted_records.size}...Complete")
+      end
+
+      # Applies strategy based on :retention_strategy parameter, or fallbacks to legacy_destroy_records.
+      #
+      # @param records [Array<ApplicationRecord, Hash, Array>] Records we want to delete or archive
+      def destroy_records!(records)
+        # TODO(lsmola) the output of this can still grow in a memory a lot, if we would delete a huge chunk of
+        # records. Will we just stream it out? Or maybe give a max amount of deleted records here?
+
+        return false unless inventory_collection.delete_allowed?
+        return if records.blank?
+
+        if inventory_collection.retention_strategy
+          ids = ids_array(records)
+          inventory_collection.store_deleted_records(ids)
+          send("#{inventory_collection.retention_strategy}_all_records!", ids)
+        else
+          legacy_destroy_records!(records)
+        end
+      end
+
+      # Convert records to list of ids in format [{:id => X}, {:id => Y}...]
+      #
+      # @param records [Array<ApplicationRecord, Hash, Array>] Records we want to delete or archive
+      # @return [Array<Hash>] Primary keys in standardized format
+      def ids_array(records)
+        if records.first.kind_of?(Hash)
+          records.map { |x| {:id => x[primary_key]} }
+        elsif records.first.kind_of?(Array)
+          records.map { |x| {:id => x[select_keys_indexes[primary_key]]} }
+        else
+          records.map { |x| {:id => x.public_send(primary_key)} }
+        end
+      end
+
+      # Archives all records
+      #
+      # @param records [Array<Hash>] Records we want to archive.
+      def archive_all_records!(records)
+        inventory_collection.model_class.where(:id => records.map { |x| x[:id] }).update_all(:archived_at => Time.now.utc)
+      end
+
+      # Destroys all records
+      #
+      # @param records [Array<Hash>] Records we want to delete.
+      def destroy_all_records!(records)
+        inventory_collection.model_class.where(:id => records.map { |x| x[:id] }).delete_all
+      end
+
+      # Deletes or sof-deletes records. If the model_class supports a custom class delete method, we will use it for
+      # batch soft-delete. This is the legacy method doing either ineffective deletion/archiving or requiring a method
+      # on a class.
+      #
+      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
+      #        to fetch ApplicationRecord objects from the DB
+      def legacy_destroy_records!(records)
+        # Is the delete_method rails standard deleting method?
+        rails_delete = %i(destroy delete).include?(inventory_collection.delete_method)
+        if !rails_delete && inventory_collection.model_class.respond_to?(inventory_collection.delete_method)
+          # We have custom delete method defined on a class, that means it supports batch destroy
+          inventory_collection.store_deleted_records(records.map { |x| {:id => record_key(x, primary_key)} })
+          inventory_collection.model_class.public_send(inventory_collection.delete_method, records.map { |x| record_key(x, primary_key) })
+        else
+          legacy_ineffective_destroy_records(records)
+        end
+      end
+
+      # Very ineffective way of deleting records, but is needed if we want to invoke hooks.
+      #
+      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
+      #        to fetch ApplicationRecord objects from the DB
+      def legacy_ineffective_destroy_records(records)
+        # We have either standard :destroy and :delete rails method, or custom instance level delete method
+        # Note: The standard :destroy and :delete rails method can't be batched because of the hooks and cascade destroy
+        ActiveRecord::Base.transaction do
+          if pure_sql_records_fetching
+            # For pure SQL fetching, we need to get the AR objects again, so we can call destroy
+            inventory_collection.model_class.where(:id => records.map { |x| record_key(x, primary_key) }).find_each do |record|
+              delete_record!(record)
+            end
+          else
+            records.each do |record|
+              delete_record!(record)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/saver/sql_helper.rb

@@ -83,6 +83,128 @@ module InventoryRefresh::SaveCollection
           "#{value}::#{sql_type}"
         end
       end
+
+      # Effective way of doing multiselect
+      #
+      # If we use "(col1, col2) IN [(a,e), (b,f), (b,e)]" it's not great, just with 10k batch, we see
+      # *** ActiveRecord::StatementInvalid Exception: PG::StatementTooComplex: ERROR:  stack depth limit exceeded
+      # HINT:  Increase the configuration parameter "max_stack_depth" (currently 2048kB), after ensuring the
+      # platform's stack depth limit is adequate.
+      #
+      # If we use "(col1 = a AND col2 = e) OR (col1 = b AND col2 = f) OR (col1 = b AND col2 = e)" with 10k batch, it
+      # takes about 6s and consumes 300MB, with 100k it takes ~1h and consume 3GB in Postgre process
+      #
+      # The best way seems to be using CTE, where the list of values we want to map is turned to 'table' and we just
+      # do RIGHT OUTER JOIN to get the complement of given identifiers. Tested on getting complement of 100k items,
+      # using 2 cols (:ems_ref and :uid_ems) from total 150k rows. It takes ~1s and 350MB in Postgre process
+      #
+      # @param manager_uuids [Array<String>, Array[Hash]] Array with manager_uuids of entities. The keys have to match
+      #        inventory_collection.manager_ref. We allow passing just array of strings, if manager_ref.size ==1, to
+      #        spare some memory
+      # @return [Arel::SelectManager] Arel for getting complement of uuids. This method modifies the passed
+      #         manager_uuids to spare some memory
+      def complement_of!(manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
+        all_attribute_keys       = inventory_collection.manager_ref
+        all_attribute_keys_array = inventory_collection.manager_ref.map(&:to_s)
+
+        active_entities     = Arel::Table.new(:active_entities)
+        active_entities_cte = Arel::Nodes::As.new(
+          active_entities,
+          Arel.sql("(#{active_entities_query(all_attribute_keys_array, manager_uuids)})")
+        )
+
+        all_entities     = Arel::Table.new(:all_entities)
+        all_entities_cte = Arel::Nodes::As.new(
+          all_entities,
+          Arel.sql("(#{all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp).select(:id, *all_attribute_keys_array).to_sql})")
+        )
+        join_condition   = all_attribute_keys.map { |key| active_entities[key].eq(all_entities[key]) }.inject(:and)
+        where_condition  = all_attribute_keys.map { |key| active_entities[key].eq(nil) }.inject(:and)
+
+        active_entities
+          .project(all_entities[:id])
+          .join(all_entities, Arel::Nodes::RightOuterJoin)
+          .on(join_condition)
+          .with(active_entities_cte, all_entities_cte)
+          .where(where_condition)
+      end
+
+      private
+
+      def all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp)
+        all_entities_query = inventory_collection.full_collection_for_comparison
+        all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive
+
+        if all_manager_uuids_scope
+          scope_keys = all_manager_uuids_scope.first.keys.map { |x| association_to_foreign_key_mapping[x.to_sym] }.map(&:to_s)
+          scope = load_scope(all_manager_uuids_scope)
+          condition = inventory_collection.build_multi_selection_condition(scope, scope_keys)
+          all_entities_query = all_entities_query.where(condition)
+        end
+
+        if all_manager_uuids_timestamp && supports_column?(:resource_timestamp)
+          all_manager_uuids_timestamp = Time.parse(all_manager_uuids_timestamp).utc
+
+          date_field = model_class.arel_table[:resource_timestamp]
+          all_entities_query = all_entities_query.where(date_field.lt(all_manager_uuids_timestamp))
+        end
+        all_entities_query
+      end
+
+      def load_scope(all_manager_uuids_scope)
+        scope_keys = all_manager_uuids_scope.first.keys.to_set
+
+        all_manager_uuids_scope.map do |cond|
+          assert_scope!(scope_keys, cond)
+
+          cond.map do |key, value|
+            foreign_key       = association_to_foreign_key_mapping[key.to_sym]
+            foreign_key_value = value.load&.id
+
+            assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
+
+            [foreign_key, foreign_key_value]
+          end.to_h
+        end
+      end
+
+      def assert_scope!(scope_keys, cond)
+        if cond.keys.to_set != scope_keys
+          raise "'#{inventory_collection}' expected keys for :all_manager_uuids_scope are #{scope_keys.to_a}, got"\
+                " #{cond.keys}. Keys must be the same for all scopes provided."
+        end
+      end
+
+      def assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
+        unless foreign_key
+          raise "'#{inventory_collection}' doesn't have relation :#{key} provided in :all_manager_uuids_scope."
+        end
+
+        unless foreign_key_value
+          raise "'#{inventory_collection}' couldn't load scope value :#{key} => #{value.inspect} provided in :all_manager_uuids_scope"
+        end
+      end
+
+      def active_entities_query(all_attribute_keys_array, manager_uuids)
+        connection = ActiveRecord::Base.connection
+
+        all_attribute_keys_array_q = all_attribute_keys_array.map { |x| quote_column_name(x) }
+        # For Postgre, only first set of values should contain the type casts
+        first_value = manager_uuids.shift.to_h
+        first_value = "(#{all_attribute_keys_array.map { |x| quote(connection, first_value[x], x, true) }.join(",")})"
+
+        # Rest of the values, without the type cast
+        values = manager_uuids.map! do |hash|
+          "(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, false) }.join(",")})"
+        end.join(",")
+
+        values = values.blank? ? first_value : [first_value, values].join(",")
+
+        <<-SQL
+          SELECT *
+          FROM   (VALUES #{values}) AS active_entities_table(#{all_attribute_keys_array_q.join(",")})
+        SQL
+      end
     end
   end
 end

data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb

@@ -30,7 +30,7 @@ module InventoryRefresh::SaveCollection
         version_attribute = if inventory_collection.parallel_safe? && supports_remote_data_timestamp?(all_attribute_keys)
                               :resource_timestamp
                             elsif inventory_collection.parallel_safe? && supports_remote_data_version?(all_attribute_keys)
-                              :resource_version
+                              :resource_counter
                             end
 
         update_query = update_query_beginning(all_attribute_keys_array)
@@ -44,6 +44,21 @@ module InventoryRefresh::SaveCollection
         update_query
       end
 
+      # Build batch update query only for passed all_attribute_keys
+      #
+      # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
+      # @param hashes [Array<Hash>] data used for building a batch update sql query
+      def build_partial_update_query(all_attribute_keys, hashes)
+        # Cache the connection for the batch
+        connection = get_connection
+
+        all_attribute_keys = (all_attribute_keys + unique_index_columns).uniq
+
+        update_query = update_query_beginning(all_attribute_keys)
+        update_query += update_query_from_values(hashes, all_attribute_keys, connection, unique_index_columns)
+        update_query
+      end
+
       private
 
       def update_query_beginning(all_attribute_keys_array)
@@ -56,7 +71,7 @@ module InventoryRefresh::SaveCollection
 
       def update_query_reset_version_columns(version_attribute)
         if version_attribute
-          attr_partial     = version_attribute.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+          attr_partial     = version_attribute.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
           attr_partial_max = "#{attr_partial}_max"
 
           # Quote the column names
@@ -72,17 +87,21 @@ module InventoryRefresh::SaveCollection
         end
       end
 
-      def update_query_from_values(hashes, all_attribute_keys_array, connection)
+      def update_query_from_values(hashes, all_attribute_keys_array, connection, matching = [:id])
         values = hashes.map! do |hash|
           "(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, true) }.join(",")})"
         end.join(",")
 
+        where_cond = matching.map do |x|
+          "updated_values.#{quote_column_name(x)} = #{q_table_name}.#{quote_column_name(x)}"
+        end.join(" AND ")
+
         <<-SQL
           FROM (
             VALUES
               #{values}
           ) AS updated_values (#{all_attribute_keys_array.map { |x| quote_column_name(x) }.join(",")})
-          WHERE updated_values.id = #{q_table_name}.id
+          WHERE #{where_cond}
         SQL
       end
 
@@ -90,7 +109,7 @@ module InventoryRefresh::SaveCollection
         if version_attribute
           # This conditional will avoid rewriting new data by old data. But we want it only when version_attribute is
           # a part of the data, since for the fake records, we just want to update ems_ref.
-          attr_partial     = version_attribute.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+          attr_partial     = version_attribute.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
           attr_partial_max = "#{attr_partial}_max"
 
           # Quote the column names

data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb

@@ -25,7 +25,7 @@ module InventoryRefresh::SaveCollection
         # Cache the connection for the batch
         connection = get_connection
         # Ignore versioning columns that are set separately
-        ignore_cols = mode == :partial ? [:resource_timestamp, :resource_version] : []
+        ignore_cols = mode == :partial ? [:resource_timestamp, :resource_counter] : []
         # Make sure we don't send a primary_key for INSERT in any form, it could break PG sequencer
         all_attribute_keys_array = all_attribute_keys.to_a - [primary_key.to_s, primary_key.to_sym] - ignore_cols
 
@@ -83,7 +83,7 @@ module InventoryRefresh::SaveCollection
 
       def insert_query_on_conflict_update(all_attribute_keys, mode, ignore_cols, column_name)
         if mode == :partial
-          ignore_cols += [:resource_timestamps, :resource_timestamps_max, :resource_versions, :resource_versions_max]
+          ignore_cols += [:resource_timestamps, :resource_timestamps_max, :resource_counters, :resource_counters_max]
         end
         ignore_cols += [:created_on, :created_at] # Lets not change created for the update clause
 
@@ -91,7 +91,7 @@ module InventoryRefresh::SaveCollection
         version_attribute = if supports_remote_data_timestamp?(all_attribute_keys)
                               :resource_timestamp
                             elsif supports_remote_data_version?(all_attribute_keys)
-                              :resource_version
+                              :resource_counter
                             end
 
         # TODO(lsmola) should we add :deleted => false to the update clause? That should handle a reconnect, without a
@@ -116,7 +116,7 @@ module InventoryRefresh::SaveCollection
       end
 
       def full_update_condition(attr_full)
-        attr_partial = attr_full.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+        attr_partial = attr_full.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
         attr_partial_max = "#{attr_partial}_max"
 
         # Quote the column names
@@ -135,11 +135,11 @@ module InventoryRefresh::SaveCollection
       end
 
       def partial_update_condition(attr_full, column_name)
-        attr_partial     = attr_full.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+        attr_partial     = attr_full.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
         attr_partial_max = "#{attr_partial}_max"
         cast             = if attr_full == :resource_timestamp
                              "timestamp"
-                           elsif attr_full == :resource_version
+                           elsif attr_full == :resource_counter
                              "integer"
                            end
 

data/lib/inventory_refresh/save_collection/sweeper.rb

@@ -0,0 +1,69 @@
+require "inventory_refresh/logging"
+require "inventory_refresh/save_collection/saver/retention_helper"
+
+module InventoryRefresh::SaveCollection
+  class Sweeper < InventoryRefresh::SaveCollection::Base
+    class << self
+      # Sweeps inactive records based on :last_seen_on and :refresh_start timestamps. All records having :last_seen_on
+      # lower than :refresh_start or nil will be archived/deleted.
+      #
+      # @param _ems [ActiveRecord] Manager owning the inventory_collections
+      # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] Array of InventoryCollection objects
+      #        for sweeping
+      # @param refresh_state [ActiveRecord] Record of :refresh_states
+      def sweep(_ems, inventory_collections, refresh_state)
+        inventory_collections.each do |inventory_collection|
+          next unless sweep_possible?(inventory_collection, refresh_state)
+
+          new(inventory_collection, refresh_state).sweep
+        end
+      end
+
+      def sweep_possible?(inventory_collection, refresh_state)
+        inventory_collection.supports_column?(:last_seen_at) && inventory_collection.parallel_safe? &&
+          inventory_collection.strategy == :local_db_find_missing_references &&
+          in_scope?(inventory_collection, refresh_state.sweep_scope)
+      end
+
+      def in_scope?(inventory_collection, sweep_scope)
+        return true unless sweep_scope
+
+        if sweep_scope.kind_of?(Array)
+          return true if sweep_scope.include?(inventory_collection&.name&.to_s)
+        end
+
+        false
+      end
+    end
+
+    include InventoryRefresh::SaveCollection::Saver::RetentionHelper
+
+    attr_reader :inventory_collection, :refresh_state, :model_class, :primary_key
+
+    def initialize(inventory_collection, refresh_state)
+      @inventory_collection = inventory_collection
+      @refresh_state        = refresh_state
+
+      @model_class            = inventory_collection.model_class
+      @primary_key            = @model_class.primary_key
+    end
+
+    def sweep
+      refresh_start = refresh_state.created_at
+      raise "Couldn't load :created_at out of RefreshState record: #{refresh_state}" unless refresh_start
+
+      table       = model_class.arel_table
+      date_field  = table[:last_seen_at]
+      all_entities_query = inventory_collection.full_collection_for_comparison
+      all_entities_query.active if inventory_collection.retention_strategy == :archive
+
+      query = all_entities_query
+              .where(date_field.lt(refresh_start)).or(all_entities_query.where(:last_seen_at => nil))
+              .select(table[:id])
+
+      query.find_in_batches do |batch|
+        destroy_records!(batch)
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_inventory.rb

@@ -1,5 +1,5 @@
-require "inventory_refresh/save_collection/recursive"
 require "inventory_refresh/save_collection/topological_sort"
+require "inventory_refresh/save_collection/sweeper"
 
 module InventoryRefresh
   class SaveInventory
@@ -11,20 +11,30 @@ module InventoryRefresh
       # @param ems [ExtManagementSystem] manager owning the inventory_collections
       # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
       #        for saving
-      def save_inventory(ems, inventory_collections, strategy = nil)
+      def save_inventory(ems, inventory_collections)
         logger.debug("#{log_header(ems)} Scanning Inventory Collections...Start")
         InventoryRefresh::InventoryCollection::Scanner.scan!(inventory_collections)
         logger.debug("#{log_header(ems)} Scanning Inventory Collections...Complete")
 
         logger.info("#{log_header(ems)} Saving EMS Inventory...")
+        InventoryRefresh::SaveCollection::TopologicalSort.save_collections(ems, inventory_collections)
+        logger.info("#{log_header(ems)} Saving EMS Inventory...Complete")
 
-        if strategy.try(:to_sym) == :recursive
-          InventoryRefresh::SaveCollection::Recursive.save_collections(ems, inventory_collections)
-        else
-          InventoryRefresh::SaveCollection::TopologicalSort.save_collections(ems, inventory_collections)
-        end
+        ems
+      end
+
+      # Sweeps inactive records based on :last_seen_at and :refresh_start timestamps. All records having :last_seen_at
+      # lower than :refresh_start or nil will be archived/deleted.
+      #
+      # @param ems [ExtManagementSystem] manager owning the inventory_collections
+      # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
+      #        for sweeping
+      # @param refresh_state [ActiveRecord] Record of :refresh_states
+      def sweep_inactive_records(ems, inventory_collections, refresh_state)
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory...")
+        InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, refresh_state)
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory...Complete")
 
-        logger.info("#{log_header(ems)} Saving EMS Inventory...Complete")
         ems
       end