inventory_refresh 0.2.2 → 0.3.0

data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb

@@ -59,7 +59,7 @@ module InventoryRefresh::SaveCollection
         skeletal_inventory_objects_index = {}
 
         inventory_collection.skeletal_primary_index.each_value do |inventory_object|
-          attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
+          attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
           index      = build_stringified_reference(attributes, unique_index_keys)
 
           skeletal_attributes_index[index]        = attributes
@@ -200,7 +200,25 @@ module InventoryRefresh::SaveCollection
         )
       end
 
+      def comparable_timestamp(timestamp)
+        # Lets cast all timestamps to to_f, rounding the time comparing precision to miliseconds, that should be
+        # enough, since we are the ones setting the record version in collector. Otherwise we will have hard time with
+        # doing equality, since the value changes going through DB (DB. cuts it at 5 decimal places)
+
+        if timestamp.kind_of?(String)
+          Time.use_zone('UTC') { Time.zone.parse(timestamp) }.to_f.round(3)
+        elsif timestamp.kind_of?(Time)
+          timestamp.in_time_zone('UTC').to_f.round(3)
+        else
+          timestamp
+        end
+      end
+
       def skeletonize_or_skip_record(record_version, hash_version, record_versions_max, inventory_object)
+        record_version      = comparable_timestamp(record_version)
+        record_versions_max = comparable_timestamp(record_versions_max) if record_versions_max
+        hash_version        = comparable_timestamp(hash_version)
+
         # Skip updating this record, because it is old
         return true if record_version && hash_version && record_version >= hash_version
 

data/lib/inventory_refresh/save_collection/saver/retention_helper.rb

@@ -3,28 +3,6 @@ module InventoryRefresh::SaveCollection
     module RetentionHelper
       private
 
-      # Deletes a complement of referenced data
-      def delete_complement
-        return unless inventory_collection.delete_allowed?
-
-        all_manager_uuids_size = inventory_collection.all_manager_uuids.size
-
-        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
-                     "#{all_manager_uuids_size}...")
-
-        query = complement_of!(inventory_collection.all_manager_uuids,
-                               inventory_collection.all_manager_uuids_scope,
-                               inventory_collection.all_manager_uuids_timestamp)
-
-        ids_of_non_active_entities = ActiveRecord::Base.connection.execute(query.to_sql).to_a
-        ids_of_non_active_entities.each_slice(10_000) do |batch|
-          destroy_records!(batch)
-        end
-
-        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
-                     "#{all_manager_uuids_size}, deleted=#{inventory_collection.deleted_records.size}...Complete")
-      end
-
       # Applies strategy based on :retention_strategy parameter, or fallbacks to legacy_destroy_records.
       #
       # @param records [Array<ApplicationRecord, Hash, Array>] Records we want to delete or archive
@@ -35,13 +13,9 @@ module InventoryRefresh::SaveCollection
         return false unless inventory_collection.delete_allowed?
         return if records.blank?
 
-        if inventory_collection.retention_strategy
-          ids = ids_array(records)
-          inventory_collection.store_deleted_records(ids)
-          send("#{inventory_collection.retention_strategy}_all_records!", ids)
-        else
-          legacy_destroy_records!(records)
-        end
+        ids = ids_array(records)
+        inventory_collection.store_deleted_records(ids)
+        send("#{inventory_collection.retention_strategy}_all_records!", ids)
       end
 
       # Convert records to list of ids in format [{:id => X}, {:id => Y}...]
@@ -71,45 +45,6 @@ module InventoryRefresh::SaveCollection
       def destroy_all_records!(records)
         inventory_collection.model_class.where(:id => records.map { |x| x[:id] }).delete_all
       end
-
-      # Deletes or sof-deletes records. If the model_class supports a custom class delete method, we will use it for
-      # batch soft-delete. This is the legacy method doing either ineffective deletion/archiving or requiring a method
-      # on a class.
-      #
-      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
-      #        to fetch ApplicationRecord objects from the DB
-      def legacy_destroy_records!(records)
-        # Is the delete_method rails standard deleting method?
-        rails_delete = %i(destroy delete).include?(inventory_collection.delete_method)
-        if !rails_delete && inventory_collection.model_class.respond_to?(inventory_collection.delete_method)
-          # We have custom delete method defined on a class, that means it supports batch destroy
-          inventory_collection.store_deleted_records(records.map { |x| {:id => record_key(x, primary_key)} })
-          inventory_collection.model_class.public_send(inventory_collection.delete_method, records.map { |x| record_key(x, primary_key) })
-        else
-          legacy_ineffective_destroy_records(records)
-        end
-      end
-
-      # Very ineffective way of deleting records, but is needed if we want to invoke hooks.
-      #
-      # @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
-      #        to fetch ApplicationRecord objects from the DB
-      def legacy_ineffective_destroy_records(records)
-        # We have either standard :destroy and :delete rails method, or custom instance level delete method
-        # Note: The standard :destroy and :delete rails method can't be batched because of the hooks and cascade destroy
-        ActiveRecord::Base.transaction do
-          if pure_sql_records_fetching
-            # For pure SQL fetching, we need to get the AR objects again, so we can call destroy
-            inventory_collection.model_class.where(:id => records.map { |x| record_key(x, primary_key) }).find_each do |record|
-              delete_record!(record)
-            end
-          else
-            records.each do |record|
-              delete_record!(record)
-            end
-          end
-        end
-      end
     end
   end
 end

data/lib/inventory_refresh/save_collection/saver/sql_helper.rb

@@ -8,9 +8,6 @@ module InventoryRefresh::SaveCollection
     module SqlHelper
       include InventoryRefresh::Logging
 
-      # TODO(lsmola) all below methods should be rewritten to arel, but we need to first extend arel to be able to do
-      # this
-
       extend ActiveSupport::Concern
 
       included do
@@ -83,128 +80,6 @@ module InventoryRefresh::SaveCollection
           "#{value}::#{sql_type}"
         end
       end
-
-      # Effective way of doing multiselect
-      #
-      # If we use "(col1, col2) IN [(a,e), (b,f), (b,e)]" it's not great, just with 10k batch, we see
-      # *** ActiveRecord::StatementInvalid Exception: PG::StatementTooComplex: ERROR:  stack depth limit exceeded
-      # HINT:  Increase the configuration parameter "max_stack_depth" (currently 2048kB), after ensuring the
-      # platform's stack depth limit is adequate.
-      #
-      # If we use "(col1 = a AND col2 = e) OR (col1 = b AND col2 = f) OR (col1 = b AND col2 = e)" with 10k batch, it
-      # takes about 6s and consumes 300MB, with 100k it takes ~1h and consume 3GB in Postgre process
-      #
-      # The best way seems to be using CTE, where the list of values we want to map is turned to 'table' and we just
-      # do RIGHT OUTER JOIN to get the complement of given identifiers. Tested on getting complement of 100k items,
-      # using 2 cols (:ems_ref and :uid_ems) from total 150k rows. It takes ~1s and 350MB in Postgre process
-      #
-      # @param manager_uuids [Array<String>, Array[Hash]] Array with manager_uuids of entities. The keys have to match
-      #        inventory_collection.manager_ref. We allow passing just array of strings, if manager_ref.size ==1, to
-      #        spare some memory
-      # @return [Arel::SelectManager] Arel for getting complement of uuids. This method modifies the passed
-      #         manager_uuids to spare some memory
-      def complement_of!(manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
-        all_attribute_keys       = inventory_collection.manager_ref
-        all_attribute_keys_array = inventory_collection.manager_ref.map(&:to_s)
-
-        active_entities     = Arel::Table.new(:active_entities)
-        active_entities_cte = Arel::Nodes::As.new(
-          active_entities,
-          Arel.sql("(#{active_entities_query(all_attribute_keys_array, manager_uuids)})")
-        )
-
-        all_entities     = Arel::Table.new(:all_entities)
-        all_entities_cte = Arel::Nodes::As.new(
-          all_entities,
-          Arel.sql("(#{all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp).select(:id, *all_attribute_keys_array).to_sql})")
-        )
-        join_condition   = all_attribute_keys.map { |key| active_entities[key].eq(all_entities[key]) }.inject(:and)
-        where_condition  = all_attribute_keys.map { |key| active_entities[key].eq(nil) }.inject(:and)
-
-        active_entities
-          .project(all_entities[:id])
-          .join(all_entities, Arel::Nodes::RightOuterJoin)
-          .on(join_condition)
-          .with(active_entities_cte, all_entities_cte)
-          .where(where_condition)
-      end
-
-      private
-
-      def all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp)
-        all_entities_query = inventory_collection.full_collection_for_comparison
-        all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive
-
-        if all_manager_uuids_scope
-          scope_keys = all_manager_uuids_scope.first.keys.map { |x| association_to_foreign_key_mapping[x.to_sym] }.map(&:to_s)
-          scope = load_scope(all_manager_uuids_scope)
-          condition = inventory_collection.build_multi_selection_condition(scope, scope_keys)
-          all_entities_query = all_entities_query.where(condition)
-        end
-
-        if all_manager_uuids_timestamp && supports_column?(:resource_timestamp)
-          all_manager_uuids_timestamp = Time.parse(all_manager_uuids_timestamp).utc
-
-          date_field = model_class.arel_table[:resource_timestamp]
-          all_entities_query = all_entities_query.where(date_field.lt(all_manager_uuids_timestamp))
-        end
-        all_entities_query
-      end
-
-      def load_scope(all_manager_uuids_scope)
-        scope_keys = all_manager_uuids_scope.first.keys.to_set
-
-        all_manager_uuids_scope.map do |cond|
-          assert_scope!(scope_keys, cond)
-
-          cond.map do |key, value|
-            foreign_key       = association_to_foreign_key_mapping[key.to_sym]
-            foreign_key_value = value.load&.id
-
-            assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
-
-            [foreign_key, foreign_key_value]
-          end.to_h
-        end
-      end
-
-      def assert_scope!(scope_keys, cond)
-        if cond.keys.to_set != scope_keys
-          raise "'#{inventory_collection}' expected keys for :all_manager_uuids_scope are #{scope_keys.to_a}, got"\
-                " #{cond.keys}. Keys must be the same for all scopes provided."
-        end
-      end
-
-      def assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
-        unless foreign_key
-          raise "'#{inventory_collection}' doesn't have relation :#{key} provided in :all_manager_uuids_scope."
-        end
-
-        unless foreign_key_value
-          raise "'#{inventory_collection}' couldn't load scope value :#{key} => #{value.inspect} provided in :all_manager_uuids_scope"
-        end
-      end
-
-      def active_entities_query(all_attribute_keys_array, manager_uuids)
-        connection = ActiveRecord::Base.connection
-
-        all_attribute_keys_array_q = all_attribute_keys_array.map { |x| quote_column_name(x) }
-        # For Postgre, only first set of values should contain the type casts
-        first_value = manager_uuids.shift.to_h
-        first_value = "(#{all_attribute_keys_array.map { |x| quote(connection, first_value[x], x, true) }.join(",")})"
-
-        # Rest of the values, without the type cast
-        values = manager_uuids.map! do |hash|
-          "(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, false) }.join(",")})"
-        end.join(",")
-
-        values = values.blank? ? first_value : [first_value, values].join(",")
-
-        <<-SQL
-          SELECT *
-          FROM   (VALUES #{values}) AS active_entities_table(#{all_attribute_keys_array_q.join(",")})
-        SQL
-      end
     end
   end
 end

data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb

@@ -27,9 +27,9 @@ module InventoryRefresh::SaveCollection
         all_attribute_keys_array << :id
 
         # If there is not version attribute, the version conditions will be ignored
-        version_attribute = if inventory_collection.parallel_safe? && supports_remote_data_timestamp?(all_attribute_keys)
+        version_attribute = if supports_remote_data_timestamp?(all_attribute_keys)
                               :resource_timestamp
-                            elsif inventory_collection.parallel_safe? && supports_remote_data_version?(all_attribute_keys)
+                            elsif supports_remote_data_version?(all_attribute_keys)
                               :resource_counter
                             end
 
@@ -130,13 +130,9 @@ module InventoryRefresh::SaveCollection
       end
 
       def update_query_returning
-        if inventory_collection.parallel_safe?
-          <<-SQL
-            RETURNING updated_values.#{quote_column_name("id")}, #{unique_index_columns.map { |x| "updated_values.#{quote_column_name(x)}" }.join(",")}
-          SQL
-        else
-          ""
-        end
+        <<-SQL
+          RETURNING updated_values.#{quote_column_name("id")}, #{unique_index_columns.map { |x| "updated_values.#{quote_column_name(x)}" }.join(",")}
+        SQL
       end
     end
   end

data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb

@@ -55,8 +55,6 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_on_conflict_behavior(all_attribute_keys, on_conflict, mode, ignore_cols, column_name)
-        return "" unless inventory_collection.parallel_safe?
-
         insert_query_on_conflict = insert_query_on_conflict_do(on_conflict)
         if on_conflict == :do_update
           insert_query_on_conflict += insert_query_on_conflict_update(all_attribute_keys, mode, ignore_cols, column_name)
@@ -94,8 +92,6 @@ module InventoryRefresh::SaveCollection
                               :resource_counter
                             end
 
-        # TODO(lsmola) should we add :deleted => false to the update clause? That should handle a reconnect, without a
-        # a need to list :deleted anywhere in the parser. We just need to check that a model has the :deleted attribute
         query = <<-SQL
           SET #{(all_attribute_keys - ignore_cols).map { |key| build_insert_set_cols(key) }.join(", ")}
         SQL
@@ -128,7 +124,7 @@ module InventoryRefresh::SaveCollection
           , #{attr_partial} = '{}', #{attr_partial_max} = NULL
 
           WHERE EXCLUDED.#{attr_full} IS NULL OR (
-            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_full} > #{q_table_name}.#{attr_full}) AND
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_full} >= #{q_table_name}.#{attr_full}) AND
             (#{q_table_name}.#{attr_partial_max} IS NULL OR EXCLUDED.#{attr_full} >= #{q_table_name}.#{attr_partial_max})
           )
         SQL
@@ -154,9 +150,9 @@ module InventoryRefresh::SaveCollection
           #{insert_query_set_jsonb_version(cast, attr_partial, attr_partial_max, column_name)}
           , #{attr_partial_max} = greatest(#{q_table_name}.#{attr_partial_max}::#{cast}, EXCLUDED.#{attr_partial_max}::#{cast})
           WHERE EXCLUDED.#{attr_partial_max} IS NULL OR (
-            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_partial_max} > #{q_table_name}.#{attr_full}) AND (
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_partial_max} >= #{q_table_name}.#{attr_full}) AND (
               (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast} IS NULL OR
-              EXCLUDED.#{attr_partial_max}::#{cast} > (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast}
+              EXCLUDED.#{attr_partial_max}::#{cast} >= (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast}
             )
           )
         SQL
@@ -183,16 +179,12 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_returning_timestamps
-        if inventory_collection.parallel_safe?
-          # For upsert, we'll return also created and updated timestamps, so we can recognize what was created and what
-          # updated
-          if inventory_collection.internal_timestamp_columns.present?
-            <<-SQL
-              , #{inventory_collection.internal_timestamp_columns.map { |x| quote_column_name(x) }.join(",")}
-            SQL
-          end
-        else
-          ""
+        # For upsert, we'll return also created and updated timestamps, so we can recognize what was created and what
+        # updated
+        if inventory_collection.internal_timestamp_columns.present?
+          <<-SQL
+            , #{inventory_collection.internal_timestamp_columns.map { |x| quote_column_name(x) }.join(",")}
+          SQL
         end
       end
     end

data/lib/inventory_refresh/save_collection/sweeper.rb

@@ -1,5 +1,7 @@
+require "inventory_refresh/exception"
 require "inventory_refresh/logging"
 require "inventory_refresh/save_collection/saver/retention_helper"
+require "inventory_refresh/inventory_collection/index/type/local_db"
 
 module InventoryRefresh::SaveCollection
   class Sweeper < InventoryRefresh::SaveCollection::Base
@@ -10,42 +12,111 @@ module InventoryRefresh::SaveCollection
       # @param _ems [ActiveRecord] Manager owning the inventory_collections
       # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] Array of InventoryCollection objects
       #        for sweeping
+      # @param sweep_scope [Array<String, Symbol, Hash>] Array of inventory collection names marking sweep. Or for
+      #        targeted sweeping it's array of hashes, where key is inventory collection name pointing to an array of
+      #        identifiers of inventory objects we want to target for sweeping.
       # @param refresh_state [ActiveRecord] Record of :refresh_states
-      def sweep(_ems, inventory_collections, refresh_state)
+      def sweep(_ems, inventory_collections, sweep_scope, refresh_state)
+        scope_set = build_scope_set(sweep_scope)
+
         inventory_collections.each do |inventory_collection|
-          next unless sweep_possible?(inventory_collection, refresh_state)
+          next unless sweep_possible?(inventory_collection, scope_set)
 
-          new(inventory_collection, refresh_state).sweep
+          new(inventory_collection, refresh_state, sweep_scope).sweep
         end
       end
 
-      def sweep_possible?(inventory_collection, refresh_state)
-        inventory_collection.supports_column?(:last_seen_at) && inventory_collection.parallel_safe? &&
-          inventory_collection.strategy == :local_db_find_missing_references &&
-          in_scope?(inventory_collection, refresh_state.sweep_scope)
+      def sweep_possible?(inventory_collection, scope_set)
+        inventory_collection.supports_column?(:last_seen_at) && in_scope?(inventory_collection, scope_set)
+      end
+
+      def in_scope?(inventory_collection, scope_set)
+        scope_set.include?(inventory_collection&.name)
       end
 
-      def in_scope?(inventory_collection, sweep_scope)
-        return true unless sweep_scope
+      def build_scope_set(sweep_scope)
+        return [] unless sweep_scope
 
         if sweep_scope.kind_of?(Array)
-          return true if sweep_scope.include?(inventory_collection&.name&.to_s)
+          sweep_scope.map(&:to_sym).to_set
+        elsif sweep_scope.kind_of?(Hash)
+          sweep_scope.keys.map(&:to_sym).to_set
         end
-
-        false
       end
     end
 
     include InventoryRefresh::SaveCollection::Saver::RetentionHelper
 
-    attr_reader :inventory_collection, :refresh_state, :model_class, :primary_key
+    attr_reader :inventory_collection, :refresh_state, :sweep_scope, :model_class, :primary_key
 
-    def initialize(inventory_collection, refresh_state)
+    delegate :inventory_object_lazy?,
+             :inventory_object?,
+             :to => :inventory_collection
+
+    def initialize(inventory_collection, refresh_state, sweep_scope)
       @inventory_collection = inventory_collection
-      @refresh_state        = refresh_state
 
-      @model_class            = inventory_collection.model_class
-      @primary_key            = @model_class.primary_key
+      @refresh_state = refresh_state
+      @sweep_scope   = sweep_scope
+
+      @model_class = inventory_collection.model_class
+      @primary_key = @model_class.primary_key
+    end
+
+    def apply_targeted_sweep_scope(all_entities_query)
+      if sweep_scope.kind_of?(Hash)
+        scope = sweep_scope[inventory_collection.name]
+        return all_entities_query if scope.nil? || scope.empty?
+
+        # Scan the scope to find all references, so we can load them from DB in batches
+        scan_sweep_scope!(scope)
+
+        scope_keys = Set.new
+        conditions = scope.map { |x| InventoryRefresh::InventoryObject.attributes_with_keys(x, inventory_collection, scope_keys) }
+        assert_conditions!(conditions, scope_keys)
+
+        all_entities_query.where(inventory_collection.build_multi_selection_condition(conditions, scope_keys))
+      else
+        all_entities_query
+      end
+    end
+
+    def loadable?(value)
+      inventory_object_lazy?(value) || inventory_object?(value)
+    end
+
+    def scan_sweep_scope!(scope)
+      scope.each do |sc|
+        sc.each_value do |value|
+          next unless loadable?(value)
+
+          value_inventory_collection = value.inventory_collection
+          value_inventory_collection.add_reference(value.reference, :key => value.key)
+        end
+      end
+    end
+
+    def assert_conditions!(conditions, scope_keys)
+      conditions.each do |cond|
+        assert_uniform_keys!(cond, scope_keys)
+        assert_non_existent_keys!(cond)
+      end
+    end
+
+    def assert_uniform_keys!(cond, scope_keys)
+      return if (diff = (scope_keys - cond.keys.to_set)).empty?
+
+      raise(InventoryRefresh::Exception::SweeperNonUniformScopeKeyFoundError,
+            "Sweeping scope for #{inventory_collection} contained non uniform keys. All keys for the"\
+            "scope must be the same, it's possible to send multiple sweeps with different key set. Missing keys"\
+            " for a scope were: #{diff.to_a}")
+    end
+
+    def assert_non_existent_keys!(cond)
+      return if (diff = (cond.keys.to_set - inventory_collection.all_column_names)).empty?
+
+      raise(InventoryRefresh::Exception::SweeperNonExistentScopeKeyFoundError,
+            "Sweeping scope for #{inventory_collection} contained keys that are not columns: #{diff.to_a}")
     end
 
     def sweep
@@ -55,7 +126,9 @@ module InventoryRefresh::SaveCollection
       table       = model_class.arel_table
       date_field  = table[:last_seen_at]
       all_entities_query = inventory_collection.full_collection_for_comparison
-      all_entities_query.active if inventory_collection.retention_strategy == :archive
+      all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive && inventory_collection.supports_column?(:archived_at)
+
+      all_entities_query = apply_targeted_sweep_scope(all_entities_query)
 
       query = all_entities_query
               .where(date_field.lt(refresh_start)).or(all_entities_query.where(:last_seen_at => nil))