inventory_refresh 0.3.5 → 1.1.0

data/lib/inventory_refresh/inventory_collection/scanner.rb

@@ -41,6 +41,7 @@ module InventoryRefresh
       # Boolean helpers the scanner uses from the :inventory_collection
       delegate :inventory_object_lazy?,
                :inventory_object?,
+               :targeted?,
                :to => :inventory_collection
 
       # Methods the scanner uses from the :inventory_collection
@@ -51,13 +52,18 @@ module InventoryRefresh
                :to => :inventory_collection
 
       # The data scanner modifies inside of the :inventory_collection
-      delegate :association,
+      delegate :all_manager_uuids_scope,
+               :association,
                :arel,
                :attribute_references,
                :custom_save_block,
                :data_collection_finalized=,
                :dependency_attributes,
+               :targeted?,
+               :targeted_scope,
                :parent,
+               :parent_inventory_collections,
+               :parent_inventory_collections=,
                :references,
                :transitive_dependency_attributes,
                :to => :inventory_collection
@@ -72,6 +78,11 @@ module InventoryRefresh
         # Scan InventoryCollection InventoryObjects and store the results inside of the InventoryCollection
         data.each do |inventory_object|
           scan_inventory_object!(inventory_object)
+
+          if targeted? && parent_inventory_collections.blank?
+            # We want to track what manager_uuids we should query from a db, for the targeted refresh
+            targeted_scope << inventory_object.reference
+          end
         end
 
         # Scan InventoryCollection skeletal data
@@ -79,12 +90,81 @@ module InventoryRefresh
           scan_inventory_object!(inventory_object)
         end
 
+        build_parent_inventory_collections!
+        scan_all_manager_uuids_scope!
+
         # Mark InventoryCollection as finalized aka. scanned
         self.data_collection_finalized = true
       end
 
       private
 
+      def scan_all_manager_uuids_scope!
+        return if all_manager_uuids_scope.nil?
+
+        all_manager_uuids_scope.each do |scope|
+          scope.each_value do |value|
+            scan_all_manager_uuids_scope_attribute!(value)
+          end
+        end
+      end
+
+      def build_parent_inventory_collections!
+        if parent_inventory_collections.nil?
+          build_parent_inventory_collection!
+        else
+          # We can't figure out what immediate parent is, we'll add parent_inventory_collections as dependencies
+          parent_inventory_collections.each { |ic_name| add_parent_inventory_collection_dependency!(ic_name) }
+        end
+
+        return if parent_inventory_collections.blank?
+
+        # Transform InventoryCollection object names to actual objects
+        self.parent_inventory_collections = parent_inventory_collections.map { |x| load_inventory_collection_by_name(x) }
+      end
+
+      def build_parent_inventory_collection!
+        return unless supports_building_inventory_collection?
+
+        if association.present? && parent.present? && associations_hash[association].present?
+          # Add immediate parent IC as dependency
+          add_parent_inventory_collection_dependency!(associations_hash[association])
+          # Add root IC in parent_inventory_collections
+          self.parent_inventory_collections = [find_parent_inventory_collection(associations_hash, inventory_collection.association)]
+        end
+      end
+
+      def supports_building_inventory_collection?
+        # Don't try to introspect ICs with custom query or saving code
+        return if arel.present? || custom_save_block.present?
+        # We support :parent_inventory_collections only for targeted mode, where all ICs are present
+        return unless targeted?
+
+        true
+      end
+
+      def add_parent_inventory_collection_dependency!(ic_name)
+        ic = load_inventory_collection_by_name(ic_name)
+        (dependency_attributes[:__parent_inventory_collections] ||= Set.new) << ic if ic
+      end
+
+      def find_parent_inventory_collection(hash, name)
+        if hash[name]
+          find_parent_inventory_collection(hash, hash[name])
+        else
+          name
+        end
+      end
+
+      def load_inventory_collection_by_name(name)
+        ic = indexed_inventory_collections[name]
+        if ic.nil? && targeted?
+          raise "Can't find InventoryCollection :#{name} referenced from #{inventory_collection}"
+        end
+
+        ic
+      end
+
       def scan_inventory_object!(inventory_object)
         inventory_object.data.each do |key, value|
           if value.kind_of?(Array)
@@ -103,8 +183,16 @@ module InventoryRefresh
         value_inventory_collection.add_reference(value.reference, :key => value.key)
       end
 
+      def scan_all_manager_uuids_scope_attribute!(value)
+        return unless loadable?(value)
+
+        add_reference(value.inventory_collection, value)
+        (dependency_attributes[:__all_manager_uuids_scope] ||= Set.new) << value.inventory_collection
+      end
+
       def scan_inventory_object_attribute!(key, value)
         return unless loadable?(value)
+
         value_inventory_collection = value.inventory_collection
 
         # Storing attributes and their dependencies
@@ -114,9 +202,9 @@ module InventoryRefresh
         # e.g. load all the referenced uuids from a DB
         add_reference(value_inventory_collection, value)
 
-        if inventory_object_lazy?(value)
+        if inventory_object_lazy?(value) && value.transitive_dependency?
           # Storing if attribute is a transitive dependency, so a lazy_find :key results in dependency
-          transitive_dependency_attributes << key if value.transitive_dependency?
+          transitive_dependency_attributes << key
         end
       end
     end

data/lib/inventory_refresh/inventory_collection/serialization.rb

@@ -3,7 +3,10 @@ require "active_support/core_ext/module/delegation"
 module InventoryRefresh
   class InventoryCollection
     class Serialization
-      delegate :build,
+      delegate :all_manager_uuids,
+               :all_manager_uuids=,
+               :build,
+               :targeted_scope,
                :data,
                :inventory_object_lazy?,
                :inventory_object?,
@@ -25,6 +28,8 @@ module InventoryRefresh
       # @param available_inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of available
       #        InventoryCollection objects
       def from_hash(inventory_objects_data, available_inventory_collections)
+        targeted_scope.merge!(inventory_objects_data["manager_uuids"].map(&:symbolize_keys!)) if inventory_objects_data["manager_uuids"]
+
         (inventory_objects_data['data'] || []).each do |inventory_object_data|
           build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
         end
@@ -32,12 +37,8 @@ module InventoryRefresh
         (inventory_objects_data['partial_data'] || []).each do |inventory_object_data|
           skeletal_primary_index.build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
         end
-      end
 
-      def sweep_scope_from_hash(sweep_scope, available_inventory_collections)
-        sweep_scope.map do |s|
-          hash_to_data(s, available_inventory_collections).symbolize_keys!
-        end
+        self.all_manager_uuids = inventory_objects_data['all_manager_uuids']
       end
 
       # Serializes InventoryCollection's data storage into Array of Hashes
@@ -46,15 +47,14 @@ module InventoryRefresh
       def to_hash
         {
           :name              => name,
+          # TODO(lsmola) we do not support nested references here, should we?
+          :manager_uuids     => targeted_scope.primary_references.values.map(&:full_reference),
+          :all_manager_uuids => all_manager_uuids,
           :data              => data.map { |x| data_to_hash(x.data) },
           :partial_data      => skeletal_primary_index.index_data.map { |x| data_to_hash(x.data) },
         }
       end
 
-      def sweep_scope_to_hash(sweep_scope)
-        sweep_scope.map { |x| data_to_hash(x) }
-      end
-
       private
 
       # Converts InventoryRefresh::InventoryObject or InventoryRefresh::InventoryObjectLazy into Hash
@@ -107,6 +107,10 @@ module InventoryRefresh
         hash.transform_values do |value|
           if value.kind_of?(Hash) && value['inventory_collection_name']
             hash_to_lazy_relation(value, available_inventory_collections, depth)
+          elsif value.kind_of?(Array) && value.first.kind_of?(Hash) && value.first['inventory_collection_name']
+            # TODO(lsmola) do we need to compact it sooner? What if first element is nil? On the other hand, we want to
+            # deprecate this Vm HABTM assignment because it's not effective
+            value.compact.map { |x| hash_to_lazy_relation(x, available_inventory_collections, depth) }
           else
             value
           end
@@ -124,6 +128,8 @@ module InventoryRefresh
         data.transform_values do |value|
           if inventory_object_lazy?(value) || inventory_object?(value)
             lazy_relation_to_hash(value, depth)
+          elsif value.kind_of?(Array) && (inventory_object_lazy?(value.compact.first) || inventory_object?(value.compact.first))
+            value.compact.map { |x| lazy_relation_to_hash(x, depth) }
           else
             value
           end

data/lib/inventory_refresh/inventory_collection.rb

@@ -71,17 +71,30 @@ module InventoryRefresh
     #   InventoryCollection, since it is already persisted into the DB.
     attr_accessor :saved
 
+    # If present, InventoryCollection switches into delete_complement mode, where it will
+    # delete every record from the DB, that is not present in this list. This is used for the batch processing,
+    # where we don't know which InventoryObject should be deleted, but we know all manager_uuids of all
+    # InventoryObject objects that exists in the provider.
+    #
+    # @return [Array, nil] nil or a list of all :manager_uuids that are present in the Provider's InventoryCollection.
+    attr_accessor :all_manager_uuids
+
+    # @return [Array, nil] Scope for applying :all_manager_uuids
+    attr_accessor :all_manager_uuids_scope
+
+    # @return [String] Timestamp in UTC before fetching :all_manager_uuids
+    attr_accessor :all_manager_uuids_timestamp
+
     # @return [Set] A set of InventoryCollection objects that depends on this InventoryCollection object.
     attr_accessor :dependees
 
-    attr_reader :model_class, :strategy, :attributes_blacklist, :attributes_whitelist, :custom_save_block, :parent,
-                :internal_attributes, :dependency_attributes, :manager_ref, :secondary_refs, :create_only,
-                :association, :complete, :update_only, :transitive_dependency_attributes, :check_changed, :arel,
-                :inventory_object_attributes, :name, :saver_strategy, :default_values,
-                :manager_ref_allowed_nil, :use_ar_object,
-                :created_records, :updated_records, :deleted_records, :retention_strategy,
-                :custom_reconnect_block, :batch_extra_attributes, :references_storage, :unconnected_edges,
-                :assert_graph_integrity
+    # @return [Array<Symbol>] @see #parent_inventory_collections documentation of InventoryCollection.new's initialize_ic_relations()
+    #   parameters
+    attr_accessor :parent_inventory_collections
+
+    attr_accessor :attributes_blacklist, :attributes_whitelist
+
+    attr_reader :model_class, :strategy, :custom_save_block, :parent, :internal_attributes, :delete_method, :dependency_attributes, :manager_ref, :create_only, :association, :complete, :update_only, :transitive_dependency_attributes, :check_changed, :arel, :inventory_object_attributes, :name, :saver_strategy, :targeted_scope, :default_values, :targeted_arel, :targeted, :manager_ref_allowed_nil, :use_ar_object, :created_records, :updated_records, :deleted_records, :retention_strategy, :custom_reconnect_block, :batch_extra_attributes, :references_storage, :unconnected_edges, :assert_graph_integrity
 
     delegate :<<,
              :build,
@@ -112,6 +125,7 @@ module InventoryRefresh
              :lazy_find_by,
              :named_ref,
              :primary_index,
+             :reindex_secondary_indexes!,
              :skeletal_primary_index,
              :to => :index_proxy
 
@@ -134,18 +148,28 @@ module InventoryRefresh
                  properties[:check_changed],
                  properties[:update_only],
                  properties[:use_ar_object],
+                 properties[:targeted],
                  properties[:assert_graph_integrity])
 
       init_strategies(properties[:strategy],
-                      properties[:retention_strategy])
+                      properties[:saver_strategy],
+                      properties[:retention_strategy],
+                      properties[:delete_method])
 
       init_references(properties[:manager_ref],
                       properties[:manager_ref_allowed_nil],
-                      properties[:secondary_refs])
+                      properties[:secondary_refs],
+                      properties[:manager_uuids])
 
-      init_ic_relations(properties[:dependency_attributes])
+      init_all_manager_uuids(properties[:all_manager_uuids],
+                             properties[:all_manager_uuids_scope],
+                             properties[:all_manager_uuids_timestamp])
 
-      init_arels(properties[:arel])
+      init_ic_relations(properties[:dependency_attributes],
+                        properties[:parent_inventory_collections])
+
+      init_arels(properties[:arel],
+                 properties[:targeted_arel])
 
       init_custom_procs(properties[:custom_save_block],
                         properties[:custom_reconnect_block])
@@ -203,16 +227,16 @@ module InventoryRefresh
 
     # @return [Array<ActiveRecord::ConnectionAdapters::IndexDefinition>] array of all unique indexes known to model
     def unique_indexes
-      @unique_indexes_cache if @unique_indexes_cache
+      return @unique_indexes if @unique_indexes
 
-      @unique_indexes_cache = model_class.connection.indexes(model_class.table_name).select(&:unique)
+      @unique_indexes = model_class.connection.indexes(model_class.table_name).select(&:unique)
 
-      if @unique_indexes_cache.blank?
+      if @unique_indexes.blank?
         raise "#{self} and its table #{model_class.table_name} must have a unique index defined, to"\
-                " be able to use saver_strategy :concurrent_safe_batch."
+              " be able to use saver_strategy :concurrent_safe_batch."
       end
 
-      @unique_indexes_cache
+      @unique_indexes
     end
 
     # Finds an index that fits the list of columns (keys) the best
@@ -222,7 +246,7 @@ module InventoryRefresh
     # @return [ActiveRecord::ConnectionAdapters::IndexDefinition] unique index fitting the keys
     def unique_index_for(keys)
       @unique_index_for_keys_cache ||= {}
-      @unique_index_for_keys_cache[keys] if @unique_index_for_keys_cache[keys]
+      return @unique_index_for_keys_cache[keys] if @unique_index_for_keys_cache[keys]
 
       # Take the uniq key having the least number of columns
       @unique_index_for_keys_cache[keys] = uniq_keys_candidates(keys).min_by { |x| x.columns.count }
@@ -239,7 +263,7 @@ module InventoryRefresh
 
       if unique_indexes.blank? || uniq_key_candidates.blank?
         raise "#{self} and its table #{model_class.table_name} must have a unique index defined "\
-                "covering columns #{keys} to be able to use saver_strategy :concurrent_safe_batch."
+              "covering columns #{keys} to be able to use saver_strategy :concurrent_safe_batch."
       end
 
       uniq_key_candidates
@@ -268,7 +292,7 @@ module InventoryRefresh
     def internal_timestamp_columns
       return @internal_timestamp_columns if @internal_timestamp_columns
 
-      @internal_timestamp_columns = %i(created_at created_on updated_at updated_on).collect do |timestamp_col|
+      @internal_timestamp_columns = %i[created_at created_on updated_at updated_on].collect do |timestamp_col|
         timestamp_col if supports_column?(timestamp_col)
       end.compact
     end
@@ -282,11 +306,6 @@ module InventoryRefresh
       @base_columns ||= (unique_index_columns + internal_columns + not_null_columns).uniq
     end
 
-    # @return [Array<Symbol>] Array of all column names on the InventoryCollection
-    def all_column_names
-      @all_column_names ||= model_class.columns.map { |x| x.name.to_sym }
-    end
-
     # @param value [Object] Object we want to test
     # @return [Boolean] true is value is kind of InventoryRefresh::InventoryObject
     def inventory_object?(value)
@@ -311,7 +330,7 @@ module InventoryRefresh
 
     # Convert manager_ref list of attributes to list of DB columns
     #
-    # @return [Array<String>] Converted manager_ref list of attributes to list of DB columns
+    # @return [Array<String>] true is processing of this InventoryCollection will be in targeted mode
     def manager_ref_to_cols
       # TODO(lsmola) this should contain the polymorphic _type, otherwise the IC with polymorphic unique key will get
       # conflicts
@@ -342,25 +361,13 @@ module InventoryRefresh
     #
     # @return [Array<Symbol>] attributes that are needed for saving of the record
     def fixed_attributes
-      not_null_attributes = []
-
       if model_class
-        # Attrs having presence validator
         presence_validators = model_class.validators.detect { |x| x.kind_of?(ActiveRecord::Validations::PresenceValidator) }
-        not_null_attributes += presence_validators.attributes if presence_validators.present?
-
-        # Column names having NOT NULL constraint
-        non_null_constraints = model_class.columns_hash.values.reject(&:null).map(&:name) - [model_class.primary_key]
-        not_null_attributes += non_null_constraints.map(&:to_sym)
-
-        # Column names having NOT NULL constraint transformed to relation names
-        not_null_attributes += non_null_constraints.map {|x| foreign_key_to_association_mapping[x]}.compact
       end
       # Attributes that has to be always on the entity, so attributes making unique index of the record + attributes
       # that have presence validation
-
       fixed_attributes = manager_ref
-      fixed_attributes += not_null_attributes.uniq
+      fixed_attributes += presence_validators.attributes if presence_validators.present?
       fixed_attributes
     end
 
@@ -422,6 +429,7 @@ module InventoryRefresh
                               :parent                => parent,
                               :arel                  => arel,
                               :strategy              => strategy,
+                              :saver_strategy        => saver_strategy,
                               :custom_save_block     => custom_save_block,
                               # We want cloned IC to be update only, since this is used for cycle resolution
                               :update_only           => true,
@@ -459,31 +467,79 @@ module InventoryRefresh
 
     # @return [Integer] default batch size for talking to the DB
     def batch_size
+      # TODO(lsmola) mode to the settings
       1000
     end
 
     # @return [Integer] default batch size for talking to the DB if not using ApplicationRecord objects
     def batch_size_pure_sql
+      # TODO(lsmola) mode to the settings
       10_000
     end
 
+    # Returns a list of stringified uuids of all scoped InventoryObjects, which is used for scoping in targeted mode
+    #
+    # @return [Array<String>] list of stringified uuids of all scoped InventoryObjects
+    def manager_uuids
+      # TODO(lsmola) LEGACY: this is still being used by :targetel_arel definitions and it expects array of strings
+      raise "This works only for :manager_ref size 1" if manager_ref.size > 1
+
+      key = manager_ref.first
+      transform_references_to_hashes(targeted_scope.primary_references).map { |x| x[key] }
+    end
+
     # Builds a multiselection conditions like (table1.a = a1 AND table2.b = b1) OR (table1.a = a2 AND table2.b = b2)
     #
     # @param hashes [Array<Hash>] data we want to use for the query
     # @param keys [Array<Symbol>] keys of attributes involved
     # @return [String] A condition usable in .where of an ActiveRecord relation
-    def build_multi_selection_condition(hashes, keys = unique_index_keys)
+    def build_multi_selection_condition(hashes, keys = manager_ref)
       arel_table = model_class.arel_table
       # We do pure SQL OR, since Arel is nesting every .or into another parentheses, otherwise this would be just
       # inject(:or) instead of to_sql with .join(" OR ")
       hashes.map { |hash| "(#{keys.map { |key| arel_table[key].eq(hash[key]) }.inject(:and).to_sql})" }.join(" OR ")
     end
 
-    # Returns iterator for the passed references and a query
-    #
-    # @return [InventoryRefresh::ApplicationRecordIterator] Iterator for the references and query
+    # @return [ActiveRecord::Relation] A relation that can fetch all data of this InventoryCollection from the DB
     def db_collection_for_comparison
-      InventoryRefresh::ApplicationRecordIterator.new(:inventory_collection => self)
+      if targeted?
+        if targeted_arel.respond_to?(:call)
+          targeted_arel.call(self)
+        elsif parent_inventory_collections.present?
+          targeted_arel_default
+        else
+          targeted_iterator_for(targeted_scope.primary_references)
+        end
+      else
+        full_collection_for_comparison
+      end
+    end
+
+    # Builds targeted query limiting the results by the :references defined in parent_inventory_collections
+    #
+    # @return [InventoryRefresh::ApplicationRecordIterator] an iterator for default targeted arel
+    def targeted_arel_default
+      if parent_inventory_collections.collect { |x| x.model_class.base_class }.uniq.count > 1
+        raise "Multiple :parent_inventory_collections with different base class are not supported by default. Write "\
+              ":targeted_arel manually, or separate [#{self}] into 2 InventoryCollection objects."
+      end
+      parent_collection = parent_inventory_collections.first
+      references        = parent_inventory_collections.map { |x| x.targeted_scope.primary_references }.reduce({}, :merge!)
+
+      parent_collection.targeted_iterator_for(references, full_collection_for_comparison)
+    end
+
+    # Gets targeted references and transforms them into list of hashes
+    #
+    # @param references [Array, InventoryRefresh::InventoryCollection::TargetedScope] passed references
+    # @return [Array<Hash>] References transformed into the array of hashes
+    def transform_references_to_hashes(references)
+      if references.kind_of?(Array)
+        # Sliced InventoryRefresh::InventoryCollection::TargetedScope
+        references.map { |x| x.second.full_reference }
+      else
+        references.values.map(&:full_reference)
+      end
     end
 
     # Builds a multiselection conditions like (table1.a = a1 AND table2.b = b1) OR (table1.a = a2 AND table2.b = b2)
@@ -492,20 +548,20 @@ module InventoryRefresh
     # @param references [Hash{String => InventoryRefresh::InventoryCollection::Reference}] passed references
     # @return [String] A condition usable in .where of an ActiveRecord relation
     def targeted_selection_for(references)
-      build_multi_selection_condition(references.map(&:second))
-    end
-
-    def select_keys
-      @select_keys ||= [@model_class.primary_key] + manager_ref_to_cols.map(&:to_s) + internal_columns.map(&:to_s)
-    end
-
-    # @return [ActiveRecord::ConnectionAdapters::AbstractAdapter] ActiveRecord connection
-    def get_connection
-      ActiveRecord::Base.connection
+      build_multi_selection_condition(transform_references_to_hashes(references))
     end
 
-    def pure_sql_record_fetching?
-      !use_ar_object?
+    # Returns iterator for the passed references and a query
+    #
+    # @param references [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Passed references
+    # @param query [ActiveRecord::Relation] relation that can fetch all data of this InventoryCollection from the DB
+    # @return [InventoryRefresh::ApplicationRecordIterator] Iterator for the references and query
+    def targeted_iterator_for(references, query = nil)
+      InventoryRefresh::ApplicationRecordIterator.new(
+        :inventory_collection => self,
+        :manager_uuids_set    => references,
+        :query                => query
+      )
     end
 
     # Builds an ActiveRecord::Relation that can fetch all the references from the DB
@@ -513,18 +569,15 @@ module InventoryRefresh
     # @param references [Hash{String => InventoryRefresh::InventoryCollection::Reference}] passed references
     # @return [ActiveRecord::Relation] relation that can fetch all the references from the DB
     def db_collection_for_comparison_for(references)
-      query = full_collection_for_comparison.where(targeted_selection_for(references))
-      if pure_sql_record_fetching?
-        return get_connection.query(query.select(*select_keys).to_sql)
-      end
-
-      query
+      full_collection_for_comparison.where(targeted_selection_for(references))
     end
 
     # @return [ActiveRecord::Relation] relation that can fetch all the references from the DB
     def full_collection_for_comparison
       return arel unless arel.nil?
+
       rel = parent.send(association)
+      rel = rel.active if rel && supports_column?(:archived_at) && retention_strategy == :archive
       rel
     end
 
@@ -541,8 +594,6 @@ module InventoryRefresh
       inventory_object_class.new(self, hash)
     end
 
-    attr_writer :attributes_blacklist, :attributes_whitelist
-
     private
 
     # Creates dynamically a subclass of InventoryRefresh::InventoryObject, that will be used per InventoryCollection
@@ -574,9 +625,16 @@ module InventoryRefresh
     def record_identity(record)
       identity = record.try(:[], :id) || record.try(:[], "id") || record.try(:id)
       raise "Cannot obtain identity of the #{record}" if identity.blank?
+
       {
         :id => identity
       }
     end
+
+    # TODO: Not used!
+    # @return [Array<Symbol>] all association attributes and foreign keys of the model class
+    def association_attributes
+      model_class.reflect_on_all_associations.map { |x| [x.name, x.foreign_key] }.flatten.compact.map(&:to_sym)
+    end
   end
 end