inventory_refresh 0.2.2 → 0.3.0

data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb

@@ -50,11 +50,6 @@ module InventoryRefresh
           dependencies.all?(&:saved?)
         end
 
-        # @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
-        def parallel_safe?
-          @parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
-        end
-
         # @return [Boolean] true if the model_class supports STI
         def supports_sti?
           @supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
@@ -81,50 +76,12 @@ module InventoryRefresh
           data_collection_finalized
         end
 
-        # @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
-        def targeted?
-          targeted
-        end
-
         # True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
         # stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
         #
         # @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
         def noop?
-          # If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
-          saving_noop? && delete_complement_noop?
-        end
-
-        # @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
-        def saving_noop?
-          saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
-        end
-
-        # @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
-        def delete_complement_noop?
-          all_manager_uuids.nil?
-        end
-
-        private
-
-        # @return true if it's a noop parent targeted InventoryCollection
-        def saving_targeted_parent_collection_noop?
-          targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
-        end
-
-        # @return true if it's a noop child targeted InventoryCollection
-        def saving_targeted_child_collection_noop?
-          targeted_noop_condition && parent_inventory_collections.present? &&
-            parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
-        end
-
-        # @return true if it's a noop full InventoryCollection refresh
-        def saving_full_collection_noop?
-          !targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
-        end
-
-        def targeted_noop_condition
-          targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
+          data.blank? && custom_save_block.nil? && skeletal_primary_index.blank?
         end
       end
     end

data/lib/inventory_refresh/inventory_collection/index/proxy.rb

@@ -64,14 +64,6 @@ module InventoryRefresh
           end
         end
 
-        def reindex_secondary_indexes!
-          data_indexes.each do |ref, index|
-            next if ref == primary_index_ref
-
-            index.reindex!
-          end
-        end
-
         def primary_index
           data_index(primary_index_ref)
         end
@@ -94,30 +86,16 @@ module InventoryRefresh
           end
         end
 
-        def find_by(manager_uuid_hash, ref: primary_index_ref)
-          # TODO(lsmola) deprecate this, it's enough to have find method
-          find(manager_uuid_hash, :ref => ref)
-        end
-
-        def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
-          # TODO(lsmola) deprecate this, it's enough to have lazy_find method
-
-          lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
-        end
-
         def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
-          # TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
-          # access the data
-          # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
           return if manager_uuid.nil?
           assert_index(manager_uuid, ref)
 
           ::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
-                                                    manager_uuid,
-                                                    :ref                         => ref,
-                                                    :key                         => key,
-                                                    :default                     => default,
-                                                    :transform_nested_lazy_finds => transform_nested_lazy_finds)
+                                                      manager_uuid,
+                                                      :ref                         => ref,
+                                                      :key                         => key,
+                                                      :default                     => default,
+                                                      :transform_nested_lazy_finds => transform_nested_lazy_finds)
         end
 
         def named_ref(ref = primary_index_ref)
@@ -132,19 +110,13 @@ module InventoryRefresh
 
         delegate :association_to_foreign_key_mapping,
                  :build_stringified_reference,
-                 :parallel_safe?,
                  :strategy,
                  :to => :inventory_collection
 
         attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
 
         def find_in_data_or_skeletal_index(reference)
-          if parallel_safe?
-            # With parallel safe strategies, we create skeletal nodes that we can look for
-            data_index_find(reference) || skeletal_index_find(reference)
-          else
-            data_index_find(reference)
-          end
+          data_index_find(reference) || skeletal_index_find(reference)
         end
 
         def skeletal_index_find(reference)

data/lib/inventory_refresh/inventory_collection/index/type/base.rb

@@ -27,14 +27,6 @@ module InventoryRefresh
             index[build_stringified_reference(inventory_object.data, attribute_names)] = inventory_object
           end
 
-          # Rebuilds the indexes for all InventoryObject objects
-          def reindex!
-            self.index = {}
-            data.each do |inventory_object|
-              store_index_for(inventory_object)
-            end
-          end
-
           # @return [Array] Returns index data
           def index_data
             index.values

data/lib/inventory_refresh/inventory_collection/references_storage.rb

@@ -45,23 +45,6 @@ module InventoryRefresh
         add_reference(reference_data)
       end
 
-      # Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
-      # it.
-      #
-      # @param references_array [Array] Array of reference objects acceptable by add_reference method.
-      # @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
-      #        index.
-      # @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
-      def merge!(references_array, ref: nil)
-        references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
-        self
-      end
-
-      # @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
-      def primary_references
-        references[primary_index_ref]
-      end
-
       # Builds a Reference object
       #
       # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference

data/lib/inventory_refresh/inventory_collection/scanner.rb

@@ -41,7 +41,6 @@ module InventoryRefresh
       # Boolean helpers the scanner uses from the :inventory_collection
       delegate :inventory_object_lazy?,
                :inventory_object?,
-               :targeted?,
                :to => :inventory_collection
 
       # Methods the scanner uses from the :inventory_collection
@@ -52,18 +51,13 @@ module InventoryRefresh
                :to => :inventory_collection
 
       # The data scanner modifies inside of the :inventory_collection
-      delegate :all_manager_uuids_scope,
-               :association,
+      delegate :association,
                :arel,
                :attribute_references,
                :custom_save_block,
                :data_collection_finalized=,
                :dependency_attributes,
-               :targeted?,
-               :targeted_scope,
                :parent,
-               :parent_inventory_collections,
-               :parent_inventory_collections=,
                :references,
                :transitive_dependency_attributes,
                :to => :inventory_collection
@@ -78,11 +72,6 @@ module InventoryRefresh
         # Scan InventoryCollection InventoryObjects and store the results inside of the InventoryCollection
         data.each do |inventory_object|
           scan_inventory_object!(inventory_object)
-
-          if targeted? && parent_inventory_collections.blank?
-            # We want to track what manager_uuids we should query from a db, for the targeted refresh
-            targeted_scope << inventory_object.reference
-          end
         end
 
         # Scan InventoryCollection skeletal data
@@ -90,80 +79,12 @@ module InventoryRefresh
           scan_inventory_object!(inventory_object)
         end
 
-        build_parent_inventory_collections!
-        scan_all_manager_uuids_scope!
-
         # Mark InventoryCollection as finalized aka. scanned
         self.data_collection_finalized = true
       end
 
       private
 
-      def scan_all_manager_uuids_scope!
-        return if all_manager_uuids_scope.nil?
-
-        all_manager_uuids_scope.each do |scope|
-          scope.each_value do |value|
-            scan_all_manager_uuids_scope_attribute!(value)
-          end
-        end
-      end
-
-      def build_parent_inventory_collections!
-        if parent_inventory_collections.nil?
-          build_parent_inventory_collection!
-        else
-          # We can't figure out what immediate parent is, we'll add parent_inventory_collections as dependencies
-          parent_inventory_collections.each { |ic_name| add_parent_inventory_collection_dependency!(ic_name) }
-        end
-
-        return if parent_inventory_collections.blank?
-
-        # Transform InventoryCollection object names to actual objects
-        self.parent_inventory_collections = parent_inventory_collections.map { |x| load_inventory_collection_by_name(x) }
-      end
-
-      def build_parent_inventory_collection!
-        return unless supports_building_inventory_collection?
-
-        if association.present? && parent.present? && associations_hash[association].present?
-          # Add immediate parent IC as dependency
-          add_parent_inventory_collection_dependency!(associations_hash[association])
-          # Add root IC in parent_inventory_collections
-          self.parent_inventory_collections = [find_parent_inventory_collection(associations_hash, inventory_collection.association)]
-        end
-      end
-
-      def supports_building_inventory_collection?
-        # Don't try to introspect ICs with custom query or saving code
-        return if arel.present? || custom_save_block.present?
-        # We support :parent_inventory_collections only for targeted mode, where all ICs are present
-        return unless targeted?
-
-        true
-      end
-
-      def add_parent_inventory_collection_dependency!(ic_name)
-        ic = load_inventory_collection_by_name(ic_name)
-        (dependency_attributes[:__parent_inventory_collections] ||= Set.new) << ic if ic
-      end
-
-      def find_parent_inventory_collection(hash, name)
-        if hash[name]
-          find_parent_inventory_collection(hash, hash[name])
-        else
-          name
-        end
-      end
-
-      def load_inventory_collection_by_name(name)
-        ic = indexed_inventory_collections[name]
-        if ic.nil?
-          raise "Can't find InventoryCollection :#{name} referenced from #{inventory_collection}" if targeted?
-        end
-        ic
-      end
-
       def scan_inventory_object!(inventory_object)
         inventory_object.data.each do |key, value|
           if value.kind_of?(Array)
@@ -182,13 +103,6 @@ module InventoryRefresh
         value_inventory_collection.add_reference(value.reference, :key => value.key)
       end
 
-      def scan_all_manager_uuids_scope_attribute!(value)
-        return unless loadable?(value)
-
-        add_reference(value.inventory_collection, value)
-        (dependency_attributes[:__all_manager_uuids_scope] ||= Set.new) << value.inventory_collection
-      end
-
       def scan_inventory_object_attribute!(key, value)
         return unless loadable?(value)
         value_inventory_collection = value.inventory_collection

data/lib/inventory_refresh/inventory_collection/serialization.rb

@@ -3,10 +3,7 @@ require "active_support/core_ext/module/delegation"
 module InventoryRefresh
   class InventoryCollection
     class Serialization
-      delegate :all_manager_uuids,
-               :all_manager_uuids=,
-               :build,
-               :targeted_scope,
+      delegate :build,
                :data,
                :inventory_object_lazy?,
                :inventory_object?,
@@ -28,8 +25,6 @@ module InventoryRefresh
       # @param available_inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of available
       #        InventoryCollection objects
       def from_hash(inventory_objects_data, available_inventory_collections)
-        targeted_scope.merge!(inventory_objects_data["manager_uuids"].map(&:symbolize_keys!)) if inventory_objects_data["manager_uuids"]
-
         (inventory_objects_data['data'] || []).each do |inventory_object_data|
           build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
         end
@@ -37,8 +32,12 @@ module InventoryRefresh
         (inventory_objects_data['partial_data'] || []).each do |inventory_object_data|
           skeletal_primary_index.build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
         end
+      end
 
-        self.all_manager_uuids = inventory_objects_data['all_manager_uuids']
+      def sweep_scope_from_hash(sweep_scope, available_inventory_collections)
+        sweep_scope.map do |s|
+          hash_to_data(s, available_inventory_collections).symbolize_keys!
+        end
       end
 
       # Serializes InventoryCollection's data storage into Array of Hashes
@@ -47,14 +46,15 @@ module InventoryRefresh
       def to_hash
         {
           :name              => name,
-          # TODO(lsmola) we do not support nested references here, should we?
-          :manager_uuids     => targeted_scope.primary_references.values.map(&:full_reference),
-          :all_manager_uuids => all_manager_uuids,
           :data              => data.map { |x| data_to_hash(x.data) },
           :partial_data      => skeletal_primary_index.index_data.map { |x| data_to_hash(x.data) },
         }
       end
 
+      def sweep_scope_to_hash(sweep_scope)
+        sweep_scope.map { |x| data_to_hash(x) }
+      end
+
       private
 
       # Converts InventoryRefresh::InventoryObject or InventoryRefresh::InventoryObjectLazy into Hash
@@ -107,10 +107,6 @@ module InventoryRefresh
         hash.transform_values do |value|
           if value.kind_of?(Hash) && value['inventory_collection_name']
             hash_to_lazy_relation(value, available_inventory_collections, depth)
-          elsif value.kind_of?(Array) && value.first.kind_of?(Hash) && value.first['inventory_collection_name']
-            # TODO(lsmola) do we need to compact it sooner? What if first element is nil? On the other hand, we want to
-            # deprecate this Vm HABTM assignment because it's not effective
-            value.compact.map { |x| hash_to_lazy_relation(x, available_inventory_collections, depth) }
           else
             value
           end
@@ -128,8 +124,6 @@ module InventoryRefresh
         data.transform_values do |value|
           if inventory_object_lazy?(value) || inventory_object?(value)
             lazy_relation_to_hash(value, depth)
-          elsif value.kind_of?(Array) && (inventory_object_lazy?(value.compact.first) || inventory_object?(value.compact.first))
-            value.compact.map { |x| lazy_relation_to_hash(x, depth) }
           else
             value
           end

data/lib/inventory_refresh/inventory_object.rb

@@ -42,64 +42,12 @@ module InventoryRefresh
     # Transforms InventoryObject object data into hash format with keys that are column names and resolves correct
     # values of the foreign keys (even the polymorphic ones)
     #
-    # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
-    # @return [Hash] Data in DB format
-    def attributes(inventory_collection_scope = nil)
-      # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
-      # different blacklist and whitelist. The generic code always passes a scope.
-      inventory_collection_scope ||= inventory_collection
-
-      attributes_for_saving = {}
-      # First transform the values
-      data.each do |key, value|
-        if !allowed?(inventory_collection_scope, key)
-          next
-        elsif value.kind_of?(Array) && value.any? { |x| loadable?(x) }
-          # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
-          # result
-          data[key] = value.compact.map(&:load).compact
-          # We can use built in _ids methods to assign array of ids into has_many relations. So e.g. the :key_pairs=
-          # relation setter will become :key_pair_ids=
-          attributes_for_saving[(key.to_s.singularize + "_ids").to_sym] = data[key].map(&:id).compact.uniq
-        elsif loadable?(value) || inventory_collection_scope.association_to_foreign_key_mapping[key]
-          # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
-          # result
-          data[key] = value.load if value.respond_to?(:load)
-          if (foreign_key = inventory_collection_scope.association_to_foreign_key_mapping[key])
-            # We have an association to fill, lets fill also the :key, cause some other InventoryObject can refer to it
-            record_id                                 = data[key].try(:id)
-            attributes_for_saving[foreign_key.to_sym] = record_id
-
-            if (foreign_type = inventory_collection_scope.association_to_foreign_type_mapping[key])
-              # If we have a polymorphic association, we need to also fill a base class name, but we want to nullify it
-              # if record_id is missing
-              base_class = data[key].try(:base_class_name) || data[key].class.try(:base_class).try(:name)
-              attributes_for_saving[foreign_type.to_sym] = record_id ? base_class : nil
-            end
-          elsif data[key].kind_of?(::InventoryRefresh::InventoryObject)
-            # We have an association to fill but not an Activerecord association, so e.g. Ancestry, lets just load
-            # it here. This way of storing ancestry is ineffective in DB call count, but RAM friendly
-            attributes_for_saving[key.to_sym] = data[key].base_class_name.constantize.find_by(:id => data[key].id)
-          else
-            # We have a normal attribute to fill
-            attributes_for_saving[key.to_sym] = data[key]
-          end
-        else
-          attributes_for_saving[key.to_sym] = value
-        end
-      end
-
-      attributes_for_saving
-    end
-
-    # Transforms InventoryObject object data into hash format with keys that are column names and resolves correct
-    # values of the foreign keys (even the polymorphic ones)
-    #
+    # @param data [Array<Object>] Array of objects that we want to map to DB table columns
     # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
     # @param all_attribute_keys [Array<Symbol>] Attribute keys we will modify based on object's data
     # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject object owning these attributes
     # @return [Hash] Data in DB format
-    def attributes_with_keys(inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
+    def self.attributes_with_keys(data, inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
       # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
       # different blacklist and whitelist. The generic code always passes a scope.
       inventory_collection_scope ||= inventory_collection
@@ -206,6 +154,36 @@ module InventoryRefresh
       end
     end
 
+    # Return true if the attribute is allowed to be saved into the DB
+    #
+    # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] InventoryCollection object owning the
+    #        attribute
+    # @param key [Symbol] attribute name
+    # @return true if the attribute is allowed to be saved into the DB
+    def self.allowed?(inventory_collection_scope, key)
+      foreign_to_association = (inventory_collection_scope.foreign_key_to_association_mapping[key] ||
+        inventory_collection_scope.foreign_type_to_association_mapping[key])
+
+      return false if inventory_collection_scope.attributes_blacklist.present? &&
+        (inventory_collection_scope.attributes_blacklist.include?(key) ||
+          (foreign_to_association && inventory_collection_scope.attributes_blacklist.include?(foreign_to_association)))
+
+      return false if inventory_collection_scope.attributes_whitelist.present? &&
+        (!inventory_collection_scope.attributes_whitelist.include?(key) &&
+          (!foreign_to_association || (foreign_to_association && inventory_collection_scope.attributes_whitelist.include?(foreign_to_association))))
+
+      true
+    end
+
+    # Return true if the object is loadable, which we determine by a list of loadable classes.
+    #
+    # @param value [Object] object we test
+    # @return true if the object is loadable
+    def self.loadable?(value)
+      value.kind_of?(::InventoryRefresh::InventoryObjectLazy) || value.kind_of?(::InventoryRefresh::InventoryObject) ||
+        value.kind_of?(::InventoryRefresh::ApplicationRecordReference)
+    end
+
     private
 
     # Assigns value based on the version attributes. If versions are specified, it asigns attribute only if it's
@@ -283,18 +261,7 @@ module InventoryRefresh
     # @param key [Symbol] attribute name
     # @return true if the attribute is allowed to be saved into the DB
     def allowed?(inventory_collection_scope, key)
-      foreign_to_association = inventory_collection_scope.foreign_key_to_association_mapping[key] ||
-                               inventory_collection_scope.foreign_type_to_association_mapping[key]
-
-      return false if inventory_collection_scope.attributes_blacklist.present? &&
-                      (inventory_collection_scope.attributes_blacklist.include?(key) ||
-                        (foreign_to_association && inventory_collection_scope.attributes_blacklist.include?(foreign_to_association)))
-
-      return false if inventory_collection_scope.attributes_whitelist.present? &&
-                      (!inventory_collection_scope.attributes_whitelist.include?(key) &&
-                        (!foreign_to_association || (foreign_to_association && inventory_collection_scope.attributes_whitelist.include?(foreign_to_association))))
-
-      true
+      self.class.allowed?(inventory_collection_scope, key)
     end
 
     # Return true if the object is loadable, which we determine by a list of loadable classes.
@@ -302,8 +269,7 @@ module InventoryRefresh
     # @param value [Object] object we test
     # @return true if the object is loadable
     def loadable?(value)
-      value.kind_of?(::InventoryRefresh::InventoryObjectLazy) || value.kind_of?(::InventoryRefresh::InventoryObject) ||
-        value.kind_of?(::InventoryRefresh::ApplicationRecordReference)
+      self.class.loadable?(value)
     end
   end
 end