inventory_refresh 0.3.5 → 1.1.0

data/lib/inventory_refresh/inventory_object.rb

@@ -42,12 +42,64 @@ module InventoryRefresh
     # Transforms InventoryObject object data into hash format with keys that are column names and resolves correct
     # values of the foreign keys (even the polymorphic ones)
     #
-    # @param data [Array<Object>] Array of objects that we want to map to DB table columns
+    # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
+    # @return [Hash] Data in DB format
+    def attributes(inventory_collection_scope = nil)
+      # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
+      # different blacklist and whitelist. The generic code always passes a scope.
+      inventory_collection_scope ||= inventory_collection
+
+      attributes_for_saving = {}
+      # First transform the values
+      data.each do |key, value|
+        if !allowed?(inventory_collection_scope, key)
+          next
+        elsif value.kind_of?(Array) && value.any? { |x| loadable?(x) }
+          # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
+          # result
+          data[key] = value.compact.map(&:load).compact
+          # We can use built in _ids methods to assign array of ids into has_many relations. So e.g. the :key_pairs=
+          # relation setter will become :key_pair_ids=
+          attributes_for_saving["#{key.to_s.singularize}_ids".to_sym] = data[key].map(&:id).compact.uniq
+        elsif loadable?(value) || inventory_collection_scope.association_to_foreign_key_mapping[key]
+          # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
+          # result
+          data[key] = value.load if value.respond_to?(:load)
+          if (foreign_key = inventory_collection_scope.association_to_foreign_key_mapping[key])
+            # We have an association to fill, lets fill also the :key, cause some other InventoryObject can refer to it
+            record_id                                 = data[key].try(:id)
+            attributes_for_saving[foreign_key.to_sym] = record_id
+
+            if (foreign_type = inventory_collection_scope.association_to_foreign_type_mapping[key])
+              # If we have a polymorphic association, we need to also fill a base class name, but we want to nullify it
+              # if record_id is missing
+              base_class = data[key].try(:base_class_name) || data[key].class.try(:base_class).try(:name)
+              attributes_for_saving[foreign_type.to_sym] = record_id ? base_class : nil
+            end
+          elsif data[key].kind_of?(::InventoryRefresh::InventoryObject)
+            # We have an association to fill but not an Activerecord association, so e.g. Ancestry, lets just load
+            # it here. This way of storing ancestry is ineffective in DB call count, but RAM friendly
+            attributes_for_saving[key.to_sym] = data[key].base_class_name.constantize.find_by(:id => data[key].id)
+          else
+            # We have a normal attribute to fill
+            attributes_for_saving[key.to_sym] = data[key]
+          end
+        else
+          attributes_for_saving[key.to_sym] = value
+        end
+      end
+
+      attributes_for_saving
+    end
+
+    # Transforms InventoryObject object data into hash format with keys that are column names and resolves correct
+    # values of the foreign keys (even the polymorphic ones)
+    #
     # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
     # @param all_attribute_keys [Array<Symbol>] Attribute keys we will modify based on object's data
     # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject object owning these attributes
     # @return [Hash] Data in DB format
-    def self.attributes_with_keys(data, inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
+    def attributes_with_keys(inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
       # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
       # different blacklist and whitelist. The generic code always passes a scope.
       inventory_collection_scope ||= inventory_collection
@@ -96,8 +148,8 @@ module InventoryRefresh
     def assign_attributes(attributes)
       attributes.each do |k, v|
         # We don't want timestamps or resource versions to be overwritten here, since those are driving the conditions
-        next if %i(resource_timestamps resource_timestamps_max resource_timestamp).include?(k)
-        next if %i(resource_counters resource_counters_max resource_counter).include?(k)
+        next if %i[resource_timestamps resource_timestamps_max resource_timestamp].include?(k)
+        next if %i[resource_counters resource_counters_max resource_counter].include?(k)
 
         if data[:resource_timestamp] && attributes[:resource_timestamp]
           assign_only_newest(:resource_timestamp, :resource_timestamps, attributes, data, k, v)
@@ -146,44 +198,14 @@ module InventoryRefresh
           end
         end
 
-        unless defined_methods.include?(attr.to_sym)
-          define_method(attr) do
-            data[attr]
-          end
+        next if defined_methods.include?(attr.to_sym)
+
+        define_method(attr) do
+          data[attr]
         end
       end
     end
 
-    # Return true if the attribute is allowed to be saved into the DB
-    #
-    # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] InventoryCollection object owning the
-    #        attribute
-    # @param key [Symbol] attribute name
-    # @return true if the attribute is allowed to be saved into the DB
-    def self.allowed?(inventory_collection_scope, key)
-      foreign_to_association = (inventory_collection_scope.foreign_key_to_association_mapping[key] ||
-        inventory_collection_scope.foreign_type_to_association_mapping[key])
-
-      return false if inventory_collection_scope.attributes_blacklist.present? &&
-        (inventory_collection_scope.attributes_blacklist.include?(key) ||
-          (foreign_to_association && inventory_collection_scope.attributes_blacklist.include?(foreign_to_association)))
-
-      return false if inventory_collection_scope.attributes_whitelist.present? &&
-        (!inventory_collection_scope.attributes_whitelist.include?(key) &&
-          (!foreign_to_association || (foreign_to_association && inventory_collection_scope.attributes_whitelist.include?(foreign_to_association))))
-
-      true
-    end
-
-    # Return true if the object is loadable, which we determine by a list of loadable classes.
-    #
-    # @param value [Object] object we test
-    # @return true if the object is loadable
-    def self.loadable?(value)
-      value.kind_of?(::InventoryRefresh::InventoryObjectLazy) || value.kind_of?(::InventoryRefresh::InventoryObject) ||
-        value.kind_of?(::InventoryRefresh::ApplicationRecordReference)
-    end
-
     private
 
     # Assigns value based on the version attributes. If versions are specified, it asigns attribute only if it's
@@ -261,7 +283,18 @@ module InventoryRefresh
     # @param key [Symbol] attribute name
     # @return true if the attribute is allowed to be saved into the DB
     def allowed?(inventory_collection_scope, key)
-      self.class.allowed?(inventory_collection_scope, key)
+      foreign_to_association = inventory_collection_scope.foreign_key_to_association_mapping[key] ||
+                               inventory_collection_scope.foreign_type_to_association_mapping[key]
+
+      return false if inventory_collection_scope.attributes_blacklist.present? &&
+                      (inventory_collection_scope.attributes_blacklist.include?(key) ||
+                        (foreign_to_association && inventory_collection_scope.attributes_blacklist.include?(foreign_to_association)))
+
+      return false if inventory_collection_scope.attributes_whitelist.present? &&
+                      (!inventory_collection_scope.attributes_whitelist.include?(key) &&
+                        (!foreign_to_association || (foreign_to_association && inventory_collection_scope.attributes_whitelist.include?(foreign_to_association))))
+
+      true
     end
 
     # Return true if the object is loadable, which we determine by a list of loadable classes.
@@ -269,7 +302,8 @@ module InventoryRefresh
     # @param value [Object] object we test
     # @return true if the object is loadable
     def loadable?(value)
-      self.class.loadable?(value)
+      value.kind_of?(::InventoryRefresh::InventoryObjectLazy) || value.kind_of?(::InventoryRefresh::InventoryObject) ||
+        value.kind_of?(::InventoryRefresh::ApplicationRecordReference)
     end
   end
 end

data/lib/inventory_refresh/inventory_object_lazy.rb

@@ -30,6 +30,7 @@ module InventoryRefresh
 
     # @return [String] stringified reference
     def to_s
+      # TODO(lsmola) do we need this method?
       stringified_reference
     end
 
@@ -70,6 +71,10 @@ module InventoryRefresh
 
     # @return [Boolean] true if the key is an association on inventory_collection_scope model class
     def association?(key)
+      # TODO(lsmola) remove this if there will be better dependency scan, probably with transitive dependencies filled
+      # in a second pass, then we can get rid of this hardcoded symbols. Right now we are not able to introspect these.
+      return true if [:parent, :genealogy_parent].include?(key)
+
       inventory_collection.dependency_attributes.key?(key) ||
         !inventory_collection.association_to_foreign_key_mapping[key].nil?
     end
@@ -95,7 +100,7 @@ module InventoryRefresh
 
     private
 
-    delegate :saved?, :saver_strategy, :skeletal_primary_index, :to => :inventory_collection
+    delegate :parallel_safe?, :saved?, :saver_strategy, :skeletal_primary_index, :targeted?, :to => :inventory_collection
     delegate :nested_secondary_index?, :primary?, :full_reference, :keys, :primary?, :to => :reference
 
     attr_writer :reference
@@ -107,18 +112,20 @@ module InventoryRefresh
     #
     # @return [InventoryRefresh::InventoryObject, NilClass] Returns pre-created InventoryObject or nil
     def skeletal_precreate!
+      # We can do skeletal pre-create only for strategies using unique indexes. Since this can build records out of
+      # the given :arel scope, we will always attempt to create the recod, so we need unique index to avoid duplication
+      # of records.
+      return unless parallel_safe?
       # Pre-create only for strategies that will be persisting data, i.e. are not saved already
       return if saved?
       # We can only do skeletal pre-create for primary index reference, since that is needed to create DB unique index
-      # and full reference must be present
-      return if !primary? || full_reference.blank?
-
-      # To avoid pre-creating invalid records all fields of a primary key must have non null value
-      # TODO(lsmola) for composite keys, it's still valid to have one of the keys nil, figure out how to allow this. We
-      # will need to scan the DB for NOT NULL constraint and allow it based on that. So we would move this check to
-      # saving code, but this will require bigger change, since having the column nil means we will have to batch it
-      # smartly, since having nil means we will need to use different unique index for the upsert/update query.
-      return if keys.any? { |x| full_reference[x].nil? }
+      return unless primary?
+      # Full reference must be present
+      return if full_reference.blank?
+
+      # To avoid pre-creating invalid records all fields of a primary key must have value
+      # TODO(lsmola) for composite keys, it's still valid to have one of the keys nil, figure out how to allow this
+      return if keys.any? { |x| full_reference[x].blank? }
 
       skeletal_primary_index.build(full_reference)
     end

data/lib/inventory_refresh/null_logger.rb

@@ -3,10 +3,10 @@ require "logger"
 module InventoryRefresh
   class NullLogger < Logger
     def initialize(*_args)
-    end 
+    end
 
     def add(*_args, &_block)
-    end 
+    end
 
     def debug?
       false

data/lib/inventory_refresh/persister.rb

@@ -3,21 +3,18 @@ module InventoryRefresh
     require 'json'
     require 'yaml'
 
-    attr_reader :manager, :collections
+    attr_reader :manager, :target, :collections
 
-    attr_accessor :refresh_state_uuid, :refresh_state_part_uuid, :refresh_time_tracking, :total_parts, :sweep_scope, :retry_count, :retry_max
-    attr_accessor :persister_started_at, :persister_finished_at,
-                  :refresh_state_part_collected_at, :refresh_state_part_sent_at,
-                  :refresh_state_started_at, :refresh_state_sent_at, :ingress_api_sent_at
+    attr_accessor :refresh_state_uuid, :refresh_state_part_uuid, :total_parts, :sweep_scope, :retry_count, :retry_max
 
     # @param manager [ManageIQ::Providers::BaseManager] A manager object
-    def initialize(manager)
+    # @param target [Object] A refresh Target object
+    def initialize(manager, target = nil)
       @manager = manager
+      @target  = target
 
       @collections = {}
 
-      self.persister_started_at = Time.now.utc.to_datetime.to_s
-
       initialize_inventory_collections
     end
 
@@ -48,6 +45,9 @@ module InventoryRefresh
                                            &block)
 
       builder.add_properties(extra_properties) if extra_properties.present?
+
+      builder.add_properties({:manager_uuids => target.try(:references, collection_name) || []}, :if_missing) if targeted?
+
       builder.evaluate_lambdas!(self)
 
       collections[collection_name] = builder.to_inventory_collection
@@ -66,7 +66,7 @@ module InventoryRefresh
     # @return [InventoryRefresh::InventoryCollection] returns a defined InventoryCollection or undefined method
     def method_missing(method_name, *arguments, &block)
       if inventory_collections_names.include?(method_name)
-        self.define_collections_reader(method_name)
+        define_collections_reader(method_name)
         send(method_name)
       else
         super
@@ -96,7 +96,7 @@ module InventoryRefresh
 
     # Returns serialized Persisted object to JSON
     # @return [String] serialized Persisted object to JSON
-    def to_json
+    def to_json(*_args)
       JSON.dump(to_hash)
     end
 
@@ -104,25 +104,21 @@ module InventoryRefresh
     def to_hash
       collections_data = collections.map do |_, collection|
         next if collection.data.blank? &&
+                collection.targeted_scope.primary_references.blank? &&
+                collection.all_manager_uuids.nil? &&
                 collection.skeletal_primary_index.index_data.blank?
 
         collection.to_hash
       end.compact
 
       {
-        :refresh_state_uuid              => refresh_state_uuid,
-        :refresh_state_part_uuid         => refresh_state_part_uuid,
-        :refresh_state_part_collected_at => refresh_state_part_collected_at,
-        :refresh_state_part_sent_at      => refresh_state_part_sent_at,
-        :refresh_state_started_at        => refresh_state_started_at,
-        :refresh_state_sent_at           => refresh_state_sent_at,
-        :ingress_api_sent_at             => ingress_api_sent_at,
-        :refresh_time_tracking           => refresh_time_tracking,
-        :retry_count                     => retry_count,
-        :retry_max                       => retry_max,
-        :total_parts                     => total_parts,
-        :sweep_scope                     => sweep_scope_to_hash(sweep_scope),
-        :collections                     => collections_data,
+        :refresh_state_uuid      => refresh_state_uuid,
+        :refresh_state_part_uuid => refresh_state_part_uuid,
+        :retry_count             => retry_count,
+        :retry_max               => retry_max,
+        :total_parts             => total_parts,
+        :sweep_scope             => sweep_scope,
+        :collections             => collections_data,
       }
     end
 
@@ -131,70 +127,33 @@ module InventoryRefresh
       #
       # @param json_data [String] input JSON data
       # @return [ManageIQ::Providers::Inventory::Persister] Persister object loaded from a passed JSON
-      def from_json(json_data, manager)
-        from_hash(JSON.parse(json_data), manager)
+      def from_json(json_data, manager, target = nil)
+        from_hash(JSON.parse(json_data), manager, target)
       end
 
       # Returns Persister object built from serialized data
       #
       # @param persister_data [Hash] serialized Persister object in hash
       # @return [ManageIQ::Providers::Inventory::Persister] Persister object built from serialized data
-      def from_hash(persister_data, manager)
-        new(manager).tap do |persister|
+      def from_hash(persister_data, manager, target = nil)
+        # TODO(lsmola) we need to pass serialized targeted scope here
+        target ||= InventoryRefresh::TargetCollection.new(:manager => manager)
+
+        new(manager, target).tap do |persister|
           persister_data['collections'].each do |collection|
             inventory_collection = persister.collections[collection['name'].try(:to_sym)]
-            raise "Unrecognized InventoryCollection name: #{collection['name']}" if inventory_collection.blank?
+            raise "Unrecognized InventoryCollection name: #{inventory_collection}" if inventory_collection.blank?
 
             inventory_collection.from_hash(collection, persister.collections)
           end
 
-          persister.refresh_state_uuid              = persister_data['refresh_state_uuid']
-          persister.refresh_state_part_uuid         = persister_data['refresh_state_part_uuid']
-          persister.refresh_state_part_collected_at = persister_data['refresh_state_part_collected_at']
-          persister.refresh_state_part_sent_at      = persister_data['refresh_state_part_sent_at']
-          persister.refresh_state_started_at        = persister_data['refresh_state_started_at']
-          persister.refresh_state_sent_at           = persister_data['refresh_state_sent_at']
-          persister.ingress_api_sent_at             = persister_data['ingress_api_sent_at']
-          persister.retry_count                     = persister_data['retry_count']
-          persister.retry_max                       = persister_data['retry_max']
-          persister.total_parts                     = persister_data['total_parts']
-          persister.sweep_scope                     = sweep_scope_from_hash(persister_data['sweep_scope'], persister.collections)
-        end
-      end
-
-      private
-
-      def assert_sweep_scope!(sweep_scope)
-        return unless sweep_scope
-
-        allowed_format_message = "Allowed format of sweep scope is Array<String> or Hash{String => Hash}, got #{sweep_scope}"
-
-        if sweep_scope.kind_of?(Array)
-          return if sweep_scope.all? { |x| x.kind_of?(String) || x.kind_of?(Symbol) }
-
-          raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
-        elsif sweep_scope.kind_of?(Hash)
-          return if sweep_scope.values.all? { |x| x.kind_of?(Array) }
-
-          raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
+          persister.refresh_state_uuid      = persister_data['refresh_state_uuid']
+          persister.refresh_state_part_uuid = persister_data['refresh_state_part_uuid']
+          persister.retry_count             = persister_data['retry_count']
+          persister.retry_max               = persister_data['retry_max']
+          persister.total_parts             = persister_data['total_parts']
+          persister.sweep_scope             = persister_data['sweep_scope']
         end
-
-        raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
-      end
-
-      def sweep_scope_from_hash(sweep_scope, available_inventory_collections)
-        assert_sweep_scope!(sweep_scope)
-
-        return sweep_scope unless sweep_scope.kind_of?(Hash)
-
-        sweep_scope.each_with_object({}) do |(k, v), obj|
-          inventory_collection = available_inventory_collections[k.try(:to_sym)]
-          raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
-
-          serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
-
-          obj[k] = serializer.sweep_scope_from_hash(v, available_inventory_collections)
-        end.symbolize_keys!
       end
     end
 
@@ -226,37 +185,28 @@ module InventoryRefresh
       nil
     end
 
-    def assert_graph_integrity?
+    def saver_strategy
+      :default
+    end
+
+    # Persisters for targeted refresh can override to true
+    def targeted?
       false
     end
 
-    def use_ar_object?
-      true
+    def assert_graph_integrity?
+      false
     end
 
     # @return [Hash] kwargs shared for all InventoryCollection objects
     def shared_options
       {
+        :saver_strategy         => saver_strategy,
         :strategy               => strategy,
+        :targeted               => targeted?,
         :parent                 => manager.presence,
         :assert_graph_integrity => assert_graph_integrity?,
-        :use_ar_object          => use_ar_object?,
       }
     end
-
-    private
-
-    def sweep_scope_to_hash(sweep_scope)
-      return sweep_scope unless sweep_scope.kind_of?(Hash)
-
-      sweep_scope.each_with_object({}) do |(k, v), obj|
-        inventory_collection = collections[k.try(:to_sym)]
-        raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
-
-        serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
-
-        obj[k] = serializer.sweep_scope_to_hash(v)
-      end
-    end
   end
 end

data/lib/inventory_refresh/save_collection/base.rb

@@ -1,5 +1,7 @@
 require "inventory_refresh/logging"
+require "inventory_refresh/save_collection/saver/batch"
 require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
+require "inventory_refresh/save_collection/saver/default"
 
 module InventoryRefresh::SaveCollection
   class Base
@@ -14,7 +16,7 @@ module InventoryRefresh::SaveCollection
         return if skip?(inventory_collection)
 
         logger.debug("----- BEGIN ----- Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
-                     " the database, for the manager: '#{ems.id}'...")
+                     " the database, for the manager: '#{ems.name}'...")
 
         if inventory_collection.custom_save_block.present?
           logger.debug("Saving collection #{inventory_collection} using a custom save block")
@@ -22,7 +24,7 @@ module InventoryRefresh::SaveCollection
         else
           save_inventory(inventory_collection)
         end
-        logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.id}'...Complete")
+        logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.name}'...Complete")
         inventory_collection.saved = true
       end