inventory_refresh 0.1.3 → 0.2.0

data/lib/inventory_refresh/inventory_object.rb

@@ -25,8 +25,13 @@ module InventoryRefresh
       reference.stringified_reference
     end
 
+    # @return [Hash] hash reference having :manager_ref keys, which can uniquely identity entity under a manager
+    def uuid
+      reference.full_reference.slice(*reference.keys).stringify_keys!
+    end
+
     # @return [InventoryRefresh::InventoryObject] returns self
-    def load
+    def load(*_args)
       self
     end
 
@@ -92,8 +97,9 @@ module InventoryRefresh
     #
     # @param inventory_collection_scope [InventoryRefresh::InventoryCollection] parent InventoryCollection object
     # @param all_attribute_keys [Array<Symbol>] Attribute keys we will modify based on object's data
+    # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject object owning these attributes
     # @return [Hash] Data in DB format
-    def attributes_with_keys(inventory_collection_scope = nil, all_attribute_keys = [])
+    def attributes_with_keys(inventory_collection_scope = nil, all_attribute_keys = [], inventory_object = nil)
       # We should explicitly pass a scope, since the inventory_object can be mapped to more InventoryCollections with
       # different blacklist and whitelist. The generic code always passes a scope.
       inventory_collection_scope ||= inventory_collection
@@ -106,7 +112,7 @@ module InventoryRefresh
         elsif loadable?(value) || inventory_collection_scope.association_to_foreign_key_mapping[key]
           # Lets fill also the original data, so other InventoryObject referring to this attribute gets the right
           # result
-          data[key] = value.load if value.respond_to?(:load)
+          data[key] = value.load(inventory_object, key) if value.respond_to?(:load)
           if (foreign_key = inventory_collection_scope.association_to_foreign_key_mapping[key])
             # We have an association to fill, lets fill also the :key, cause some other InventoryObject can refer to it
             record_id = data[key].try(:id)
@@ -143,12 +149,12 @@ module InventoryRefresh
       attributes.each do |k, v|
         # We don't want timestamps or resource versions to be overwritten here, since those are driving the conditions
         next if %i(resource_timestamps resource_timestamps_max resource_timestamp).include?(k)
-        next if %i(resource_versions resource_versions_max resource_version).include?(k)
+        next if %i(resource_counters resource_counters_max resource_counter).include?(k)
 
         if data[:resource_timestamp] && attributes[:resource_timestamp]
           assign_only_newest(:resource_timestamp, :resource_timestamps, attributes, data, k, v)
-        elsif data[:resource_version] && attributes[:resource_version]
-          assign_only_newest(:resource_version, :resource_versions, attributes, data, k, v)
+        elsif data[:resource_counter] && attributes[:resource_counter]
+          assign_only_newest(:resource_counter, :resource_counters, attributes, data, k, v)
         else
           public_send("#{k}=", v)
         end
@@ -156,8 +162,8 @@ module InventoryRefresh
 
       if attributes[:resource_timestamp]
         assign_full_row_version_attr(:resource_timestamp, attributes, data)
-      elsif attributes[:resource_version]
-        assign_full_row_version_attr(:resource_version, attributes, data)
+      elsif attributes[:resource_counter]
+        assign_full_row_version_attr(:resource_counter, attributes, data)
       end
 
       self
@@ -206,9 +212,9 @@ module InventoryRefresh
     # newer than existing attribute.
     #
     # @param full_row_version_attr [Symbol] Attr name for full rows, allowed values are
-    #        [:resource_timestamp, :resource_version]
+    #        [:resource_timestamp, :resource_counter]
     # @param partial_row_version_attr [Symbol] Attr name for partial rows, allowed values are
-    #        [:resource_timestamps, :resource_versions]
+    #        [:resource_timestamps, :resource_counters]
     # @param attributes [Hash] New attributes we are assigning
     # @param data [Hash] Existing attributes of the InventoryObject
     # @param k [Symbol] Name of the attribute we are assigning
@@ -247,7 +253,7 @@ module InventoryRefresh
     # Assigns attribute representing version of the whole row
     #
     # @param full_row_version_attr [Symbol] Attr name for full rows, allowed values are
-    #        [:resource_timestamp, :resource_version]
+    #        [:resource_timestamp, :resource_counter]
     # @param attributes [Hash] New attributes we are assigning
     # @param data [Hash] Existing attributes of the InventoryObject
     def assign_full_row_version_attr(full_row_version_attr, attributes, data)

data/lib/inventory_refresh/inventory_object_lazy.rb

@@ -42,12 +42,14 @@ module InventoryRefresh
       "InventoryObjectLazy:('#{self}', #{inventory_collection}#{suffix})"
     end
 
+    # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject object owning this relation
+    # @param inventory_object_key [Symbol] InventoryObject object's attribute pointing to this relation
     # @return [InventoryRefresh::InventoryObject, Object] InventoryRefresh::InventoryObject instance or an attribute
     #         on key
-    def load
+    def load(inventory_object = nil, inventory_object_key = nil)
       transform_nested_secondary_indexes! if transform_nested_lazy_finds && nested_secondary_index?
 
-      key ? load_object_with_key : load_object
+      load_object(inventory_object, inventory_object_key)
     end
 
     # return [Boolean] true if the Lazy object is causing a dependency, Lazy link is always a dependency if no :key
@@ -128,15 +130,15 @@ module InventoryRefresh
       skeletal_primary_index.build(full_reference)
     end
 
+    # @param loaded_object [InventoryRefresh::InventoryObject, NilClass] Loaded object or nil if object wasn't found
     # @return [Object] value found or :key or default value if the value is nil
-    def load_object_with_key
+    def load_object_with_key(loaded_object)
       # TODO(lsmola) Log error if we are accessing path that is present in blacklist or not present in whitelist
-      found = inventory_collection.find(reference)
-      if found.present?
-        if found.try(:data).present?
-          found.data[key] || default
+      if loaded_object.present?
+        if loaded_object.try(:data).present?
+          loaded_object.data[key] || default
         else
-          found.public_send(key) || default
+          loaded_object.public_send(key) || default
         end
       else
         default
@@ -144,8 +146,16 @@ module InventoryRefresh
     end
 
     # @return [InventoryRefresh::InventoryObject, NilClass] InventoryRefresh::InventoryObject instance or nil if not found
-    def load_object
-      inventory_collection.find(reference)
+    def load_object(inventory_object = nil, inventory_object_key = nil)
+      loaded_object = inventory_collection.find(reference)
+
+      if inventory_object && inventory_object_key && !loaded_object && reference.loadable?
+        # Object was not loaded, but the reference is pointing to something, lets return it as edge that should've
+        # been loaded.
+        inventory_object.inventory_collection.store_unconnected_edges(inventory_object, inventory_object_key, self)
+      end
+
+      key ? load_object_with_key(loaded_object) : loaded_object
     end
   end
 end

data/lib/inventory_refresh/persister.rb

@@ -0,0 +1,212 @@
+module InventoryRefresh
+  class Persister
+    require 'json'
+    require 'yaml'
+
+    attr_reader :manager, :target, :collections
+
+    attr_accessor :refresh_state_uuid, :refresh_state_part_uuid, :total_parts, :sweep_scope, :retry_count, :retry_max
+
+    # @param manager [ManageIQ::Providers::BaseManager] A manager object
+    # @param target [Object] A refresh Target object
+    def initialize(manager, target = nil)
+      @manager = manager
+      @target  = target
+
+      @collections = {}
+
+      initialize_inventory_collections
+    end
+
+    # Interface for creating InventoryCollection under @collections
+    #
+    # @param builder_class    [ManageIQ::Providers::Inventory::Persister::Builder] or subclasses
+    # @param collection_name  [Symbol || Array] used as InventoryCollection:association
+    # @param extra_properties [Hash]   props from InventoryCollection.initialize list
+    #         - adds/overwrites properties added by builder
+    #
+    # @param settings [Hash] builder settings
+    #         - @see ManageIQ::Providers::Inventory::Persister::Builder.default_options
+    #         - @see make_builder_settings()
+    #
+    # @example
+    #   add_collection(:vms, ManageIQ::Providers::Inventory::Persister::Builder::CloudManager) do |builder|
+    #     builder.add_properties(
+    #       :strategy => :local_db_cache_all,
+    #     )
+    #   )
+    #
+    # @see documentation https://github.com/ManageIQ/guides/tree/master/providers/persister/inventory_collections.md
+    #
+    def add_collection(collection_name, builder_class = inventory_collection_builder, extra_properties = {}, settings = {}, &block)
+      builder = builder_class.prepare_data(collection_name,
+                                           self.class,
+                                           builder_settings(settings),
+                                           &block)
+
+      builder.add_properties(extra_properties) if extra_properties.present?
+
+      builder.add_properties({:manager_uuids => target.try(:references, collection_name) || []}, :if_missing) if targeted?
+
+      builder.evaluate_lambdas!(self)
+
+      collections[collection_name] = builder.to_inventory_collection
+    end
+
+    # @return [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects of the persister
+    def inventory_collections
+      collections.values
+    end
+
+    # @return [Array<Symbol>] array of InventoryCollection object names of the persister
+    def inventory_collections_names
+      collections.keys
+    end
+
+    # @return [InventoryRefresh::InventoryCollection] returns a defined InventoryCollection or undefined method
+    def method_missing(method_name, *arguments, &block)
+      if inventory_collections_names.include?(method_name)
+        self.define_collections_reader(method_name)
+        send(method_name)
+      else
+        super
+      end
+    end
+
+    # @return [Boolean] true if InventoryCollection with passed method_name name is defined
+    def respond_to_missing?(method_name, _include_private = false)
+      inventory_collections_names.include?(method_name) || super
+    end
+
+    # Defines a new attr reader returning InventoryCollection object
+    def define_collections_reader(collection_key)
+      define_singleton_method(collection_key) do
+        collections[collection_key]
+      end
+    end
+
+    def inventory_collection_builder
+      ::InventoryRefresh::InventoryCollection::Builder
+    end
+
+    # Persists InventoryCollection objects into the DB
+    def persist!
+      InventoryRefresh::SaveInventory.save_inventory(manager, inventory_collections)
+    end
+
+    # Returns serialized Persisted object to JSON
+    # @return [String] serialized Persisted object to JSON
+    def to_json
+      JSON.dump(to_hash)
+    end
+
+    # @return [Hash] entire Persister object serialized to hash
+    def to_hash
+      collections_data = collections.map do |_, collection|
+        next if collection.data.blank? &&
+                collection.targeted_scope.primary_references.blank? &&
+                collection.all_manager_uuids.nil? &&
+                collection.skeletal_primary_index.index_data.blank?
+
+        collection.to_hash
+      end.compact
+
+      {
+        :refresh_state_uuid      => refresh_state_uuid,
+        :refresh_state_part_uuid => refresh_state_part_uuid,
+        :retry_count             => retry_count,
+        :retry_max               => retry_max,
+        :total_parts             => total_parts,
+        :sweep_scope             => sweep_scope,
+        :collections             => collections_data,
+      }
+    end
+
+    class << self
+      # Returns Persister object loaded from a passed JSON
+      #
+      # @param json_data [String] input JSON data
+      # @return [ManageIQ::Providers::Inventory::Persister] Persister object loaded from a passed JSON
+      def from_json(json_data, manager, target = nil)
+        from_hash(JSON.parse(json_data), manager, target)
+      end
+
+      # Returns Persister object built from serialized data
+      #
+      # @param persister_data [Hash] serialized Persister object in hash
+      # @return [ManageIQ::Providers::Inventory::Persister] Persister object built from serialized data
+      def from_hash(persister_data, manager, target = nil)
+        # TODO(lsmola) we need to pass serialized targeted scope here
+        target ||= InventoryRefresh::TargetCollection.new(:manager => manager)
+
+        new(manager, target).tap do |persister|
+          persister_data['collections'].each do |collection|
+            inventory_collection = persister.collections[collection['name'].try(:to_sym)]
+            raise "Unrecognized InventoryCollection name: #{inventory_collection}" if inventory_collection.blank?
+
+            inventory_collection.from_hash(collection, persister.collections)
+          end
+
+          persister.refresh_state_uuid      = persister_data['refresh_state_uuid']
+          persister.refresh_state_part_uuid = persister_data['refresh_state_part_uuid']
+          persister.retry_count             = persister_data['retry_count']
+          persister.retry_max               = persister_data['retry_max']
+          persister.total_parts             = persister_data['total_parts']
+          persister.sweep_scope             = persister_data['sweep_scope']
+        end
+      end
+    end
+
+    protected
+
+    def initialize_inventory_collections
+      # can be implemented in a subclass
+    end
+
+    # @param extra_settings [Hash]
+    #   :auto_inventory_attributes
+    #     - auto creates inventory_object_attributes from target model_class setters
+    #     - attributes used in InventoryObject.add_attributes
+    #   :without_model_class
+    #     - if false and no model_class derived or specified, throws exception
+    #     - doesn't try to derive model class automatically
+    #     - @see method ManageIQ::Providers::Inventory::Persister::Builder.auto_model_class
+    def builder_settings(extra_settings = {})
+      opts = inventory_collection_builder.default_options
+
+      opts[:shared_properties]         = shared_options
+      opts[:auto_inventory_attributes] = true
+      opts[:without_model_class]       = false
+
+      opts.merge(extra_settings)
+    end
+
+    def strategy
+      nil
+    end
+
+    def saver_strategy
+      :default
+    end
+
+    # Persisters for targeted refresh can override to true
+    def targeted?
+      false
+    end
+
+    def assert_graph_integrity?
+      false
+    end
+
+    # @return [Hash] kwargs shared for all InventoryCollection objects
+    def shared_options
+      {
+        :saver_strategy         => saver_strategy,
+        :strategy               => strategy,
+        :targeted               => targeted?,
+        :parent                 => manager.presence,
+        :assert_graph_integrity => assert_graph_integrity?,
+      }
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/base.rb

@@ -1,6 +1,5 @@
 require "inventory_refresh/logging"
 require "inventory_refresh/save_collection/saver/batch"
-require "inventory_refresh/save_collection/saver/concurrent_safe"
 require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
 require "inventory_refresh/save_collection/saver/default"
 
@@ -14,7 +13,9 @@ module InventoryRefresh::SaveCollection
       # @param ems [ExtManagementSystem] manger owning the InventoryCollection object
       # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we want to save
       def save_inventory_object_inventory(ems, inventory_collection)
-        logger.debug("Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
+        return if skip?(inventory_collection)
+
+        logger.debug("----- BEGIN ----- Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
                      " the database, for the manager: '#{ems.name}'...")
 
         if inventory_collection.custom_save_block.present?
@@ -23,12 +24,26 @@ module InventoryRefresh::SaveCollection
         else
           save_inventory(inventory_collection)
         end
-        logger.debug("Saving collection #{inventory_collection}, for the manager: '#{ems.name}'...Complete")
+        logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.name}'...Complete")
         inventory_collection.saved = true
       end
 
       private
 
+      # Returns true and sets collection as saved, if the collection should be skipped.
+      #
+      # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we want to save
+      # @return [Boolean] True if processing of the collection should be skipped
+      def skip?(inventory_collection)
+        if inventory_collection.noop?
+          logger.debug("Skipping #{inventory_collection} processing because it will do no operation.")
+          inventory_collection.saved = true
+          return true
+        end
+
+        false
+      end
+
       # Saves one InventoryCollection object into the DB using a configured saver_strategy class.
       #
       # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we want to save

data/lib/inventory_refresh/save_collection/saver/base.rb

@@ -24,7 +24,7 @@ module InventoryRefresh::SaveCollection
         @arel_primary_key       = @model_class.arel_attribute(@primary_key)
         @unique_index_keys      = inventory_collection.unique_index_keys
         @unique_index_keys_to_s = inventory_collection.manager_ref_to_cols.map(&:to_s)
-        @select_keys            = [@primary_key] + @unique_index_keys_to_s
+        @select_keys            = [@primary_key] + @unique_index_keys_to_s + internal_columns.map(&:to_s)
         @unique_db_primary_keys = Set.new
         @unique_db_indexes      = Set.new
 
@@ -77,18 +77,23 @@ module InventoryRefresh::SaveCollection
       def save_inventory_collection!
         # If we have a targeted InventoryCollection that wouldn't do anything, quickly skip it
         return if inventory_collection.noop?
-        # If we want to use delete_complement strategy using :all_manager_uuids attribute, we are skipping any other
-        # job. We want to do 1 :delete_complement job at 1 time, to keep to memory down.
-        return delete_complement if inventory_collection.all_manager_uuids.present?
 
-        save!(association)
+        # Delete_complement strategy using :all_manager_uuids attribute
+        delete_complement unless inventory_collection.delete_complement_noop?
+
+        # Create/Update/Archive/Delete records based on InventoryCollection data and scope
+        save!(association) unless inventory_collection.saving_noop?
       end
 
       protected
 
       attr_reader :inventory_collection, :association
 
-      delegate :build_stringified_reference, :build_stringified_reference_for_record, :to => :inventory_collection
+      delegate :build_stringified_reference,
+               :build_stringified_reference_for_record,
+               :resource_version_column,
+               :internal_columns,
+               :to => :inventory_collection
 
       # Applies serialize method for each relevant attribute, which will cast the value to the right type.
       #
@@ -126,6 +131,8 @@ module InventoryRefresh::SaveCollection
                   :batch_size, :batch_size_for_persisting, :model_class, :serializable_keys, :deserializable_keys, :pg_types, :table_name,
                   :q_table_name
 
+      delegate :supports_column?, :to => :inventory_collection
+
       # Saves the InventoryCollection
       #
       # @param association [Symbol] An existing association on manager
@@ -200,27 +207,7 @@ module InventoryRefresh::SaveCollection
 
       # Deletes a complement of referenced data
       def delete_complement
-        return unless inventory_collection.delete_allowed?
-
-        all_manager_uuids_size = inventory_collection.all_manager_uuids.size
-
-        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
-                     "#{all_manager_uuids_size}...")
-        deleted_counter = 0
-
-        inventory_collection.db_collection_for_comparison_for_complement_of(
-          inventory_collection.all_manager_uuids
-        ).find_in_batches do |batch|
-          ActiveRecord::Base.transaction do
-            batch.each do |record|
-              record.public_send(inventory_collection.delete_method)
-              deleted_counter += 1
-            end
-          end
-        end
-
-        logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
-                     "#{all_manager_uuids_size}, deleted=#{deleted_counter}...Complete")
+        raise(":delete_complement method is supported only for :saver_strategy => [:batch, :concurrent_safe_batch]")
       end
 
       # Deletes/soft-deletes a given record
@@ -299,8 +286,8 @@ module InventoryRefresh::SaveCollection
       # @param update_time [Time] data hash
       def assign_attributes_for_update!(hash, update_time)
         hash[:type]         = model_class.name if supports_sti? && hash[:type].nil?
-        hash[:updated_on]   = update_time if supports_updated_on?
-        hash[:updated_at]   = update_time if supports_updated_at?
+        hash[:updated_on]   = update_time if supports_column?(:updated_on)
+        hash[:updated_at]   = update_time if supports_column?(:updated_at)
       end
 
       # Enriches data hash with timestamp and type columns
@@ -308,8 +295,8 @@ module InventoryRefresh::SaveCollection
       # @param hash [Hash] data hash
       # @param create_time [Time] data hash
       def assign_attributes_for_create!(hash, create_time)
-        hash[:created_on]   = create_time if supports_created_on?
-        hash[:created_at]   = create_time if supports_created_at?
+        hash[:created_on]   = create_time if supports_column?(:created_on)
+        hash[:created_at]   = create_time if supports_column?(:created_at)
         assign_attributes_for_update!(hash, create_time)
       end
 
@@ -343,49 +330,25 @@ module InventoryRefresh::SaveCollection
         @supports_sti_cache ||= inventory_collection.supports_sti?
       end
 
-      # @return [Boolean] true if the model_class has created_on column
-      def supports_created_on?
-        @supports_created_on_cache ||= inventory_collection.supports_created_on?
-      end
-
-      # @return [Boolean] true if the model_class has updated_on column
-      def supports_updated_on?
-        @supports_updated_on_cache ||= inventory_collection.supports_updated_on?
-      end
-
-      # @return [Boolean] true if the model_class has created_at column
-      def supports_created_at?
-        @supports_created_at_cache ||= inventory_collection.supports_created_at?
-      end
-
-      # @return [Boolean] true if the model_class has updated_at column
-      def supports_updated_at?
-        @supports_updated_at_cache ||= inventory_collection.supports_updated_at?
-      end
-
       # @return [Boolean] true if any serializable keys are present
       def serializable_keys?
         @serializable_keys_bool_cache ||= serializable_keys.present?
       end
 
-      # @return [Boolean] true if the model_class has resource_timestamp column
+      # @return [Boolean] true if the keys we are saving have resource_timestamp column
       def supports_remote_data_timestamp?(all_attribute_keys)
         all_attribute_keys.include?(:resource_timestamp) # include? on Set is O(1)
       end
 
-      # @return [Boolean] true if the model_class has resource_version column
+      # @return [Boolean] true if the keys we are saving have resource_counter column
       def supports_remote_data_version?(all_attribute_keys)
-        all_attribute_keys.include?(:resource_version) # include? on Set is O(1)
-      end
-
-      # @return [Boolean] true if the model_class has resource_timestamps column
-      def supports_resource_timestamps_max?
-        @supports_resource_timestamps_max_cache ||= inventory_collection.supports_resource_timestamps_max?
+        all_attribute_keys.include?(:resource_counter) # include? on Set is O(1)
       end
 
-      # @return [Boolean] true if the model_class has resource_versions column
-      def supports_resource_versions_max?
-        @supports_resource_versions_max_cache ||= inventory_collection.supports_resource_versions_max?
+      # @return [Boolean] true if the keys we are saving have resource_version column, which solves for a quick check
+      #                   if the record was modified
+      def supports_resource_version?(all_attribute_keys)
+        all_attribute_keys.include?(resource_version_column) # include? on Set is O(1)
       end
     end
   end