inventory_refresh 0.3.3 → 1.0.0

data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb

@@ -23,13 +23,13 @@ module InventoryRefresh::SaveCollection
         connection = get_connection
 
         # We want to ignore create timestamps when updating
-        all_attribute_keys_array = all_attribute_keys.to_a.delete_if { |x| %i(created_at created_on).include?(x) }
+        all_attribute_keys_array = all_attribute_keys.to_a.delete_if { |x| %i[created_at created_on].include?(x) }
         all_attribute_keys_array << :id
 
         # If there is not version attribute, the version conditions will be ignored
-        version_attribute = if supports_remote_data_timestamp?(all_attribute_keys)
+        version_attribute = if inventory_collection.parallel_safe? && supports_remote_data_timestamp?(all_attribute_keys)
                               :resource_timestamp
-                            elsif supports_remote_data_version?(all_attribute_keys)
+                            elsif inventory_collection.parallel_safe? && supports_remote_data_version?(all_attribute_keys)
                               :resource_counter
                             end
 
@@ -130,9 +130,13 @@ module InventoryRefresh::SaveCollection
       end
 
       def update_query_returning
-        <<-SQL
-          RETURNING updated_values.#{quote_column_name("id")}, #{unique_index_columns.map { |x| "updated_values.#{quote_column_name(x)}" }.join(",")}
-        SQL
+        if inventory_collection.parallel_safe?
+          <<-SQL
+            RETURNING updated_values.#{quote_column_name("id")}, #{unique_index_columns.map { |x| "updated_values.#{quote_column_name(x)}" }.join(",")}
+          SQL
+        else
+          ""
+        end
       end
     end
   end

data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb

@@ -19,7 +19,7 @@ module InventoryRefresh::SaveCollection
       #        columns of a row, :partial is when we save only few columns, so a partial row.
       # @param on_conflict [Symbol, NilClass] defines behavior on conflict with unique index constraint, allowed values
       #        are :do_update, :do_nothing, nil
-      def build_insert_query(all_attribute_keys, hashes, on_conflict: nil, mode:, column_name: nil)
+      def build_insert_query(all_attribute_keys, hashes, mode:, on_conflict: nil, column_name: nil)
         logger.debug("Building insert query for #{inventory_collection} of size #{inventory_collection.size}...")
 
         # Cache the connection for the batch
@@ -55,6 +55,8 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_on_conflict_behavior(all_attribute_keys, on_conflict, mode, ignore_cols, column_name)
+        return "" unless inventory_collection.parallel_safe?
+
         insert_query_on_conflict = insert_query_on_conflict_do(on_conflict)
         if on_conflict == :do_update
           insert_query_on_conflict += insert_query_on_conflict_update(all_attribute_keys, mode, ignore_cols, column_name)
@@ -63,11 +65,12 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_on_conflict_do(on_conflict)
-        if on_conflict == :do_nothing
+        case on_conflict
+        when :do_nothing
           <<-SQL
             ON CONFLICT DO NOTHING
           SQL
-        elsif on_conflict == :do_update
+        when :do_update
           index_where_condition = unique_index_for(unique_index_keys).where
           where_to_sql          = index_where_condition ? "WHERE #{index_where_condition}" : ""
 
@@ -92,6 +95,8 @@ module InventoryRefresh::SaveCollection
                               :resource_counter
                             end
 
+        # TODO(lsmola) should we add :deleted => false to the update clause? That should handle a reconnect, without a
+        # a need to list :deleted anywhere in the parser. We just need to check that a model has the :deleted attribute
         query = <<-SQL
           SET #{(all_attribute_keys - ignore_cols).map { |key| build_insert_set_cols(key) }.join(", ")}
         SQL
@@ -103,10 +108,12 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_on_conflict_update_mode(mode, version_attribute, column_name)
-        if mode == :full
+        case mode
+        when :full
           full_update_condition(version_attribute)
-        elsif mode == :partial
+        when :partial
           raise "Column name must be provided" unless column_name
+
           partial_update_condition(version_attribute, column_name)
         end
       end
@@ -124,7 +131,7 @@ module InventoryRefresh::SaveCollection
           , #{attr_partial} = '{}', #{attr_partial_max} = NULL
 
           WHERE EXCLUDED.#{attr_full} IS NULL OR (
-            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_full} >= #{q_table_name}.#{attr_full}) AND
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_full} > #{q_table_name}.#{attr_full}) AND
             (#{q_table_name}.#{attr_partial_max} IS NULL OR EXCLUDED.#{attr_full} >= #{q_table_name}.#{attr_partial_max})
           )
         SQL
@@ -133,9 +140,10 @@ module InventoryRefresh::SaveCollection
       def partial_update_condition(attr_full, column_name)
         attr_partial     = attr_full.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
         attr_partial_max = "#{attr_partial}_max"
-        cast             = if attr_full == :resource_timestamp
+        cast             = case attr_full
+                           when :resource_timestamp
                              "timestamp"
-                           elsif attr_full == :resource_counter
+                           when :resource_counter
                              "integer"
                            end
 
@@ -150,9 +158,9 @@ module InventoryRefresh::SaveCollection
           #{insert_query_set_jsonb_version(cast, attr_partial, attr_partial_max, column_name)}
           , #{attr_partial_max} = greatest(#{q_table_name}.#{attr_partial_max}::#{cast}, EXCLUDED.#{attr_partial_max}::#{cast})
           WHERE EXCLUDED.#{attr_partial_max} IS NULL OR (
-            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_partial_max} >= #{q_table_name}.#{attr_full}) AND (
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_partial_max} > #{q_table_name}.#{attr_full}) AND (
               (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast} IS NULL OR
-              EXCLUDED.#{attr_partial_max}::#{cast} >= (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast}
+              EXCLUDED.#{attr_partial_max}::#{cast} > (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast}
             )
           )
         SQL
@@ -179,12 +187,16 @@ module InventoryRefresh::SaveCollection
       end
 
       def insert_query_returning_timestamps
-        # For upsert, we'll return also created and updated timestamps, so we can recognize what was created and what
-        # updated
-        if inventory_collection.internal_timestamp_columns.present?
-          <<-SQL
-            , #{inventory_collection.internal_timestamp_columns.map { |x| quote_column_name(x) }.join(",")}
-          SQL
+        if inventory_collection.parallel_safe?
+          # For upsert, we'll return also created and updated timestamps, so we can recognize what was created and what
+          # updated
+          if inventory_collection.internal_timestamp_columns.present?
+            <<-SQL
+              , #{inventory_collection.internal_timestamp_columns.map { |x| quote_column_name(x) }.join(",")}
+            SQL
+          end
+        else
+          ""
         end
       end
     end

data/lib/inventory_refresh/save_collection/sweeper.rb

@@ -1,7 +1,5 @@
-require "inventory_refresh/exception"
 require "inventory_refresh/logging"
 require "inventory_refresh/save_collection/saver/retention_helper"
-require "inventory_refresh/inventory_collection/index/type/local_db"
 
 module InventoryRefresh::SaveCollection
   class Sweeper < InventoryRefresh::SaveCollection::Base
@@ -12,111 +10,39 @@ module InventoryRefresh::SaveCollection
       # @param _ems [ActiveRecord] Manager owning the inventory_collections
       # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] Array of InventoryCollection objects
       #        for sweeping
-      # @param sweep_scope [Array<String, Symbol, Hash>] Array of inventory collection names marking sweep. Or for
-      #        targeted sweeping it's array of hashes, where key is inventory collection name pointing to an array of
-      #        identifiers of inventory objects we want to target for sweeping.
       # @param refresh_state [ActiveRecord] Record of :refresh_states
-      def sweep(_ems, inventory_collections, sweep_scope, refresh_state)
-        scope_set = build_scope_set(sweep_scope)
-
+      def sweep(_ems, inventory_collections, refresh_state)
         inventory_collections.each do |inventory_collection|
-          next unless sweep_possible?(inventory_collection, scope_set)
+          next unless sweep_possible?(inventory_collection, refresh_state)
 
-          new(inventory_collection, refresh_state, sweep_scope).sweep
+          new(inventory_collection, refresh_state).sweep
         end
       end
 
-      def sweep_possible?(inventory_collection, scope_set)
-        inventory_collection.supports_column?(:last_seen_at) && in_scope?(inventory_collection, scope_set)
-      end
-
-      def in_scope?(inventory_collection, scope_set)
-        scope_set.include?(inventory_collection&.name)
+      def sweep_possible?(inventory_collection, refresh_state)
+        inventory_collection.supports_column?(:last_seen_at) && inventory_collection.parallel_safe? &&
+          inventory_collection.strategy == :local_db_find_missing_references &&
+          in_scope?(inventory_collection, refresh_state.sweep_scope)
       end
 
-      def build_scope_set(sweep_scope)
-        return [] unless sweep_scope
+      def in_scope?(inventory_collection, sweep_scope)
+        return true unless sweep_scope
+        return true if sweep_scope.kind_of?(Array) && sweep_scope.include?(inventory_collection&.name&.to_s)
 
-        if sweep_scope.kind_of?(Array)
-          sweep_scope.map(&:to_sym).to_set
-        elsif sweep_scope.kind_of?(Hash)
-          sweep_scope.keys.map(&:to_sym).to_set
-        end
+        false
       end
     end
 
     include InventoryRefresh::SaveCollection::Saver::RetentionHelper
 
-    attr_reader :inventory_collection, :refresh_state, :sweep_scope, :model_class, :primary_key
+    attr_reader :inventory_collection, :refresh_state, :model_class, :primary_key
 
-    delegate :inventory_object_lazy?,
-             :inventory_object?,
-             :to => :inventory_collection
-
-    def initialize(inventory_collection, refresh_state, sweep_scope)
+    def initialize(inventory_collection, refresh_state)
       @inventory_collection = inventory_collection
+      @refresh_state        = refresh_state
 
-      @refresh_state = refresh_state
-      @sweep_scope   = sweep_scope
-
-      @model_class = inventory_collection.model_class
-      @primary_key = @model_class.primary_key
-    end
-
-    def apply_targeted_sweep_scope(all_entities_query)
-      if sweep_scope.kind_of?(Hash)
-        scope = sweep_scope[inventory_collection.name]
-        return all_entities_query if scope.nil? || scope.empty?
-
-        # Scan the scope to find all references, so we can load them from DB in batches
-        scan_sweep_scope!(scope)
-
-        scope_keys = Set.new
-        conditions = scope.map { |x| InventoryRefresh::InventoryObject.attributes_with_keys(x, inventory_collection, scope_keys) }
-        assert_conditions!(conditions, scope_keys)
-
-        all_entities_query.where(inventory_collection.build_multi_selection_condition(conditions, scope_keys))
-      else
-        all_entities_query
-      end
-    end
-
-    def loadable?(value)
-      inventory_object_lazy?(value) || inventory_object?(value)
-    end
-
-    def scan_sweep_scope!(scope)
-      scope.each do |sc|
-        sc.each_value do |value|
-          next unless loadable?(value)
-
-          value_inventory_collection = value.inventory_collection
-          value_inventory_collection.add_reference(value.reference, :key => value.key)
-        end
-      end
-    end
-
-    def assert_conditions!(conditions, scope_keys)
-      conditions.each do |cond|
-        assert_uniform_keys!(cond, scope_keys)
-        assert_non_existent_keys!(cond)
-      end
-    end
-
-    def assert_uniform_keys!(cond, scope_keys)
-      return if (diff = (scope_keys - cond.keys.to_set)).empty?
-
-      raise(InventoryRefresh::Exception::SweeperNonUniformScopeKeyFoundError,
-            "Sweeping scope for #{inventory_collection} contained non uniform keys. All keys for the"\
-            "scope must be the same, it's possible to send multiple sweeps with different key set. Missing keys"\
-            " for a scope were: #{diff.to_a}")
-    end
-
-    def assert_non_existent_keys!(cond)
-      return if (diff = (cond.keys.to_set - inventory_collection.all_column_names)).empty?
-
-      raise(InventoryRefresh::Exception::SweeperNonExistentScopeKeyFoundError,
-            "Sweeping scope for #{inventory_collection} contained keys that are not columns: #{diff.to_a}")
+      @model_class            = inventory_collection.model_class
+      @primary_key            = @model_class.primary_key
     end
 
     def sweep
@@ -126,9 +52,7 @@ module InventoryRefresh::SaveCollection
       table       = model_class.arel_table
       date_field  = table[:last_seen_at]
       all_entities_query = inventory_collection.full_collection_for_comparison
-      all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive && inventory_collection.supports_column?(:archived_at)
-
-      all_entities_query = apply_targeted_sweep_scope(all_entities_query)
+      all_entities_query.active if inventory_collection.retention_strategy == :archive
 
       query = all_entities_query
               .where(date_field.lt(refresh_start)).or(all_entities_query.where(:last_seen_at => nil))

data/lib/inventory_refresh/save_collection/topological_sort.rb

@@ -17,21 +17,21 @@ module InventoryRefresh::SaveCollection
 
         layers = InventoryRefresh::Graph::TopologicalSort.new(graph).topological_sort
 
-        logger.debug("Saving manager #{ems.id}...")
+        logger.debug("Saving manager #{ems.name}...")
 
-        sorted_graph_log = "Topological sorting of manager #{ems.id} resulted in these layers processable in parallel:\n"
+        sorted_graph_log = "Topological sorting of manager #{ems.name} resulted in these layers processable in parallel:\n"
         sorted_graph_log += graph.to_graphviz(:layers => layers)
         logger.debug(sorted_graph_log)
 
         layers.each_with_index do |layer, index|
-          logger.debug("Saving manager #{ems.id} | Layer #{index}")
+          logger.debug("Saving manager #{ems.name} | Layer #{index}")
           layer.each do |inventory_collection|
             save_inventory_object_inventory(ems, inventory_collection) unless inventory_collection.saved?
           end
-          logger.debug("Saved manager #{ems.id} | Layer #{index}")
+          logger.debug("Saved manager #{ems.name} | Layer #{index}")
         end
 
-        logger.debug("Saving manager #{ems.id}...Complete")
+        logger.debug("Saving manager #{ems.name}...Complete")
       end
     end
   end

data/lib/inventory_refresh/save_inventory.rb

@@ -29,18 +29,11 @@ module InventoryRefresh
       # @param ems [ExtManagementSystem] manager owning the inventory_collections
       # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
       #        for sweeping
-      # @param sweep_scope [Array<String, Symbol, Hash>] Array of inventory collection names marking sweep. Or for
-      #        targeted sweeping it's array of hashes, where key is inventory collection name pointing to an array of
-      #        identifiers of inventory objects we want to target for sweeping.
       # @param refresh_state [ActiveRecord] Record of :refresh_states
-      def sweep_inactive_records(ems, inventory_collections, sweep_scope, refresh_state)
-        inventory_collections.each do |inventory_collection|
-          inventory_collection.strategy = :local_db_find_references
-        end
-
-        logger.info("#{log_header(ems)} Sweeping EMS Inventory with scope #{sweep_scope} and date #{refresh_state.created_at} ...")
-        InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, sweep_scope, refresh_state)
-        logger.info("#{log_header(ems)} Sweeping EMS Inventory with scope #{sweep_scope} and date #{refresh_state.created_at}...Complete")
+      def sweep_inactive_records(ems, inventory_collections, refresh_state)
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory...")
+        InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, refresh_state)
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory...Complete")
 
         ems
       end
@@ -50,7 +43,7 @@ module InventoryRefresh
       # @param ems [ExtManagementSystem] manager owning the inventory_collections
       # @return [String] helper string for logging
       def log_header(ems)
-        "EMS: [#{ems.id}]"
+        "EMS: [#{ems.name}], id: [#{ems.id}]"
       end
     end
   end

data/lib/inventory_refresh/target.rb

@@ -0,0 +1,73 @@
+module InventoryRefresh
+  class Target
+    attr_reader :association, :manager_ref, :event_id, :options
+
+    # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
+    #                             should be the same of association of a counterpart InventoryCollection object
+    # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
+    #                           The keys should be the same as the keys of the counterpart InventoryObject
+    # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    # @param manager_id [Integer] A primary key of the Manager owning the Target
+    # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
+    # @param options [Hash] A free form options hash
+    def initialize(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
+      raise "Provide either :manager or :manager_id argument" if manager.nil? && manager_id.nil?
+
+      @manager     = manager
+      @manager_id  = manager_id
+      @association = association
+      @manager_ref = manager_ref
+      @event_id    = event_id
+      @options     = options
+    end
+
+    # A Rails recommended interface for deserializing an object
+    # @return [InventoryRefresh::Target] InventoryRefresh::Target instance
+    def self.load(*args)
+      new(*args)
+    end
+
+    # A Rails recommended interface for serializing an object
+    #
+    # @param obj [InventoryRefresh::Target] InventoryRefresh::Target instance we want to serialize
+    # @return [Hash] serialized object
+    def self.dump(obj)
+      obj.dump
+    end
+
+    # Returns a serialized InventoryRefresh::Target object. This can be used to initialize a new object, then the object
+    # target acts the same as the object InventoryRefresh::Target.new(target.serialize)
+    #
+    # @return [Hash] serialized object
+    def dump
+      {
+        :manager_id  => manager_id,
+        :association => association,
+        :manager_ref => manager_ref,
+        :event_id    => event_id,
+        :options     => options
+      }
+    end
+
+    alias id dump
+    alias name manager_ref
+
+    # @return [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    def manager
+      @manager || ManageIQ::Providers::BaseManager.find(@manager_id)
+    end
+
+    # @return [Integer] A primary key of the Manager owning the Target
+    def manager_id
+      @manager_id || manager.try(:id)
+    end
+
+    # Loads InventoryRefresh::Target ApplicationRecord representation from our DB, this requires that InventoryRefresh::Target
+    # has been refreshed, otherwise the AR object can be missing.
+    #
+    # @return [ApplicationRecord] A InventoryRefresh::Target loaded from the database as AR object
+    def load_from_db
+      manager.public_send(association).find_by(manager_ref)
+    end
+  end
+end

data/lib/inventory_refresh/target_collection.rb

@@ -0,0 +1,92 @@
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class TargetCollection
+    attr_reader :targets
+
+    delegate :<<, :to => :targets
+
+    # @param manager [ManageIQ::Providers::BaseManager] manager owning the TargetCollection
+    # @param manager_id [Integer] primary key of manager owning the TargetCollection
+    # @param event [EmsEvent] EmsEvent associated with the TargetCollection
+    # @param targets [Array<InventoryRefresh::Target, ApplicationRecord>] Array of InventoryRefresh::Target objects or
+    #                ApplicationRecord objects
+    def initialize(manager: nil, manager_id: nil, event: nil, targets: [])
+      @manager    = manager
+      @manager_id = manager_id
+      @event      = event
+      @targets    = targets
+    end
+
+    # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
+    #                             should be the same of association of a counterpart InventoryCollection object
+    # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
+    #                           The keys should be the same as the keys of the counterpart InventoryObject
+    # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    # @param manager_id [Integer] A primary key of the Manager owning the Target
+    # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
+    # @param options [Hash] A free form options hash
+    def add_target(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
+      self << InventoryRefresh::Target.new(:association => association,
+                                           :manager_ref => manager_ref,
+                                           :manager     => manager || @manager,
+                                           :manager_id  => manager_id || @manager_id || @manager.try(:id),
+                                           :event_id    => event_id || @event.try(:id),
+                                           :options     => options)
+    end
+
+    # @return [String] A String containing a summary
+    def name
+      "Collection of #{targets.size} targets"
+    end
+
+    # @return [String] A String containing an id of each target in the TargetCollection
+    def id
+      "Collection of targets with id: #{targets.collect(&:name)}"
+    end
+
+    # Returns targets in a format:
+    #   {
+    #     :vms => {:ems_ref => Set.new(["vm_ref_1", "vm_ref2"])},
+    #     :network_ports => {:ems_ref => Set.new(["network_port_1", "network_port2"])
+    #   }
+    #
+    # Then we can quickly access all objects affected by:
+    #   NetworkPort.where(target_collection.manager_refs_by_association[:network_ports].to_a) =>
+    #     return AR objects with ems_refs ["network_port_1", "network_port2"]
+    # And we can get a list of ids for the API query by:
+    #   target_collection.manager_refs_by_association[:network_ports][:ems_ref].to_a =>
+    #     ["network_port_1", "network_port2"]
+    #
+    # Only targets of a type InventoryRefresh::Target are processed, any other targets present should be converted to
+    # InventoryRefresh::Target, e.g. in the Inventory::Collector code.
+    def manager_refs_by_association
+      @manager_refs_by_association ||= targets.select { |x| x.kind_of?(InventoryRefresh::Target) }.each_with_object({}) do |x, obj|
+        if obj[x.association].blank?
+          obj[x.association] = x.manager_ref.each_with_object({}) { |(key, value), hash| hash[key] = Set.new([value]) }
+        else
+          obj[x.association].each do |key, value|
+            value << x.manager_ref[key]
+          end
+        end
+      end
+    end
+
+    # Resets the cached @manager_refs_by_association to enforce reload when calling :manager_refs_by_association method
+    def manager_refs_by_association_reset
+      @manager_refs_by_association = nil
+    end
+
+    # Returns list of ems_refs
+    # @return [Array<String>]
+    def references(collection)
+      manager_refs_by_association.try(:[], collection).try(:[], :ems_ref)&.to_a || []
+    end
+
+    # Returns list of names
+    # @return [Array<String>]
+    def name_references(collection)
+      manager_refs_by_association.try(:[], collection).try(:[], :name)&.to_a || []
+    end
+  end
+end

data/lib/inventory_refresh/version.rb

@@ -1,3 +1,3 @@
 module InventoryRefresh
-  VERSION = "0.3.3".freeze
+  VERSION = "1.0.0".freeze
 end

data/lib/inventory_refresh.rb

@@ -6,4 +6,6 @@ require "inventory_refresh/logging"
 require "inventory_refresh/null_logger"
 require "inventory_refresh/persister"
 require "inventory_refresh/save_inventory"
+require "inventory_refresh/target"
+require "inventory_refresh/target_collection"
 require "inventory_refresh/version"