inventory_refresh 0.2.2 → 0.3.0

data/lib/inventory_refresh/save_collection/topological_sort.rb

@@ -17,21 +17,21 @@ module InventoryRefresh::SaveCollection
 
         layers = InventoryRefresh::Graph::TopologicalSort.new(graph).topological_sort
 
-        logger.debug("Saving manager #{ems.name}...")
+        logger.debug("Saving manager #{ems.id}...")
 
-        sorted_graph_log = "Topological sorting of manager #{ems.name} resulted in these layers processable in parallel:\n"
+        sorted_graph_log = "Topological sorting of manager #{ems.id} resulted in these layers processable in parallel:\n"
         sorted_graph_log += graph.to_graphviz(:layers => layers)
         logger.debug(sorted_graph_log)
 
         layers.each_with_index do |layer, index|
-          logger.debug("Saving manager #{ems.name} | Layer #{index}")
+          logger.debug("Saving manager #{ems.id} | Layer #{index}")
           layer.each do |inventory_collection|
             save_inventory_object_inventory(ems, inventory_collection) unless inventory_collection.saved?
           end
-          logger.debug("Saved manager #{ems.name} | Layer #{index}")
+          logger.debug("Saved manager #{ems.id} | Layer #{index}")
         end
 
-        logger.debug("Saving manager #{ems.name}...Complete")
+        logger.debug("Saving manager #{ems.id}...Complete")
       end
     end
   end

data/lib/inventory_refresh/save_inventory.rb

@@ -29,11 +29,18 @@ module InventoryRefresh
       # @param ems [ExtManagementSystem] manager owning the inventory_collections
       # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
       #        for sweeping
+      # @param sweep_scope [Array<String, Symbol, Hash>] Array of inventory collection names marking sweep. Or for
+      #        targeted sweeping it's array of hashes, where key is inventory collection name pointing to an array of
+      #        identifiers of inventory objects we want to target for sweeping.
       # @param refresh_state [ActiveRecord] Record of :refresh_states
-      def sweep_inactive_records(ems, inventory_collections, refresh_state)
-        logger.info("#{log_header(ems)} Sweeping EMS Inventory...")
-        InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, refresh_state)
-        logger.info("#{log_header(ems)} Sweeping EMS Inventory...Complete")
+      def sweep_inactive_records(ems, inventory_collections, sweep_scope, refresh_state)
+        inventory_collections.each do |inventory_collection|
+          inventory_collection.strategy = :local_db_find_references
+        end
+
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory with scope #{sweep_scope} and date #{refresh_state.created_at} ...")
+        InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, sweep_scope, refresh_state)
+        logger.info("#{log_header(ems)} Sweeping EMS Inventory with scope #{sweep_scope} and date #{refresh_state.created_at}...Complete")
 
         ems
       end
@@ -43,7 +50,7 @@ module InventoryRefresh
       # @param ems [ExtManagementSystem] manager owning the inventory_collections
       # @return [String] helper string for logging
       def log_header(ems)
-        "EMS: [#{ems.name}], id: [#{ems.id}]"
+        "EMS: [#{ems.id}]"
       end
     end
   end

data/lib/inventory_refresh/version.rb

@@ -1,3 +1,3 @@
 module InventoryRefresh
-  VERSION = "0.2.2".freeze
+  VERSION = "0.3.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: inventory_refresh
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-14 00:00:00.000000000 Z
+date: 2019-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -28,22 +28,16 @@ dependencies:
   name: more_core_extensions
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.5'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.5'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -78,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
   name: factory_bot
   requirement: !ruby/object:Gem::Requirement
@@ -128,34 +122,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop-performance
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -198,6 +164,7 @@ files:
 - lib/inventory_refresh.rb
 - lib/inventory_refresh/application_record_iterator.rb
 - lib/inventory_refresh/application_record_reference.rb
+- lib/inventory_refresh/exception.rb
 - lib/inventory_refresh/graph.rb
 - lib/inventory_refresh/graph/topological_sort.rb
 - lib/inventory_refresh/inventory_collection.rb
@@ -225,9 +192,7 @@ files:
 - lib/inventory_refresh/persister.rb
 - lib/inventory_refresh/save_collection/base.rb
 - lib/inventory_refresh/save_collection/saver/base.rb
-- lib/inventory_refresh/save_collection/saver/batch.rb
 - lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb
-- lib/inventory_refresh/save_collection/saver/default.rb
 - lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb
 - lib/inventory_refresh/save_collection/saver/retention_helper.rb
 - lib/inventory_refresh/save_collection/saver/sql_helper.rb
@@ -236,8 +201,6 @@ files:
 - lib/inventory_refresh/save_collection/sweeper.rb
 - lib/inventory_refresh/save_collection/topological_sort.rb
 - lib/inventory_refresh/save_inventory.rb
-- lib/inventory_refresh/target.rb
-- lib/inventory_refresh/target_collection.rb
 - lib/inventory_refresh/version.rb
 homepage: https://github.com/ManageIQ/inventory_refresh
 licenses:
@@ -258,7 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
+rubyforge_project: 
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Topological Inventory Persister

data/lib/inventory_refresh/save_collection/saver/batch.rb

@@ -1,17 +0,0 @@
-require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
-
-module InventoryRefresh::SaveCollection
-  module Saver
-    class Batch < InventoryRefresh::SaveCollection::Saver::ConcurrentSafeBatch
-      private
-
-      # Just returning manager ref transformed to column names, for strategies that do not expect to have unique DB
-      # indexes.
-      #
-      # @return [Array<Symbol>] manager ref transformed to column names
-      def unique_index_columns
-        inventory_collection.manager_ref_to_cols.map(&:to_sym)
-      end
-    end
-  end
-end

data/lib/inventory_refresh/save_collection/saver/default.rb

@@ -1,57 +0,0 @@
-require "inventory_refresh/save_collection/saver/base"
-
-module InventoryRefresh::SaveCollection
-  module Saver
-    class Default < InventoryRefresh::SaveCollection::Saver::Base
-      private
-
-      # Updates the passed record with hash data and stores primary key value into inventory_object.
-      #
-      # @param record [ApplicationRecord] record we want to update in DB
-      # @param hash [Hash] data we want to update the record with
-      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
-      #        key value
-      def update_record!(record, hash, inventory_object)
-        record.assign_attributes(hash.except(:id))
-        if !inventory_collection.check_changed? || record.changed?
-          record.save
-          inventory_collection.store_updated_records(record)
-        end
-
-        inventory_object.id = record.id
-      end
-
-      # Creates a new record in the DB using the passed hash data
-      #
-      # @param hash [Hash] hash with data we want to persist to DB
-      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
-      #        key value
-      def create_record!(hash, inventory_object)
-        record = inventory_collection.model_class.create!(hash.except(:id))
-        inventory_collection.store_created_records(record)
-
-        inventory_object.id = record.id
-      end
-
-      # Asserts we do not have duplicate records in the DB. If the record is duplicate we will destroy it.
-      #
-      # @param record [ApplicationRecord] record we want to update in DB
-      # @param index [String] manager_uuid of the record
-      # @return [Boolean] false if the record is duplicate
-      def assert_unique_record(record, index)
-        # TODO(lsmola) can go away once we indexed our DB with unique indexes
-        if unique_db_indexes.include?(index) # Include on Set is O(1)
-          # We have a duplicate in the DB, destroy it. A find_each method does automatically .order(:id => :asc)
-          # so we always keep the oldest record in the case of duplicates.
-          logger.warn("A duplicate record was detected and destroyed, inventory_collection: "\
-                      "'#{inventory_collection}', record: '#{record}', duplicate_index: '#{index}'")
-          record.destroy
-          return false
-        else
-          unique_db_indexes << index
-        end
-        true
-      end
-    end
-  end
-end

data/lib/inventory_refresh/target.rb

@@ -1,73 +0,0 @@
-module InventoryRefresh
-  class Target
-    attr_reader :association, :manager_ref, :event_id, :options
-
-    # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
-    #                             should be the same of association of a counterpart InventoryCollection object
-    # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
-    #                           The keys should be the same as the keys of the counterpart InventoryObject
-    # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
-    # @param manager_id [Integer] A primary key of the Manager owning the Target
-    # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
-    # @param options [Hash] A free form options hash
-    def initialize(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
-      raise "Provide either :manager or :manager_id argument" if manager.nil? && manager_id.nil?
-
-      @manager     = manager
-      @manager_id  = manager_id
-      @association = association
-      @manager_ref = manager_ref
-      @event_id    = event_id
-      @options     = options
-    end
-
-    # A Rails recommended interface for deserializing an object
-    # @return [InventoryRefresh::Target] InventoryRefresh::Target instance
-    def self.load(*args)
-      new(*args)
-    end
-
-    # A Rails recommended interface for serializing an object
-    #
-    # @param obj [InventoryRefresh::Target] InventoryRefresh::Target instance we want to serialize
-    # @return [Hash] serialized object
-    def self.dump(obj)
-      obj.dump
-    end
-
-    # Returns a serialized InventoryRefresh::Target object. This can be used to initialize a new object, then the object
-    # target acts the same as the object InventoryRefresh::Target.new(target.serialize)
-    #
-    # @return [Hash] serialized object
-    def dump
-      {
-        :manager_id  => manager_id,
-        :association => association,
-        :manager_ref => manager_ref,
-        :event_id    => event_id,
-        :options     => options
-      }
-    end
-
-    alias id dump
-    alias name manager_ref
-
-    # @return [ManageIQ::Providers::BaseManager] The Manager owning the Target
-    def manager
-      @manager || ManageIQ::Providers::BaseManager.find(@manager_id)
-    end
-
-    # @return [Integer] A primary key of the Manager owning the Target
-    def manager_id
-      @manager_id || manager.try(:id)
-    end
-
-    # Loads InventoryRefresh::Target ApplicationRecord representation from our DB, this requires that InventoryRefresh::Target
-    # has been refreshed, otherwise the AR object can be missing.
-    #
-    # @return [ApplicationRecord] A InventoryRefresh::Target loaded from the database as AR object
-    def load_from_db
-      manager.public_send(association).find_by(manager_ref)
-    end
-  end
-end

data/lib/inventory_refresh/target_collection.rb

@@ -1,92 +0,0 @@
-require "active_support/core_ext/module/delegation"
-
-module InventoryRefresh
-  class TargetCollection
-    attr_reader :targets
-
-    delegate :<<, :to => :targets
-
-    # @param manager [ManageIQ::Providers::BaseManager] manager owning the TargetCollection
-    # @param manager_id [Integer] primary key of manager owning the TargetCollection
-    # @param event [EmsEvent] EmsEvent associated with the TargetCollection
-    # @param targets [Array<InventoryRefresh::Target, ApplicationRecord>] Array of InventoryRefresh::Target objects or
-    #                ApplicationRecord objects
-    def initialize(manager: nil, manager_id: nil, event: nil, targets: [])
-      @manager    = manager
-      @manager_id = manager_id
-      @event      = event
-      @targets    = targets
-    end
-
-    # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
-    #                             should be the same of association of a counterpart InventoryCollection object
-    # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
-    #                           The keys should be the same as the keys of the counterpart InventoryObject
-    # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
-    # @param manager_id [Integer] A primary key of the Manager owning the Target
-    # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
-    # @param options [Hash] A free form options hash
-    def add_target(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
-      self << InventoryRefresh::Target.new(:association => association,
-                                         :manager_ref => manager_ref,
-                                         :manager     => manager || @manager,
-                                         :manager_id  => manager_id || @manager_id || @manager.try(:id),
-                                         :event_id    => event_id || @event.try(:id),
-                                         :options     => options)
-    end
-
-    # @return [String] A String containing a summary
-    def name
-      "Collection of #{targets.size} targets"
-    end
-
-    # @return [String] A String containing an id of each target in the TargetCollection
-    def id
-      "Collection of targets with id: #{targets.collect(&:name)}"
-    end
-
-    # Returns targets in a format:
-    #   {
-    #     :vms => {:ems_ref => Set.new(["vm_ref_1", "vm_ref2"])},
-    #     :network_ports => {:ems_ref => Set.new(["network_port_1", "network_port2"])
-    #   }
-    #
-    # Then we can quickly access all objects affected by:
-    #   NetworkPort.where(target_collection.manager_refs_by_association[:network_ports].to_a) =>
-    #     return AR objects with ems_refs ["network_port_1", "network_port2"]
-    # And we can get a list of ids for the API query by:
-    #   target_collection.manager_refs_by_association[:network_ports][:ems_ref].to_a =>
-    #     ["network_port_1", "network_port2"]
-    #
-    # Only targets of a type InventoryRefresh::Target are processed, any other targets present should be converted to
-    # InventoryRefresh::Target, e.g. in the Inventory::Collector code.
-    def manager_refs_by_association
-      @manager_refs_by_association ||= targets.select { |x| x.kind_of?(InventoryRefresh::Target) }.each_with_object({}) do |x, obj|
-        if obj[x.association].blank?
-          obj[x.association] = x.manager_ref.each_with_object({}) { |(key, value), hash| hash[key] = Set.new([value]) }
-        else
-          obj[x.association].each do |key, value|
-            value << x.manager_ref[key]
-          end
-        end
-      end
-    end
-
-    # Resets the cached @manager_refs_by_association to enforce reload when calling :manager_refs_by_association method
-    def manager_refs_by_association_reset
-      @manager_refs_by_association = nil
-    end
-
-    # Returns list of ems_refs
-    # @return [Array<String>]
-    def references(collection)
-      manager_refs_by_association.try(:[], collection).try(:[], :ems_ref)&.to_a || []
-    end
-
-    # Returns list of names
-    # @return [Array<String>]
-    def name_references(collection)
-      manager_refs_by_association.try(:[], collection).try(:[], :name)&.to_a || []
-    end
-  end
-end