inventory_refresh 0.0.1

data/lib/inventory_refresh/save_collection/topological_sort.rb

@@ -0,0 +1,38 @@
+require "inventory_refresh/graph"
+require "inventory_refresh/inventory_collection/graph"
+require "inventory_refresh/save_collection/base"
+
+module InventoryRefresh::SaveCollection
+  class TopologicalSort < InventoryRefresh::SaveCollection::Base
+    class << self
+      # Saves the passed InventoryCollection objects by doing a topology sort of the graph, then going layer by layer
+      # and saving InventoryCollection object in each layer.
+      #
+      # @param ems [ExtManagementSystem] manager owning the inventory_collections
+      # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
+      #        for saving
+      def save_collections(ems, inventory_collections)
+        graph = InventoryRefresh::InventoryCollection::Graph.new(inventory_collections)
+        graph.build_directed_acyclic_graph!
+
+        layers = InventoryRefresh::Graph::TopologicalSort.new(graph).topological_sort
+
+        #_log.debug("Saving manager #{ems.name}...")
+
+        #sorted_graph_log = "Topological sorting of manager #{ems.name} resulted in these layers processable in parallel:\n"
+        #sorted_graph_log += graph.to_graphviz(:layers => layers)
+        #_log.debug(sorted_graph_log)
+
+        layers.each_with_index do |layer, index|
+          #_log.debug("Saving manager #{ems.name} | Layer #{index}")
+          layer.each do |inventory_collection|
+            save_inventory_object_inventory(ems, inventory_collection) unless inventory_collection.saved?
+          end
+          #_log.debug("Saved manager #{ems.name} | Layer #{index}")
+        end
+
+        #_log.debug("Saving manager #{ems.name}...Complete")
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_inventory.rb

@@ -0,0 +1,38 @@
+require "inventory_refresh/save_collection/recursive"
+require "inventory_refresh/save_collection/topological_sort"
+
+module InventoryRefresh
+  class SaveInventory
+    class << self
+      # Saves the passed InventoryCollection objects
+      #
+      # @param ems [ExtManagementSystem] manager owning the inventory_collections
+      # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
+      #        for saving
+      def save_inventory(ems, inventory_collections, strategy = nil)
+        #_log.debug("#{log_header(ems)} Scanning Inventory Collections...Start")
+        InventoryRefresh::InventoryCollection::Scanner.scan!(inventory_collections)
+        #_log.debug("#{log_header(ems)} Scanning Inventory Collections...Complete")
+
+        #_log.info("#{log_header(ems)} Saving EMS Inventory...")
+
+        if strategy.try(:to_sym) == :recursive
+          InventoryRefresh::SaveCollection::Recursive.save_collections(ems, inventory_collections)
+        else
+          InventoryRefresh::SaveCollection::TopologicalSort.save_collections(ems, inventory_collections)
+        end
+
+        #_log.info("#{log_header(ems)} Saving EMS Inventory...Complete")
+        ems
+      end
+
+      private
+
+      # @param ems [ExtManagementSystem] manager owning the inventory_collections
+      # @return [String] helper string for logging
+      def log_header(ems)
+        "EMS: [#{ems.name}], id: [#{ems.id}]"
+      end
+    end
+  end
+end

data/lib/inventory_refresh/target.rb

@@ -0,0 +1,73 @@
+module InventoryRefresh
+  class Target
+    attr_reader :association, :manager_ref, :event_id, :options
+
+    # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
+    #                             should be the same of association of a counterpart InventoryCollection object
+    # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
+    #                           The keys should be the same as the keys of the counterpart InventoryObject
+    # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    # @param manager_id [Integer] A primary key of the Manager owning the Target
+    # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
+    # @param options [Hash] A free form options hash
+    def initialize(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
+      raise "Provide either :manager or :manager_id argument" if manager.nil? && manager_id.nil?
+
+      @manager     = manager
+      @manager_id  = manager_id
+      @association = association
+      @manager_ref = manager_ref
+      @event_id    = event_id
+      @options     = options
+    end
+
+    # A Rails recommended interface for deserializing an object
+    # @return [InventoryRefresh::Target] InventoryRefresh::Target instance
+    def self.load(*args)
+      new(*args)
+    end
+
+    # A Rails recommended interface for serializing an object
+    #
+    # @param obj [InventoryRefresh::Target] InventoryRefresh::Target instance we want to serialize
+    # @return [Hash] serialized object
+    def self.dump(obj)
+      obj.dump
+    end
+
+    # Returns a serialized InventoryRefresh::Target object. This can be used to initialize a new object, then the object
+    # target acts the same as the object InventoryRefresh::Target.new(target.serialize)
+    #
+    # @return [Hash] serialized object
+    def dump
+      {
+        :manager_id  => manager_id,
+        :association => association,
+        :manager_ref => manager_ref,
+        :event_id    => event_id,
+        :options     => options
+      }
+    end
+
+    alias id dump
+    alias name manager_ref
+
+    # @return [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    def manager
+      @manager || ManageIQ::Providers::BaseManager.find(@manager_id)
+    end
+
+    # @return [Integer] A primary key of the Manager owning the Target
+    def manager_id
+      @manager_id || manager.try(:id)
+    end
+
+    # Loads InventoryRefresh::Target ApplicationRecord representation from our DB, this requires that InventoryRefresh::Target
+    # has been refreshed, otherwise the AR object can be missing.
+    #
+    # @return [ApplicationRecord] A InventoryRefresh::Target loaded from the database as AR object
+    def load_from_db
+      manager.public_send(association).find_by(manager_ref)
+    end
+  end
+end

data/lib/inventory_refresh/target_collection.rb

@@ -0,0 +1,80 @@
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class TargetCollection
+    attr_reader :targets
+
+    delegate :<<, :to => :targets
+
+    # @param manager [ManageIQ::Providers::BaseManager] manager owning the TargetCollection
+    # @param manager_id [Integer] primary key of manager owning the TargetCollection
+    # @param event [EmsEvent] EmsEvent associated with the TargetCollection
+    # @param targets [Array<InventoryRefresh::Target, ApplicationRecord>] Array of InventoryRefresh::Target objects or
+    #                ApplicationRecord objects
+    def initialize(manager: nil, manager_id: nil, event: nil, targets: [])
+      @manager    = manager
+      @manager_id = manager_id
+      @event      = event
+      @targets    = targets
+    end
+
+    # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
+    #                             should be the same of association of a counterpart InventoryCollection object
+    # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
+    #                           The keys should be the same as the keys of the counterpart InventoryObject
+    # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
+    # @param manager_id [Integer] A primary key of the Manager owning the Target
+    # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
+    # @param options [Hash] A free form options hash
+    def add_target(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
+      self << InventoryRefresh::Target.new(:association => association,
+                                         :manager_ref => manager_ref,
+                                         :manager     => manager || @manager,
+                                         :manager_id  => manager_id || @manager_id || @manager.try(:id),
+                                         :event_id    => event_id || @event.try(:id),
+                                         :options     => options)
+    end
+
+    # @return [String] A String containing a summary
+    def name
+      "Collection of #{targets.size} targets"
+    end
+
+    # @return [String] A String containing an id of each target in the TargetCollection
+    def id
+      "Collection of targets with id: #{targets.collect(&:name)}"
+    end
+
+    # Returns targets in a format:
+    #   {
+    #     :vms => {:ems_ref => Set.new(["vm_ref_1", "vm_ref2"])},
+    #     :network_ports => {:ems_ref => Set.new(["network_port_1", "network_port2"])
+    #   }
+    #
+    # Then we can quickly access all objects affected by:
+    #   NetworkPort.where(target_collection.manager_refs_by_association[:network_ports].to_a) =>
+    #     return AR objects with ems_refs ["network_port_1", "network_port2"]
+    # And we can get a list of ids for the API query by:
+    #   target_collection.manager_refs_by_association[:network_ports][:ems_ref].to_a =>
+    #     ["network_port_1", "network_port2"]
+    #
+    # Only targets of a type InventoryRefresh::Target are processed, any other targets present should be converted to
+    # InventoryRefresh::Target, e.g. in the Inventory::Collector code.
+    def manager_refs_by_association
+      @manager_refs_by_association ||= targets.select { |x| x.kind_of?(InventoryRefresh::Target) }.each_with_object({}) do |x, obj|
+        if obj[x.association].blank?
+          obj[x.association] = x.manager_ref.each_with_object({}) { |(key, value), hash| hash[key] = Set.new([value]) }
+        else
+          obj[x.association].each do |key, value|
+            value << x.manager_ref[key]
+          end
+        end
+      end
+    end
+
+    # Resets the cached @manager_refs_by_association to enforce reload when calling :manager_refs_by_association method
+    def manager_refs_by_association_reset
+      @manager_refs_by_association = nil
+    end
+  end
+end

data/lib/inventory_refresh/version.rb

@@ -0,0 +1,3 @@
+module InventoryRefresh
+  VERSION = "0.0.1".freeze
+end

data/tools/ci/create_db_user.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+psql -c "CREATE USER root SUPERUSER PASSWORD 'smartvm';" -U postgres

metadata

@@ -0,0 +1,207 @@
+--- !ruby/object:Gem::Specification
+name: inventory_refresh
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- ManageIQ Developers
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-09-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.6
+- !ruby/object:Gem::Dependency
+  name: more_core_extensions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+- !ruby/object:Gem::Dependency
+  name: ancestry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: factory_girl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.5.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Topological Inventory Persister
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".codeclimate.yml"
+- ".gitignore"
+- ".rspec"
+- ".rspec_ci"
+- ".rubocop.yml"
+- ".rubocop_cc.yml"
+- ".rubocop_local.yml"
+- ".travis.yml"
+- ".yamllint"
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- inventory_refresh.gemspec
+- lib/inventory_refresh.rb
+- lib/inventory_refresh/application_record_iterator.rb
+- lib/inventory_refresh/application_record_reference.rb
+- lib/inventory_refresh/graph.rb
+- lib/inventory_refresh/graph/topological_sort.rb
+- lib/inventory_refresh/inventory_collection.rb
+- lib/inventory_refresh/inventory_collection/data_storage.rb
+- lib/inventory_refresh/inventory_collection/graph.rb
+- lib/inventory_refresh/inventory_collection/index/proxy.rb
+- lib/inventory_refresh/inventory_collection/index/type/base.rb
+- lib/inventory_refresh/inventory_collection/index/type/data.rb
+- lib/inventory_refresh/inventory_collection/index/type/local_db.rb
+- lib/inventory_refresh/inventory_collection/index/type/skeletal.rb
+- lib/inventory_refresh/inventory_collection/reference.rb
+- lib/inventory_refresh/inventory_collection/references_storage.rb
+- lib/inventory_refresh/inventory_collection/scanner.rb
+- lib/inventory_refresh/inventory_collection/serialization.rb
+- lib/inventory_refresh/inventory_object.rb
+- lib/inventory_refresh/inventory_object_lazy.rb
+- lib/inventory_refresh/save_collection/base.rb
+- lib/inventory_refresh/save_collection/recursive.rb
+- lib/inventory_refresh/save_collection/saver/base.rb
+- lib/inventory_refresh/save_collection/saver/batch.rb
+- lib/inventory_refresh/save_collection/saver/concurrent_safe.rb
+- lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb
+- lib/inventory_refresh/save_collection/saver/default.rb
+- lib/inventory_refresh/save_collection/saver/sql_helper.rb
+- lib/inventory_refresh/save_collection/saver/sql_helper_update.rb
+- lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb
+- lib/inventory_refresh/save_collection/topological_sort.rb
+- lib/inventory_refresh/save_inventory.rb
+- lib/inventory_refresh/target.rb
+- lib/inventory_refresh/target_collection.rb
+- lib/inventory_refresh/version.rb
+- tools/ci/create_db_user.sh
+homepage: https://github.com/ManageIQ/inventory_refresh
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: Topological Inventory Persister
+test_files: []