inventory_refresh 0.3.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 562e3672dadb805f4cd063f1ebd6151ee3defa02a4d72653d4d90984080d5825
-  data.tar.gz: 303f14059268fa274404e1ca6757f76c65612dd5b75919029f712d812c27717d
+  metadata.gz: 8d788fa837920653d4f68b418a41687dcfb5555da780854c371654b93cf0fbaa
+  data.tar.gz: 516547253906ef09089351d6e007ebaac0160aaadee73a38782aa598f284e396
 SHA512:
-  metadata.gz: 46c6e8b9bbeed08a8ca7860cac0e37a8d957335aa472f267445347f70bd3cf984b5d4eb3ef8f80ff1ed332bfd2b74c696568c3d8472d4bb0d5c4d9f3e65b51f5
-  data.tar.gz: 1c5839b48a77df2edeb6307e79559c49b00d3971be5637d50d752d85249c2b730ecf01880151e57c3fd9788d22c9188b91f310fcd64c0cbe2565aa145420e09b
+  metadata.gz: c6b1fcf9600b6f9ff22da8cc5226f7c6395a7c35f9b9b62f9c0a2edb51158e1d3eaa59b1c1400b1e1ccf2f2bf1ff7bfa5901db04326afc490d569715daf73ad9
+  data.tar.gz: 50b1acc7a4b10a5a8a678b16fd06aa9c809068054d9b469391e877e668e18340ca998ab8485346d5f51cd0f682582fcd9ef516df2f88e273c1a401f45fe31241

data/.codeclimate.yml

@@ -1,18 +1,21 @@
----
-exclude_paths:
-- ".git/"
-- "**.xml"
-- "**.yaml"
-- "**.yml"
-- "locale/"
-- "spec/"
-- "tools/"
-engines:
+prepare:
+  fetch:
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_base.yml
+    path: ".rubocop_base.yml"
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_cc_base.yml
+    path: ".rubocop_cc_base.yml"
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/base.yml
+    path: styles/base.yml
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/cc_base.yml
+    path: styles/cc_base.yml
+plugins:
+  rubocop:
+    enabled: true
+    config: ".rubocop_cc.yml"
+    channel: rubocop-0-82
   brakeman:
-    # very slow :sad_panda:
     enabled: false
   bundler-audit:
-    # requires Gemfile.lock
     enabled: false
   csslint:
     enabled: false
@@ -24,25 +27,17 @@ engines:
       - javascript
   eslint:
     enabled: false
-    channel: "eslint-3"
+    channel: eslint-3
   fixme:
-    # let's enable later
     enabled: false
   markdownlint:
-    # let's enable later
     enabled: false
-  rubocop:
-    enabled: true
-    config: '.rubocop_cc.yml'
-    channel: rubocop-0-69
-prepare:
-  fetch:
-  - url: "https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml"
-    path: ".rubocop_base.yml"
-  - url: "https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_cc_base.yml"
-    path: ".rubocop_cc_base.yml"
-ratings:
-  paths:
-  - Gemfile.lock
-  - "**.rake"
-  - "**.rb"
+exclude_patterns:
+- ".git/"
+- "**.xml"
+- "**.yaml"
+- "**.yml"
+- locale/
+- spec/
+- tools/
+version: '2'

data/.github/workflows/ci.yaml

@@ -0,0 +1,47 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+        - '2.5'
+        - '2.6'
+        - '2.7'
+        rails-version:
+        - '5.2'
+        - '6.0'
+        - '6.1'
+    services:
+      postgres:
+        image: manageiq/postgresql:10
+        env:
+          POSTGRESQL_USER: root
+          POSTGRESQL_PASSWORD: smartvm
+          POSTGRESQL_DATABASE: inventory_refresh_dummy_test
+        options: --health-cmd pg_isready --health-interval 2s --health-timeout 5s --health-retries 5
+        ports:
+        - 5432:5432
+    env:
+      TEST_RAILS_VERSION: ${{ matrix.rails-version }}
+      PGHOST: localhost
+      PGPASSWORD: smartvm
+      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+    - name: Prepare tests
+      run: bundle exec rake spec:setup
+    - name: Run tests
+      run: bundle exec rake
+    - name: Report code coverage
+      if: ${{ github.ref == 'refs/heads/master' && matrix.ruby-version == '2.7' && matrix.rails-version == '6.1' }}
+      continue-on-error: true
+      uses: paambaati/codeclimate-action@v3.0.0

data/.rubocop.yml

@@ -1,4 +1,4 @@
 inherit_from:
-- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
-# put all local rubocop config into .rubocop_local.yml as it will be loaded by .rubocop_cc.yml as well
-- .rubocop_local.yml
+- ".rubocop_local.yml"
+inherit_gem:
+  manageiq-style: ".rubocop_base.yml"

data/.rubocop_cc.yml

@@ -1,5 +1,4 @@
 inherit_from:
-# this is downloaded by .codeclimate.yml
-- .rubocop_base.yml
-- .rubocop_cc_base.yml
-- .rubocop_local.yml
+- ".rubocop_base.yml"
+- ".rubocop_cc_base.yml"
+- ".rubocop_local.yml"

data/.rubocop_local.yml

@@ -1,2 +1,5 @@
-# GlobalVars:
-#   AllowedVariables:
+AllCops:
+  Exclude:
+  - spec/schema.rb
+Rails:
+  Enabled: false

data/.whitesource

@@ -0,0 +1,3 @@
+{
+  "settingsInheritedFrom": "ManageIQ/whitesource-config@master"
+}

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## [Unreleased]
+
+## [1.0.0] - 2022-02-09
+### Changed
+- Run rubocop -A (#102)
+- Switch from travis to github actions (#104, #105)
+
+### Added
+- **BREAKING** Add back support for non-concurrent-safe batch strategies (#101)
+- Add support for Rails 6.1 (#103)
+- Add bundler-inject (#106)
+
+### Fixed
+- Fix InventoryCollection missing cache returns (#107)

data/Gemfile

@@ -1,10 +1,16 @@
 source "https://rubygems.org"
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+plugin "bundler-inject", "~> 2.0"
+require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundler-inject") rescue nil
 
 # Specify your gem's dependencies in inventory_refresh.gemspec
 gemspec
 
-# Load other additional Gemfiles
-#   Developers can create a file ending in .rb under bundler.d/ to specify additional development dependencies
-Dir.glob(File.join(__dir__, 'bundler.d/*.rb')).each { |f| eval_gemfile(File.expand_path(f, __dir__)) }
+case ENV['TEST_RAILS_VERSION']
+when "5.2"
+  gem "activerecord", "~>5.2.6"
+when "6.0"
+  gem "activerecord", "~>6.0.4"
+when "6.1"
+  gem "activerecord", "~>6.1.4"
+end

data/README.md

@@ -1,10 +1,9 @@
 # InventoryRefresh
 
 [![Gem Version](https://badge.fury.io/rb/inventory_refresh.svg)](http://badge.fury.io/rb/inventory_refresh)
-[![Build Status](https://travis-ci.org/ManageIQ/inventory_refresh.svg)](https://travis-ci.org/ManageIQ/inventory_refresh)
+[![CI](https://github.com/ManageIQ/inventory_refresh/actions/workflows/ci.yaml/badge.svg)](https://github.com/ManageIQ/inventory_refresh/actions/workflows/ci.yaml)
 [![Code Climate](https://codeclimate.com/github/ManageIQ/inventory_refresh.svg)](https://codeclimate.com/github/ManageIQ/inventory_refresh)
 [![Test Coverage](https://codeclimate.com/github/ManageIQ/inventory_refresh/badges/coverage.svg)](https://codeclimate.com/github/ManageIQ/inventory_refresh/coverage)
-[![Security](https://hakiri.io/github/ManageIQ/inventory_refresh/master.svg)](https://hakiri.io/github/ManageIQ/inventory_refresh/master)
 
 [![Chat](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ManageIQ/manageiq/providers?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 

data/Rakefile

@@ -12,7 +12,7 @@ namespace :spec do
 
   task :db_load_schema do
     require "active_record"
-    with_connection(test_database_name) { load File.join(__dir__, %w{spec schema.rb}) }
+    with_connection(test_database_name) { load File.join(__dir__, %w[spec schema.rb]) }
   end
 
   desc "Setup test database"
@@ -20,7 +20,7 @@ namespace :spec do
 
   def connection_spec
     require 'yaml'
-    @connection_spec ||= YAML.load_file(File.join(__dir__, %w(config database.yml)))
+    @connection_spec ||= YAML.load_file(File.join(__dir__, %w[config database.yml]))
   end
 
   def test_database_name

data/inventory_refresh.gemspec

@@ -1,5 +1,4 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "inventory_refresh/version"
 
@@ -8,30 +7,30 @@ Gem::Specification.new do |spec|
   spec.version       = InventoryRefresh::VERSION
   spec.authors       = ["ManageIQ Developers"]
 
-  spec.summary       = %q{Topological Inventory Persister}
-  spec.description   = %q{Topological Inventory Persister}
+  spec.summary       = 'Topological Inventory Persister'
+  spec.description   = 'Topological Inventory Persister'
   spec.homepage      = "https://github.com/ManageIQ/inventory_refresh"
   spec.licenses      = ["Apache-2.0"]
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
-  spec.add_dependency "activerecord", "~> 5.0"
-  spec.add_dependency "more_core_extensions", "~> 3.5"
+  spec.add_dependency "activerecord", ">=5.0", "< 7.0"
+  spec.add_dependency "more_core_extensions", ">=3.5", "< 5"
   spec.add_dependency "pg", "> 0"
 
   spec.add_development_dependency "ancestry"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "factory_bot", "~> 4.11.1"
+  spec.add_development_dependency "manageiq-style"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "rubocop"
-  spec.add_development_dependency "rubocop-performance"
-  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov", ">= 0.21.2"
 end

data/lib/inventory_refresh/application_record_iterator.rb

@@ -1,12 +1,20 @@
 module InventoryRefresh
   class ApplicationRecordIterator
-    attr_reader :inventory_collection
+    attr_reader :inventory_collection, :manager_uuids_set, :iterator, :query
 
-    # An iterator that can fetch batches of the AR objects based on a set of attribute_indexes
+    # An iterator that can fetch batches of the AR objects based on a set of manager refs, or just mimics AR relation
+    # when given an iterator. Or given query, acts as iterator by selecting batches.
     #
     # @param inventory_collection [InventoryRefresh::InventoryCollection] Inventory collection owning the iterator
-    def initialize(inventory_collection: nil)
+    # @param manager_uuids_set [Array<InventoryRefresh::InventoryCollection::Reference>] Array of references we want to
+    #        fetch from the DB
+    # @param iterator [Proc] Block based iterator
+    # @query query [ActiveRecord::Relation] Existing query we want to use for querying the db
+    def initialize(inventory_collection: nil, manager_uuids_set: nil, iterator: nil, query: nil)
       @inventory_collection = inventory_collection
+      @manager_uuids_set    = manager_uuids_set
+      @iterator             = iterator
+      @query                = query
     end
 
     # Iterator that mimics find_in_batches of ActiveRecord::Relation. This iterator serves for making more optimized query
@@ -17,22 +25,27 @@ module InventoryRefresh
     # and relation.where(:id => 500ids)
     #
     # @param batch_size [Integer] A batch size we want to fetch from DB
-    # @param attributes_index [Hash{String => Hash}] Indexed hash with data we will be saving
     # @yield Code processing the batches
-    def find_in_batches(batch_size: 1000, attributes_index: {})
-      attributes_index.each_slice(batch_size) do |batch|
-        yield(inventory_collection.db_collection_for_comparison_for(batch))
+    def find_in_batches(batch_size: 1000, &block)
+      if iterator
+        iterator.call(&block)
+      elsif query
+        manager_uuids_set.each_slice(batch_size) do |batch|
+          yield(query.where(inventory_collection.targeted_selection_for(batch)))
+        end
+      else
+        manager_uuids_set.each_slice(batch_size) do |batch|
+          yield(inventory_collection.db_collection_for_comparison_for(batch))
+        end
       end
     end
 
     # Iterator that mimics find_each of ActiveRecord::Relation using find_in_batches (see #find_in_batches)
     #
     # @yield Code processing the batches
-    def find_each(attributes_index: {})
-      find_in_batches(:attributes_index => attributes_index) do |batch|
-        batch.each do |item|
-          yield(item)
-        end
+    def find_each(&block)
+      find_in_batches do |batch|
+        batch.each(&block)
       end
     end
   end

data/lib/inventory_refresh/graph/topological_sort.rb

@@ -8,30 +8,30 @@ module InventoryRefresh
         @graph = graph
       end
 
-        ################################################################################################################
-        # Topological sort of the graph of the DTO collections to find the right order of saving DTO collections and
-        # identify what DTO collections can be saved in parallel.
-        # Does not mutate graph.
-        #
-        # @return [Array<Array>] Array of arrays(layers) of InventoryCollection objects
-        ################################################################################################################
-        # The expected input here is the directed acyclic Graph G (inventory_collections), consisting of Vertices(Nodes) V and
-        # Edges E:
-        # G = (V, E)
-        #
-        # The directed edge is defined as (u, v), where u is the dependency of v, i.e. arrow comes from u to v:
-        # (u, v) ∈ E and  u,v ∈ V
-        #
-        # S0 is a layer that has no dependencies:
-        # S0 = { v ∈ V ∣ ∀u ∈ V.(u,v) ∉ E}
-        #
-        # Si+1 is a layer whose dependencies are in the sum of the previous layers from i to 0, cannot write
-        # mathematical sum using U in text editor, so there is an alternative format using _(sum)
-        # Si+1 = { v ∈ V ∣ ∀u ∈ V.(u,v) ∈ E → u ∈ _(sum(S0..Si))_ }
-        #
-        # Then each Si can have their Vertices(DTO collections) processed in parallel. This algorithm cannot
-        # identify independent sub-graphs inside of the layers Si, so we can make the processing even more effective.
-        ################################################################################################################
+      ################################################################################################################
+      # Topological sort of the graph of the DTO collections to find the right order of saving DTO collections and
+      # identify what DTO collections can be saved in parallel.
+      # Does not mutate graph.
+      #
+      # @return [Array<Array>] Array of arrays(layers) of InventoryCollection objects
+      ################################################################################################################
+      # The expected input here is the directed acyclic Graph G (inventory_collections), consisting of Vertices(Nodes) V and
+      # Edges E:
+      # G = (V, E)
+      #
+      # The directed edge is defined as (u, v), where u is the dependency of v, i.e. arrow comes from u to v:
+      # (u, v) ∈ E and  u,v ∈ V
+      #
+      # S0 is a layer that has no dependencies:
+      # S0 = { v ∈ V ∣ ∀u ∈ V.(u,v) ∉ E}
+      #
+      # Si+1 is a layer whose dependencies are in the sum of the previous layers from i to 0, cannot write
+      # mathematical sum using U in text editor, so there is an alternative format using _(sum)
+      # Si+1 = { v ∈ V ∣ ∀u ∈ V.(u,v) ∈ E → u ∈ _(sum(S0..Si))_ }
+      #
+      # Then each Si can have their Vertices(DTO collections) processed in parallel. This algorithm cannot
+      # identify independent sub-graphs inside of the layers Si, so we can make the processing even more effective.
+      ################################################################################################################
       def topological_sort
         nodes          = graph.nodes.dup
         edges          = graph.edges
@@ -45,14 +45,12 @@ module InventoryRefresh
           max_depth -= 1
           if max_depth <= 0
             message = "Max depth reached while doing topological sort, your graph probably contains a cycle"
-            #logger.error("#{message}:\n#{graph.to_graphviz}")
             raise "#{message} (see log)"
           end
 
           set, nodes = nodes.partition { |v| edges.select { |e| e.second == v }.all? { |e| sets.flatten.include?(e.first) } }
           if set.blank?
             message = "Blank dependency set while doing topological sort, your graph probably contains a cycle"
-            #logger.error("#{message}:\n#{graph.to_graphviz}")
             raise "#{message} (see log)"
           end
 

data/lib/inventory_refresh/graph.rb

@@ -37,7 +37,7 @@ module InventoryRefresh
         s << "  #{node_names[from]} -> #{node_names[to]};"
       end
       s << "}"
-      s.join("\n") + "\n"
+      "#{s.join("\n")}\n"
     end
 
     protected
@@ -147,7 +147,7 @@ module InventoryRefresh
       nodes.group_by { |n| n.respond_to?(:name) ? n.name.to_s : n.to_s }.each do |base_name, ns|
         ns.each_with_index do |n, i|
           name = ns.size == 1 ? base_name : "#{base_name}_#{i}"
-          name = '"' + name.gsub(/["\\]/) { |c| "\\" + c } + '"' unless name =~ /^[A-Za-z0-9_]+$/
+          name = "\"#{name.gsub(/["\\]/) { |c| "\\#{c}" }}\"" unless /^[A-Za-z0-9_]+$/.match?(name)
           node_names[n] = name
         end
       end

data/lib/inventory_refresh/inventory_collection/builder.rb

@@ -4,17 +4,39 @@ module InventoryRefresh
       class MissingModelClassError < StandardError; end
 
       def self.allowed_properties
-        %i(arel                         association
-           attributes_blacklist         attributes_whitelist    batch_extra_attributes
-           complete                     create_only             custom_save_block
-           custom_reconnect_block       default_values
-           dependency_attributes        check_changed           inventory_object_attributes
-           manager_ref                  manager_ref_allowed_nil
-           model_class                  name                    parent
-           retention_strategy           strategy
-           secondary_refs
-           update_only                  use_ar_object
-           assert_graph_integrity).to_set
+        %i[
+          all_manager_uuids
+          arel
+          association
+          attributes_blacklist
+          attributes_whitelist
+          batch_extra_attributes
+          complete
+          create_only
+          custom_save_block
+          custom_reconnect_block
+          default_values
+          delete_method
+          dependency_attributes
+          check_changed
+          inventory_object_attributes
+          manager_ref
+          manager_ref_allowed_nil
+          manager_uuids
+          model_class
+          name
+          parent
+          parent_inventory_collections
+          retention_strategy
+          strategy
+          saver_strategy
+          secondary_refs
+          targeted
+          targeted_arel
+          update_only
+          use_ar_object
+          assert_graph_integrity
+        ].to_set
       end
 
       def allowed_properties
@@ -76,8 +98,8 @@ module InventoryRefresh
 
         send(@name.to_sym) if @name.respond_to?(:to_sym) && respond_to?(@name.to_sym)
 
-        if @properties[:model_class].nil?
-          add_properties(:model_class => auto_model_class) unless @options[:without_model_class]
+        if @properties[:model_class].nil? && !(@options[:without_model_class])
+          add_properties(:model_class => auto_model_class)
         end
       end
 
@@ -177,7 +199,7 @@ module InventoryRefresh
 
       def assert_allowed_property(name)
         unless allowed_properties.include?(name)
-          raise "InventoryCollection property :#{name} is not allowed. Allowed properties are:\n#{self.allowed_properties.to_a.map(&:to_s).join(', ')}"
+          raise "InventoryCollection property :#{name} is not allowed. Allowed properties are:\n#{allowed_properties.to_a.map(&:to_s).join(', ')}"
         end
       end
 
@@ -216,7 +238,7 @@ module InventoryRefresh
       def auto_inventory_attributes
         return if @properties[:model_class].nil?
 
-        (@properties[:model_class].new.methods - ar_base_class.methods).grep(/^[\w]+?\=$/).collect do |setter|
+        (@properties[:model_class].new.methods - ar_base_class.methods).grep(/^\w+?=$/).collect do |setter|
           setter.to_s[0..setter.length - 2].to_sym
         end
       end

data/lib/inventory_refresh/inventory_collection/data_storage.rb

@@ -70,6 +70,15 @@ module InventoryRefresh
         build(hash)
       end
 
+      # Finds InventoryObject.
+      #
+      # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the InventoryCollection
+      # @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
+      def find_in_data(hash)
+        _hash, _uuid, inventory_object = primary_index_scan(hash)
+        inventory_object
+      end
+
       # Finds of builds a new InventoryObject. By building it, we also put in into the InventoryCollection's storage.
       #
       # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the