inventory_refresh 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +0 -1
- data/.travis.yml +6 -8
- data/inventory_refresh.gemspec +3 -5
- data/lib/inventory_refresh/application_record_iterator.rb +9 -26
- data/lib/inventory_refresh/exception.rb +8 -0
- data/lib/inventory_refresh/inventory_collection/builder.rb +6 -6
- data/lib/inventory_refresh/inventory_collection/data_storage.rb +0 -9
- data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +35 -144
- data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +1 -44
- data/lib/inventory_refresh/inventory_collection/index/proxy.rb +6 -34
- data/lib/inventory_refresh/inventory_collection/index/type/base.rb +0 -8
- data/lib/inventory_refresh/inventory_collection/references_storage.rb +0 -17
- data/lib/inventory_refresh/inventory_collection/scanner.rb +1 -87
- data/lib/inventory_refresh/inventory_collection/serialization.rb +10 -16
- data/lib/inventory_refresh/inventory_collection.rb +36 -110
- data/lib/inventory_refresh/inventory_object.rb +34 -68
- data/lib/inventory_refresh/inventory_object_lazy.rb +10 -17
- data/lib/inventory_refresh/persister.rb +63 -29
- data/lib/inventory_refresh/save_collection/base.rb +2 -4
- data/lib/inventory_refresh/save_collection/saver/base.rb +8 -108
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +48 -126
- data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +19 -1
- data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +3 -68
- data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +0 -125
- data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +5 -9
- data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +9 -17
- data/lib/inventory_refresh/save_collection/sweeper.rb +91 -18
- data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
- data/lib/inventory_refresh/save_inventory.rb +12 -5
- data/lib/inventory_refresh/version.rb +1 -1
- data/lib/inventory_refresh.rb +0 -2
- metadata +15 -57
- data/lib/inventory_refresh/save_collection/saver/batch.rb +0 -17
- data/lib/inventory_refresh/save_collection/saver/default.rb +0 -57
- data/lib/inventory_refresh/target.rb +0 -73
- data/lib/inventory_refresh/target_collection.rb +0 -92
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 666554cf2d3928670cd6772706360d2c2523fc0475b63f2aead1e7e2bec675d7
|
4
|
+
data.tar.gz: 992aa863fdd2d020878f3bd18e0c83d7bf37aa79d6363814e10fdc5cd311a35a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1ce760c31989eaa2ec3d230031c176007953b9d5790d7d5e0c85d7b62e43f6d7d45cbe88651a70df3ca77c11f4d7b5587348e2ba0d951add5d8d76a90ba9d0e0
|
7
|
+
data.tar.gz: 4084a11ccd8ab92142e44399159290d5b14be05aa111f5c681571c6ba034a4ecb55587f0d3c2dda9d0ef765e4c5aefa9f2ea9e4183c27b81d4547b3719ae8393
|
data/.codeclimate.yml
CHANGED
data/.travis.yml
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
|
1
|
+
dist: xenial
|
2
2
|
sudo: false
|
3
3
|
language: ruby
|
4
4
|
cache: bundler
|
5
5
|
rvm:
|
6
|
-
- 2.
|
7
|
-
- 2.6
|
6
|
+
- 2.4.4
|
7
|
+
- 2.3.6
|
8
8
|
addons:
|
9
|
-
postgresql: '
|
9
|
+
postgresql: '10'
|
10
10
|
env:
|
11
11
|
global:
|
12
12
|
- CC_TEST_REPORTER_ID=1ef1a3a3d007395b11083d634a6fdac1e3d979b6428c39d2cd8d58556cdd68f7
|
@@ -14,12 +14,10 @@ before_install:
|
|
14
14
|
- gem install bundler -v 2.0.1
|
15
15
|
- psql -c "CREATE USER root SUPERUSER PASSWORD 'smartvm';" -U postgres
|
16
16
|
before_script:
|
17
|
-
|
18
|
-
|
17
|
+
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
|
18
|
+
> ./cc-test-reporter
|
19
19
|
- chmod +x ./cc-test-reporter
|
20
20
|
- "./cc-test-reporter before-build"
|
21
|
-
# inventory_refresh
|
22
21
|
- bundle exec rake spec:setup
|
23
22
|
after_script:
|
24
|
-
# Code Climate
|
25
23
|
- "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"
|
data/inventory_refresh.gemspec
CHANGED
@@ -22,16 +22,14 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
23
|
spec.require_paths = ["lib"]
|
24
24
|
|
25
|
-
spec.add_dependency "activerecord", "
|
26
|
-
spec.add_dependency "more_core_extensions", "
|
25
|
+
spec.add_dependency "activerecord", "~> 5.0"
|
26
|
+
spec.add_dependency "more_core_extensions", "~> 3.5"
|
27
27
|
spec.add_dependency "pg", "> 0"
|
28
28
|
|
29
29
|
spec.add_development_dependency "ancestry"
|
30
|
-
spec.add_development_dependency "bundler", "~> 2.0"
|
30
|
+
spec.add_development_dependency "bundler", "~> 2.0.1"
|
31
31
|
spec.add_development_dependency "factory_bot", "~> 4.11.1"
|
32
32
|
spec.add_development_dependency "rake", "~> 10.0"
|
33
33
|
spec.add_development_dependency "rspec", "~> 3.0"
|
34
|
-
spec.add_development_dependency "rubocop"
|
35
|
-
spec.add_development_dependency "rubocop-performance"
|
36
34
|
spec.add_development_dependency "simplecov"
|
37
35
|
end
|
@@ -1,20 +1,12 @@
|
|
1
1
|
module InventoryRefresh
|
2
2
|
class ApplicationRecordIterator
|
3
|
-
attr_reader :inventory_collection
|
3
|
+
attr_reader :inventory_collection
|
4
4
|
|
5
|
-
# An iterator that can fetch batches of the AR objects based on a set of
|
6
|
-
# when given an iterator. Or given query, acts as iterator by selecting batches.
|
5
|
+
# An iterator that can fetch batches of the AR objects based on a set of attribute_indexes
|
7
6
|
#
|
8
7
|
# @param inventory_collection [InventoryRefresh::InventoryCollection] Inventory collection owning the iterator
|
9
|
-
|
10
|
-
# fetch from the DB
|
11
|
-
# @param iterator [Proc] Block based iterator
|
12
|
-
# @query query [ActiveRecord::Relation] Existing query we want to use for querying the db
|
13
|
-
def initialize(inventory_collection: nil, manager_uuids_set: nil, iterator: nil, query: nil)
|
8
|
+
def initialize(inventory_collection: nil)
|
14
9
|
@inventory_collection = inventory_collection
|
15
|
-
@manager_uuids_set = manager_uuids_set
|
16
|
-
@iterator = iterator
|
17
|
-
@query = query
|
18
10
|
end
|
19
11
|
|
20
12
|
# Iterator that mimics find_in_batches of ActiveRecord::Relation. This iterator serves for making more optimized query
|
@@ -25,28 +17,19 @@ module InventoryRefresh
|
|
25
17
|
# and relation.where(:id => 500ids)
|
26
18
|
#
|
27
19
|
# @param batch_size [Integer] A batch size we want to fetch from DB
|
20
|
+
# @param attributes_index [Hash{String => Hash}] Indexed hash with data we will be saving
|
28
21
|
# @yield Code processing the batches
|
29
|
-
def find_in_batches(batch_size: 1000)
|
30
|
-
|
31
|
-
|
32
|
-
yield(batch)
|
33
|
-
end
|
34
|
-
elsif query
|
35
|
-
manager_uuids_set.each_slice(batch_size) do |batch|
|
36
|
-
yield(query.where(inventory_collection.targeted_selection_for(batch)))
|
37
|
-
end
|
38
|
-
else
|
39
|
-
manager_uuids_set.each_slice(batch_size) do |batch|
|
40
|
-
yield(inventory_collection.db_collection_for_comparison_for(batch))
|
41
|
-
end
|
22
|
+
def find_in_batches(batch_size: 1000, attributes_index: {})
|
23
|
+
attributes_index.each_slice(batch_size) do |batch|
|
24
|
+
yield(inventory_collection.db_collection_for_comparison_for(batch))
|
42
25
|
end
|
43
26
|
end
|
44
27
|
|
45
28
|
# Iterator that mimics find_each of ActiveRecord::Relation using find_in_batches (see #find_in_batches)
|
46
29
|
#
|
47
30
|
# @yield Code processing the batches
|
48
|
-
def find_each
|
49
|
-
find_in_batches do |batch|
|
31
|
+
def find_each(attributes_index: {})
|
32
|
+
find_in_batches(:attributes_index => attributes_index) do |batch|
|
50
33
|
batch.each do |item|
|
51
34
|
yield(item)
|
52
35
|
end
|
@@ -0,0 +1,8 @@
|
|
1
|
+
module InventoryRefresh
|
2
|
+
module Exception
|
3
|
+
class SweeperError < StandardError; end
|
4
|
+
class SweeperNonExistentScopeKeyFoundError < SweeperError; end
|
5
|
+
class SweeperNonUniformScopeKeyFoundError < SweeperError; end
|
6
|
+
class SweeperScopeBadFormat < SweeperError; end
|
7
|
+
end
|
8
|
+
end
|
@@ -4,16 +4,16 @@ module InventoryRefresh
|
|
4
4
|
class MissingModelClassError < StandardError; end
|
5
5
|
|
6
6
|
def self.allowed_properties
|
7
|
-
%i(
|
7
|
+
%i(arel association
|
8
8
|
attributes_blacklist attributes_whitelist batch_extra_attributes
|
9
9
|
complete create_only custom_save_block
|
10
|
-
custom_reconnect_block default_values
|
10
|
+
custom_reconnect_block default_values
|
11
11
|
dependency_attributes check_changed inventory_object_attributes
|
12
|
-
manager_ref manager_ref_allowed_nil
|
12
|
+
manager_ref manager_ref_allowed_nil
|
13
13
|
model_class name parent
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
retention_strategy strategy
|
15
|
+
secondary_refs
|
16
|
+
update_only use_ar_object
|
17
17
|
assert_graph_integrity).to_set
|
18
18
|
end
|
19
19
|
|
@@ -70,15 +70,6 @@ module InventoryRefresh
|
|
70
70
|
build(hash)
|
71
71
|
end
|
72
72
|
|
73
|
-
# Finds InventoryObject.
|
74
|
-
#
|
75
|
-
# @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the InventoryCollection
|
76
|
-
# @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
|
77
|
-
def find_in_data(hash)
|
78
|
-
_hash, _uuid, inventory_object = primary_index_scan(hash)
|
79
|
-
inventory_object
|
80
|
-
end
|
81
|
-
|
82
73
|
# Finds of builds a new InventoryObject. By building it, we also put in into the InventoryCollection's storage.
|
83
74
|
#
|
84
75
|
# @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the
|
@@ -35,26 +35,14 @@ module InventoryRefresh
|
|
35
35
|
# - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
|
36
36
|
# the DB. Then if we reference an object that is not present, it will
|
37
37
|
# load them from the db using :local_db_find_references strategy.
|
38
|
-
# @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
|
39
|
-
# Allowed saver strategies are:
|
40
|
-
# - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
|
41
|
-
# since it will lead to non consistent data.
|
42
|
-
# - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
|
43
|
-
# concurrently, since it will lead to non consistent data.
|
44
|
-
# - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
|
45
|
-
# atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
|
46
|
-
# executed as batched SQL queries, instead of sending 1 query per record.
|
47
38
|
# @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
|
48
39
|
# - :destroy => Will destroy the inactive records.
|
49
40
|
# - :archive => Will archive the inactive records by setting :archived_at timestamp.
|
50
|
-
|
51
|
-
|
52
|
-
# :model_class.
|
53
|
-
def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
|
54
|
-
@saver_strategy = process_saver_strategy(saver_strategy)
|
41
|
+
def init_strategies(strategy, retention_strategy)
|
42
|
+
@saver_strategy = :concurrent_safe_batch
|
55
43
|
@strategy = process_strategy(strategy)
|
44
|
+
# TODO(lsmola) why don't we just set this strategy based on :archived_at column? Lets do that
|
56
45
|
@retention_strategy = process_retention_strategy(retention_strategy)
|
57
|
-
@delete_method = delete_method || :destroy
|
58
46
|
end
|
59
47
|
|
60
48
|
# @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
|
@@ -67,42 +55,10 @@ module InventoryRefresh
|
|
67
55
|
# avoid this usecase by a proper modeling.
|
68
56
|
# Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
|
69
57
|
# @param secondary_refs [Hash] TODO
|
70
|
-
|
71
|
-
# this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
|
72
|
-
# case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
|
73
|
-
# custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
|
74
|
-
def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
|
58
|
+
def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs)
|
75
59
|
@manager_ref = manager_ref || %i(ems_ref)
|
76
60
|
@manager_ref_allowed_nil = manager_ref_allowed_nil || []
|
77
61
|
@secondary_refs = secondary_refs || {}
|
78
|
-
@manager_uuids = manager_uuids || []
|
79
|
-
end
|
80
|
-
|
81
|
-
# @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
|
82
|
-
# having this parameter defined will invoke only :delete_method on a complement of this set, making sure
|
83
|
-
# the DB has only this set of data after. This :attribute serves for deleting of top level
|
84
|
-
# InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
|
85
|
-
# child collections is already handled by the scope of the parent_inventory_collections and using Rails
|
86
|
-
# :dependent => :destroy,
|
87
|
-
# @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
|
88
|
-
# all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
|
89
|
-
# region.
|
90
|
-
# If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
|
91
|
-
# scope itself. E.g. sending a list of all active regions, we will delete complement entities not
|
92
|
-
# belonging to those regions.
|
93
|
-
# Example:
|
94
|
-
# :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
|
95
|
-
# :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
|
96
|
-
#
|
97
|
-
# Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
|
98
|
-
# regions X and Y.
|
99
|
-
# @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
|
100
|
-
# all_manager_uuids. Meaning we won't be archiving any newer entities.
|
101
|
-
def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
|
102
|
-
# TODO(lsmola) Should we refactor this to use references too?
|
103
|
-
@all_manager_uuids = all_manager_uuids
|
104
|
-
@all_manager_uuids_scope = all_manager_uuids_scope
|
105
|
-
@all_manager_uuids_timestamp = all_manager_uuids_timestamp
|
106
62
|
end
|
107
63
|
|
108
64
|
# @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
|
@@ -119,48 +75,9 @@ module InventoryRefresh
|
|
119
75
|
# This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
|
120
76
|
# will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
|
121
77
|
# are saved.
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
#
|
126
|
-
# Example:
|
127
|
-
# taking inventory collections :vms and :disks (local disks), if we write that:
|
128
|
-
# inventory_collection = InventoryCollection.new({
|
129
|
-
# :model_class => ::Disk,
|
130
|
-
# :association => :disks,
|
131
|
-
# :manager_ref => [:vm, :location]
|
132
|
-
# :parent_inventory_collection => [:vms],
|
133
|
-
# })
|
134
|
-
#
|
135
|
-
# Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
|
136
|
-
# points:
|
137
|
-
# 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
|
138
|
-
# from vms to disks table).
|
139
|
-
# 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
|
140
|
-
# a Vm subquery
|
141
|
-
# 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
|
142
|
-
# only under 1 Vm. (In current models, this modeled going through Hardware model)
|
143
|
-
# 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
|
144
|
-
#
|
145
|
-
# Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
|
146
|
-
# implications:
|
147
|
-
# 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
|
148
|
-
# archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
|
149
|
-
# 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
|
150
|
-
# we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
|
151
|
-
# archived/removed before the Vm, while we always want to archive the VM first.
|
152
|
-
# 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
|
153
|
-
# we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
|
154
|
-
# always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
|
155
|
-
# :manager_uuids are mutually exclusive attributes)
|
156
|
-
# 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
|
157
|
-
# While this information can be introspected from the data, it creates a scope for create&update&delete,
|
158
|
-
# which means it has to work with no data provided (causing delete all). So with no data we cannot
|
159
|
-
# introspect anything.
|
160
|
-
def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
|
161
|
-
@dependency_attributes = dependency_attributes || {}
|
162
|
-
@dependees = Set.new
|
163
|
-
@parent_inventory_collections = parent_inventory_collections
|
78
|
+
def init_ic_relations(dependency_attributes)
|
79
|
+
@dependency_attributes = dependency_attributes || {}
|
80
|
+
@dependees = Set.new
|
164
81
|
end
|
165
82
|
|
166
83
|
# @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
|
@@ -175,18 +92,13 @@ module InventoryRefresh
|
|
175
92
|
# @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
|
176
93
|
# it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
|
177
94
|
# for the batch saver strategy.
|
178
|
-
|
179
|
-
# :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
|
180
|
-
def init_flags(complete, create_only, check_changed,
|
181
|
-
update_only, use_ar_object, targeted,
|
182
|
-
assert_graph_integrity)
|
95
|
+
def init_flags(complete, create_only, check_changed, update_only, use_ar_object, assert_graph_integrity)
|
183
96
|
@complete = complete.nil? ? true : complete
|
184
97
|
@create_only = create_only.nil? ? false : create_only
|
185
98
|
@check_changed = check_changed.nil? ? true : check_changed
|
186
99
|
@saved = false
|
187
100
|
@update_only = update_only.nil? ? false : update_only
|
188
101
|
@use_ar_object = use_ar_object || false
|
189
|
-
@targeted = !!targeted
|
190
102
|
@assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
|
191
103
|
end
|
192
104
|
|
@@ -230,7 +142,7 @@ module InventoryRefresh
|
|
230
142
|
@attributes_whitelist = Set.new
|
231
143
|
@batch_extra_attributes = batch_extra_attributes || []
|
232
144
|
@inventory_object_attributes = inventory_object_attributes
|
233
|
-
@internal_attributes = %i(__feedback_edge_set_parent
|
145
|
+
@internal_attributes = %i(__feedback_edge_set_parent)
|
234
146
|
@transitive_dependency_attributes = Set.new
|
235
147
|
|
236
148
|
blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
|
@@ -240,7 +152,6 @@ module InventoryRefresh
|
|
240
152
|
def init_storages
|
241
153
|
@data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
|
242
154
|
@references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
|
243
|
-
@targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
|
244
155
|
end
|
245
156
|
|
246
157
|
# @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
|
@@ -248,35 +159,18 @@ module InventoryRefresh
|
|
248
159
|
# doing create/update/delete.
|
249
160
|
#
|
250
161
|
# Example:
|
251
|
-
#
|
252
|
-
#
|
253
|
-
#
|
254
|
-
#
|
255
|
-
#
|
256
|
-
#
|
257
|
-
#
|
258
|
-
#
|
259
|
-
# @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
|
260
|
-
# we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
|
261
|
-
# manager_uuids available.
|
262
|
-
# Example:
|
263
|
-
# targeted_arel = lambda do |inventory_collection|
|
264
|
-
# # Getting ems_refs of parent :vms and :miq_templates
|
265
|
-
# manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
|
266
|
-
# inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
|
267
|
-
# 'vms' => {:ems_ref => manager_uuids}
|
162
|
+
# add_collection(:cross_link_vms) do |builder|
|
163
|
+
# builder.add_properties(
|
164
|
+
# :arel => Vm.where(:tenant => manager.tenant),
|
165
|
+
# :association => nil,
|
166
|
+
# :model_class => Vm,
|
167
|
+
# :name => :cross_link_vms,
|
168
|
+
# :manager_ref => [:uid_ems],
|
169
|
+
# :strategy => :local_db_find_references,
|
268
170
|
# )
|
269
171
|
# end
|
270
|
-
|
271
|
-
|
272
|
-
# :model_class => ::Hardware,
|
273
|
-
# :association => :hardwares,
|
274
|
-
# :parent_inventory_collection => [:vms, :miq_templates],
|
275
|
-
# :targeted_arel => targeted_arel,
|
276
|
-
# })
|
277
|
-
def init_arels(arel, targeted_arel)
|
278
|
-
@arel = arel
|
279
|
-
@targeted_arel = targeted_arel
|
172
|
+
def init_arels(arel)
|
173
|
+
@arel = arel
|
280
174
|
end
|
281
175
|
|
282
176
|
# @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
|
@@ -289,7 +183,7 @@ module InventoryRefresh
|
|
289
183
|
# hash = inventory_object.attributes # Loads possible dependencies into saveable hash
|
290
184
|
# obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
|
291
185
|
# # queries, avoid this, this is just a simple example :-)
|
292
|
-
# obj.
|
186
|
+
# obj.update_attributes(hash) if obj
|
293
187
|
# obj ||= SomeModel.create(hash)
|
294
188
|
# inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
|
295
189
|
# primary key back to the InventoryObject
|
@@ -392,23 +286,6 @@ module InventoryRefresh
|
|
392
286
|
@deleted_records = []
|
393
287
|
end
|
394
288
|
|
395
|
-
# Processes passed saver strategy
|
396
|
-
#
|
397
|
-
# @param saver_strategy [Symbol] Passed saver strategy
|
398
|
-
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
399
|
-
def process_saver_strategy(saver_strategy)
|
400
|
-
return :default unless saver_strategy
|
401
|
-
|
402
|
-
saver_strategy = saver_strategy.to_sym
|
403
|
-
case saver_strategy
|
404
|
-
when :default, :batch, :concurrent_safe_batch
|
405
|
-
saver_strategy
|
406
|
-
else
|
407
|
-
raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
|
408
|
-
":default, :batch and :concurrent_safe_batch"
|
409
|
-
end
|
410
|
-
end
|
411
|
-
|
412
289
|
# Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
|
413
290
|
#
|
414
291
|
# @param strategy_name [Symbol] Passed saver strategy
|
@@ -434,12 +311,26 @@ module InventoryRefresh
|
|
434
311
|
strategy_name
|
435
312
|
end
|
436
313
|
|
314
|
+
# Saves passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
|
315
|
+
#
|
316
|
+
# @param strategy [Symbol] Passed saver strategy
|
317
|
+
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
318
|
+
def strategy=(strategy)
|
319
|
+
@strategy = process_strategy(strategy)
|
320
|
+
end
|
321
|
+
|
437
322
|
# Processes passed retention strategy
|
438
323
|
#
|
439
324
|
# @param retention_strategy [Symbol] Passed retention strategy
|
440
325
|
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
441
326
|
def process_retention_strategy(retention_strategy)
|
442
|
-
|
327
|
+
unless retention_strategy
|
328
|
+
if supports_column?(:archived_at)
|
329
|
+
return :archive
|
330
|
+
else
|
331
|
+
return :destroy
|
332
|
+
end
|
333
|
+
end
|
443
334
|
|
444
335
|
retention_strategy = retention_strategy.to_sym
|
445
336
|
case retention_strategy
|
@@ -50,11 +50,6 @@ module InventoryRefresh
|
|
50
50
|
dependencies.all?(&:saved?)
|
51
51
|
end
|
52
52
|
|
53
|
-
# @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
|
54
|
-
def parallel_safe?
|
55
|
-
@parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
|
56
|
-
end
|
57
|
-
|
58
53
|
# @return [Boolean] true if the model_class supports STI
|
59
54
|
def supports_sti?
|
60
55
|
@supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
|
@@ -81,50 +76,12 @@ module InventoryRefresh
|
|
81
76
|
data_collection_finalized
|
82
77
|
end
|
83
78
|
|
84
|
-
# @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
|
85
|
-
def targeted?
|
86
|
-
targeted
|
87
|
-
end
|
88
|
-
|
89
79
|
# True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
|
90
80
|
# stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
|
91
81
|
#
|
92
82
|
# @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
|
93
83
|
def noop?
|
94
|
-
|
95
|
-
saving_noop? && delete_complement_noop?
|
96
|
-
end
|
97
|
-
|
98
|
-
# @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
|
99
|
-
def saving_noop?
|
100
|
-
saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
|
101
|
-
end
|
102
|
-
|
103
|
-
# @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
|
104
|
-
def delete_complement_noop?
|
105
|
-
all_manager_uuids.nil?
|
106
|
-
end
|
107
|
-
|
108
|
-
private
|
109
|
-
|
110
|
-
# @return true if it's a noop parent targeted InventoryCollection
|
111
|
-
def saving_targeted_parent_collection_noop?
|
112
|
-
targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
|
113
|
-
end
|
114
|
-
|
115
|
-
# @return true if it's a noop child targeted InventoryCollection
|
116
|
-
def saving_targeted_child_collection_noop?
|
117
|
-
targeted_noop_condition && parent_inventory_collections.present? &&
|
118
|
-
parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
|
119
|
-
end
|
120
|
-
|
121
|
-
# @return true if it's a noop full InventoryCollection refresh
|
122
|
-
def saving_full_collection_noop?
|
123
|
-
!targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
|
124
|
-
end
|
125
|
-
|
126
|
-
def targeted_noop_condition
|
127
|
-
targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
|
84
|
+
data.blank? && custom_save_block.nil? && skeletal_primary_index.blank?
|
128
85
|
end
|
129
86
|
end
|
130
87
|
end
|
@@ -64,14 +64,6 @@ module InventoryRefresh
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
-
def reindex_secondary_indexes!
|
68
|
-
data_indexes.each do |ref, index|
|
69
|
-
next if ref == primary_index_ref
|
70
|
-
|
71
|
-
index.reindex!
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
67
|
def primary_index
|
76
68
|
data_index(primary_index_ref)
|
77
69
|
end
|
@@ -94,30 +86,16 @@ module InventoryRefresh
|
|
94
86
|
end
|
95
87
|
end
|
96
88
|
|
97
|
-
def find_by(manager_uuid_hash, ref: primary_index_ref)
|
98
|
-
# TODO(lsmola) deprecate this, it's enough to have find method
|
99
|
-
find(manager_uuid_hash, :ref => ref)
|
100
|
-
end
|
101
|
-
|
102
|
-
def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
|
103
|
-
# TODO(lsmola) deprecate this, it's enough to have lazy_find method
|
104
|
-
|
105
|
-
lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
|
106
|
-
end
|
107
|
-
|
108
89
|
def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
|
109
|
-
# TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
|
110
|
-
# access the data
|
111
|
-
# TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
|
112
90
|
return if manager_uuid.nil?
|
113
91
|
assert_index(manager_uuid, ref)
|
114
92
|
|
115
93
|
::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
94
|
+
manager_uuid,
|
95
|
+
:ref => ref,
|
96
|
+
:key => key,
|
97
|
+
:default => default,
|
98
|
+
:transform_nested_lazy_finds => transform_nested_lazy_finds)
|
121
99
|
end
|
122
100
|
|
123
101
|
def named_ref(ref = primary_index_ref)
|
@@ -132,19 +110,13 @@ module InventoryRefresh
|
|
132
110
|
|
133
111
|
delegate :association_to_foreign_key_mapping,
|
134
112
|
:build_stringified_reference,
|
135
|
-
:parallel_safe?,
|
136
113
|
:strategy,
|
137
114
|
:to => :inventory_collection
|
138
115
|
|
139
116
|
attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
|
140
117
|
|
141
118
|
def find_in_data_or_skeletal_index(reference)
|
142
|
-
|
143
|
-
# With parallel safe strategies, we create skeletal nodes that we can look for
|
144
|
-
data_index_find(reference) || skeletal_index_find(reference)
|
145
|
-
else
|
146
|
-
data_index_find(reference)
|
147
|
-
end
|
119
|
+
data_index_find(reference) || skeletal_index_find(reference)
|
148
120
|
end
|
149
121
|
|
150
122
|
def skeletal_index_find(reference)
|
@@ -27,14 +27,6 @@ module InventoryRefresh
|
|
27
27
|
index[build_stringified_reference(inventory_object.data, attribute_names)] = inventory_object
|
28
28
|
end
|
29
29
|
|
30
|
-
# Rebuilds the indexes for all InventoryObject objects
|
31
|
-
def reindex!
|
32
|
-
self.index = {}
|
33
|
-
data.each do |inventory_object|
|
34
|
-
store_index_for(inventory_object)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
30
|
# @return [Array] Returns index data
|
39
31
|
def index_data
|
40
32
|
index.values
|
@@ -45,23 +45,6 @@ module InventoryRefresh
|
|
45
45
|
add_reference(reference_data)
|
46
46
|
end
|
47
47
|
|
48
|
-
# Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
|
49
|
-
# it.
|
50
|
-
#
|
51
|
-
# @param references_array [Array] Array of reference objects acceptable by add_reference method.
|
52
|
-
# @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
|
53
|
-
# index.
|
54
|
-
# @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
|
55
|
-
def merge!(references_array, ref: nil)
|
56
|
-
references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
|
57
|
-
self
|
58
|
-
end
|
59
|
-
|
60
|
-
# @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
|
61
|
-
def primary_references
|
62
|
-
references[primary_index_ref]
|
63
|
-
end
|
64
|
-
|
65
48
|
# Builds a Reference object
|
66
49
|
#
|
67
50
|
# @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference
|