inventory_refresh 0.2.3 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.codeclimate.yml +0 -1
- data/.travis.yml +6 -8
- data/inventory_refresh.gemspec +3 -5
- data/lib/inventory_refresh/application_record_iterator.rb +9 -26
- data/lib/inventory_refresh/exception.rb +8 -0
- data/lib/inventory_refresh/inventory_collection/builder.rb +6 -6
- data/lib/inventory_refresh/inventory_collection/data_storage.rb +0 -9
- data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +35 -144
- data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +1 -44
- data/lib/inventory_refresh/inventory_collection/index/proxy.rb +6 -34
- data/lib/inventory_refresh/inventory_collection/index/type/base.rb +0 -8
- data/lib/inventory_refresh/inventory_collection/references_storage.rb +0 -17
- data/lib/inventory_refresh/inventory_collection/scanner.rb +1 -87
- data/lib/inventory_refresh/inventory_collection/serialization.rb +10 -16
- data/lib/inventory_refresh/inventory_collection.rb +36 -110
- data/lib/inventory_refresh/inventory_object.rb +34 -68
- data/lib/inventory_refresh/inventory_object_lazy.rb +10 -17
- data/lib/inventory_refresh/persister.rb +63 -29
- data/lib/inventory_refresh/save_collection/base.rb +2 -4
- data/lib/inventory_refresh/save_collection/saver/base.rb +8 -108
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +48 -126
- data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +19 -1
- data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +3 -68
- data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +0 -125
- data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +5 -9
- data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +9 -17
- data/lib/inventory_refresh/save_collection/sweeper.rb +91 -18
- data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
- data/lib/inventory_refresh/save_inventory.rb +12 -5
- data/lib/inventory_refresh/version.rb +1 -1
- data/lib/inventory_refresh.rb +0 -2
- metadata +15 -57
- data/lib/inventory_refresh/save_collection/saver/batch.rb +0 -17
- data/lib/inventory_refresh/save_collection/saver/default.rb +0 -57
- data/lib/inventory_refresh/target.rb +0 -73
- data/lib/inventory_refresh/target_collection.rb +0 -92
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 666554cf2d3928670cd6772706360d2c2523fc0475b63f2aead1e7e2bec675d7
|
4
|
+
data.tar.gz: 992aa863fdd2d020878f3bd18e0c83d7bf37aa79d6363814e10fdc5cd311a35a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1ce760c31989eaa2ec3d230031c176007953b9d5790d7d5e0c85d7b62e43f6d7d45cbe88651a70df3ca77c11f4d7b5587348e2ba0d951add5d8d76a90ba9d0e0
|
7
|
+
data.tar.gz: 4084a11ccd8ab92142e44399159290d5b14be05aa111f5c681571c6ba034a4ecb55587f0d3c2dda9d0ef765e4c5aefa9f2ea9e4183c27b81d4547b3719ae8393
|
data/.codeclimate.yml
CHANGED
data/.travis.yml
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
|
1
|
+
dist: xenial
|
2
2
|
sudo: false
|
3
3
|
language: ruby
|
4
4
|
cache: bundler
|
5
5
|
rvm:
|
6
|
-
- 2.
|
7
|
-
- 2.6
|
6
|
+
- 2.4.4
|
7
|
+
- 2.3.6
|
8
8
|
addons:
|
9
|
-
postgresql: '
|
9
|
+
postgresql: '10'
|
10
10
|
env:
|
11
11
|
global:
|
12
12
|
- CC_TEST_REPORTER_ID=1ef1a3a3d007395b11083d634a6fdac1e3d979b6428c39d2cd8d58556cdd68f7
|
@@ -14,12 +14,10 @@ before_install:
|
|
14
14
|
- gem install bundler -v 2.0.1
|
15
15
|
- psql -c "CREATE USER root SUPERUSER PASSWORD 'smartvm';" -U postgres
|
16
16
|
before_script:
|
17
|
-
|
18
|
-
|
17
|
+
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
|
18
|
+
> ./cc-test-reporter
|
19
19
|
- chmod +x ./cc-test-reporter
|
20
20
|
- "./cc-test-reporter before-build"
|
21
|
-
# inventory_refresh
|
22
21
|
- bundle exec rake spec:setup
|
23
22
|
after_script:
|
24
|
-
# Code Climate
|
25
23
|
- "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"
|
data/inventory_refresh.gemspec
CHANGED
@@ -22,16 +22,14 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
23
|
spec.require_paths = ["lib"]
|
24
24
|
|
25
|
-
spec.add_dependency "activerecord", "
|
26
|
-
spec.add_dependency "more_core_extensions", "
|
25
|
+
spec.add_dependency "activerecord", "~> 5.0"
|
26
|
+
spec.add_dependency "more_core_extensions", "~> 3.5"
|
27
27
|
spec.add_dependency "pg", "> 0"
|
28
28
|
|
29
29
|
spec.add_development_dependency "ancestry"
|
30
|
-
spec.add_development_dependency "bundler", "~> 2.0"
|
30
|
+
spec.add_development_dependency "bundler", "~> 2.0.1"
|
31
31
|
spec.add_development_dependency "factory_bot", "~> 4.11.1"
|
32
32
|
spec.add_development_dependency "rake", "~> 10.0"
|
33
33
|
spec.add_development_dependency "rspec", "~> 3.0"
|
34
|
-
spec.add_development_dependency "rubocop"
|
35
|
-
spec.add_development_dependency "rubocop-performance"
|
36
34
|
spec.add_development_dependency "simplecov"
|
37
35
|
end
|
@@ -1,20 +1,12 @@
|
|
1
1
|
module InventoryRefresh
|
2
2
|
class ApplicationRecordIterator
|
3
|
-
attr_reader :inventory_collection
|
3
|
+
attr_reader :inventory_collection
|
4
4
|
|
5
|
-
# An iterator that can fetch batches of the AR objects based on a set of
|
6
|
-
# when given an iterator. Or given query, acts as iterator by selecting batches.
|
5
|
+
# An iterator that can fetch batches of the AR objects based on a set of attribute_indexes
|
7
6
|
#
|
8
7
|
# @param inventory_collection [InventoryRefresh::InventoryCollection] Inventory collection owning the iterator
|
9
|
-
|
10
|
-
# fetch from the DB
|
11
|
-
# @param iterator [Proc] Block based iterator
|
12
|
-
# @query query [ActiveRecord::Relation] Existing query we want to use for querying the db
|
13
|
-
def initialize(inventory_collection: nil, manager_uuids_set: nil, iterator: nil, query: nil)
|
8
|
+
def initialize(inventory_collection: nil)
|
14
9
|
@inventory_collection = inventory_collection
|
15
|
-
@manager_uuids_set = manager_uuids_set
|
16
|
-
@iterator = iterator
|
17
|
-
@query = query
|
18
10
|
end
|
19
11
|
|
20
12
|
# Iterator that mimics find_in_batches of ActiveRecord::Relation. This iterator serves for making more optimized query
|
@@ -25,28 +17,19 @@ module InventoryRefresh
|
|
25
17
|
# and relation.where(:id => 500ids)
|
26
18
|
#
|
27
19
|
# @param batch_size [Integer] A batch size we want to fetch from DB
|
20
|
+
# @param attributes_index [Hash{String => Hash}] Indexed hash with data we will be saving
|
28
21
|
# @yield Code processing the batches
|
29
|
-
def find_in_batches(batch_size: 1000)
|
30
|
-
|
31
|
-
|
32
|
-
yield(batch)
|
33
|
-
end
|
34
|
-
elsif query
|
35
|
-
manager_uuids_set.each_slice(batch_size) do |batch|
|
36
|
-
yield(query.where(inventory_collection.targeted_selection_for(batch)))
|
37
|
-
end
|
38
|
-
else
|
39
|
-
manager_uuids_set.each_slice(batch_size) do |batch|
|
40
|
-
yield(inventory_collection.db_collection_for_comparison_for(batch))
|
41
|
-
end
|
22
|
+
def find_in_batches(batch_size: 1000, attributes_index: {})
|
23
|
+
attributes_index.each_slice(batch_size) do |batch|
|
24
|
+
yield(inventory_collection.db_collection_for_comparison_for(batch))
|
42
25
|
end
|
43
26
|
end
|
44
27
|
|
45
28
|
# Iterator that mimics find_each of ActiveRecord::Relation using find_in_batches (see #find_in_batches)
|
46
29
|
#
|
47
30
|
# @yield Code processing the batches
|
48
|
-
def find_each
|
49
|
-
find_in_batches do |batch|
|
31
|
+
def find_each(attributes_index: {})
|
32
|
+
find_in_batches(:attributes_index => attributes_index) do |batch|
|
50
33
|
batch.each do |item|
|
51
34
|
yield(item)
|
52
35
|
end
|
@@ -0,0 +1,8 @@
|
|
1
|
+
module InventoryRefresh
|
2
|
+
module Exception
|
3
|
+
class SweeperError < StandardError; end
|
4
|
+
class SweeperNonExistentScopeKeyFoundError < SweeperError; end
|
5
|
+
class SweeperNonUniformScopeKeyFoundError < SweeperError; end
|
6
|
+
class SweeperScopeBadFormat < SweeperError; end
|
7
|
+
end
|
8
|
+
end
|
@@ -4,16 +4,16 @@ module InventoryRefresh
|
|
4
4
|
class MissingModelClassError < StandardError; end
|
5
5
|
|
6
6
|
def self.allowed_properties
|
7
|
-
%i(
|
7
|
+
%i(arel association
|
8
8
|
attributes_blacklist attributes_whitelist batch_extra_attributes
|
9
9
|
complete create_only custom_save_block
|
10
|
-
custom_reconnect_block default_values
|
10
|
+
custom_reconnect_block default_values
|
11
11
|
dependency_attributes check_changed inventory_object_attributes
|
12
|
-
manager_ref manager_ref_allowed_nil
|
12
|
+
manager_ref manager_ref_allowed_nil
|
13
13
|
model_class name parent
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
retention_strategy strategy
|
15
|
+
secondary_refs
|
16
|
+
update_only use_ar_object
|
17
17
|
assert_graph_integrity).to_set
|
18
18
|
end
|
19
19
|
|
@@ -70,15 +70,6 @@ module InventoryRefresh
|
|
70
70
|
build(hash)
|
71
71
|
end
|
72
72
|
|
73
|
-
# Finds InventoryObject.
|
74
|
-
#
|
75
|
-
# @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the InventoryCollection
|
76
|
-
# @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
|
77
|
-
def find_in_data(hash)
|
78
|
-
_hash, _uuid, inventory_object = primary_index_scan(hash)
|
79
|
-
inventory_object
|
80
|
-
end
|
81
|
-
|
82
73
|
# Finds of builds a new InventoryObject. By building it, we also put in into the InventoryCollection's storage.
|
83
74
|
#
|
84
75
|
# @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the
|
@@ -35,26 +35,14 @@ module InventoryRefresh
|
|
35
35
|
# - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
|
36
36
|
# the DB. Then if we reference an object that is not present, it will
|
37
37
|
# load them from the db using :local_db_find_references strategy.
|
38
|
-
# @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
|
39
|
-
# Allowed saver strategies are:
|
40
|
-
# - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
|
41
|
-
# since it will lead to non consistent data.
|
42
|
-
# - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
|
43
|
-
# concurrently, since it will lead to non consistent data.
|
44
|
-
# - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
|
45
|
-
# atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
|
46
|
-
# executed as batched SQL queries, instead of sending 1 query per record.
|
47
38
|
# @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
|
48
39
|
# - :destroy => Will destroy the inactive records.
|
49
40
|
# - :archive => Will archive the inactive records by setting :archived_at timestamp.
|
50
|
-
|
51
|
-
|
52
|
-
# :model_class.
|
53
|
-
def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
|
54
|
-
@saver_strategy = process_saver_strategy(saver_strategy)
|
41
|
+
def init_strategies(strategy, retention_strategy)
|
42
|
+
@saver_strategy = :concurrent_safe_batch
|
55
43
|
@strategy = process_strategy(strategy)
|
44
|
+
# TODO(lsmola) why don't we just set this strategy based on :archived_at column? Lets do that
|
56
45
|
@retention_strategy = process_retention_strategy(retention_strategy)
|
57
|
-
@delete_method = delete_method || :destroy
|
58
46
|
end
|
59
47
|
|
60
48
|
# @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
|
@@ -67,42 +55,10 @@ module InventoryRefresh
|
|
67
55
|
# avoid this usecase by a proper modeling.
|
68
56
|
# Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
|
69
57
|
# @param secondary_refs [Hash] TODO
|
70
|
-
|
71
|
-
# this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
|
72
|
-
# case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
|
73
|
-
# custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
|
74
|
-
def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
|
58
|
+
def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs)
|
75
59
|
@manager_ref = manager_ref || %i(ems_ref)
|
76
60
|
@manager_ref_allowed_nil = manager_ref_allowed_nil || []
|
77
61
|
@secondary_refs = secondary_refs || {}
|
78
|
-
@manager_uuids = manager_uuids || []
|
79
|
-
end
|
80
|
-
|
81
|
-
# @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
|
82
|
-
# having this parameter defined will invoke only :delete_method on a complement of this set, making sure
|
83
|
-
# the DB has only this set of data after. This :attribute serves for deleting of top level
|
84
|
-
# InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
|
85
|
-
# child collections is already handled by the scope of the parent_inventory_collections and using Rails
|
86
|
-
# :dependent => :destroy,
|
87
|
-
# @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
|
88
|
-
# all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
|
89
|
-
# region.
|
90
|
-
# If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
|
91
|
-
# scope itself. E.g. sending a list of all active regions, we will delete complement entities not
|
92
|
-
# belonging to those regions.
|
93
|
-
# Example:
|
94
|
-
# :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
|
95
|
-
# :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
|
96
|
-
#
|
97
|
-
# Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
|
98
|
-
# regions X and Y.
|
99
|
-
# @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
|
100
|
-
# all_manager_uuids. Meaning we won't be archiving any newer entities.
|
101
|
-
def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
|
102
|
-
# TODO(lsmola) Should we refactor this to use references too?
|
103
|
-
@all_manager_uuids = all_manager_uuids
|
104
|
-
@all_manager_uuids_scope = all_manager_uuids_scope
|
105
|
-
@all_manager_uuids_timestamp = all_manager_uuids_timestamp
|
106
62
|
end
|
107
63
|
|
108
64
|
# @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
|
@@ -119,48 +75,9 @@ module InventoryRefresh
|
|
119
75
|
# This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
|
120
76
|
# will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
|
121
77
|
# are saved.
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
#
|
126
|
-
# Example:
|
127
|
-
# taking inventory collections :vms and :disks (local disks), if we write that:
|
128
|
-
# inventory_collection = InventoryCollection.new({
|
129
|
-
# :model_class => ::Disk,
|
130
|
-
# :association => :disks,
|
131
|
-
# :manager_ref => [:vm, :location]
|
132
|
-
# :parent_inventory_collection => [:vms],
|
133
|
-
# })
|
134
|
-
#
|
135
|
-
# Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
|
136
|
-
# points:
|
137
|
-
# 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
|
138
|
-
# from vms to disks table).
|
139
|
-
# 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
|
140
|
-
# a Vm subquery
|
141
|
-
# 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
|
142
|
-
# only under 1 Vm. (In current models, this modeled going through Hardware model)
|
143
|
-
# 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
|
144
|
-
#
|
145
|
-
# Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
|
146
|
-
# implications:
|
147
|
-
# 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
|
148
|
-
# archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
|
149
|
-
# 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
|
150
|
-
# we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
|
151
|
-
# archived/removed before the Vm, while we always want to archive the VM first.
|
152
|
-
# 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
|
153
|
-
# we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
|
154
|
-
# always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
|
155
|
-
# :manager_uuids are mutually exclusive attributes)
|
156
|
-
# 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
|
157
|
-
# While this information can be introspected from the data, it creates a scope for create&update&delete,
|
158
|
-
# which means it has to work with no data provided (causing delete all). So with no data we cannot
|
159
|
-
# introspect anything.
|
160
|
-
def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
|
161
|
-
@dependency_attributes = dependency_attributes || {}
|
162
|
-
@dependees = Set.new
|
163
|
-
@parent_inventory_collections = parent_inventory_collections
|
78
|
+
def init_ic_relations(dependency_attributes)
|
79
|
+
@dependency_attributes = dependency_attributes || {}
|
80
|
+
@dependees = Set.new
|
164
81
|
end
|
165
82
|
|
166
83
|
# @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
|
@@ -175,18 +92,13 @@ module InventoryRefresh
|
|
175
92
|
# @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
|
176
93
|
# it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
|
177
94
|
# for the batch saver strategy.
|
178
|
-
|
179
|
-
# :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
|
180
|
-
def init_flags(complete, create_only, check_changed,
|
181
|
-
update_only, use_ar_object, targeted,
|
182
|
-
assert_graph_integrity)
|
95
|
+
def init_flags(complete, create_only, check_changed, update_only, use_ar_object, assert_graph_integrity)
|
183
96
|
@complete = complete.nil? ? true : complete
|
184
97
|
@create_only = create_only.nil? ? false : create_only
|
185
98
|
@check_changed = check_changed.nil? ? true : check_changed
|
186
99
|
@saved = false
|
187
100
|
@update_only = update_only.nil? ? false : update_only
|
188
101
|
@use_ar_object = use_ar_object || false
|
189
|
-
@targeted = !!targeted
|
190
102
|
@assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
|
191
103
|
end
|
192
104
|
|
@@ -230,7 +142,7 @@ module InventoryRefresh
|
|
230
142
|
@attributes_whitelist = Set.new
|
231
143
|
@batch_extra_attributes = batch_extra_attributes || []
|
232
144
|
@inventory_object_attributes = inventory_object_attributes
|
233
|
-
@internal_attributes = %i(__feedback_edge_set_parent
|
145
|
+
@internal_attributes = %i(__feedback_edge_set_parent)
|
234
146
|
@transitive_dependency_attributes = Set.new
|
235
147
|
|
236
148
|
blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
|
@@ -240,7 +152,6 @@ module InventoryRefresh
|
|
240
152
|
def init_storages
|
241
153
|
@data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
|
242
154
|
@references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
|
243
|
-
@targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
|
244
155
|
end
|
245
156
|
|
246
157
|
# @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
|
@@ -248,35 +159,18 @@ module InventoryRefresh
|
|
248
159
|
# doing create/update/delete.
|
249
160
|
#
|
250
161
|
# Example:
|
251
|
-
#
|
252
|
-
#
|
253
|
-
#
|
254
|
-
#
|
255
|
-
#
|
256
|
-
#
|
257
|
-
#
|
258
|
-
#
|
259
|
-
# @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
|
260
|
-
# we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
|
261
|
-
# manager_uuids available.
|
262
|
-
# Example:
|
263
|
-
# targeted_arel = lambda do |inventory_collection|
|
264
|
-
# # Getting ems_refs of parent :vms and :miq_templates
|
265
|
-
# manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
|
266
|
-
# inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
|
267
|
-
# 'vms' => {:ems_ref => manager_uuids}
|
162
|
+
# add_collection(:cross_link_vms) do |builder|
|
163
|
+
# builder.add_properties(
|
164
|
+
# :arel => Vm.where(:tenant => manager.tenant),
|
165
|
+
# :association => nil,
|
166
|
+
# :model_class => Vm,
|
167
|
+
# :name => :cross_link_vms,
|
168
|
+
# :manager_ref => [:uid_ems],
|
169
|
+
# :strategy => :local_db_find_references,
|
268
170
|
# )
|
269
171
|
# end
|
270
|
-
|
271
|
-
|
272
|
-
# :model_class => ::Hardware,
|
273
|
-
# :association => :hardwares,
|
274
|
-
# :parent_inventory_collection => [:vms, :miq_templates],
|
275
|
-
# :targeted_arel => targeted_arel,
|
276
|
-
# })
|
277
|
-
def init_arels(arel, targeted_arel)
|
278
|
-
@arel = arel
|
279
|
-
@targeted_arel = targeted_arel
|
172
|
+
def init_arels(arel)
|
173
|
+
@arel = arel
|
280
174
|
end
|
281
175
|
|
282
176
|
# @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
|
@@ -289,7 +183,7 @@ module InventoryRefresh
|
|
289
183
|
# hash = inventory_object.attributes # Loads possible dependencies into saveable hash
|
290
184
|
# obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
|
291
185
|
# # queries, avoid this, this is just a simple example :-)
|
292
|
-
# obj.
|
186
|
+
# obj.update_attributes(hash) if obj
|
293
187
|
# obj ||= SomeModel.create(hash)
|
294
188
|
# inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
|
295
189
|
# primary key back to the InventoryObject
|
@@ -392,23 +286,6 @@ module InventoryRefresh
|
|
392
286
|
@deleted_records = []
|
393
287
|
end
|
394
288
|
|
395
|
-
# Processes passed saver strategy
|
396
|
-
#
|
397
|
-
# @param saver_strategy [Symbol] Passed saver strategy
|
398
|
-
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
399
|
-
def process_saver_strategy(saver_strategy)
|
400
|
-
return :default unless saver_strategy
|
401
|
-
|
402
|
-
saver_strategy = saver_strategy.to_sym
|
403
|
-
case saver_strategy
|
404
|
-
when :default, :batch, :concurrent_safe_batch
|
405
|
-
saver_strategy
|
406
|
-
else
|
407
|
-
raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
|
408
|
-
":default, :batch and :concurrent_safe_batch"
|
409
|
-
end
|
410
|
-
end
|
411
|
-
|
412
289
|
# Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
|
413
290
|
#
|
414
291
|
# @param strategy_name [Symbol] Passed saver strategy
|
@@ -434,12 +311,26 @@ module InventoryRefresh
|
|
434
311
|
strategy_name
|
435
312
|
end
|
436
313
|
|
314
|
+
# Saves passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
|
315
|
+
#
|
316
|
+
# @param strategy [Symbol] Passed saver strategy
|
317
|
+
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
318
|
+
def strategy=(strategy)
|
319
|
+
@strategy = process_strategy(strategy)
|
320
|
+
end
|
321
|
+
|
437
322
|
# Processes passed retention strategy
|
438
323
|
#
|
439
324
|
# @param retention_strategy [Symbol] Passed retention strategy
|
440
325
|
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
441
326
|
def process_retention_strategy(retention_strategy)
|
442
|
-
|
327
|
+
unless retention_strategy
|
328
|
+
if supports_column?(:archived_at)
|
329
|
+
return :archive
|
330
|
+
else
|
331
|
+
return :destroy
|
332
|
+
end
|
333
|
+
end
|
443
334
|
|
444
335
|
retention_strategy = retention_strategy.to_sym
|
445
336
|
case retention_strategy
|
@@ -50,11 +50,6 @@ module InventoryRefresh
|
|
50
50
|
dependencies.all?(&:saved?)
|
51
51
|
end
|
52
52
|
|
53
|
-
# @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
|
54
|
-
def parallel_safe?
|
55
|
-
@parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
|
56
|
-
end
|
57
|
-
|
58
53
|
# @return [Boolean] true if the model_class supports STI
|
59
54
|
def supports_sti?
|
60
55
|
@supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
|
@@ -81,50 +76,12 @@ module InventoryRefresh
|
|
81
76
|
data_collection_finalized
|
82
77
|
end
|
83
78
|
|
84
|
-
# @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
|
85
|
-
def targeted?
|
86
|
-
targeted
|
87
|
-
end
|
88
|
-
|
89
79
|
# True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
|
90
80
|
# stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
|
91
81
|
#
|
92
82
|
# @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
|
93
83
|
def noop?
|
94
|
-
|
95
|
-
saving_noop? && delete_complement_noop?
|
96
|
-
end
|
97
|
-
|
98
|
-
# @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
|
99
|
-
def saving_noop?
|
100
|
-
saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
|
101
|
-
end
|
102
|
-
|
103
|
-
# @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
|
104
|
-
def delete_complement_noop?
|
105
|
-
all_manager_uuids.nil?
|
106
|
-
end
|
107
|
-
|
108
|
-
private
|
109
|
-
|
110
|
-
# @return true if it's a noop parent targeted InventoryCollection
|
111
|
-
def saving_targeted_parent_collection_noop?
|
112
|
-
targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
|
113
|
-
end
|
114
|
-
|
115
|
-
# @return true if it's a noop child targeted InventoryCollection
|
116
|
-
def saving_targeted_child_collection_noop?
|
117
|
-
targeted_noop_condition && parent_inventory_collections.present? &&
|
118
|
-
parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
|
119
|
-
end
|
120
|
-
|
121
|
-
# @return true if it's a noop full InventoryCollection refresh
|
122
|
-
def saving_full_collection_noop?
|
123
|
-
!targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
|
124
|
-
end
|
125
|
-
|
126
|
-
def targeted_noop_condition
|
127
|
-
targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
|
84
|
+
data.blank? && custom_save_block.nil? && skeletal_primary_index.blank?
|
128
85
|
end
|
129
86
|
end
|
130
87
|
end
|
@@ -64,14 +64,6 @@ module InventoryRefresh
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
-
def reindex_secondary_indexes!
|
68
|
-
data_indexes.each do |ref, index|
|
69
|
-
next if ref == primary_index_ref
|
70
|
-
|
71
|
-
index.reindex!
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
67
|
def primary_index
|
76
68
|
data_index(primary_index_ref)
|
77
69
|
end
|
@@ -94,30 +86,16 @@ module InventoryRefresh
|
|
94
86
|
end
|
95
87
|
end
|
96
88
|
|
97
|
-
def find_by(manager_uuid_hash, ref: primary_index_ref)
|
98
|
-
# TODO(lsmola) deprecate this, it's enough to have find method
|
99
|
-
find(manager_uuid_hash, :ref => ref)
|
100
|
-
end
|
101
|
-
|
102
|
-
def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
|
103
|
-
# TODO(lsmola) deprecate this, it's enough to have lazy_find method
|
104
|
-
|
105
|
-
lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
|
106
|
-
end
|
107
|
-
|
108
89
|
def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
|
109
|
-
# TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
|
110
|
-
# access the data
|
111
|
-
# TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
|
112
90
|
return if manager_uuid.nil?
|
113
91
|
assert_index(manager_uuid, ref)
|
114
92
|
|
115
93
|
::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
94
|
+
manager_uuid,
|
95
|
+
:ref => ref,
|
96
|
+
:key => key,
|
97
|
+
:default => default,
|
98
|
+
:transform_nested_lazy_finds => transform_nested_lazy_finds)
|
121
99
|
end
|
122
100
|
|
123
101
|
def named_ref(ref = primary_index_ref)
|
@@ -132,19 +110,13 @@ module InventoryRefresh
|
|
132
110
|
|
133
111
|
delegate :association_to_foreign_key_mapping,
|
134
112
|
:build_stringified_reference,
|
135
|
-
:parallel_safe?,
|
136
113
|
:strategy,
|
137
114
|
:to => :inventory_collection
|
138
115
|
|
139
116
|
attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
|
140
117
|
|
141
118
|
def find_in_data_or_skeletal_index(reference)
|
142
|
-
|
143
|
-
# With parallel safe strategies, we create skeletal nodes that we can look for
|
144
|
-
data_index_find(reference) || skeletal_index_find(reference)
|
145
|
-
else
|
146
|
-
data_index_find(reference)
|
147
|
-
end
|
119
|
+
data_index_find(reference) || skeletal_index_find(reference)
|
148
120
|
end
|
149
121
|
|
150
122
|
def skeletal_index_find(reference)
|
@@ -27,14 +27,6 @@ module InventoryRefresh
|
|
27
27
|
index[build_stringified_reference(inventory_object.data, attribute_names)] = inventory_object
|
28
28
|
end
|
29
29
|
|
30
|
-
# Rebuilds the indexes for all InventoryObject objects
|
31
|
-
def reindex!
|
32
|
-
self.index = {}
|
33
|
-
data.each do |inventory_object|
|
34
|
-
store_index_for(inventory_object)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
30
|
# @return [Array] Returns index data
|
39
31
|
def index_data
|
40
32
|
index.values
|
@@ -45,23 +45,6 @@ module InventoryRefresh
|
|
45
45
|
add_reference(reference_data)
|
46
46
|
end
|
47
47
|
|
48
|
-
# Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
|
49
|
-
# it.
|
50
|
-
#
|
51
|
-
# @param references_array [Array] Array of reference objects acceptable by add_reference method.
|
52
|
-
# @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
|
53
|
-
# index.
|
54
|
-
# @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
|
55
|
-
def merge!(references_array, ref: nil)
|
56
|
-
references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
|
57
|
-
self
|
58
|
-
end
|
59
|
-
|
60
|
-
# @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
|
61
|
-
def primary_references
|
62
|
-
references[primary_index_ref]
|
63
|
-
end
|
64
|
-
|
65
48
|
# Builds a Reference object
|
66
49
|
#
|
67
50
|
# @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference
|