inventory_refresh 0.1.3 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +6 -0
- data/Gemfile +4 -0
- data/bundler.d/.gitkeep +0 -0
- data/inventory_refresh.gemspec +4 -4
- data/lib/inventory_refresh/inventory_collection/builder.rb +249 -0
- data/lib/inventory_refresh/inventory_collection/graph.rb +0 -15
- data/lib/inventory_refresh/inventory_collection/helpers/associations_helper.rb +80 -0
- data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +456 -0
- data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +132 -0
- data/lib/inventory_refresh/inventory_collection/helpers.rb +6 -0
- data/lib/inventory_refresh/inventory_collection/index/type/skeletal.rb +5 -5
- data/lib/inventory_refresh/inventory_collection/reference.rb +4 -0
- data/lib/inventory_refresh/inventory_collection/scanner.rb +111 -18
- data/lib/inventory_refresh/inventory_collection/serialization.rb +7 -7
- data/lib/inventory_refresh/inventory_collection/unconnected_edge.rb +19 -0
- data/lib/inventory_refresh/inventory_collection.rb +114 -649
- data/lib/inventory_refresh/inventory_object.rb +17 -11
- data/lib/inventory_refresh/inventory_object_lazy.rb +20 -10
- data/lib/inventory_refresh/persister.rb +212 -0
- data/lib/inventory_refresh/save_collection/base.rb +18 -3
- data/lib/inventory_refresh/save_collection/saver/base.rb +25 -62
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +73 -225
- data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +226 -0
- data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +115 -0
- data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +122 -0
- data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +24 -5
- data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +6 -6
- data/lib/inventory_refresh/save_collection/sweeper.rb +69 -0
- data/lib/inventory_refresh/save_inventory.rb +18 -8
- data/lib/inventory_refresh/target_collection.rb +12 -0
- data/lib/inventory_refresh/version.rb +1 -1
- data/lib/inventory_refresh.rb +1 -0
- metadata +24 -15
- data/lib/inventory_refresh/save_collection/recursive.rb +0 -52
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe.rb +0 -71
@@ -0,0 +1,456 @@
|
|
1
|
+
require_relative "../helpers"
|
2
|
+
|
3
|
+
module InventoryRefresh
|
4
|
+
class InventoryCollection
|
5
|
+
module Helpers
|
6
|
+
module InitializeHelper
|
7
|
+
# @param association [Symbol] A Rails association callable on a :parent attribute is used for comparing with the
|
8
|
+
# objects in the DB, to decide if the InventoryObjects will be created/deleted/updated or used for obtaining
|
9
|
+
# the data from a DB, if a DB strategy is used. It returns objects of the :model_class class or its sub STI.
|
10
|
+
# @param model_class [Class] A class of an ApplicationRecord model, that we want to persist into the DB or load from
|
11
|
+
# the DB.
|
12
|
+
# @param name [Symbol] A unique name of the InventoryCollection under a Persister. If not provided, the :association
|
13
|
+
# attribute is used. If :association is nil as well, the :name will be inferred from the :model_class.
|
14
|
+
# @param parent [ApplicationRecord] An ApplicationRecord object that has a callable :association method returning
|
15
|
+
# the objects of a :model_class.
|
16
|
+
def init_basic_properties(association, model_class, name, parent)
|
17
|
+
@association = association
|
18
|
+
@model_class = model_class
|
19
|
+
@name = name || association || model_class.to_s.demodulize.tableize
|
20
|
+
@parent = parent || nil
|
21
|
+
end
|
22
|
+
|
23
|
+
# @param strategy [Symbol] A strategy of the InventoryCollection that will be used for saving/loading of the
|
24
|
+
# InventoryObject objects.
|
25
|
+
# Allowed strategies are:
|
26
|
+
# - nil => InventoryObject objects of the InventoryCollection will be saved to the DB, only these objects
|
27
|
+
# will be referable from the other InventoryCollection objects.
|
28
|
+
# - :local_db_cache_all => Loads InventoryObject objects from the database, it loads all the objects that
|
29
|
+
# are a result of a [<:parent>.<:association>, :arel] taking
|
30
|
+
# first defined in this order. This strategy will not save any objects in the DB.
|
31
|
+
# - :local_db_find_references => Loads InventoryObject objects from the database, it loads only objects that
|
32
|
+
# were referenced by the other InventoryCollections using a filtered result
|
33
|
+
# of a [<:parent>.<:association>, :arel] taking first
|
34
|
+
# defined in this order. This strategy will not save any objects in the DB.
|
35
|
+
# - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
|
36
|
+
# the DB. Then if we reference an object that is not present, it will
|
37
|
+
# load them from the db using :local_db_find_references strategy.
|
38
|
+
# @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
|
39
|
+
# Allowed saver strategies are:
|
40
|
+
# - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
|
41
|
+
# since it will lead to non consistent data.
|
42
|
+
# - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
|
43
|
+
# concurrently, since it will lead to non consistent data.
|
44
|
+
# - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
|
45
|
+
# atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
|
46
|
+
# executed as batched SQL queries, instead of sending 1 query per record.
|
47
|
+
# @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
|
48
|
+
# - :destroy => Will destroy the inactive records.
|
49
|
+
# - :archive => Will archive the inactive records by setting :archived_at timestamp.
|
50
|
+
# @param delete_method [Symbol] A delete method that will be used for deleting of the InventoryObject, if the
|
51
|
+
# object is marked for deletion. A default is :destroy, the instance method must be defined on the
|
52
|
+
# :model_class.
|
53
|
+
def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
|
54
|
+
@saver_strategy = process_saver_strategy(saver_strategy)
|
55
|
+
@strategy = process_strategy(strategy)
|
56
|
+
@retention_strategy = process_retention_strategy(retention_strategy)
|
57
|
+
@delete_method = delete_method || :destroy
|
58
|
+
end
|
59
|
+
|
60
|
+
# @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
|
61
|
+
# InventoryCollection. Using these keys, we need to be able to uniquely identify each of the InventoryObject
|
62
|
+
# objects inside.
|
63
|
+
# @param manager_ref_allowed_nil [Array] Array of symbols having manager_ref columns, that are a foreign key an can
|
64
|
+
# be nil. Given the table are shared by many providers, it can happen, that the table is used only partially.
|
65
|
+
# Then it can happen we want to allow certain foreign keys to be nil, while being sure the referential
|
66
|
+
# integrity is not broken. Of course the DB Foreign Key can't be created in this case, so we should try to
|
67
|
+
# avoid this usecase by a proper modeling.
|
68
|
+
# Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
|
69
|
+
# @param secondary_refs [Hash] TODO
|
70
|
+
# @param manager_uuids [Array|Proc] Array of manager_uuids of the InventoryObjects we want to create/update/delete. Using
|
71
|
+
# this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
|
72
|
+
# case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
|
73
|
+
# custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
|
74
|
+
def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
|
75
|
+
@manager_ref = manager_ref || %i(ems_ref)
|
76
|
+
@manager_ref_allowed_nil = manager_ref_allowed_nil || []
|
77
|
+
@secondary_refs = secondary_refs || {}
|
78
|
+
@manager_uuids = manager_uuids || []
|
79
|
+
end
|
80
|
+
|
81
|
+
# @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
|
82
|
+
# having this parameter defined will invoke only :delete_method on a complement of this set, making sure
|
83
|
+
# the DB has only this set of data after. This :attribute serves for deleting of top level
|
84
|
+
# InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
|
85
|
+
# child collections is already handled by the scope of the parent_inventory_collections and using Rails
|
86
|
+
# :dependent => :destroy,
|
87
|
+
# @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
|
88
|
+
# all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
|
89
|
+
# region.
|
90
|
+
# If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
|
91
|
+
# scope itself. E.g. sending a list of all active regions, we will delete complement entities not
|
92
|
+
# belonging to those regions.
|
93
|
+
# Example:
|
94
|
+
# :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
|
95
|
+
# :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
|
96
|
+
#
|
97
|
+
# Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
|
98
|
+
# regions X and Y.
|
99
|
+
# @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
|
100
|
+
# all_manager_uuids. Meaning we won't be archiving any newer entities.
|
101
|
+
def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
|
102
|
+
# TODO(lsmola) Should we refactor this to use references too?
|
103
|
+
@all_manager_uuids = all_manager_uuids
|
104
|
+
@all_manager_uuids_scope = all_manager_uuids_scope
|
105
|
+
@all_manager_uuids_timestamp = all_manager_uuids_timestamp
|
106
|
+
end
|
107
|
+
|
108
|
+
# @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
|
109
|
+
# by manually place the InventoryCollection into the graph, to make sure the saving is invoked after the
|
110
|
+
# dependencies were saved. The dependencies itself are InventoryCollection objects. For a common use-cases
|
111
|
+
# we do not need to define dependencies manually, since those are inferred automatically by scanning of the
|
112
|
+
# data.
|
113
|
+
#
|
114
|
+
# Example:
|
115
|
+
# :dependency_attributes => {
|
116
|
+
# :orchestration_stacks => [collections[:orchestration_stacks]],
|
117
|
+
# :orchestration_stacks_resources => [collections[:orchestration_stacks_resources]]
|
118
|
+
# }
|
119
|
+
# This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
|
120
|
+
# will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
|
121
|
+
# are saved.
|
122
|
+
# @param parent_inventory_collections [Array] Array of symbols having a name pointing to the
|
123
|
+
# InventoryRefresh::InventoryCollection objects, that serve as parents to this InventoryCollection. There are
|
124
|
+
# several scenarios to consider, when deciding if InventoryCollection has parent collections, see the example.
|
125
|
+
#
|
126
|
+
# Example:
|
127
|
+
# taking inventory collections :vms and :disks (local disks), if we write that:
|
128
|
+
# inventory_collection = InventoryCollection.new({
|
129
|
+
# :model_class => ::Disk,
|
130
|
+
# :association => :disks,
|
131
|
+
# :manager_ref => [:vm, :location]
|
132
|
+
# :parent_inventory_collection => [:vms],
|
133
|
+
# })
|
134
|
+
#
|
135
|
+
# Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
|
136
|
+
# points:
|
137
|
+
# 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
|
138
|
+
# from vms to disks table).
|
139
|
+
# 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
|
140
|
+
# a Vm subquery
|
141
|
+
# 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
|
142
|
+
# only under 1 Vm. (In current models, this modeled going through Hardware model)
|
143
|
+
# 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
|
144
|
+
#
|
145
|
+
# Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
|
146
|
+
# implications:
|
147
|
+
# 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
|
148
|
+
# archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
|
149
|
+
# 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
|
150
|
+
# we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
|
151
|
+
# archived/removed before the Vm, while we always want to archive the VM first.
|
152
|
+
# 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
|
153
|
+
# we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
|
154
|
+
# always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
|
155
|
+
# :manager_uuids are mutually exclusive attributes)
|
156
|
+
# 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
|
157
|
+
# While this information can be introspected from the data, it creates a scope for create&update&delete,
|
158
|
+
# which means it has to work with no data provided (causing delete all). So with no data we cannot
|
159
|
+
# introspect anything.
|
160
|
+
def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
|
161
|
+
@dependency_attributes = dependency_attributes || {}
|
162
|
+
@dependees = Set.new
|
163
|
+
@parent_inventory_collections = parent_inventory_collections
|
164
|
+
end
|
165
|
+
|
166
|
+
# @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
|
167
|
+
# we can create/update/delete the InventoryObject objects. If :complete is false we will only do
|
168
|
+
# create/update without delete.
|
169
|
+
# @param create_only [Boolean] TODO
|
170
|
+
# @param check_changed [Boolean] By default true. If true, before updating the InventoryObject, we call Rails
|
171
|
+
# 'changed?' method. This can optimize speed of updates heavily, but it can fail to recognize the change for
|
172
|
+
# e.g. Ancestry and Relationship based columns. If false, we always update the InventoryObject.
|
173
|
+
# @param update_only [Boolean] By default false. If true we only update the InventoryObject objects, if false we do
|
174
|
+
# create/update/delete.
|
175
|
+
# @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
|
176
|
+
# it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
|
177
|
+
# for the batch saver strategy.
|
178
|
+
# @param targeted [Boolean] True if the collection is targeted, in that case it will be leveraging :manager_uuids
|
179
|
+
# :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
|
180
|
+
def init_flags(complete, create_only, check_changed,
|
181
|
+
update_only, use_ar_object, targeted,
|
182
|
+
assert_graph_integrity)
|
183
|
+
@complete = complete.nil? ? true : complete
|
184
|
+
@create_only = create_only.nil? ? false : create_only
|
185
|
+
@check_changed = check_changed.nil? ? true : check_changed
|
186
|
+
@saved = false
|
187
|
+
@update_only = update_only.nil? ? false : update_only
|
188
|
+
@use_ar_object = use_ar_object || false
|
189
|
+
@targeted = !!targeted
|
190
|
+
@assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
|
191
|
+
end
|
192
|
+
|
193
|
+
# @param attributes_blacklist [Array] Attributes we do not want to include into saving. We cannot blacklist an
|
194
|
+
# attribute that is needed for saving of the object.
|
195
|
+
# Note: attributes_blacklist is also used for internal resolving of the cycles in the graph.
|
196
|
+
#
|
197
|
+
# In the Example2 of the <param custom_save_block>, we have a custom saving code, that saves a :parent
|
198
|
+
# attribute of the OrchestrationStack. That means we don't want that attribute saved as a part of
|
199
|
+
# InventoryCollection for OrchestrationStack, so we would set :attributes_blacklist => [:parent]. Then the
|
200
|
+
# :parent will be ignored while saving.
|
201
|
+
# @param attributes_whitelist [Array] Same usage as the :attributes_blacklist, but defining full set of attributes
|
202
|
+
# that should be saved. Attributes that are part of :manager_ref and needed validations are automatically
|
203
|
+
# added.
|
204
|
+
# @param inventory_object_attributes [Array] Array of attribute names that will be exposed as readers/writers on the
|
205
|
+
# InventoryObject objects inside.
|
206
|
+
#
|
207
|
+
# Example: Given
|
208
|
+
# inventory_collection = InventoryCollection.new({
|
209
|
+
# :model_class => ::Vm,
|
210
|
+
# :arel => @ems.vms,
|
211
|
+
# :inventory_object_attributes => [:name, :label]
|
212
|
+
# })
|
213
|
+
# And building the inventory_object like:
|
214
|
+
# inventory_object = inventory_collection.build(:ems_ref => "vm1", :name => "vm1")
|
215
|
+
# We can use inventory_object_attributes as setters and getters:
|
216
|
+
# inventory_object.name = "Name"
|
217
|
+
# inventory_object.label = inventory_object.name
|
218
|
+
# Which would be equivalent to less nicer way:
|
219
|
+
# inventory_object[:name] = "Name"
|
220
|
+
# inventory_object[:label] = inventory_object[:name]
|
221
|
+
# So by using inventory_object_attributes, we will be guarding the allowed attributes and will have an
|
222
|
+
# explicit list of allowed attributes, that can be used also for documentation purposes.
|
223
|
+
# @param batch_extra_attributes [Array] Array of symbols marking which extra attributes we want to store into the
|
224
|
+
# db. These extra attributes might be a product of :use_ar_object assignment and we need to specify them
|
225
|
+
# manually, if we want to use a batch saving strategy and we have models that populate attributes as a side
|
226
|
+
# effect.
|
227
|
+
def init_model_attributes(attributes_blacklist, attributes_whitelist,
|
228
|
+
inventory_object_attributes, batch_extra_attributes)
|
229
|
+
@attributes_blacklist = Set.new
|
230
|
+
@attributes_whitelist = Set.new
|
231
|
+
@batch_extra_attributes = batch_extra_attributes || []
|
232
|
+
@inventory_object_attributes = inventory_object_attributes
|
233
|
+
@internal_attributes = %i(__feedback_edge_set_parent __parent_inventory_collections __all_manager_uuids_scope)
|
234
|
+
@transitive_dependency_attributes = Set.new
|
235
|
+
|
236
|
+
blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
|
237
|
+
whitelist_attributes!(attributes_whitelist) if attributes_whitelist.present?
|
238
|
+
end
|
239
|
+
|
240
|
+
def init_storages
|
241
|
+
@data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
|
242
|
+
@references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
|
243
|
+
@targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
|
244
|
+
end
|
245
|
+
|
246
|
+
# @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
|
247
|
+
# we can provide Arel directly to say what records should be compared to check if InventoryObject will be
|
248
|
+
# doing create/update/delete.
|
249
|
+
#
|
250
|
+
# Example:
|
251
|
+
# for a targeted refresh, we want to delete/update/create only a list of vms specified with a list of
|
252
|
+
# ems_refs:
|
253
|
+
# :arel => manager.vms.where(:ems_ref => manager_refs)
|
254
|
+
# Then we want to do the same for the hardwares of only those vms:
|
255
|
+
# :arel => manager.hardwares.joins(:vm_or_template).where(
|
256
|
+
# 'vms' => {:ems_ref => manager_refs}
|
257
|
+
# )
|
258
|
+
# And etc. for the other Vm related records.
|
259
|
+
# @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
|
260
|
+
# we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
|
261
|
+
# manager_uuids available.
|
262
|
+
# Example:
|
263
|
+
# targeted_arel = lambda do |inventory_collection|
|
264
|
+
# # Getting ems_refs of parent :vms and :miq_templates
|
265
|
+
# manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
|
266
|
+
# inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
|
267
|
+
# 'vms' => {:ems_ref => manager_uuids}
|
268
|
+
# )
|
269
|
+
# end
|
270
|
+
#
|
271
|
+
# inventory_collection = InventoryCollection.new({
|
272
|
+
# :model_class => ::Hardware,
|
273
|
+
# :association => :hardwares,
|
274
|
+
# :parent_inventory_collection => [:vms, :miq_templates],
|
275
|
+
# :targeted_arel => targeted_arel,
|
276
|
+
# })
|
277
|
+
def init_arels(arel, targeted_arel)
|
278
|
+
@arel = arel
|
279
|
+
@targeted_arel = targeted_arel
|
280
|
+
end
|
281
|
+
|
282
|
+
# @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
|
283
|
+
# to just save every InventoryObject inside by the defined rules and default saving algorithm.
|
284
|
+
#
|
285
|
+
# Example1 - saving SomeModel in my own ineffective way :-) :
|
286
|
+
#
|
287
|
+
# custom_save = lambda do |_ems, inventory_collection|
|
288
|
+
# inventory_collection.each |inventory_object| do
|
289
|
+
# hash = inventory_object.attributes # Loads possible dependencies into saveable hash
|
290
|
+
# obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
|
291
|
+
# # queries, avoid this, this is just a simple example :-)
|
292
|
+
# obj.update_attributes(hash) if obj
|
293
|
+
# obj ||= SomeModel.create(hash)
|
294
|
+
# inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
|
295
|
+
# primary key back to the InventoryObject
|
296
|
+
# end
|
297
|
+
#
|
298
|
+
# Example2 - saving parent OrchestrationStack in a more effective way, than the default saving algorithm can
|
299
|
+
# achieve. Ancestry gem requires an ActiveRecord object for association and is not defined as a proper
|
300
|
+
# ActiveRecord association. That leads in N+1 queries in the default saving algorithm, so we can do better
|
301
|
+
# with custom saving for now. The InventoryCollection is defined as a custom dependencies processor,
|
302
|
+
# without its own :model_class and InventoryObjects inside:
|
303
|
+
#
|
304
|
+
# InventoryRefresh::InventoryCollection.new({
|
305
|
+
# :association => :orchestration_stack_ancestry,
|
306
|
+
# :custom_save_block => orchestration_stack_ancestry_save_block,
|
307
|
+
# :dependency_attributes => {
|
308
|
+
# :orchestration_stacks => [collections[:orchestration_stacks]],
|
309
|
+
# :orchestration_stacks_resources => [collections[:orchestration_stacks_resources]]
|
310
|
+
# }
|
311
|
+
# })
|
312
|
+
#
|
313
|
+
# And the lambda is defined as:
|
314
|
+
#
|
315
|
+
# orchestration_stack_ancestry_save_block = lambda do |_ems, inventory_collection|
|
316
|
+
# stacks_inventory_collection = inventory_collection.dependency_attributes[:orchestration_stacks].try(:first)
|
317
|
+
#
|
318
|
+
# return if stacks_inventory_collection.blank?
|
319
|
+
#
|
320
|
+
# stacks_parents = stacks_inventory_collection.data.each_with_object({}) do |x, obj|
|
321
|
+
# parent_id = x.data[:parent].load.try(:id)
|
322
|
+
# obj[x.id] = parent_id if parent_id
|
323
|
+
# end
|
324
|
+
#
|
325
|
+
# model_class = stacks_inventory_collection.model_class
|
326
|
+
#
|
327
|
+
# stacks_parents_indexed = model_class
|
328
|
+
# .select([:id, :ancestry])
|
329
|
+
# .where(:id => stacks_parents.values).find_each.index_by(&:id)
|
330
|
+
#
|
331
|
+
# model_class
|
332
|
+
# .select([:id, :ancestry])
|
333
|
+
# .where(:id => stacks_parents.keys).find_each do |stack|
|
334
|
+
# parent = stacks_parents_indexed[stacks_parents[stack.id]]
|
335
|
+
# stack.update_attribute(:parent, parent)
|
336
|
+
# end
|
337
|
+
# end
|
338
|
+
# @param custom_reconnect_block [Proc] A custom lambda for reconnect logic of previously disconnected records
|
339
|
+
#
|
340
|
+
# Example - Reconnect disconnected Vms
|
341
|
+
# InventoryRefresh::InventoryCollection.new({
|
342
|
+
# :association => :orchestration_stack_ancestry,
|
343
|
+
# :custom_reconnect_block => vms_custom_reconnect_block,
|
344
|
+
# })
|
345
|
+
#
|
346
|
+
# And the lambda is defined as:
|
347
|
+
#
|
348
|
+
# vms_custom_reconnect_block = lambda do |inventory_collection, inventory_objects_index, attributes_index|
|
349
|
+
# inventory_objects_index.each_slice(1000) do |batch|
|
350
|
+
# Vm.where(:ems_ref => batch.map(&:second).map(&:manager_uuid)).each do |record|
|
351
|
+
# index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record)
|
352
|
+
#
|
353
|
+
# # We need to delete the record from the inventory_objects_index and attributes_index, otherwise it
|
354
|
+
# # would be sent for create.
|
355
|
+
# inventory_object = inventory_objects_index.delete(index)
|
356
|
+
# hash = attributes_index.delete(index)
|
357
|
+
#
|
358
|
+
# record.assign_attributes(hash.except(:id, :type))
|
359
|
+
# if !inventory_collection.check_changed? || record.changed?
|
360
|
+
# record.save!
|
361
|
+
# inventory_collection.store_updated_records(record)
|
362
|
+
# end
|
363
|
+
#
|
364
|
+
# inventory_object.id = record.id
|
365
|
+
# end
|
366
|
+
# end
|
367
|
+
def init_custom_procs(custom_save_block, custom_reconnect_block)
|
368
|
+
@custom_save_block = custom_save_block
|
369
|
+
@custom_reconnect_block = custom_reconnect_block
|
370
|
+
end
|
371
|
+
|
372
|
+
# @param default_values [Hash] A hash of an attributes that will be added to every inventory object created by
|
373
|
+
# inventory_collection.build(hash)
|
374
|
+
#
|
375
|
+
# Example: Given
|
376
|
+
# inventory_collection = InventoryCollection.new({
|
377
|
+
# :model_class => ::Vm,
|
378
|
+
# :arel => @ems.vms,
|
379
|
+
# :default_values => {:ems_id => 10}
|
380
|
+
# })
|
381
|
+
# And building the inventory_object like:
|
382
|
+
# inventory_object = inventory_collection.build(:ems_ref => "vm_1", :name => "vm1")
|
383
|
+
# The inventory_object.data will look like:
|
384
|
+
# {:ems_ref => "vm_1", :name => "vm1", :ems_id => 10}
|
385
|
+
def init_data(default_values)
|
386
|
+
@default_values = default_values || {}
|
387
|
+
end
|
388
|
+
|
389
|
+
def init_changed_records_stats
|
390
|
+
@created_records = []
|
391
|
+
@updated_records = []
|
392
|
+
@deleted_records = []
|
393
|
+
end
|
394
|
+
|
395
|
+
# Processes passed saver strategy
|
396
|
+
#
|
397
|
+
# @param saver_strategy [Symbol] Passed saver strategy
|
398
|
+
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
399
|
+
def process_saver_strategy(saver_strategy)
|
400
|
+
return :default unless saver_strategy
|
401
|
+
|
402
|
+
saver_strategy = saver_strategy.to_sym
|
403
|
+
case saver_strategy
|
404
|
+
when :default, :batch, :concurrent_safe_batch
|
405
|
+
saver_strategy
|
406
|
+
else
|
407
|
+
raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
|
408
|
+
":default, :batch and :concurrent_safe_batch"
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
412
|
+
# Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
|
413
|
+
#
|
414
|
+
# @param strategy_name [Symbol] Passed saver strategy
|
415
|
+
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
416
|
+
def process_strategy(strategy_name)
|
417
|
+
self.data_collection_finalized = false
|
418
|
+
|
419
|
+
return unless strategy_name
|
420
|
+
|
421
|
+
strategy_name = strategy_name.to_sym
|
422
|
+
case strategy_name
|
423
|
+
when :local_db_cache_all
|
424
|
+
self.data_collection_finalized = true
|
425
|
+
self.saved = true
|
426
|
+
when :local_db_find_references
|
427
|
+
self.saved = true
|
428
|
+
when :local_db_find_missing_references
|
429
|
+
nil
|
430
|
+
else
|
431
|
+
raise "Unknown InventoryCollection strategy: :#{strategy_name}, allowed strategies are :local_db_cache_all, "\
|
432
|
+
":local_db_find_references and :local_db_find_missing_references."
|
433
|
+
end
|
434
|
+
strategy_name
|
435
|
+
end
|
436
|
+
|
437
|
+
# Processes passed retention strategy
|
438
|
+
#
|
439
|
+
# @param retention_strategy [Symbol] Passed retention strategy
|
440
|
+
# @return [Symbol] Returns back the passed strategy if supported, or raises exception
|
441
|
+
def process_retention_strategy(retention_strategy)
|
442
|
+
return unless retention_strategy
|
443
|
+
|
444
|
+
retention_strategy = retention_strategy.to_sym
|
445
|
+
case retention_strategy
|
446
|
+
when :destroy, :archive
|
447
|
+
retention_strategy
|
448
|
+
else
|
449
|
+
raise "Unknown InventoryCollection retention strategy: :#{retention_strategy}, allowed strategies are "\
|
450
|
+
":destroy and :archive"
|
451
|
+
end
|
452
|
+
end
|
453
|
+
end
|
454
|
+
end
|
455
|
+
end
|
456
|
+
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require_relative "../helpers"
|
2
|
+
|
3
|
+
module InventoryRefresh
|
4
|
+
class InventoryCollection
|
5
|
+
module Helpers
|
6
|
+
module QuestionsHelper
|
7
|
+
# @return [Boolean] true means we want to call .changed? on every ActiveRecord object before saving it
|
8
|
+
def check_changed?
|
9
|
+
check_changed
|
10
|
+
end
|
11
|
+
|
12
|
+
# @return [Boolean] true means we want to use ActiveRecord object for writing attributes and we want to perform
|
13
|
+
# casting on all columns
|
14
|
+
def use_ar_object?
|
15
|
+
use_ar_object
|
16
|
+
end
|
17
|
+
|
18
|
+
# @return [Boolean] true means the data is not complete, leading to only creating and updating data
|
19
|
+
def complete?
|
20
|
+
complete
|
21
|
+
end
|
22
|
+
|
23
|
+
# @return [Boolean] true means we want to only update data
|
24
|
+
def update_only?
|
25
|
+
update_only
|
26
|
+
end
|
27
|
+
|
28
|
+
# @return [Boolean] true means we will delete/soft-delete data
|
29
|
+
def delete_allowed?
|
30
|
+
complete? && !update_only?
|
31
|
+
end
|
32
|
+
|
33
|
+
# @return [Boolean] true means we will delete/soft-delete data
|
34
|
+
def create_allowed?
|
35
|
+
!update_only?
|
36
|
+
end
|
37
|
+
|
38
|
+
# @return [Boolean] true means that only create of new data is allowed
|
39
|
+
def create_only?
|
40
|
+
create_only
|
41
|
+
end
|
42
|
+
|
43
|
+
# @return [Boolean] true if the whole InventoryCollection object has all data persisted
|
44
|
+
def saved?
|
45
|
+
saved
|
46
|
+
end
|
47
|
+
|
48
|
+
# @return [Boolean] true if all dependencies have all data persisted
|
49
|
+
def saveable?
|
50
|
+
dependencies.all?(&:saved?)
|
51
|
+
end
|
52
|
+
|
53
|
+
# @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
|
54
|
+
def parallel_safe?
|
55
|
+
@parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
|
56
|
+
end
|
57
|
+
|
58
|
+
# @return [Boolean] true if the model_class supports STI
|
59
|
+
def supports_sti?
|
60
|
+
@supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
|
61
|
+
@supports_sti_cache
|
62
|
+
end
|
63
|
+
|
64
|
+
# @param column_name [Symbol, String]
|
65
|
+
# @return [Boolean] true if the model_class supports given column
|
66
|
+
def supports_column?(column_name)
|
67
|
+
@supported_cols_cache ||= {}
|
68
|
+
return @supported_cols_cache[column_name.to_sym] unless @supported_cols_cache[column_name.to_sym].nil?
|
69
|
+
|
70
|
+
include_col = model_class.column_names.include?(column_name.to_s)
|
71
|
+
if %w(created_on created_at updated_on updated_at).include?(column_name.to_s)
|
72
|
+
include_col &&= ActiveRecord::Base.record_timestamps
|
73
|
+
end
|
74
|
+
|
75
|
+
@supported_cols_cache[column_name.to_sym] = include_col
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Boolean] true if no more data will be added to this InventoryCollection object, that usually happens
|
79
|
+
# after the parsing step is finished
|
80
|
+
def data_collection_finalized?
|
81
|
+
data_collection_finalized
|
82
|
+
end
|
83
|
+
|
84
|
+
# @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
|
85
|
+
def targeted?
|
86
|
+
targeted
|
87
|
+
end
|
88
|
+
|
89
|
+
# True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
|
90
|
+
# stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
|
91
|
+
#
|
92
|
+
# @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
|
93
|
+
def noop?
|
94
|
+
# If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
|
95
|
+
saving_noop? && delete_complement_noop?
|
96
|
+
end
|
97
|
+
|
98
|
+
# @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
|
99
|
+
def saving_noop?
|
100
|
+
saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
|
101
|
+
end
|
102
|
+
|
103
|
+
# @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
|
104
|
+
def delete_complement_noop?
|
105
|
+
all_manager_uuids.nil?
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
|
110
|
+
# @return true if it's a noop parent targeted InventoryCollection
|
111
|
+
def saving_targeted_parent_collection_noop?
|
112
|
+
targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
|
113
|
+
end
|
114
|
+
|
115
|
+
# @return true if it's a noop child targeted InventoryCollection
|
116
|
+
def saving_targeted_child_collection_noop?
|
117
|
+
targeted_noop_condition && parent_inventory_collections.present? &&
|
118
|
+
parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
|
119
|
+
end
|
120
|
+
|
121
|
+
# @return true if it's a noop full InventoryCollection refresh
|
122
|
+
def saving_full_collection_noop?
|
123
|
+
!targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
|
124
|
+
end
|
125
|
+
|
126
|
+
def targeted_noop_condition
|
127
|
+
targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -87,19 +87,19 @@ module InventoryRefresh
|
|
87
87
|
#
|
88
88
|
# @param attributes [Hash] Attributes we want to extend with version related attributes
|
89
89
|
def fill_versions!(attributes)
|
90
|
-
if inventory_collection.
|
90
|
+
if inventory_collection.supports_column?(:resource_timestamps_max) && attributes[:resource_timestamp]
|
91
91
|
fill_specific_version_attr(:resource_timestamps, :resource_timestamp, attributes)
|
92
|
-
elsif inventory_collection.
|
93
|
-
fill_specific_version_attr(:
|
92
|
+
elsif inventory_collection.supports_column?(:resource_counters_max) && attributes[:resource_counter]
|
93
|
+
fill_specific_version_attr(:resource_counters, :resource_counter, attributes)
|
94
94
|
end
|
95
95
|
end
|
96
96
|
|
97
97
|
# Add specific versions columns into the passed attributes
|
98
98
|
#
|
99
99
|
# @param partial_row_version_attr [Symbol] Attr name for partial rows, allowed values are
|
100
|
-
# [:resource_timestamps, :
|
100
|
+
# [:resource_timestamps, :resource_counters]
|
101
101
|
# @param full_row_version_attr [Symbol] Attr name for full rows, allowed values are
|
102
|
-
# [:resource_timestamp, :
|
102
|
+
# [:resource_timestamp, :resource_counter]
|
103
103
|
# @param attributes [Hash] Attributes we want to extend with version related attributes
|
104
104
|
def fill_specific_version_attr(partial_row_version_attr, full_row_version_attr, attributes)
|
105
105
|
# We have to symbolize, since serializing persistor makes these strings
|