inventory_refresh 0.3.6 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +25 -30
  3. data/.github/workflows/ci.yaml +47 -0
  4. data/.rubocop.yml +3 -3
  5. data/.rubocop_cc.yml +3 -4
  6. data/.rubocop_local.yml +5 -2
  7. data/.whitesource +3 -0
  8. data/CHANGELOG.md +19 -0
  9. data/Gemfile +10 -4
  10. data/README.md +1 -2
  11. data/Rakefile +2 -2
  12. data/inventory_refresh.gemspec +8 -9
  13. data/lib/inventory_refresh/application_record_iterator.rb +25 -12
  14. data/lib/inventory_refresh/graph/topological_sort.rb +24 -26
  15. data/lib/inventory_refresh/graph.rb +2 -2
  16. data/lib/inventory_refresh/inventory_collection/builder.rb +37 -15
  17. data/lib/inventory_refresh/inventory_collection/data_storage.rb +9 -0
  18. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +147 -38
  19. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +48 -4
  20. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +35 -3
  21. data/lib/inventory_refresh/inventory_collection/index/type/base.rb +8 -0
  22. data/lib/inventory_refresh/inventory_collection/index/type/local_db.rb +2 -0
  23. data/lib/inventory_refresh/inventory_collection/index/type/skeletal.rb +1 -0
  24. data/lib/inventory_refresh/inventory_collection/reference.rb +1 -0
  25. data/lib/inventory_refresh/inventory_collection/references_storage.rb +17 -0
  26. data/lib/inventory_refresh/inventory_collection/scanner.rb +91 -3
  27. data/lib/inventory_refresh/inventory_collection/serialization.rb +16 -10
  28. data/lib/inventory_refresh/inventory_collection.rb +122 -64
  29. data/lib/inventory_refresh/inventory_object.rb +74 -40
  30. data/lib/inventory_refresh/inventory_object_lazy.rb +17 -10
  31. data/lib/inventory_refresh/null_logger.rb +2 -2
  32. data/lib/inventory_refresh/persister.rb +43 -93
  33. data/lib/inventory_refresh/save_collection/base.rb +4 -2
  34. data/lib/inventory_refresh/save_collection/saver/base.rb +114 -15
  35. data/lib/inventory_refresh/save_collection/saver/batch.rb +17 -0
  36. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +129 -51
  37. data/lib/inventory_refresh/save_collection/saver/default.rb +57 -0
  38. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +2 -19
  39. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +68 -3
  40. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +125 -0
  41. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +10 -6
  42. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +28 -16
  43. data/lib/inventory_refresh/save_collection/sweeper.rb +17 -93
  44. data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
  45. data/lib/inventory_refresh/save_inventory.rb +5 -12
  46. data/lib/inventory_refresh/target.rb +73 -0
  47. data/lib/inventory_refresh/target_collection.rb +92 -0
  48. data/lib/inventory_refresh/version.rb +1 -1
  49. data/lib/inventory_refresh.rb +2 -0
  50. metadata +34 -37
  51. data/.travis.yml +0 -23
  52. data/lib/inventory_refresh/exception.rb +0 -8
@@ -35,14 +35,26 @@ module InventoryRefresh
35
35
  # - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
36
36
  # the DB. Then if we reference an object that is not present, it will
37
37
  # load them from the db using :local_db_find_references strategy.
38
+ # @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
39
+ # Allowed saver strategies are:
40
+ # - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
41
+ # since it will lead to non consistent data.
42
+ # - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
43
+ # concurrently, since it will lead to non consistent data.
44
+ # - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
45
+ # atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
46
+ # executed as batched SQL queries, instead of sending 1 query per record.
38
47
  # @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
39
48
  # - :destroy => Will destroy the inactive records.
40
49
  # - :archive => Will archive the inactive records by setting :archived_at timestamp.
41
- def init_strategies(strategy, retention_strategy)
42
- @saver_strategy = :concurrent_safe_batch
50
+ # @param delete_method [Symbol] A delete method that will be used for deleting of the InventoryObject, if the
51
+ # object is marked for deletion. A default is :destroy, the instance method must be defined on the
52
+ # :model_class.
53
+ def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
54
+ @saver_strategy = process_saver_strategy(saver_strategy)
43
55
  @strategy = process_strategy(strategy)
44
- # TODO(lsmola) why don't we just set this strategy based on :archived_at column? Lets do that
45
56
  @retention_strategy = process_retention_strategy(retention_strategy)
57
+ @delete_method = delete_method || :destroy
46
58
  end
47
59
 
48
60
  # @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
@@ -55,10 +67,42 @@ module InventoryRefresh
55
67
  # avoid this usecase by a proper modeling.
56
68
  # Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
57
69
  # @param secondary_refs [Hash] TODO
58
- def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs)
59
- @manager_ref = manager_ref || %i(ems_ref)
70
+ # @param manager_uuids [Array|Proc] Array of manager_uuids of the InventoryObjects we want to create/update/delete. Using
71
+ # this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
72
+ # case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
73
+ # custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
74
+ def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
75
+ @manager_ref = manager_ref || %i[ems_ref]
60
76
  @manager_ref_allowed_nil = manager_ref_allowed_nil || []
61
77
  @secondary_refs = secondary_refs || {}
78
+ @manager_uuids = manager_uuids || []
79
+ end
80
+
81
+ # @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
82
+ # having this parameter defined will invoke only :delete_method on a complement of this set, making sure
83
+ # the DB has only this set of data after. This :attribute serves for deleting of top level
84
+ # InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
85
+ # child collections is already handled by the scope of the parent_inventory_collections and using Rails
86
+ # :dependent => :destroy,
87
+ # @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
88
+ # all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
89
+ # region.
90
+ # If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
91
+ # scope itself. E.g. sending a list of all active regions, we will delete complement entities not
92
+ # belonging to those regions.
93
+ # Example:
94
+ # :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
95
+ # :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
96
+ #
97
+ # Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
98
+ # regions X and Y.
99
+ # @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
100
+ # all_manager_uuids. Meaning we won't be archiving any newer entities.
101
+ def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
102
+ # TODO(lsmola) Should we refactor this to use references too?
103
+ @all_manager_uuids = all_manager_uuids
104
+ @all_manager_uuids_scope = all_manager_uuids_scope
105
+ @all_manager_uuids_timestamp = all_manager_uuids_timestamp
62
106
  end
63
107
 
64
108
  # @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
@@ -75,9 +119,48 @@ module InventoryRefresh
75
119
  # This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
76
120
  # will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
77
121
  # are saved.
78
- def init_ic_relations(dependency_attributes)
79
- @dependency_attributes = dependency_attributes || {}
80
- @dependees = Set.new
122
+ # @param parent_inventory_collections [Array] Array of symbols having a name pointing to the
123
+ # InventoryRefresh::InventoryCollection objects, that serve as parents to this InventoryCollection. There are
124
+ # several scenarios to consider, when deciding if InventoryCollection has parent collections, see the example.
125
+ #
126
+ # Example:
127
+ # taking inventory collections :vms and :disks (local disks), if we write that:
128
+ # inventory_collection = InventoryCollection.new({
129
+ # :model_class => ::Disk,
130
+ # :association => :disks,
131
+ # :manager_ref => [:vm, :location]
132
+ # :parent_inventory_collection => [:vms],
133
+ # })
134
+ #
135
+ # Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
136
+ # points:
137
+ # 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
138
+ # from vms to disks table).
139
+ # 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
140
+ # a Vm subquery
141
+ # 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
142
+ # only under 1 Vm. (In current models, this modeled going through Hardware model)
143
+ # 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
144
+ #
145
+ # Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
146
+ # implications:
147
+ # 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
148
+ # archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
149
+ # 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
150
+ # we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
151
+ # archived/removed before the Vm, while we always want to archive the VM first.
152
+ # 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
153
+ # we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
154
+ # always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
155
+ # :manager_uuids are mutually exclusive attributes)
156
+ # 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
157
+ # While this information can be introspected from the data, it creates a scope for create&update&delete,
158
+ # which means it has to work with no data provided (causing delete all). So with no data we cannot
159
+ # introspect anything.
160
+ def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
161
+ @dependency_attributes = dependency_attributes || {}
162
+ @dependees = Set.new
163
+ @parent_inventory_collections = parent_inventory_collections
81
164
  end
82
165
 
83
166
  # @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
@@ -92,13 +175,18 @@ module InventoryRefresh
92
175
  # @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
93
176
  # it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
94
177
  # for the batch saver strategy.
95
- def init_flags(complete, create_only, check_changed, update_only, use_ar_object, assert_graph_integrity)
178
+ # @param targeted [Boolean] True if the collection is targeted, in that case it will be leveraging :manager_uuids
179
+ # :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
180
+ def init_flags(complete, create_only, check_changed,
181
+ update_only, use_ar_object, targeted,
182
+ assert_graph_integrity)
96
183
  @complete = complete.nil? ? true : complete
97
184
  @create_only = create_only.nil? ? false : create_only
98
185
  @check_changed = check_changed.nil? ? true : check_changed
99
186
  @saved = false
100
187
  @update_only = update_only.nil? ? false : update_only
101
188
  @use_ar_object = use_ar_object || false
189
+ @targeted = !!targeted
102
190
  @assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
103
191
  end
104
192
 
@@ -142,7 +230,7 @@ module InventoryRefresh
142
230
  @attributes_whitelist = Set.new
143
231
  @batch_extra_attributes = batch_extra_attributes || []
144
232
  @inventory_object_attributes = inventory_object_attributes
145
- @internal_attributes = %i(__feedback_edge_set_parent)
233
+ @internal_attributes = %i[__feedback_edge_set_parent __parent_inventory_collections __all_manager_uuids_scope]
146
234
  @transitive_dependency_attributes = Set.new
147
235
 
148
236
  blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
@@ -152,6 +240,7 @@ module InventoryRefresh
152
240
  def init_storages
153
241
  @data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
154
242
  @references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
243
+ @targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
155
244
  end
156
245
 
157
246
  # @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
@@ -159,18 +248,35 @@ module InventoryRefresh
159
248
  # doing create/update/delete.
160
249
  #
161
250
  # Example:
162
- # add_collection(:cross_link_vms) do |builder|
163
- # builder.add_properties(
164
- # :arel => Vm.where(:tenant => manager.tenant),
165
- # :association => nil,
166
- # :model_class => Vm,
167
- # :name => :cross_link_vms,
168
- # :manager_ref => [:uid_ems],
169
- # :strategy => :local_db_find_references,
251
+ # for a targeted refresh, we want to delete/update/create only a list of vms specified with a list of
252
+ # ems_refs:
253
+ # :arel => manager.vms.where(:ems_ref => manager_refs)
254
+ # Then we want to do the same for the hardwares of only those vms:
255
+ # :arel => manager.hardwares.joins(:vm_or_template).where(
256
+ # 'vms' => {:ems_ref => manager_refs}
257
+ # )
258
+ # And etc. for the other Vm related records.
259
+ # @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
260
+ # we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
261
+ # manager_uuids available.
262
+ # Example:
263
+ # targeted_arel = lambda do |inventory_collection|
264
+ # # Getting ems_refs of parent :vms and :miq_templates
265
+ # manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
266
+ # inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
267
+ # 'vms' => {:ems_ref => manager_uuids}
170
268
  # )
171
269
  # end
172
- def init_arels(arel)
173
- @arel = arel
270
+ #
271
+ # inventory_collection = InventoryCollection.new({
272
+ # :model_class => ::Hardware,
273
+ # :association => :hardwares,
274
+ # :parent_inventory_collection => [:vms, :miq_templates],
275
+ # :targeted_arel => targeted_arel,
276
+ # })
277
+ def init_arels(arel, targeted_arel)
278
+ @arel = arel
279
+ @targeted_arel = targeted_arel
174
280
  end
175
281
 
176
282
  # @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
@@ -183,7 +289,7 @@ module InventoryRefresh
183
289
  # hash = inventory_object.attributes # Loads possible dependencies into saveable hash
184
290
  # obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
185
291
  # # queries, avoid this, this is just a simple example :-)
186
- # obj.update_attributes(hash) if obj
292
+ # obj.update(hash) if obj
187
293
  # obj ||= SomeModel.create(hash)
188
294
  # inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
189
295
  # primary key back to the InventoryObject
@@ -286,6 +392,23 @@ module InventoryRefresh
286
392
  @deleted_records = []
287
393
  end
288
394
 
395
+ # Processes passed saver strategy
396
+ #
397
+ # @param saver_strategy [Symbol] Passed saver strategy
398
+ # @return [Symbol] Returns back the passed strategy if supported, or raises exception
399
+ def process_saver_strategy(saver_strategy)
400
+ return :default unless saver_strategy
401
+
402
+ saver_strategy = saver_strategy.to_sym
403
+ case saver_strategy
404
+ when :default, :batch, :concurrent_safe_batch
405
+ saver_strategy
406
+ else
407
+ raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
408
+ ":default, :batch and :concurrent_safe_batch"
409
+ end
410
+ end
411
+
289
412
  # Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
290
413
  #
291
414
  # @param strategy_name [Symbol] Passed saver strategy
@@ -306,31 +429,17 @@ module InventoryRefresh
306
429
  nil
307
430
  else
308
431
  raise "Unknown InventoryCollection strategy: :#{strategy_name}, allowed strategies are :local_db_cache_all, "\
309
- ":local_db_find_references and :local_db_find_missing_references."
432
+ ":local_db_find_references and :local_db_find_missing_references."
310
433
  end
311
434
  strategy_name
312
435
  end
313
436
 
314
- # Saves passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
315
- #
316
- # @param strategy [Symbol] Passed saver strategy
317
- # @return [Symbol] Returns back the passed strategy if supported, or raises exception
318
- def strategy=(strategy)
319
- @strategy = process_strategy(strategy)
320
- end
321
-
322
437
  # Processes passed retention strategy
323
438
  #
324
439
  # @param retention_strategy [Symbol] Passed retention strategy
325
440
  # @return [Symbol] Returns back the passed strategy if supported, or raises exception
326
441
  def process_retention_strategy(retention_strategy)
327
- unless retention_strategy
328
- if supports_column?(:archived_at)
329
- return :archive
330
- else
331
- return :destroy
332
- end
333
- end
442
+ return unless retention_strategy
334
443
 
335
444
  retention_strategy = retention_strategy.to_sym
336
445
  case retention_strategy
@@ -338,7 +447,7 @@ module InventoryRefresh
338
447
  retention_strategy
339
448
  else
340
449
  raise "Unknown InventoryCollection retention strategy: :#{retention_strategy}, allowed strategies are "\
341
- ":destroy and :archive"
450
+ ":destroy and :archive"
342
451
  end
343
452
  end
344
453
  end
@@ -50,10 +50,16 @@ module InventoryRefresh
50
50
  dependencies.all?(&:saved?)
51
51
  end
52
52
 
53
+ # @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
54
+ def parallel_safe?
55
+ return @parallel_safe unless @parallel_safe.nil?
56
+ @parallel_safe = %i[concurrent_safe concurrent_safe_batch].include?(saver_strategy)
57
+ end
58
+
53
59
  # @return [Boolean] true if the model_class supports STI
54
60
  def supports_sti?
55
- @supports_sti_cache = model_class&.column_names.to_a.include?("type") if @supports_sti_cache.nil?
56
- @supports_sti_cache
61
+ return @supports_sti unless @supports_sti.nil?
62
+ @supports_sti = model_class&.column_names.to_a.include?("type")
57
63
  end
58
64
 
59
65
  # @param column_name [Symbol, String]
@@ -63,7 +69,7 @@ module InventoryRefresh
63
69
  return @supported_cols_cache[column_name.to_sym] unless @supported_cols_cache[column_name.to_sym].nil?
64
70
 
65
71
  include_col = model_class&.column_names.to_a.include?(column_name.to_s)
66
- if %w(created_on created_at updated_on updated_at).include?(column_name.to_s)
72
+ if %w[created_on created_at updated_on updated_at].include?(column_name.to_s)
67
73
  include_col &&= ActiveRecord::Base.record_timestamps
68
74
  end
69
75
 
@@ -76,12 +82,50 @@ module InventoryRefresh
76
82
  data_collection_finalized
77
83
  end
78
84
 
85
+ # @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
86
+ def targeted?
87
+ targeted
88
+ end
89
+
79
90
  # True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
80
91
  # stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
81
92
  #
82
93
  # @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
83
94
  def noop?
84
- data.blank? && custom_save_block.nil? && skeletal_primary_index.blank?
95
+ # If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
96
+ saving_noop? && delete_complement_noop?
97
+ end
98
+
99
+ # @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
100
+ def saving_noop?
101
+ saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
102
+ end
103
+
104
+ # @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
105
+ def delete_complement_noop?
106
+ all_manager_uuids.nil?
107
+ end
108
+
109
+ private
110
+
111
+ # @return true if it's a noop parent targeted InventoryCollection
112
+ def saving_targeted_parent_collection_noop?
113
+ targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
114
+ end
115
+
116
+ # @return true if it's a noop child targeted InventoryCollection
117
+ def saving_targeted_child_collection_noop?
118
+ targeted_noop_condition && parent_inventory_collections.present? &&
119
+ parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
120
+ end
121
+
122
+ # @return true if it's a noop full InventoryCollection refresh
123
+ def saving_full_collection_noop?
124
+ !targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
125
+ end
126
+
127
+ def targeted_noop_condition
128
+ targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
85
129
  end
86
130
  end
87
131
  end
@@ -59,11 +59,19 @@ module InventoryRefresh
59
59
  end
60
60
 
61
61
  def build_secondary_indexes_for(inventory_object)
62
- secondary_refs.keys.each do |ref|
62
+ secondary_refs.each_key do |ref|
63
63
  data_index(ref).store_index_for(inventory_object)
64
64
  end
65
65
  end
66
66
 
67
+ def reindex_secondary_indexes!
68
+ data_indexes.each do |ref, index|
69
+ next if ref == primary_index_ref
70
+
71
+ index.reindex!
72
+ end
73
+ end
74
+
67
75
  def primary_index
68
76
  data_index(primary_index_ref)
69
77
  end
@@ -72,6 +80,7 @@ module InventoryRefresh
72
80
  # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
73
81
  # TODO(lsmola) this method should return lazy too, the rest of the finders should be deprecated
74
82
  return if reference.nil?
83
+
75
84
  assert_index(reference, ref)
76
85
 
77
86
  reference = inventory_collection.build_reference(reference, ref)
@@ -86,8 +95,23 @@ module InventoryRefresh
86
95
  end
87
96
  end
88
97
 
98
+ def find_by(manager_uuid_hash, ref: primary_index_ref)
99
+ # TODO(lsmola) deprecate this, it's enough to have find method
100
+ find(manager_uuid_hash, :ref => ref)
101
+ end
102
+
103
+ def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
104
+ # TODO(lsmola) deprecate this, it's enough to have lazy_find method
105
+
106
+ lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
107
+ end
108
+
89
109
  def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
110
+ # TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
111
+ # access the data
112
+ # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
90
113
  return if manager_uuid.nil?
114
+
91
115
  assert_index(manager_uuid, ref)
92
116
 
93
117
  ::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
@@ -110,13 +134,19 @@ module InventoryRefresh
110
134
 
111
135
  delegate :association_to_foreign_key_mapping,
112
136
  :build_stringified_reference,
137
+ :parallel_safe?,
113
138
  :strategy,
114
139
  :to => :inventory_collection
115
140
 
116
141
  attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
117
142
 
118
143
  def find_in_data_or_skeletal_index(reference)
119
- data_index_find(reference) || skeletal_index_find(reference)
144
+ if parallel_safe?
145
+ # With parallel safe strategies, we create skeletal nodes that we can look for
146
+ data_index_find(reference) || skeletal_index_find(reference)
147
+ else
148
+ data_index_find(reference)
149
+ end
120
150
  end
121
151
 
122
152
  def skeletal_index_find(reference)
@@ -154,6 +184,7 @@ module InventoryRefresh
154
184
  next unless association_to_foreign_key_mapping[key]
155
185
  # Skip if data on key are nil or InventoryObject or InventoryObjectLazy
156
186
  next if data[key].nil? || data[key].kind_of?(InventoryRefresh::InventoryObject) || data[key].kind_of?(InventoryRefresh::InventoryObjectLazy)
187
+
157
188
  # Raise error since relation must be nil or InventoryObject or InventoryObjectLazy
158
189
  raise "Wrong index for key :#{key}, the value must be of type Nil or InventoryObject or InventoryObjectLazy, got: #{data[key]}"
159
190
  end
@@ -179,6 +210,7 @@ module InventoryRefresh
179
210
  unless required_index_keys_present?(manager_uuid.keys, ref)
180
211
  raise "Finder has missing keys for index :#{ref}, missing indexes are: #{missing_keys(manager_uuid.keys, ref)}"
181
212
  end
213
+
182
214
  # Test that keys, that are relations, are nil or InventoryObject or InventoryObjectlazy class
183
215
  assert_relation_keys(manager_uuid, ref)
184
216
  else
@@ -196,7 +228,7 @@ module InventoryRefresh
196
228
  assert_relation_keys({named_ref(ref).first => manager_uuid}, ref)
197
229
  end
198
230
  rescue => e
199
- #_log.error("Error when asserting index: #{manager_uuid}, with ref: #{ref} of: #{inventory_collection}")
231
+ logger.error("Error when asserting index: #{manager_uuid}, with ref: #{ref} of: #{inventory_collection}")
200
232
  raise e
201
233
  end
202
234
  end
@@ -27,6 +27,14 @@ module InventoryRefresh
27
27
  index[build_stringified_reference(inventory_object.data, attribute_names)] = inventory_object
28
28
  end
29
29
 
30
+ # Rebuilds the indexes for all InventoryObject objects
31
+ def reindex!
32
+ self.index = {}
33
+ data.each do |inventory_object|
34
+ store_index_for(inventory_object)
35
+ end
36
+ end
37
+
30
38
  # @return [Array] Returns index data
31
39
  def index_data
32
40
  index.values
@@ -31,6 +31,7 @@ module InventoryRefresh
31
31
  return index[reference.stringified_reference]
32
32
  else
33
33
  return index[reference.stringified_reference] if index && index[reference.stringified_reference]
34
+
34
35
  # We haven't found the reference, lets add it to the list of references and load it
35
36
  add_reference(reference)
36
37
  end
@@ -271,6 +272,7 @@ module InventoryRefresh
271
272
  # and find used in parser
272
273
  # TODO(lsmola) the last usage of this should be lazy_find_by with :key specified, maybe we can get rid of this?
273
274
  next unless (foreign_key = association_to_foreign_key_mapping[ref])
275
+
274
276
  base_class_name = attributes[association_to_foreign_type_mapping[ref].try(:to_sym)] || association_to_base_class_mapping[ref]
275
277
  id = attributes[foreign_key.to_sym]
276
278
  attributes[ref] = InventoryRefresh::ApplicationRecordReference.new(base_class_name, id)
@@ -47,6 +47,7 @@ module InventoryRefresh
47
47
  def skeletonize_primary_index(index_value)
48
48
  inventory_object = primary_index.delete(index_value)
49
49
  return unless inventory_object
50
+
50
51
  fill_versions!(inventory_object.data)
51
52
 
52
53
  index[index_value] = inventory_object
@@ -92,6 +92,7 @@ module InventoryRefresh
92
92
  data
93
93
  else
94
94
  raise "Please provide Hash as a reference, :manager_ref count includes more than 1 attribute. keys: #{keys}, data: #{data}" if keys.size > 1
95
+
95
96
  {keys.first => data}
96
97
  end
97
98
  end
@@ -45,6 +45,23 @@ module InventoryRefresh
45
45
  add_reference(reference_data)
46
46
  end
47
47
 
48
+ # Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
49
+ # it.
50
+ #
51
+ # @param references_array [Array] Array of reference objects acceptable by add_reference method.
52
+ # @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
53
+ # index.
54
+ # @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
55
+ def merge!(references_array, ref: nil)
56
+ references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
57
+ self
58
+ end
59
+
60
+ # @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
61
+ def primary_references
62
+ references[primary_index_ref]
63
+ end
64
+
48
65
  # Builds a Reference object
49
66
  #
50
67
  # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference