inventory_refresh 0.3.5 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +25 -30
  3. data/.github/workflows/ci.yaml +58 -0
  4. data/.rubocop.yml +3 -3
  5. data/.rubocop_cc.yml +3 -4
  6. data/.rubocop_local.yml +5 -2
  7. data/.whitesource +3 -0
  8. data/CHANGELOG.md +19 -0
  9. data/Gemfile +10 -4
  10. data/README.md +1 -2
  11. data/Rakefile +2 -2
  12. data/inventory_refresh.gemspec +9 -10
  13. data/lib/inventory_refresh/application_record_iterator.rb +25 -12
  14. data/lib/inventory_refresh/graph/topological_sort.rb +24 -26
  15. data/lib/inventory_refresh/graph.rb +2 -2
  16. data/lib/inventory_refresh/inventory_collection/builder.rb +37 -15
  17. data/lib/inventory_refresh/inventory_collection/data_storage.rb +9 -0
  18. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +147 -38
  19. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +48 -4
  20. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +35 -3
  21. data/lib/inventory_refresh/inventory_collection/index/type/base.rb +8 -0
  22. data/lib/inventory_refresh/inventory_collection/index/type/local_db.rb +2 -0
  23. data/lib/inventory_refresh/inventory_collection/index/type/skeletal.rb +1 -0
  24. data/lib/inventory_refresh/inventory_collection/reference.rb +1 -0
  25. data/lib/inventory_refresh/inventory_collection/references_storage.rb +17 -0
  26. data/lib/inventory_refresh/inventory_collection/scanner.rb +91 -3
  27. data/lib/inventory_refresh/inventory_collection/serialization.rb +16 -10
  28. data/lib/inventory_refresh/inventory_collection.rb +122 -64
  29. data/lib/inventory_refresh/inventory_object.rb +74 -40
  30. data/lib/inventory_refresh/inventory_object_lazy.rb +17 -10
  31. data/lib/inventory_refresh/null_logger.rb +2 -2
  32. data/lib/inventory_refresh/persister.rb +43 -93
  33. data/lib/inventory_refresh/save_collection/base.rb +4 -2
  34. data/lib/inventory_refresh/save_collection/saver/base.rb +114 -15
  35. data/lib/inventory_refresh/save_collection/saver/batch.rb +17 -0
  36. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +129 -51
  37. data/lib/inventory_refresh/save_collection/saver/default.rb +57 -0
  38. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +2 -19
  39. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +68 -3
  40. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +125 -0
  41. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +10 -6
  42. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +28 -16
  43. data/lib/inventory_refresh/save_collection/sweeper.rb +17 -93
  44. data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
  45. data/lib/inventory_refresh/save_inventory.rb +5 -12
  46. data/lib/inventory_refresh/target.rb +73 -0
  47. data/lib/inventory_refresh/target_collection.rb +92 -0
  48. data/lib/inventory_refresh/version.rb +1 -1
  49. data/lib/inventory_refresh.rb +2 -0
  50. metadata +42 -39
  51. data/.travis.yml +0 -23
  52. data/lib/inventory_refresh/exception.rb +0 -8
@@ -35,14 +35,26 @@ module InventoryRefresh
35
35
  # - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
36
36
  # the DB. Then if we reference an object that is not present, it will
37
37
  # load them from the db using :local_db_find_references strategy.
38
+ # @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
39
+ # Allowed saver strategies are:
40
+ # - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
41
+ # since it will lead to non consistent data.
42
+ # - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
43
+ # concurrently, since it will lead to non consistent data.
44
+ # - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
45
+ # atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
46
+ # executed as batched SQL queries, instead of sending 1 query per record.
38
47
  # @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
39
48
  # - :destroy => Will destroy the inactive records.
40
49
  # - :archive => Will archive the inactive records by setting :archived_at timestamp.
41
- def init_strategies(strategy, retention_strategy)
42
- @saver_strategy = :concurrent_safe_batch
50
+ # @param delete_method [Symbol] A delete method that will be used for deleting of the InventoryObject, if the
51
+ # object is marked for deletion. A default is :destroy, the instance method must be defined on the
52
+ # :model_class.
53
+ def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
54
+ @saver_strategy = process_saver_strategy(saver_strategy)
43
55
  @strategy = process_strategy(strategy)
44
- # TODO(lsmola) why don't we just set this strategy based on :archived_at column? Lets do that
45
56
  @retention_strategy = process_retention_strategy(retention_strategy)
57
+ @delete_method = delete_method || :destroy
46
58
  end
47
59
 
48
60
  # @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
@@ -55,10 +67,42 @@ module InventoryRefresh
55
67
  # avoid this usecase by a proper modeling.
56
68
  # Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
57
69
  # @param secondary_refs [Hash] TODO
58
- def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs)
59
- @manager_ref = manager_ref || %i(ems_ref)
70
+ # @param manager_uuids [Array|Proc] Array of manager_uuids of the InventoryObjects we want to create/update/delete. Using
71
+ # this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
72
+ # case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
73
+ # custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
74
+ def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
75
+ @manager_ref = manager_ref || %i[ems_ref]
60
76
  @manager_ref_allowed_nil = manager_ref_allowed_nil || []
61
77
  @secondary_refs = secondary_refs || {}
78
+ @manager_uuids = manager_uuids || []
79
+ end
80
+
81
+ # @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
82
+ # having this parameter defined will invoke only :delete_method on a complement of this set, making sure
83
+ # the DB has only this set of data after. This :attribute serves for deleting of top level
84
+ # InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
85
+ # child collections is already handled by the scope of the parent_inventory_collections and using Rails
86
+ # :dependent => :destroy,
87
+ # @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
88
+ # all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
89
+ # region.
90
+ # If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
91
+ # scope itself. E.g. sending a list of all active regions, we will delete complement entities not
92
+ # belonging to those regions.
93
+ # Example:
94
+ # :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
95
+ # :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
96
+ #
97
+ # Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
98
+ # regions X and Y.
99
+ # @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
100
+ # all_manager_uuids. Meaning we won't be archiving any newer entities.
101
+ def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
102
+ # TODO(lsmola) Should we refactor this to use references too?
103
+ @all_manager_uuids = all_manager_uuids
104
+ @all_manager_uuids_scope = all_manager_uuids_scope
105
+ @all_manager_uuids_timestamp = all_manager_uuids_timestamp
62
106
  end
63
107
 
64
108
  # @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
@@ -75,9 +119,48 @@ module InventoryRefresh
75
119
  # This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
76
120
  # will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
77
121
  # are saved.
78
- def init_ic_relations(dependency_attributes)
79
- @dependency_attributes = dependency_attributes || {}
80
- @dependees = Set.new
122
+ # @param parent_inventory_collections [Array] Array of symbols having a name pointing to the
123
+ # InventoryRefresh::InventoryCollection objects, that serve as parents to this InventoryCollection. There are
124
+ # several scenarios to consider, when deciding if InventoryCollection has parent collections, see the example.
125
+ #
126
+ # Example:
127
+ # taking inventory collections :vms and :disks (local disks), if we write that:
128
+ # inventory_collection = InventoryCollection.new({
129
+ # :model_class => ::Disk,
130
+ # :association => :disks,
131
+ # :manager_ref => [:vm, :location]
132
+ # :parent_inventory_collection => [:vms],
133
+ # })
134
+ #
135
+ # Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
136
+ # points:
137
+ # 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
138
+ # from vms to disks table).
139
+ # 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
140
+ # a Vm subquery
141
+ # 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
142
+ # only under 1 Vm. (In current models, this modeled going through Hardware model)
143
+ # 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
144
+ #
145
+ # Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
146
+ # implications:
147
+ # 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
148
+ # archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
149
+ # 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
150
+ # we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
151
+ # archived/removed before the Vm, while we always want to archive the VM first.
152
+ # 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
153
+ # we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
154
+ # always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
155
+ # :manager_uuids are mutually exclusive attributes)
156
+ # 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
157
+ # While this information can be introspected from the data, it creates a scope for create&update&delete,
158
+ # which means it has to work with no data provided (causing delete all). So with no data we cannot
159
+ # introspect anything.
160
+ def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
161
+ @dependency_attributes = dependency_attributes || {}
162
+ @dependees = Set.new
163
+ @parent_inventory_collections = parent_inventory_collections
81
164
  end
82
165
 
83
166
  # @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
@@ -92,13 +175,18 @@ module InventoryRefresh
92
175
  # @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
93
176
  # it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
94
177
  # for the batch saver strategy.
95
- def init_flags(complete, create_only, check_changed, update_only, use_ar_object, assert_graph_integrity)
178
+ # @param targeted [Boolean] True if the collection is targeted, in that case it will be leveraging :manager_uuids
179
+ # :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
180
+ def init_flags(complete, create_only, check_changed,
181
+ update_only, use_ar_object, targeted,
182
+ assert_graph_integrity)
96
183
  @complete = complete.nil? ? true : complete
97
184
  @create_only = create_only.nil? ? false : create_only
98
185
  @check_changed = check_changed.nil? ? true : check_changed
99
186
  @saved = false
100
187
  @update_only = update_only.nil? ? false : update_only
101
188
  @use_ar_object = use_ar_object || false
189
+ @targeted = !!targeted
102
190
  @assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
103
191
  end
104
192
 
@@ -142,7 +230,7 @@ module InventoryRefresh
142
230
  @attributes_whitelist = Set.new
143
231
  @batch_extra_attributes = batch_extra_attributes || []
144
232
  @inventory_object_attributes = inventory_object_attributes
145
- @internal_attributes = %i(__feedback_edge_set_parent)
233
+ @internal_attributes = %i[__feedback_edge_set_parent __parent_inventory_collections __all_manager_uuids_scope]
146
234
  @transitive_dependency_attributes = Set.new
147
235
 
148
236
  blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
@@ -152,6 +240,7 @@ module InventoryRefresh
152
240
  def init_storages
153
241
  @data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
154
242
  @references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
243
+ @targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
155
244
  end
156
245
 
157
246
  # @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
@@ -159,18 +248,35 @@ module InventoryRefresh
159
248
  # doing create/update/delete.
160
249
  #
161
250
  # Example:
162
- # add_collection(:cross_link_vms) do |builder|
163
- # builder.add_properties(
164
- # :arel => Vm.where(:tenant => manager.tenant),
165
- # :association => nil,
166
- # :model_class => Vm,
167
- # :name => :cross_link_vms,
168
- # :manager_ref => [:uid_ems],
169
- # :strategy => :local_db_find_references,
251
+ # for a targeted refresh, we want to delete/update/create only a list of vms specified with a list of
252
+ # ems_refs:
253
+ # :arel => manager.vms.where(:ems_ref => manager_refs)
254
+ # Then we want to do the same for the hardwares of only those vms:
255
+ # :arel => manager.hardwares.joins(:vm_or_template).where(
256
+ # 'vms' => {:ems_ref => manager_refs}
257
+ # )
258
+ # And etc. for the other Vm related records.
259
+ # @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
260
+ # we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
261
+ # manager_uuids available.
262
+ # Example:
263
+ # targeted_arel = lambda do |inventory_collection|
264
+ # # Getting ems_refs of parent :vms and :miq_templates
265
+ # manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
266
+ # inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
267
+ # 'vms' => {:ems_ref => manager_uuids}
170
268
  # )
171
269
  # end
172
- def init_arels(arel)
173
- @arel = arel
270
+ #
271
+ # inventory_collection = InventoryCollection.new({
272
+ # :model_class => ::Hardware,
273
+ # :association => :hardwares,
274
+ # :parent_inventory_collection => [:vms, :miq_templates],
275
+ # :targeted_arel => targeted_arel,
276
+ # })
277
+ def init_arels(arel, targeted_arel)
278
+ @arel = arel
279
+ @targeted_arel = targeted_arel
174
280
  end
175
281
 
176
282
  # @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
@@ -183,7 +289,7 @@ module InventoryRefresh
183
289
  # hash = inventory_object.attributes # Loads possible dependencies into saveable hash
184
290
  # obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
185
291
  # # queries, avoid this, this is just a simple example :-)
186
- # obj.update_attributes(hash) if obj
292
+ # obj.update(hash) if obj
187
293
  # obj ||= SomeModel.create(hash)
188
294
  # inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
189
295
  # primary key back to the InventoryObject
@@ -286,6 +392,23 @@ module InventoryRefresh
286
392
  @deleted_records = []
287
393
  end
288
394
 
395
+ # Processes passed saver strategy
396
+ #
397
+ # @param saver_strategy [Symbol] Passed saver strategy
398
+ # @return [Symbol] Returns back the passed strategy if supported, or raises exception
399
+ def process_saver_strategy(saver_strategy)
400
+ return :default unless saver_strategy
401
+
402
+ saver_strategy = saver_strategy.to_sym
403
+ case saver_strategy
404
+ when :default, :batch, :concurrent_safe_batch
405
+ saver_strategy
406
+ else
407
+ raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
408
+ ":default, :batch and :concurrent_safe_batch"
409
+ end
410
+ end
411
+
289
412
  # Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
290
413
  #
291
414
  # @param strategy_name [Symbol] Passed saver strategy
@@ -306,31 +429,17 @@ module InventoryRefresh
306
429
  nil
307
430
  else
308
431
  raise "Unknown InventoryCollection strategy: :#{strategy_name}, allowed strategies are :local_db_cache_all, "\
309
- ":local_db_find_references and :local_db_find_missing_references."
432
+ ":local_db_find_references and :local_db_find_missing_references."
310
433
  end
311
434
  strategy_name
312
435
  end
313
436
 
314
- # Saves passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
315
- #
316
- # @param strategy [Symbol] Passed saver strategy
317
- # @return [Symbol] Returns back the passed strategy if supported, or raises exception
318
- def strategy=(strategy)
319
- @strategy = process_strategy(strategy)
320
- end
321
-
322
437
  # Processes passed retention strategy
323
438
  #
324
439
  # @param retention_strategy [Symbol] Passed retention strategy
325
440
  # @return [Symbol] Returns back the passed strategy if supported, or raises exception
326
441
  def process_retention_strategy(retention_strategy)
327
- unless retention_strategy
328
- if supports_column?(:archived_at)
329
- return :archive
330
- else
331
- return :destroy
332
- end
333
- end
442
+ return unless retention_strategy
334
443
 
335
444
  retention_strategy = retention_strategy.to_sym
336
445
  case retention_strategy
@@ -338,7 +447,7 @@ module InventoryRefresh
338
447
  retention_strategy
339
448
  else
340
449
  raise "Unknown InventoryCollection retention strategy: :#{retention_strategy}, allowed strategies are "\
341
- ":destroy and :archive"
450
+ ":destroy and :archive"
342
451
  end
343
452
  end
344
453
  end
@@ -50,10 +50,16 @@ module InventoryRefresh
50
50
  dependencies.all?(&:saved?)
51
51
  end
52
52
 
53
+ # @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
54
+ def parallel_safe?
55
+ return @parallel_safe unless @parallel_safe.nil?
56
+ @parallel_safe = %i[concurrent_safe concurrent_safe_batch].include?(saver_strategy)
57
+ end
58
+
53
59
  # @return [Boolean] true if the model_class supports STI
54
60
  def supports_sti?
55
- @supports_sti_cache = model_class&.column_names.to_a.include?("type") if @supports_sti_cache.nil?
56
- @supports_sti_cache
61
+ return @supports_sti unless @supports_sti.nil?
62
+ @supports_sti = model_class&.column_names.to_a.include?("type")
57
63
  end
58
64
 
59
65
  # @param column_name [Symbol, String]
@@ -63,7 +69,7 @@ module InventoryRefresh
63
69
  return @supported_cols_cache[column_name.to_sym] unless @supported_cols_cache[column_name.to_sym].nil?
64
70
 
65
71
  include_col = model_class&.column_names.to_a.include?(column_name.to_s)
66
- if %w(created_on created_at updated_on updated_at).include?(column_name.to_s)
72
+ if %w[created_on created_at updated_on updated_at].include?(column_name.to_s)
67
73
  include_col &&= ActiveRecord::Base.record_timestamps
68
74
  end
69
75
 
@@ -76,12 +82,50 @@ module InventoryRefresh
76
82
  data_collection_finalized
77
83
  end
78
84
 
85
+ # @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
86
+ def targeted?
87
+ targeted
88
+ end
89
+
79
90
  # True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
80
91
  # stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
81
92
  #
82
93
  # @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
83
94
  def noop?
84
- data.blank? && custom_save_block.nil? && skeletal_primary_index.blank?
95
+ # If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
96
+ saving_noop? && delete_complement_noop?
97
+ end
98
+
99
+ # @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
100
+ def saving_noop?
101
+ saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
102
+ end
103
+
104
+ # @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
105
+ def delete_complement_noop?
106
+ all_manager_uuids.nil?
107
+ end
108
+
109
+ private
110
+
111
+ # @return true if it's a noop parent targeted InventoryCollection
112
+ def saving_targeted_parent_collection_noop?
113
+ targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
114
+ end
115
+
116
+ # @return true if it's a noop child targeted InventoryCollection
117
+ def saving_targeted_child_collection_noop?
118
+ targeted_noop_condition && parent_inventory_collections.present? &&
119
+ parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
120
+ end
121
+
122
+ # @return true if it's a noop full InventoryCollection refresh
123
+ def saving_full_collection_noop?
124
+ !targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
125
+ end
126
+
127
+ def targeted_noop_condition
128
+ targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
85
129
  end
86
130
  end
87
131
  end
@@ -59,11 +59,19 @@ module InventoryRefresh
59
59
  end
60
60
 
61
61
  def build_secondary_indexes_for(inventory_object)
62
- secondary_refs.keys.each do |ref|
62
+ secondary_refs.each_key do |ref|
63
63
  data_index(ref).store_index_for(inventory_object)
64
64
  end
65
65
  end
66
66
 
67
+ def reindex_secondary_indexes!
68
+ data_indexes.each do |ref, index|
69
+ next if ref == primary_index_ref
70
+
71
+ index.reindex!
72
+ end
73
+ end
74
+
67
75
  def primary_index
68
76
  data_index(primary_index_ref)
69
77
  end
@@ -72,6 +80,7 @@ module InventoryRefresh
72
80
  # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
73
81
  # TODO(lsmola) this method should return lazy too, the rest of the finders should be deprecated
74
82
  return if reference.nil?
83
+
75
84
  assert_index(reference, ref)
76
85
 
77
86
  reference = inventory_collection.build_reference(reference, ref)
@@ -86,8 +95,23 @@ module InventoryRefresh
86
95
  end
87
96
  end
88
97
 
98
+ def find_by(manager_uuid_hash, ref: primary_index_ref)
99
+ # TODO(lsmola) deprecate this, it's enough to have find method
100
+ find(manager_uuid_hash, :ref => ref)
101
+ end
102
+
103
+ def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
104
+ # TODO(lsmola) deprecate this, it's enough to have lazy_find method
105
+
106
+ lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
107
+ end
108
+
89
109
  def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
110
+ # TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
111
+ # access the data
112
+ # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
90
113
  return if manager_uuid.nil?
114
+
91
115
  assert_index(manager_uuid, ref)
92
116
 
93
117
  ::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
@@ -110,13 +134,19 @@ module InventoryRefresh
110
134
 
111
135
  delegate :association_to_foreign_key_mapping,
112
136
  :build_stringified_reference,
137
+ :parallel_safe?,
113
138
  :strategy,
114
139
  :to => :inventory_collection
115
140
 
116
141
  attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
117
142
 
118
143
  def find_in_data_or_skeletal_index(reference)
119
- data_index_find(reference) || skeletal_index_find(reference)
144
+ if parallel_safe?
145
+ # With parallel safe strategies, we create skeletal nodes that we can look for
146
+ data_index_find(reference) || skeletal_index_find(reference)
147
+ else
148
+ data_index_find(reference)
149
+ end
120
150
  end
121
151
 
122
152
  def skeletal_index_find(reference)
@@ -154,6 +184,7 @@ module InventoryRefresh
154
184
  next unless association_to_foreign_key_mapping[key]
155
185
  # Skip if data on key are nil or InventoryObject or InventoryObjectLazy
156
186
  next if data[key].nil? || data[key].kind_of?(InventoryRefresh::InventoryObject) || data[key].kind_of?(InventoryRefresh::InventoryObjectLazy)
187
+
157
188
  # Raise error since relation must be nil or InventoryObject or InventoryObjectLazy
158
189
  raise "Wrong index for key :#{key}, the value must be of type Nil or InventoryObject or InventoryObjectLazy, got: #{data[key]}"
159
190
  end
@@ -179,6 +210,7 @@ module InventoryRefresh
179
210
  unless required_index_keys_present?(manager_uuid.keys, ref)
180
211
  raise "Finder has missing keys for index :#{ref}, missing indexes are: #{missing_keys(manager_uuid.keys, ref)}"
181
212
  end
213
+
182
214
  # Test that keys, that are relations, are nil or InventoryObject or InventoryObjectlazy class
183
215
  assert_relation_keys(manager_uuid, ref)
184
216
  else
@@ -196,7 +228,7 @@ module InventoryRefresh
196
228
  assert_relation_keys({named_ref(ref).first => manager_uuid}, ref)
197
229
  end
198
230
  rescue => e
199
- #_log.error("Error when asserting index: #{manager_uuid}, with ref: #{ref} of: #{inventory_collection}")
231
+ logger.error("Error when asserting index: #{manager_uuid}, with ref: #{ref} of: #{inventory_collection}")
200
232
  raise e
201
233
  end
202
234
  end
@@ -27,6 +27,14 @@ module InventoryRefresh
27
27
  index[build_stringified_reference(inventory_object.data, attribute_names)] = inventory_object
28
28
  end
29
29
 
30
+ # Rebuilds the indexes for all InventoryObject objects
31
+ def reindex!
32
+ self.index = {}
33
+ data.each do |inventory_object|
34
+ store_index_for(inventory_object)
35
+ end
36
+ end
37
+
30
38
  # @return [Array] Returns index data
31
39
  def index_data
32
40
  index.values
@@ -31,6 +31,7 @@ module InventoryRefresh
31
31
  return index[reference.stringified_reference]
32
32
  else
33
33
  return index[reference.stringified_reference] if index && index[reference.stringified_reference]
34
+
34
35
  # We haven't found the reference, lets add it to the list of references and load it
35
36
  add_reference(reference)
36
37
  end
@@ -271,6 +272,7 @@ module InventoryRefresh
271
272
  # and find used in parser
272
273
  # TODO(lsmola) the last usage of this should be lazy_find_by with :key specified, maybe we can get rid of this?
273
274
  next unless (foreign_key = association_to_foreign_key_mapping[ref])
275
+
274
276
  base_class_name = attributes[association_to_foreign_type_mapping[ref].try(:to_sym)] || association_to_base_class_mapping[ref]
275
277
  id = attributes[foreign_key.to_sym]
276
278
  attributes[ref] = InventoryRefresh::ApplicationRecordReference.new(base_class_name, id)
@@ -47,6 +47,7 @@ module InventoryRefresh
47
47
  def skeletonize_primary_index(index_value)
48
48
  inventory_object = primary_index.delete(index_value)
49
49
  return unless inventory_object
50
+
50
51
  fill_versions!(inventory_object.data)
51
52
 
52
53
  index[index_value] = inventory_object
@@ -92,6 +92,7 @@ module InventoryRefresh
92
92
  data
93
93
  else
94
94
  raise "Please provide Hash as a reference, :manager_ref count includes more than 1 attribute. keys: #{keys}, data: #{data}" if keys.size > 1
95
+
95
96
  {keys.first => data}
96
97
  end
97
98
  end
@@ -45,6 +45,23 @@ module InventoryRefresh
45
45
  add_reference(reference_data)
46
46
  end
47
47
 
48
+ # Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
49
+ # it.
50
+ #
51
+ # @param references_array [Array] Array of reference objects acceptable by add_reference method.
52
+ # @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
53
+ # index.
54
+ # @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
55
+ def merge!(references_array, ref: nil)
56
+ references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
57
+ self
58
+ end
59
+
60
+ # @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
61
+ def primary_references
62
+ references[primary_index_ref]
63
+ end
64
+
48
65
  # Builds a Reference object
49
66
  #
50
67
  # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference