inventory_refresh 0.2.3 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +0 -1
  3. data/.travis.yml +6 -8
  4. data/inventory_refresh.gemspec +3 -5
  5. data/lib/inventory_refresh/application_record_iterator.rb +9 -26
  6. data/lib/inventory_refresh/exception.rb +8 -0
  7. data/lib/inventory_refresh/inventory_collection/builder.rb +6 -6
  8. data/lib/inventory_refresh/inventory_collection/data_storage.rb +0 -9
  9. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +35 -144
  10. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +1 -44
  11. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +6 -34
  12. data/lib/inventory_refresh/inventory_collection/index/type/base.rb +0 -8
  13. data/lib/inventory_refresh/inventory_collection/references_storage.rb +0 -17
  14. data/lib/inventory_refresh/inventory_collection/scanner.rb +1 -87
  15. data/lib/inventory_refresh/inventory_collection/serialization.rb +10 -16
  16. data/lib/inventory_refresh/inventory_collection.rb +36 -110
  17. data/lib/inventory_refresh/inventory_object.rb +34 -68
  18. data/lib/inventory_refresh/inventory_object_lazy.rb +10 -17
  19. data/lib/inventory_refresh/persister.rb +63 -29
  20. data/lib/inventory_refresh/save_collection/base.rb +2 -4
  21. data/lib/inventory_refresh/save_collection/saver/base.rb +8 -108
  22. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +48 -126
  23. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +19 -1
  24. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +3 -68
  25. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +0 -125
  26. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +5 -9
  27. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +9 -17
  28. data/lib/inventory_refresh/save_collection/sweeper.rb +91 -18
  29. data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
  30. data/lib/inventory_refresh/save_inventory.rb +12 -5
  31. data/lib/inventory_refresh/version.rb +1 -1
  32. data/lib/inventory_refresh.rb +0 -2
  33. metadata +15 -57
  34. data/lib/inventory_refresh/save_collection/saver/batch.rb +0 -17
  35. data/lib/inventory_refresh/save_collection/saver/default.rb +0 -57
  36. data/lib/inventory_refresh/target.rb +0 -73
  37. data/lib/inventory_refresh/target_collection.rb +0 -92
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e7a23c438a97181e679731cb18607916042f0a7f86b0d1d6992c18cf0bba4b0c
4
- data.tar.gz: baf5ace688468d39bac2fcb853509d18627349b8bd9d6699f79a2ac0b010b175
3
+ metadata.gz: 666554cf2d3928670cd6772706360d2c2523fc0475b63f2aead1e7e2bec675d7
4
+ data.tar.gz: 992aa863fdd2d020878f3bd18e0c83d7bf37aa79d6363814e10fdc5cd311a35a
5
5
  SHA512:
6
- metadata.gz: 1bd6e088f308ea571c6d98b4c6d67504f9479a24b6c1a019d4d219e20d0ed864eff30851e3470e4cb729b95b527b7a687e59d21651f13a7fe026ae95df372514
7
- data.tar.gz: 85b3003558c8916af43ab407c93619b2685328a9cefc5e03eabddf9ffbba68b222b40a33715243872258e2c8c175cf1675fe5143beb9d61d6b12f6874f4eb1b1
6
+ metadata.gz: 1ce760c31989eaa2ec3d230031c176007953b9d5790d7d5e0c85d7b62e43f6d7d45cbe88651a70df3ca77c11f4d7b5587348e2ba0d951add5d8d76a90ba9d0e0
7
+ data.tar.gz: 4084a11ccd8ab92142e44399159290d5b14be05aa111f5c681571c6ba034a4ecb55587f0d3c2dda9d0ef765e4c5aefa9f2ea9e4183c27b81d4547b3719ae8393
data/.codeclimate.yml CHANGED
@@ -34,7 +34,6 @@ engines:
34
34
  rubocop:
35
35
  enabled: true
36
36
  config: '.rubocop_cc.yml'
37
- channel: rubocop-0-69
38
37
  prepare:
39
38
  fetch:
40
39
  - url: "https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml"
data/.travis.yml CHANGED
@@ -1,12 +1,12 @@
1
- ---
1
+ dist: xenial
2
2
  sudo: false
3
3
  language: ruby
4
4
  cache: bundler
5
5
  rvm:
6
- - 2.5.7
7
- - 2.6.5
6
+ - 2.4.4
7
+ - 2.3.6
8
8
  addons:
9
- postgresql: '9.5'
9
+ postgresql: '10'
10
10
  env:
11
11
  global:
12
12
  - CC_TEST_REPORTER_ID=1ef1a3a3d007395b11083d634a6fdac1e3d979b6428c39d2cd8d58556cdd68f7
@@ -14,12 +14,10 @@ before_install:
14
14
  - gem install bundler -v 2.0.1
15
15
  - psql -c "CREATE USER root SUPERUSER PASSWORD 'smartvm';" -U postgres
16
16
  before_script:
17
- # Code Climate
18
- - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
17
+ - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
18
+ > ./cc-test-reporter
19
19
  - chmod +x ./cc-test-reporter
20
20
  - "./cc-test-reporter before-build"
21
- # inventory_refresh
22
21
  - bundle exec rake spec:setup
23
22
  after_script:
24
- # Code Climate
25
23
  - "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"
@@ -22,16 +22,14 @@ Gem::Specification.new do |spec|
22
22
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
23
23
  spec.require_paths = ["lib"]
24
24
 
25
- spec.add_dependency "activerecord", ">=5.0", "< 6.1"
26
- spec.add_dependency "more_core_extensions", ">=3.5", "< 5"
25
+ spec.add_dependency "activerecord", "~> 5.0"
26
+ spec.add_dependency "more_core_extensions", "~> 3.5"
27
27
  spec.add_dependency "pg", "> 0"
28
28
 
29
29
  spec.add_development_dependency "ancestry"
30
- spec.add_development_dependency "bundler", "~> 2.0"
30
+ spec.add_development_dependency "bundler", "~> 2.0.1"
31
31
  spec.add_development_dependency "factory_bot", "~> 4.11.1"
32
32
  spec.add_development_dependency "rake", "~> 10.0"
33
33
  spec.add_development_dependency "rspec", "~> 3.0"
34
- spec.add_development_dependency "rubocop"
35
- spec.add_development_dependency "rubocop-performance"
36
34
  spec.add_development_dependency "simplecov"
37
35
  end
@@ -1,20 +1,12 @@
1
1
  module InventoryRefresh
2
2
  class ApplicationRecordIterator
3
- attr_reader :inventory_collection, :manager_uuids_set, :iterator, :query
3
+ attr_reader :inventory_collection
4
4
 
5
- # An iterator that can fetch batches of the AR objects based on a set of manager refs, or just mimics AR relation
6
- # when given an iterator. Or given query, acts as iterator by selecting batches.
5
+ # An iterator that can fetch batches of the AR objects based on a set of attribute_indexes
7
6
  #
8
7
  # @param inventory_collection [InventoryRefresh::InventoryCollection] Inventory collection owning the iterator
9
- # @param manager_uuids_set [Array<InventoryRefresh::InventoryCollection::Reference>] Array of references we want to
10
- # fetch from the DB
11
- # @param iterator [Proc] Block based iterator
12
- # @query query [ActiveRecord::Relation] Existing query we want to use for querying the db
13
- def initialize(inventory_collection: nil, manager_uuids_set: nil, iterator: nil, query: nil)
8
+ def initialize(inventory_collection: nil)
14
9
  @inventory_collection = inventory_collection
15
- @manager_uuids_set = manager_uuids_set
16
- @iterator = iterator
17
- @query = query
18
10
  end
19
11
 
20
12
  # Iterator that mimics find_in_batches of ActiveRecord::Relation. This iterator serves for making more optimized query
@@ -25,28 +17,19 @@ module InventoryRefresh
25
17
  # and relation.where(:id => 500ids)
26
18
  #
27
19
  # @param batch_size [Integer] A batch size we want to fetch from DB
20
+ # @param attributes_index [Hash{String => Hash}] Indexed hash with data we will be saving
28
21
  # @yield Code processing the batches
29
- def find_in_batches(batch_size: 1000)
30
- if iterator
31
- iterator.call do |batch|
32
- yield(batch)
33
- end
34
- elsif query
35
- manager_uuids_set.each_slice(batch_size) do |batch|
36
- yield(query.where(inventory_collection.targeted_selection_for(batch)))
37
- end
38
- else
39
- manager_uuids_set.each_slice(batch_size) do |batch|
40
- yield(inventory_collection.db_collection_for_comparison_for(batch))
41
- end
22
+ def find_in_batches(batch_size: 1000, attributes_index: {})
23
+ attributes_index.each_slice(batch_size) do |batch|
24
+ yield(inventory_collection.db_collection_for_comparison_for(batch))
42
25
  end
43
26
  end
44
27
 
45
28
  # Iterator that mimics find_each of ActiveRecord::Relation using find_in_batches (see #find_in_batches)
46
29
  #
47
30
  # @yield Code processing the batches
48
- def find_each
49
- find_in_batches do |batch|
31
+ def find_each(attributes_index: {})
32
+ find_in_batches(:attributes_index => attributes_index) do |batch|
50
33
  batch.each do |item|
51
34
  yield(item)
52
35
  end
@@ -0,0 +1,8 @@
1
+ module InventoryRefresh
2
+ module Exception
3
+ class SweeperError < StandardError; end
4
+ class SweeperNonExistentScopeKeyFoundError < SweeperError; end
5
+ class SweeperNonUniformScopeKeyFoundError < SweeperError; end
6
+ class SweeperScopeBadFormat < SweeperError; end
7
+ end
8
+ end
@@ -4,16 +4,16 @@ module InventoryRefresh
4
4
  class MissingModelClassError < StandardError; end
5
5
 
6
6
  def self.allowed_properties
7
- %i(all_manager_uuids arel association
7
+ %i(arel association
8
8
  attributes_blacklist attributes_whitelist batch_extra_attributes
9
9
  complete create_only custom_save_block
10
- custom_reconnect_block default_values delete_method
10
+ custom_reconnect_block default_values
11
11
  dependency_attributes check_changed inventory_object_attributes
12
- manager_ref manager_ref_allowed_nil manager_uuids
12
+ manager_ref manager_ref_allowed_nil
13
13
  model_class name parent
14
- parent_inventory_collections retention_strategy strategy
15
- saver_strategy secondary_refs targeted
16
- targeted_arel update_only use_ar_object
14
+ retention_strategy strategy
15
+ secondary_refs
16
+ update_only use_ar_object
17
17
  assert_graph_integrity).to_set
18
18
  end
19
19
 
@@ -70,15 +70,6 @@ module InventoryRefresh
70
70
  build(hash)
71
71
  end
72
72
 
73
- # Finds InventoryObject.
74
- #
75
- # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the InventoryCollection
76
- # @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
77
- def find_in_data(hash)
78
- _hash, _uuid, inventory_object = primary_index_scan(hash)
79
- inventory_object
80
- end
81
-
82
73
  # Finds of builds a new InventoryObject. By building it, we also put in into the InventoryCollection's storage.
83
74
  #
84
75
  # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the
@@ -35,26 +35,14 @@ module InventoryRefresh
35
35
  # - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
36
36
  # the DB. Then if we reference an object that is not present, it will
37
37
  # load them from the db using :local_db_find_references strategy.
38
- # @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
39
- # Allowed saver strategies are:
40
- # - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
41
- # since it will lead to non consistent data.
42
- # - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
43
- # concurrently, since it will lead to non consistent data.
44
- # - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
45
- # atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
46
- # executed as batched SQL queries, instead of sending 1 query per record.
47
38
  # @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
48
39
  # - :destroy => Will destroy the inactive records.
49
40
  # - :archive => Will archive the inactive records by setting :archived_at timestamp.
50
- # @param delete_method [Symbol] A delete method that will be used for deleting of the InventoryObject, if the
51
- # object is marked for deletion. A default is :destroy, the instance method must be defined on the
52
- # :model_class.
53
- def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
54
- @saver_strategy = process_saver_strategy(saver_strategy)
41
+ def init_strategies(strategy, retention_strategy)
42
+ @saver_strategy = :concurrent_safe_batch
55
43
  @strategy = process_strategy(strategy)
44
+ # TODO(lsmola) why don't we just set this strategy based on :archived_at column? Lets do that
56
45
  @retention_strategy = process_retention_strategy(retention_strategy)
57
- @delete_method = delete_method || :destroy
58
46
  end
59
47
 
60
48
  # @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
@@ -67,42 +55,10 @@ module InventoryRefresh
67
55
  # avoid this usecase by a proper modeling.
68
56
  # Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
69
57
  # @param secondary_refs [Hash] TODO
70
- # @param manager_uuids [Array|Proc] Array of manager_uuids of the InventoryObjects we want to create/update/delete. Using
71
- # this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
72
- # case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
73
- # custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
74
- def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
58
+ def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs)
75
59
  @manager_ref = manager_ref || %i(ems_ref)
76
60
  @manager_ref_allowed_nil = manager_ref_allowed_nil || []
77
61
  @secondary_refs = secondary_refs || {}
78
- @manager_uuids = manager_uuids || []
79
- end
80
-
81
- # @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
82
- # having this parameter defined will invoke only :delete_method on a complement of this set, making sure
83
- # the DB has only this set of data after. This :attribute serves for deleting of top level
84
- # InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
85
- # child collections is already handled by the scope of the parent_inventory_collections and using Rails
86
- # :dependent => :destroy,
87
- # @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
88
- # all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
89
- # region.
90
- # If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
91
- # scope itself. E.g. sending a list of all active regions, we will delete complement entities not
92
- # belonging to those regions.
93
- # Example:
94
- # :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
95
- # :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
96
- #
97
- # Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
98
- # regions X and Y.
99
- # @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
100
- # all_manager_uuids. Meaning we won't be archiving any newer entities.
101
- def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
102
- # TODO(lsmola) Should we refactor this to use references too?
103
- @all_manager_uuids = all_manager_uuids
104
- @all_manager_uuids_scope = all_manager_uuids_scope
105
- @all_manager_uuids_timestamp = all_manager_uuids_timestamp
106
62
  end
107
63
 
108
64
  # @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
@@ -119,48 +75,9 @@ module InventoryRefresh
119
75
  # This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
120
76
  # will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
121
77
  # are saved.
122
- # @param parent_inventory_collections [Array] Array of symbols having a name pointing to the
123
- # InventoryRefresh::InventoryCollection objects, that serve as parents to this InventoryCollection. There are
124
- # several scenarios to consider, when deciding if InventoryCollection has parent collections, see the example.
125
- #
126
- # Example:
127
- # taking inventory collections :vms and :disks (local disks), if we write that:
128
- # inventory_collection = InventoryCollection.new({
129
- # :model_class => ::Disk,
130
- # :association => :disks,
131
- # :manager_ref => [:vm, :location]
132
- # :parent_inventory_collection => [:vms],
133
- # })
134
- #
135
- # Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
136
- # points:
137
- # 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
138
- # from vms to disks table).
139
- # 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
140
- # a Vm subquery
141
- # 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
142
- # only under 1 Vm. (In current models, this modeled going through Hardware model)
143
- # 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
144
- #
145
- # Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
146
- # implications:
147
- # 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
148
- # archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
149
- # 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
150
- # we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
151
- # archived/removed before the Vm, while we always want to archive the VM first.
152
- # 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
153
- # we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
154
- # always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
155
- # :manager_uuids are mutually exclusive attributes)
156
- # 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
157
- # While this information can be introspected from the data, it creates a scope for create&update&delete,
158
- # which means it has to work with no data provided (causing delete all). So with no data we cannot
159
- # introspect anything.
160
- def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
161
- @dependency_attributes = dependency_attributes || {}
162
- @dependees = Set.new
163
- @parent_inventory_collections = parent_inventory_collections
78
+ def init_ic_relations(dependency_attributes)
79
+ @dependency_attributes = dependency_attributes || {}
80
+ @dependees = Set.new
164
81
  end
165
82
 
166
83
  # @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
@@ -175,18 +92,13 @@ module InventoryRefresh
175
92
  # @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
176
93
  # it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
177
94
  # for the batch saver strategy.
178
- # @param targeted [Boolean] True if the collection is targeted, in that case it will be leveraging :manager_uuids
179
- # :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
180
- def init_flags(complete, create_only, check_changed,
181
- update_only, use_ar_object, targeted,
182
- assert_graph_integrity)
95
+ def init_flags(complete, create_only, check_changed, update_only, use_ar_object, assert_graph_integrity)
183
96
  @complete = complete.nil? ? true : complete
184
97
  @create_only = create_only.nil? ? false : create_only
185
98
  @check_changed = check_changed.nil? ? true : check_changed
186
99
  @saved = false
187
100
  @update_only = update_only.nil? ? false : update_only
188
101
  @use_ar_object = use_ar_object || false
189
- @targeted = !!targeted
190
102
  @assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
191
103
  end
192
104
 
@@ -230,7 +142,7 @@ module InventoryRefresh
230
142
  @attributes_whitelist = Set.new
231
143
  @batch_extra_attributes = batch_extra_attributes || []
232
144
  @inventory_object_attributes = inventory_object_attributes
233
- @internal_attributes = %i(__feedback_edge_set_parent __parent_inventory_collections __all_manager_uuids_scope)
145
+ @internal_attributes = %i(__feedback_edge_set_parent)
234
146
  @transitive_dependency_attributes = Set.new
235
147
 
236
148
  blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
@@ -240,7 +152,6 @@ module InventoryRefresh
240
152
  def init_storages
241
153
  @data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
242
154
  @references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
243
- @targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
244
155
  end
245
156
 
246
157
  # @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
@@ -248,35 +159,18 @@ module InventoryRefresh
248
159
  # doing create/update/delete.
249
160
  #
250
161
  # Example:
251
- # for a targeted refresh, we want to delete/update/create only a list of vms specified with a list of
252
- # ems_refs:
253
- # :arel => manager.vms.where(:ems_ref => manager_refs)
254
- # Then we want to do the same for the hardwares of only those vms:
255
- # :arel => manager.hardwares.joins(:vm_or_template).where(
256
- # 'vms' => {:ems_ref => manager_refs}
257
- # )
258
- # And etc. for the other Vm related records.
259
- # @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
260
- # we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
261
- # manager_uuids available.
262
- # Example:
263
- # targeted_arel = lambda do |inventory_collection|
264
- # # Getting ems_refs of parent :vms and :miq_templates
265
- # manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
266
- # inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
267
- # 'vms' => {:ems_ref => manager_uuids}
162
+ # add_collection(:cross_link_vms) do |builder|
163
+ # builder.add_properties(
164
+ # :arel => Vm.where(:tenant => manager.tenant),
165
+ # :association => nil,
166
+ # :model_class => Vm,
167
+ # :name => :cross_link_vms,
168
+ # :manager_ref => [:uid_ems],
169
+ # :strategy => :local_db_find_references,
268
170
  # )
269
171
  # end
270
- #
271
- # inventory_collection = InventoryCollection.new({
272
- # :model_class => ::Hardware,
273
- # :association => :hardwares,
274
- # :parent_inventory_collection => [:vms, :miq_templates],
275
- # :targeted_arel => targeted_arel,
276
- # })
277
- def init_arels(arel, targeted_arel)
278
- @arel = arel
279
- @targeted_arel = targeted_arel
172
+ def init_arels(arel)
173
+ @arel = arel
280
174
  end
281
175
 
282
176
  # @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
@@ -289,7 +183,7 @@ module InventoryRefresh
289
183
  # hash = inventory_object.attributes # Loads possible dependencies into saveable hash
290
184
  # obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
291
185
  # # queries, avoid this, this is just a simple example :-)
292
- # obj.update(hash) if obj
186
+ # obj.update_attributes(hash) if obj
293
187
  # obj ||= SomeModel.create(hash)
294
188
  # inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
295
189
  # primary key back to the InventoryObject
@@ -392,23 +286,6 @@ module InventoryRefresh
392
286
  @deleted_records = []
393
287
  end
394
288
 
395
- # Processes passed saver strategy
396
- #
397
- # @param saver_strategy [Symbol] Passed saver strategy
398
- # @return [Symbol] Returns back the passed strategy if supported, or raises exception
399
- def process_saver_strategy(saver_strategy)
400
- return :default unless saver_strategy
401
-
402
- saver_strategy = saver_strategy.to_sym
403
- case saver_strategy
404
- when :default, :batch, :concurrent_safe_batch
405
- saver_strategy
406
- else
407
- raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
408
- ":default, :batch and :concurrent_safe_batch"
409
- end
410
- end
411
-
412
289
  # Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
413
290
  #
414
291
  # @param strategy_name [Symbol] Passed saver strategy
@@ -434,12 +311,26 @@ module InventoryRefresh
434
311
  strategy_name
435
312
  end
436
313
 
314
+ # Saves passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
315
+ #
316
+ # @param strategy [Symbol] Passed saver strategy
317
+ # @return [Symbol] Returns back the passed strategy if supported, or raises exception
318
+ def strategy=(strategy)
319
+ @strategy = process_strategy(strategy)
320
+ end
321
+
437
322
  # Processes passed retention strategy
438
323
  #
439
324
  # @param retention_strategy [Symbol] Passed retention strategy
440
325
  # @return [Symbol] Returns back the passed strategy if supported, or raises exception
441
326
  def process_retention_strategy(retention_strategy)
442
- return unless retention_strategy
327
+ unless retention_strategy
328
+ if supports_column?(:archived_at)
329
+ return :archive
330
+ else
331
+ return :destroy
332
+ end
333
+ end
443
334
 
444
335
  retention_strategy = retention_strategy.to_sym
445
336
  case retention_strategy
@@ -50,11 +50,6 @@ module InventoryRefresh
50
50
  dependencies.all?(&:saved?)
51
51
  end
52
52
 
53
- # @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
54
- def parallel_safe?
55
- @parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
56
- end
57
-
58
53
  # @return [Boolean] true if the model_class supports STI
59
54
  def supports_sti?
60
55
  @supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
@@ -81,50 +76,12 @@ module InventoryRefresh
81
76
  data_collection_finalized
82
77
  end
83
78
 
84
- # @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
85
- def targeted?
86
- targeted
87
- end
88
-
89
79
  # True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
90
80
  # stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
91
81
  #
92
82
  # @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
93
83
  def noop?
94
- # If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
95
- saving_noop? && delete_complement_noop?
96
- end
97
-
98
- # @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
99
- def saving_noop?
100
- saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
101
- end
102
-
103
- # @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
104
- def delete_complement_noop?
105
- all_manager_uuids.nil?
106
- end
107
-
108
- private
109
-
110
- # @return true if it's a noop parent targeted InventoryCollection
111
- def saving_targeted_parent_collection_noop?
112
- targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
113
- end
114
-
115
- # @return true if it's a noop child targeted InventoryCollection
116
- def saving_targeted_child_collection_noop?
117
- targeted_noop_condition && parent_inventory_collections.present? &&
118
- parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
119
- end
120
-
121
- # @return true if it's a noop full InventoryCollection refresh
122
- def saving_full_collection_noop?
123
- !targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
124
- end
125
-
126
- def targeted_noop_condition
127
- targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
84
+ data.blank? && custom_save_block.nil? && skeletal_primary_index.blank?
128
85
  end
129
86
  end
130
87
  end
@@ -64,14 +64,6 @@ module InventoryRefresh
64
64
  end
65
65
  end
66
66
 
67
- def reindex_secondary_indexes!
68
- data_indexes.each do |ref, index|
69
- next if ref == primary_index_ref
70
-
71
- index.reindex!
72
- end
73
- end
74
-
75
67
  def primary_index
76
68
  data_index(primary_index_ref)
77
69
  end
@@ -94,30 +86,16 @@ module InventoryRefresh
94
86
  end
95
87
  end
96
88
 
97
- def find_by(manager_uuid_hash, ref: primary_index_ref)
98
- # TODO(lsmola) deprecate this, it's enough to have find method
99
- find(manager_uuid_hash, :ref => ref)
100
- end
101
-
102
- def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
103
- # TODO(lsmola) deprecate this, it's enough to have lazy_find method
104
-
105
- lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
106
- end
107
-
108
89
  def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
109
- # TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
110
- # access the data
111
- # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
112
90
  return if manager_uuid.nil?
113
91
  assert_index(manager_uuid, ref)
114
92
 
115
93
  ::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
116
- manager_uuid,
117
- :ref => ref,
118
- :key => key,
119
- :default => default,
120
- :transform_nested_lazy_finds => transform_nested_lazy_finds)
94
+ manager_uuid,
95
+ :ref => ref,
96
+ :key => key,
97
+ :default => default,
98
+ :transform_nested_lazy_finds => transform_nested_lazy_finds)
121
99
  end
122
100
 
123
101
  def named_ref(ref = primary_index_ref)
@@ -132,19 +110,13 @@ module InventoryRefresh
132
110
 
133
111
  delegate :association_to_foreign_key_mapping,
134
112
  :build_stringified_reference,
135
- :parallel_safe?,
136
113
  :strategy,
137
114
  :to => :inventory_collection
138
115
 
139
116
  attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
140
117
 
141
118
  def find_in_data_or_skeletal_index(reference)
142
- if parallel_safe?
143
- # With parallel safe strategies, we create skeletal nodes that we can look for
144
- data_index_find(reference) || skeletal_index_find(reference)
145
- else
146
- data_index_find(reference)
147
- end
119
+ data_index_find(reference) || skeletal_index_find(reference)
148
120
  end
149
121
 
150
122
  def skeletal_index_find(reference)
@@ -27,14 +27,6 @@ module InventoryRefresh
27
27
  index[build_stringified_reference(inventory_object.data, attribute_names)] = inventory_object
28
28
  end
29
29
 
30
- # Rebuilds the indexes for all InventoryObject objects
31
- def reindex!
32
- self.index = {}
33
- data.each do |inventory_object|
34
- store_index_for(inventory_object)
35
- end
36
- end
37
-
38
30
  # @return [Array] Returns index data
39
31
  def index_data
40
32
  index.values
@@ -45,23 +45,6 @@ module InventoryRefresh
45
45
  add_reference(reference_data)
46
46
  end
47
47
 
48
- # Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
49
- # it.
50
- #
51
- # @param references_array [Array] Array of reference objects acceptable by add_reference method.
52
- # @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
53
- # index.
54
- # @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
55
- def merge!(references_array, ref: nil)
56
- references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
57
- self
58
- end
59
-
60
- # @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
61
- def primary_references
62
- references[primary_index_ref]
63
- end
64
-
65
48
  # Builds a Reference object
66
49
  #
67
50
  # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference