inventory_refresh 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +0 -1
  3. data/.travis.yml +6 -8
  4. data/inventory_refresh.gemspec +3 -5
  5. data/lib/inventory_refresh/application_record_iterator.rb +9 -26
  6. data/lib/inventory_refresh/exception.rb +8 -0
  7. data/lib/inventory_refresh/inventory_collection/builder.rb +6 -6
  8. data/lib/inventory_refresh/inventory_collection/data_storage.rb +0 -9
  9. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +35 -144
  10. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +1 -44
  11. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +6 -34
  12. data/lib/inventory_refresh/inventory_collection/index/type/base.rb +0 -8
  13. data/lib/inventory_refresh/inventory_collection/references_storage.rb +0 -17
  14. data/lib/inventory_refresh/inventory_collection/scanner.rb +1 -87
  15. data/lib/inventory_refresh/inventory_collection/serialization.rb +10 -16
  16. data/lib/inventory_refresh/inventory_collection.rb +36 -110
  17. data/lib/inventory_refresh/inventory_object.rb +34 -68
  18. data/lib/inventory_refresh/inventory_object_lazy.rb +10 -17
  19. data/lib/inventory_refresh/persister.rb +63 -29
  20. data/lib/inventory_refresh/save_collection/base.rb +2 -4
  21. data/lib/inventory_refresh/save_collection/saver/base.rb +8 -108
  22. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +48 -126
  23. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +19 -1
  24. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +3 -68
  25. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +0 -125
  26. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +5 -9
  27. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +9 -17
  28. data/lib/inventory_refresh/save_collection/sweeper.rb +91 -18
  29. data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
  30. data/lib/inventory_refresh/save_inventory.rb +12 -5
  31. data/lib/inventory_refresh/version.rb +1 -1
  32. data/lib/inventory_refresh.rb +0 -2
  33. metadata +15 -57
  34. data/lib/inventory_refresh/save_collection/saver/batch.rb +0 -17
  35. data/lib/inventory_refresh/save_collection/saver/default.rb +0 -57
  36. data/lib/inventory_refresh/target.rb +0 -73
  37. data/lib/inventory_refresh/target_collection.rb +0 -92
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e7a23c438a97181e679731cb18607916042f0a7f86b0d1d6992c18cf0bba4b0c
4
- data.tar.gz: baf5ace688468d39bac2fcb853509d18627349b8bd9d6699f79a2ac0b010b175
3
+ metadata.gz: 666554cf2d3928670cd6772706360d2c2523fc0475b63f2aead1e7e2bec675d7
4
+ data.tar.gz: 992aa863fdd2d020878f3bd18e0c83d7bf37aa79d6363814e10fdc5cd311a35a
5
5
  SHA512:
6
- metadata.gz: 1bd6e088f308ea571c6d98b4c6d67504f9479a24b6c1a019d4d219e20d0ed864eff30851e3470e4cb729b95b527b7a687e59d21651f13a7fe026ae95df372514
7
- data.tar.gz: 85b3003558c8916af43ab407c93619b2685328a9cefc5e03eabddf9ffbba68b222b40a33715243872258e2c8c175cf1675fe5143beb9d61d6b12f6874f4eb1b1
6
+ metadata.gz: 1ce760c31989eaa2ec3d230031c176007953b9d5790d7d5e0c85d7b62e43f6d7d45cbe88651a70df3ca77c11f4d7b5587348e2ba0d951add5d8d76a90ba9d0e0
7
+ data.tar.gz: 4084a11ccd8ab92142e44399159290d5b14be05aa111f5c681571c6ba034a4ecb55587f0d3c2dda9d0ef765e4c5aefa9f2ea9e4183c27b81d4547b3719ae8393
data/.codeclimate.yml CHANGED
@@ -34,7 +34,6 @@ engines:
34
34
  rubocop:
35
35
  enabled: true
36
36
  config: '.rubocop_cc.yml'
37
- channel: rubocop-0-69
38
37
  prepare:
39
38
  fetch:
40
39
  - url: "https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml"
data/.travis.yml CHANGED
@@ -1,12 +1,12 @@
1
- ---
1
+ dist: xenial
2
2
  sudo: false
3
3
  language: ruby
4
4
  cache: bundler
5
5
  rvm:
6
- - 2.5.7
7
- - 2.6.5
6
+ - 2.4.4
7
+ - 2.3.6
8
8
  addons:
9
- postgresql: '9.5'
9
+ postgresql: '10'
10
10
  env:
11
11
  global:
12
12
  - CC_TEST_REPORTER_ID=1ef1a3a3d007395b11083d634a6fdac1e3d979b6428c39d2cd8d58556cdd68f7
@@ -14,12 +14,10 @@ before_install:
14
14
  - gem install bundler -v 2.0.1
15
15
  - psql -c "CREATE USER root SUPERUSER PASSWORD 'smartvm';" -U postgres
16
16
  before_script:
17
- # Code Climate
18
- - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
17
+ - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
18
+ > ./cc-test-reporter
19
19
  - chmod +x ./cc-test-reporter
20
20
  - "./cc-test-reporter before-build"
21
- # inventory_refresh
22
21
  - bundle exec rake spec:setup
23
22
  after_script:
24
- # Code Climate
25
23
  - "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"
@@ -22,16 +22,14 @@ Gem::Specification.new do |spec|
22
22
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
23
23
  spec.require_paths = ["lib"]
24
24
 
25
- spec.add_dependency "activerecord", ">=5.0", "< 6.1"
26
- spec.add_dependency "more_core_extensions", ">=3.5", "< 5"
25
+ spec.add_dependency "activerecord", "~> 5.0"
26
+ spec.add_dependency "more_core_extensions", "~> 3.5"
27
27
  spec.add_dependency "pg", "> 0"
28
28
 
29
29
  spec.add_development_dependency "ancestry"
30
- spec.add_development_dependency "bundler", "~> 2.0"
30
+ spec.add_development_dependency "bundler", "~> 2.0.1"
31
31
  spec.add_development_dependency "factory_bot", "~> 4.11.1"
32
32
  spec.add_development_dependency "rake", "~> 10.0"
33
33
  spec.add_development_dependency "rspec", "~> 3.0"
34
- spec.add_development_dependency "rubocop"
35
- spec.add_development_dependency "rubocop-performance"
36
34
  spec.add_development_dependency "simplecov"
37
35
  end
@@ -1,20 +1,12 @@
1
1
  module InventoryRefresh
2
2
  class ApplicationRecordIterator
3
- attr_reader :inventory_collection, :manager_uuids_set, :iterator, :query
3
+ attr_reader :inventory_collection
4
4
 
5
- # An iterator that can fetch batches of the AR objects based on a set of manager refs, or just mimics AR relation
6
- # when given an iterator. Or given query, acts as iterator by selecting batches.
5
+ # An iterator that can fetch batches of the AR objects based on a set of attribute_indexes
7
6
  #
8
7
  # @param inventory_collection [InventoryRefresh::InventoryCollection] Inventory collection owning the iterator
9
- # @param manager_uuids_set [Array<InventoryRefresh::InventoryCollection::Reference>] Array of references we want to
10
- # fetch from the DB
11
- # @param iterator [Proc] Block based iterator
12
- # @query query [ActiveRecord::Relation] Existing query we want to use for querying the db
13
- def initialize(inventory_collection: nil, manager_uuids_set: nil, iterator: nil, query: nil)
8
+ def initialize(inventory_collection: nil)
14
9
  @inventory_collection = inventory_collection
15
- @manager_uuids_set = manager_uuids_set
16
- @iterator = iterator
17
- @query = query
18
10
  end
19
11
 
20
12
  # Iterator that mimics find_in_batches of ActiveRecord::Relation. This iterator serves for making more optimized query
@@ -25,28 +17,19 @@ module InventoryRefresh
25
17
  # and relation.where(:id => 500ids)
26
18
  #
27
19
  # @param batch_size [Integer] A batch size we want to fetch from DB
20
+ # @param attributes_index [Hash{String => Hash}] Indexed hash with data we will be saving
28
21
  # @yield Code processing the batches
29
- def find_in_batches(batch_size: 1000)
30
- if iterator
31
- iterator.call do |batch|
32
- yield(batch)
33
- end
34
- elsif query
35
- manager_uuids_set.each_slice(batch_size) do |batch|
36
- yield(query.where(inventory_collection.targeted_selection_for(batch)))
37
- end
38
- else
39
- manager_uuids_set.each_slice(batch_size) do |batch|
40
- yield(inventory_collection.db_collection_for_comparison_for(batch))
41
- end
22
+ def find_in_batches(batch_size: 1000, attributes_index: {})
23
+ attributes_index.each_slice(batch_size) do |batch|
24
+ yield(inventory_collection.db_collection_for_comparison_for(batch))
42
25
  end
43
26
  end
44
27
 
45
28
  # Iterator that mimics find_each of ActiveRecord::Relation using find_in_batches (see #find_in_batches)
46
29
  #
47
30
  # @yield Code processing the batches
48
- def find_each
49
- find_in_batches do |batch|
31
+ def find_each(attributes_index: {})
32
+ find_in_batches(:attributes_index => attributes_index) do |batch|
50
33
  batch.each do |item|
51
34
  yield(item)
52
35
  end
@@ -0,0 +1,8 @@
1
+ module InventoryRefresh
2
+ module Exception
3
+ class SweeperError < StandardError; end
4
+ class SweeperNonExistentScopeKeyFoundError < SweeperError; end
5
+ class SweeperNonUniformScopeKeyFoundError < SweeperError; end
6
+ class SweeperScopeBadFormat < SweeperError; end
7
+ end
8
+ end
@@ -4,16 +4,16 @@ module InventoryRefresh
4
4
  class MissingModelClassError < StandardError; end
5
5
 
6
6
  def self.allowed_properties
7
- %i(all_manager_uuids arel association
7
+ %i(arel association
8
8
  attributes_blacklist attributes_whitelist batch_extra_attributes
9
9
  complete create_only custom_save_block
10
- custom_reconnect_block default_values delete_method
10
+ custom_reconnect_block default_values
11
11
  dependency_attributes check_changed inventory_object_attributes
12
- manager_ref manager_ref_allowed_nil manager_uuids
12
+ manager_ref manager_ref_allowed_nil
13
13
  model_class name parent
14
- parent_inventory_collections retention_strategy strategy
15
- saver_strategy secondary_refs targeted
16
- targeted_arel update_only use_ar_object
14
+ retention_strategy strategy
15
+ secondary_refs
16
+ update_only use_ar_object
17
17
  assert_graph_integrity).to_set
18
18
  end
19
19
 
@@ -70,15 +70,6 @@ module InventoryRefresh
70
70
  build(hash)
71
71
  end
72
72
 
73
- # Finds InventoryObject.
74
- #
75
- # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the InventoryCollection
76
- # @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
77
- def find_in_data(hash)
78
- _hash, _uuid, inventory_object = primary_index_scan(hash)
79
- inventory_object
80
- end
81
-
82
73
  # Finds of builds a new InventoryObject. By building it, we also put in into the InventoryCollection's storage.
83
74
  #
84
75
  # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the
@@ -35,26 +35,14 @@ module InventoryRefresh
35
35
  # - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
36
36
  # the DB. Then if we reference an object that is not present, it will
37
37
  # load them from the db using :local_db_find_references strategy.
38
- # @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
39
- # Allowed saver strategies are:
40
- # - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
41
- # since it will lead to non consistent data.
42
- # - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
43
- # concurrently, since it will lead to non consistent data.
44
- # - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
45
- # atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
46
- # executed as batched SQL queries, instead of sending 1 query per record.
47
38
  # @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
48
39
  # - :destroy => Will destroy the inactive records.
49
40
  # - :archive => Will archive the inactive records by setting :archived_at timestamp.
50
- # @param delete_method [Symbol] A delete method that will be used for deleting of the InventoryObject, if the
51
- # object is marked for deletion. A default is :destroy, the instance method must be defined on the
52
- # :model_class.
53
- def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
54
- @saver_strategy = process_saver_strategy(saver_strategy)
41
+ def init_strategies(strategy, retention_strategy)
42
+ @saver_strategy = :concurrent_safe_batch
55
43
  @strategy = process_strategy(strategy)
44
+ # TODO(lsmola) why don't we just set this strategy based on :archived_at column? Lets do that
56
45
  @retention_strategy = process_retention_strategy(retention_strategy)
57
- @delete_method = delete_method || :destroy
58
46
  end
59
47
 
60
48
  # @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
@@ -67,42 +55,10 @@ module InventoryRefresh
67
55
  # avoid this usecase by a proper modeling.
68
56
  # Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
69
57
  # @param secondary_refs [Hash] TODO
70
- # @param manager_uuids [Array|Proc] Array of manager_uuids of the InventoryObjects we want to create/update/delete. Using
71
- # this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
72
- # case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
73
- # custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
74
- def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
58
+ def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs)
75
59
  @manager_ref = manager_ref || %i(ems_ref)
76
60
  @manager_ref_allowed_nil = manager_ref_allowed_nil || []
77
61
  @secondary_refs = secondary_refs || {}
78
- @manager_uuids = manager_uuids || []
79
- end
80
-
81
- # @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
82
- # having this parameter defined will invoke only :delete_method on a complement of this set, making sure
83
- # the DB has only this set of data after. This :attribute serves for deleting of top level
84
- # InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
85
- # child collections is already handled by the scope of the parent_inventory_collections and using Rails
86
- # :dependent => :destroy,
87
- # @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
88
- # all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
89
- # region.
90
- # If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
91
- # scope itself. E.g. sending a list of all active regions, we will delete complement entities not
92
- # belonging to those regions.
93
- # Example:
94
- # :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
95
- # :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
96
- #
97
- # Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
98
- # regions X and Y.
99
- # @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
100
- # all_manager_uuids. Meaning we won't be archiving any newer entities.
101
- def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
102
- # TODO(lsmola) Should we refactor this to use references too?
103
- @all_manager_uuids = all_manager_uuids
104
- @all_manager_uuids_scope = all_manager_uuids_scope
105
- @all_manager_uuids_timestamp = all_manager_uuids_timestamp
106
62
  end
107
63
 
108
64
  # @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
@@ -119,48 +75,9 @@ module InventoryRefresh
119
75
  # This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
120
76
  # will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
121
77
  # are saved.
122
- # @param parent_inventory_collections [Array] Array of symbols having a name pointing to the
123
- # InventoryRefresh::InventoryCollection objects, that serve as parents to this InventoryCollection. There are
124
- # several scenarios to consider, when deciding if InventoryCollection has parent collections, see the example.
125
- #
126
- # Example:
127
- # taking inventory collections :vms and :disks (local disks), if we write that:
128
- # inventory_collection = InventoryCollection.new({
129
- # :model_class => ::Disk,
130
- # :association => :disks,
131
- # :manager_ref => [:vm, :location]
132
- # :parent_inventory_collection => [:vms],
133
- # })
134
- #
135
- # Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
136
- # points:
137
- # 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
138
- # from vms to disks table).
139
- # 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
140
- # a Vm subquery
141
- # 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
142
- # only under 1 Vm. (In current models, this modeled going through Hardware model)
143
- # 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
144
- #
145
- # Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
146
- # implications:
147
- # 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
148
- # archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
149
- # 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
150
- # we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
151
- # archived/removed before the Vm, while we always want to archive the VM first.
152
- # 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
153
- # we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
154
- # always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
155
- # :manager_uuids are mutually exclusive attributes)
156
- # 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
157
- # While this information can be introspected from the data, it creates a scope for create&update&delete,
158
- # which means it has to work with no data provided (causing delete all). So with no data we cannot
159
- # introspect anything.
160
- def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
161
- @dependency_attributes = dependency_attributes || {}
162
- @dependees = Set.new
163
- @parent_inventory_collections = parent_inventory_collections
78
+ def init_ic_relations(dependency_attributes)
79
+ @dependency_attributes = dependency_attributes || {}
80
+ @dependees = Set.new
164
81
  end
165
82
 
166
83
  # @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
@@ -175,18 +92,13 @@ module InventoryRefresh
175
92
  # @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
176
93
  # it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
177
94
  # for the batch saver strategy.
178
- # @param targeted [Boolean] True if the collection is targeted, in that case it will be leveraging :manager_uuids
179
- # :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
180
- def init_flags(complete, create_only, check_changed,
181
- update_only, use_ar_object, targeted,
182
- assert_graph_integrity)
95
+ def init_flags(complete, create_only, check_changed, update_only, use_ar_object, assert_graph_integrity)
183
96
  @complete = complete.nil? ? true : complete
184
97
  @create_only = create_only.nil? ? false : create_only
185
98
  @check_changed = check_changed.nil? ? true : check_changed
186
99
  @saved = false
187
100
  @update_only = update_only.nil? ? false : update_only
188
101
  @use_ar_object = use_ar_object || false
189
- @targeted = !!targeted
190
102
  @assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
191
103
  end
192
104
 
@@ -230,7 +142,7 @@ module InventoryRefresh
230
142
  @attributes_whitelist = Set.new
231
143
  @batch_extra_attributes = batch_extra_attributes || []
232
144
  @inventory_object_attributes = inventory_object_attributes
233
- @internal_attributes = %i(__feedback_edge_set_parent __parent_inventory_collections __all_manager_uuids_scope)
145
+ @internal_attributes = %i(__feedback_edge_set_parent)
234
146
  @transitive_dependency_attributes = Set.new
235
147
 
236
148
  blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
@@ -240,7 +152,6 @@ module InventoryRefresh
240
152
  def init_storages
241
153
  @data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
242
154
  @references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
243
- @targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
244
155
  end
245
156
 
246
157
  # @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
@@ -248,35 +159,18 @@ module InventoryRefresh
248
159
  # doing create/update/delete.
249
160
  #
250
161
  # Example:
251
- # for a targeted refresh, we want to delete/update/create only a list of vms specified with a list of
252
- # ems_refs:
253
- # :arel => manager.vms.where(:ems_ref => manager_refs)
254
- # Then we want to do the same for the hardwares of only those vms:
255
- # :arel => manager.hardwares.joins(:vm_or_template).where(
256
- # 'vms' => {:ems_ref => manager_refs}
257
- # )
258
- # And etc. for the other Vm related records.
259
- # @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
260
- # we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
261
- # manager_uuids available.
262
- # Example:
263
- # targeted_arel = lambda do |inventory_collection|
264
- # # Getting ems_refs of parent :vms and :miq_templates
265
- # manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
266
- # inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
267
- # 'vms' => {:ems_ref => manager_uuids}
162
+ # add_collection(:cross_link_vms) do |builder|
163
+ # builder.add_properties(
164
+ # :arel => Vm.where(:tenant => manager.tenant),
165
+ # :association => nil,
166
+ # :model_class => Vm,
167
+ # :name => :cross_link_vms,
168
+ # :manager_ref => [:uid_ems],
169
+ # :strategy => :local_db_find_references,
268
170
  # )
269
171
  # end
270
- #
271
- # inventory_collection = InventoryCollection.new({
272
- # :model_class => ::Hardware,
273
- # :association => :hardwares,
274
- # :parent_inventory_collection => [:vms, :miq_templates],
275
- # :targeted_arel => targeted_arel,
276
- # })
277
- def init_arels(arel, targeted_arel)
278
- @arel = arel
279
- @targeted_arel = targeted_arel
172
+ def init_arels(arel)
173
+ @arel = arel
280
174
  end
281
175
 
282
176
  # @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
@@ -289,7 +183,7 @@ module InventoryRefresh
289
183
  # hash = inventory_object.attributes # Loads possible dependencies into saveable hash
290
184
  # obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
291
185
  # # queries, avoid this, this is just a simple example :-)
292
- # obj.update(hash) if obj
186
+ # obj.update_attributes(hash) if obj
293
187
  # obj ||= SomeModel.create(hash)
294
188
  # inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
295
189
  # primary key back to the InventoryObject
@@ -392,23 +286,6 @@ module InventoryRefresh
392
286
  @deleted_records = []
393
287
  end
394
288
 
395
- # Processes passed saver strategy
396
- #
397
- # @param saver_strategy [Symbol] Passed saver strategy
398
- # @return [Symbol] Returns back the passed strategy if supported, or raises exception
399
- def process_saver_strategy(saver_strategy)
400
- return :default unless saver_strategy
401
-
402
- saver_strategy = saver_strategy.to_sym
403
- case saver_strategy
404
- when :default, :batch, :concurrent_safe_batch
405
- saver_strategy
406
- else
407
- raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
408
- ":default, :batch and :concurrent_safe_batch"
409
- end
410
- end
411
-
412
289
  # Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
413
290
  #
414
291
  # @param strategy_name [Symbol] Passed saver strategy
@@ -434,12 +311,26 @@ module InventoryRefresh
434
311
  strategy_name
435
312
  end
436
313
 
314
+ # Saves passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
315
+ #
316
+ # @param strategy [Symbol] Passed saver strategy
317
+ # @return [Symbol] Returns back the passed strategy if supported, or raises exception
318
+ def strategy=(strategy)
319
+ @strategy = process_strategy(strategy)
320
+ end
321
+
437
322
  # Processes passed retention strategy
438
323
  #
439
324
  # @param retention_strategy [Symbol] Passed retention strategy
440
325
  # @return [Symbol] Returns back the passed strategy if supported, or raises exception
441
326
  def process_retention_strategy(retention_strategy)
442
- return unless retention_strategy
327
+ unless retention_strategy
328
+ if supports_column?(:archived_at)
329
+ return :archive
330
+ else
331
+ return :destroy
332
+ end
333
+ end
443
334
 
444
335
  retention_strategy = retention_strategy.to_sym
445
336
  case retention_strategy
@@ -50,11 +50,6 @@ module InventoryRefresh
50
50
  dependencies.all?(&:saved?)
51
51
  end
52
52
 
53
- # @return [Boolean] true if we are using a saver strategy that allows saving in parallel processes
54
- def parallel_safe?
55
- @parallel_safe_cache ||= %i(concurrent_safe concurrent_safe_batch).include?(saver_strategy)
56
- end
57
-
58
53
  # @return [Boolean] true if the model_class supports STI
59
54
  def supports_sti?
60
55
  @supports_sti_cache = model_class.column_names.include?("type") if @supports_sti_cache.nil?
@@ -81,50 +76,12 @@ module InventoryRefresh
81
76
  data_collection_finalized
82
77
  end
83
78
 
84
- # @return [Boolean] true is processing of this InventoryCollection will be in targeted mode
85
- def targeted?
86
- targeted
87
- end
88
-
89
79
  # True if processing of this InventoryCollection object would lead to no operations. Then we use this marker to
90
80
  # stop processing of the InventoryCollector object very soon, to avoid a lot of unnecessary Db queries, etc.
91
81
  #
92
82
  # @return [Boolean] true if processing of this InventoryCollection object would lead to no operations.
93
83
  def noop?
94
- # If this InventoryCollection doesn't do anything. it can easily happen for targeted/batched strategies.
95
- saving_noop? && delete_complement_noop?
96
- end
97
-
98
- # @return [Boolean] true if processing InventoryCollection will not lead to any created/updated/deleted record
99
- def saving_noop?
100
- saving_targeted_parent_collection_noop? || saving_targeted_child_collection_noop? || saving_full_collection_noop?
101
- end
102
-
103
- # @return true if processing InventoryCollection will not lead to deleting the complement of passed ids
104
- def delete_complement_noop?
105
- all_manager_uuids.nil?
106
- end
107
-
108
- private
109
-
110
- # @return true if it's a noop parent targeted InventoryCollection
111
- def saving_targeted_parent_collection_noop?
112
- targeted_noop_condition && parent_inventory_collections.blank? && targeted_scope.primary_references.blank?
113
- end
114
-
115
- # @return true if it's a noop child targeted InventoryCollection
116
- def saving_targeted_child_collection_noop?
117
- targeted_noop_condition && parent_inventory_collections.present? &&
118
- parent_inventory_collections.all? { |x| x.targeted_scope.primary_references.blank? }
119
- end
120
-
121
- # @return true if it's a noop full InventoryCollection refresh
122
- def saving_full_collection_noop?
123
- !targeted? && data.blank? && !delete_allowed? && skeletal_primary_index.blank?
124
- end
125
-
126
- def targeted_noop_condition
127
- targeted? && custom_save_block.nil? && skeletal_primary_index.blank?
84
+ data.blank? && custom_save_block.nil? && skeletal_primary_index.blank?
128
85
  end
129
86
  end
130
87
  end
@@ -64,14 +64,6 @@ module InventoryRefresh
64
64
  end
65
65
  end
66
66
 
67
- def reindex_secondary_indexes!
68
- data_indexes.each do |ref, index|
69
- next if ref == primary_index_ref
70
-
71
- index.reindex!
72
- end
73
- end
74
-
75
67
  def primary_index
76
68
  data_index(primary_index_ref)
77
69
  end
@@ -94,30 +86,16 @@ module InventoryRefresh
94
86
  end
95
87
  end
96
88
 
97
- def find_by(manager_uuid_hash, ref: primary_index_ref)
98
- # TODO(lsmola) deprecate this, it's enough to have find method
99
- find(manager_uuid_hash, :ref => ref)
100
- end
101
-
102
- def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
103
- # TODO(lsmola) deprecate this, it's enough to have lazy_find method
104
-
105
- lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
106
- end
107
-
108
89
  def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
109
- # TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
110
- # access the data
111
- # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
112
90
  return if manager_uuid.nil?
113
91
  assert_index(manager_uuid, ref)
114
92
 
115
93
  ::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
116
- manager_uuid,
117
- :ref => ref,
118
- :key => key,
119
- :default => default,
120
- :transform_nested_lazy_finds => transform_nested_lazy_finds)
94
+ manager_uuid,
95
+ :ref => ref,
96
+ :key => key,
97
+ :default => default,
98
+ :transform_nested_lazy_finds => transform_nested_lazy_finds)
121
99
  end
122
100
 
123
101
  def named_ref(ref = primary_index_ref)
@@ -132,19 +110,13 @@ module InventoryRefresh
132
110
 
133
111
  delegate :association_to_foreign_key_mapping,
134
112
  :build_stringified_reference,
135
- :parallel_safe?,
136
113
  :strategy,
137
114
  :to => :inventory_collection
138
115
 
139
116
  attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
140
117
 
141
118
  def find_in_data_or_skeletal_index(reference)
142
- if parallel_safe?
143
- # With parallel safe strategies, we create skeletal nodes that we can look for
144
- data_index_find(reference) || skeletal_index_find(reference)
145
- else
146
- data_index_find(reference)
147
- end
119
+ data_index_find(reference) || skeletal_index_find(reference)
148
120
  end
149
121
 
150
122
  def skeletal_index_find(reference)
@@ -27,14 +27,6 @@ module InventoryRefresh
27
27
  index[build_stringified_reference(inventory_object.data, attribute_names)] = inventory_object
28
28
  end
29
29
 
30
- # Rebuilds the indexes for all InventoryObject objects
31
- def reindex!
32
- self.index = {}
33
- data.each do |inventory_object|
34
- store_index_for(inventory_object)
35
- end
36
- end
37
-
38
30
  # @return [Array] Returns index data
39
31
  def index_data
40
32
  index.values
@@ -45,23 +45,6 @@ module InventoryRefresh
45
45
  add_reference(reference_data)
46
46
  end
47
47
 
48
- # Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
49
- # it.
50
- #
51
- # @param references_array [Array] Array of reference objects acceptable by add_reference method.
52
- # @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
53
- # index.
54
- # @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
55
- def merge!(references_array, ref: nil)
56
- references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
57
- self
58
- end
59
-
60
- # @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
61
- def primary_references
62
- references[primary_index_ref]
63
- end
64
-
65
48
  # Builds a Reference object
66
49
  #
67
50
  # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference