inventory_refresh 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +0 -1
  3. data/.travis.yml +6 -8
  4. data/inventory_refresh.gemspec +3 -5
  5. data/lib/inventory_refresh/application_record_iterator.rb +9 -26
  6. data/lib/inventory_refresh/exception.rb +8 -0
  7. data/lib/inventory_refresh/inventory_collection/builder.rb +6 -6
  8. data/lib/inventory_refresh/inventory_collection/data_storage.rb +0 -9
  9. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +35 -144
  10. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +1 -44
  11. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +6 -34
  12. data/lib/inventory_refresh/inventory_collection/index/type/base.rb +0 -8
  13. data/lib/inventory_refresh/inventory_collection/references_storage.rb +0 -17
  14. data/lib/inventory_refresh/inventory_collection/scanner.rb +1 -87
  15. data/lib/inventory_refresh/inventory_collection/serialization.rb +10 -16
  16. data/lib/inventory_refresh/inventory_collection.rb +36 -110
  17. data/lib/inventory_refresh/inventory_object.rb +34 -68
  18. data/lib/inventory_refresh/inventory_object_lazy.rb +10 -17
  19. data/lib/inventory_refresh/persister.rb +63 -29
  20. data/lib/inventory_refresh/save_collection/base.rb +2 -4
  21. data/lib/inventory_refresh/save_collection/saver/base.rb +8 -108
  22. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +48 -126
  23. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +19 -1
  24. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +3 -68
  25. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +0 -125
  26. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +5 -9
  27. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +9 -17
  28. data/lib/inventory_refresh/save_collection/sweeper.rb +91 -18
  29. data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
  30. data/lib/inventory_refresh/save_inventory.rb +12 -5
  31. data/lib/inventory_refresh/version.rb +1 -1
  32. data/lib/inventory_refresh.rb +0 -2
  33. metadata +15 -57
  34. data/lib/inventory_refresh/save_collection/saver/batch.rb +0 -17
  35. data/lib/inventory_refresh/save_collection/saver/default.rb +0 -57
  36. data/lib/inventory_refresh/target.rb +0 -73
  37. data/lib/inventory_refresh/target_collection.rb +0 -92
@@ -3,15 +3,13 @@ module InventoryRefresh
3
3
  require 'json'
4
4
  require 'yaml'
5
5
 
6
- attr_reader :manager, :target, :collections
6
+ attr_reader :manager, :collections
7
7
 
8
8
  attr_accessor :refresh_state_uuid, :refresh_state_part_uuid, :total_parts, :sweep_scope, :retry_count, :retry_max
9
9
 
10
10
  # @param manager [ManageIQ::Providers::BaseManager] A manager object
11
- # @param target [Object] A refresh Target object
12
- def initialize(manager, target = nil)
11
+ def initialize(manager)
13
12
  @manager = manager
14
- @target = target
15
13
 
16
14
  @collections = {}
17
15
 
@@ -45,9 +43,6 @@ module InventoryRefresh
45
43
  &block)
46
44
 
47
45
  builder.add_properties(extra_properties) if extra_properties.present?
48
-
49
- builder.add_properties({:manager_uuids => target.try(:references, collection_name) || []}, :if_missing) if targeted?
50
-
51
46
  builder.evaluate_lambdas!(self)
52
47
 
53
48
  collections[collection_name] = builder.to_inventory_collection
@@ -104,8 +99,6 @@ module InventoryRefresh
104
99
  def to_hash
105
100
  collections_data = collections.map do |_, collection|
106
101
  next if collection.data.blank? &&
107
- collection.targeted_scope.primary_references.blank? &&
108
- collection.all_manager_uuids.nil? &&
109
102
  collection.skeletal_primary_index.index_data.blank?
110
103
 
111
104
  collection.to_hash
@@ -117,7 +110,7 @@ module InventoryRefresh
117
110
  :retry_count => retry_count,
118
111
  :retry_max => retry_max,
119
112
  :total_parts => total_parts,
120
- :sweep_scope => sweep_scope,
113
+ :sweep_scope => sweep_scope_to_hash(sweep_scope),
121
114
  :collections => collections_data,
122
115
  }
123
116
  end
@@ -127,22 +120,19 @@ module InventoryRefresh
127
120
  #
128
121
  # @param json_data [String] input JSON data
129
122
  # @return [ManageIQ::Providers::Inventory::Persister] Persister object loaded from a passed JSON
130
- def from_json(json_data, manager, target = nil)
131
- from_hash(JSON.parse(json_data), manager, target)
123
+ def from_json(json_data, manager)
124
+ from_hash(JSON.parse(json_data), manager)
132
125
  end
133
126
 
134
127
  # Returns Persister object built from serialized data
135
128
  #
136
129
  # @param persister_data [Hash] serialized Persister object in hash
137
130
  # @return [ManageIQ::Providers::Inventory::Persister] Persister object built from serialized data
138
- def from_hash(persister_data, manager, target = nil)
139
- # TODO(lsmola) we need to pass serialized targeted scope here
140
- target ||= InventoryRefresh::TargetCollection.new(:manager => manager)
141
-
142
- new(manager, target).tap do |persister|
131
+ def from_hash(persister_data, manager)
132
+ new(manager).tap do |persister|
143
133
  persister_data['collections'].each do |collection|
144
134
  inventory_collection = persister.collections[collection['name'].try(:to_sym)]
145
- raise "Unrecognized InventoryCollection name: #{inventory_collection}" if inventory_collection.blank?
135
+ raise "Unrecognized InventoryCollection name: #{collection['name']}" if inventory_collection.blank?
146
136
 
147
137
  inventory_collection.from_hash(collection, persister.collections)
148
138
  end
@@ -152,9 +142,44 @@ module InventoryRefresh
152
142
  persister.retry_count = persister_data['retry_count']
153
143
  persister.retry_max = persister_data['retry_max']
154
144
  persister.total_parts = persister_data['total_parts']
155
- persister.sweep_scope = persister_data['sweep_scope']
145
+ persister.sweep_scope = sweep_scope_from_hash(persister_data['sweep_scope'], persister.collections)
156
146
  end
157
147
  end
148
+
149
+ private
150
+
151
+ def assert_sweep_scope!(sweep_scope)
152
+ return unless sweep_scope
153
+
154
+ allowed_format_message = "Allowed format of sweep scope is Array<String> or Hash{String => Hash}, got #{sweep_scope}"
155
+
156
+ if sweep_scope.kind_of?(Array)
157
+ return if sweep_scope.all? { |x| x.kind_of?(String) || x.kind_of?(Symbol) }
158
+
159
+ raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
160
+ elsif sweep_scope.kind_of?(Hash)
161
+ return if sweep_scope.values.all? { |x| x.kind_of?(Array) }
162
+
163
+ raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
164
+ end
165
+
166
+ raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
167
+ end
168
+
169
+ def sweep_scope_from_hash(sweep_scope, available_inventory_collections)
170
+ assert_sweep_scope!(sweep_scope)
171
+
172
+ return sweep_scope unless sweep_scope.kind_of?(Hash)
173
+
174
+ sweep_scope.each_with_object({}) do |(k, v), obj|
175
+ inventory_collection = available_inventory_collections[k.try(:to_sym)]
176
+ raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
177
+
178
+ serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
179
+
180
+ obj[k] = serializer.sweep_scope_from_hash(v, available_inventory_collections)
181
+ end.symbolize_keys!
182
+ end
158
183
  end
159
184
 
160
185
  protected
@@ -185,28 +210,37 @@ module InventoryRefresh
185
210
  nil
186
211
  end
187
212
 
188
- def saver_strategy
189
- :default
190
- end
191
-
192
- # Persisters for targeted refresh can override to true
193
- def targeted?
213
+ def assert_graph_integrity?
194
214
  false
195
215
  end
196
216
 
197
- def assert_graph_integrity?
198
- false
217
+ def use_ar_object?
218
+ true
199
219
  end
200
220
 
201
221
  # @return [Hash] kwargs shared for all InventoryCollection objects
202
222
  def shared_options
203
223
  {
204
- :saver_strategy => saver_strategy,
205
224
  :strategy => strategy,
206
- :targeted => targeted?,
207
225
  :parent => manager.presence,
208
226
  :assert_graph_integrity => assert_graph_integrity?,
227
+ :use_ar_object => use_ar_object?,
209
228
  }
210
229
  end
230
+
231
+ private
232
+
233
+ def sweep_scope_to_hash(sweep_scope)
234
+ return sweep_scope unless sweep_scope.kind_of?(Hash)
235
+
236
+ sweep_scope.each_with_object({}) do |(k, v), obj|
237
+ inventory_collection = collections[k.try(:to_sym)]
238
+ raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
239
+
240
+ serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
241
+
242
+ obj[k] = serializer.sweep_scope_to_hash(v)
243
+ end
244
+ end
211
245
  end
212
246
  end
@@ -1,7 +1,5 @@
1
1
  require "inventory_refresh/logging"
2
- require "inventory_refresh/save_collection/saver/batch"
3
2
  require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
4
- require "inventory_refresh/save_collection/saver/default"
5
3
 
6
4
  module InventoryRefresh::SaveCollection
7
5
  class Base
@@ -16,7 +14,7 @@ module InventoryRefresh::SaveCollection
16
14
  return if skip?(inventory_collection)
17
15
 
18
16
  logger.debug("----- BEGIN ----- Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
19
- " the database, for the manager: '#{ems.name}'...")
17
+ " the database, for the manager: '#{ems.id}'...")
20
18
 
21
19
  if inventory_collection.custom_save_block.present?
22
20
  logger.debug("Saving collection #{inventory_collection} using a custom save block")
@@ -24,7 +22,7 @@ module InventoryRefresh::SaveCollection
24
22
  else
25
23
  save_inventory(inventory_collection)
26
24
  end
27
- logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.name}'...Complete")
25
+ logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.id}'...Complete")
28
26
  inventory_collection.saved = true
29
27
  end
30
28
 
@@ -12,8 +12,6 @@ module InventoryRefresh::SaveCollection
12
12
  # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we will be saving
13
13
  def initialize(inventory_collection)
14
14
  @inventory_collection = inventory_collection
15
- # TODO(lsmola) do I need to reload every time? Also it should be enough to clear the associations.
16
- inventory_collection.parent.reload if inventory_collection.parent
17
15
  @association = inventory_collection.db_collection_for_comparison
18
16
 
19
17
  # Private attrs
@@ -28,14 +26,10 @@ module InventoryRefresh::SaveCollection
28
26
  @unique_db_primary_keys = Set.new
29
27
  @unique_db_indexes = Set.new
30
28
 
31
- # Right now ApplicationRecordIterator in association is used for targeted refresh. Given the small amount of
32
- # records flowing through there, we probably don't need to optimize that association to fetch a pure SQL.
33
- @pure_sql_records_fetching = !inventory_collection.use_ar_object? && !@association.kind_of?(InventoryRefresh::ApplicationRecordIterator)
34
-
35
29
  @batch_size_for_persisting = inventory_collection.batch_size_pure_sql
30
+ @batch_size = inventory_collection.use_ar_object? ? @batch_size_for_persisting : inventory_collection.batch_size
36
31
 
37
- @batch_size = @pure_sql_records_fetching ? @batch_size_for_persisting : inventory_collection.batch_size
38
- @record_key_method = @pure_sql_records_fetching ? :pure_sql_record_key : :ar_record_key
32
+ @record_key_method = inventory_collection.pure_sql_record_fetching? ? :pure_sql_record_key : :ar_record_key
39
33
  @select_keys_indexes = @select_keys.each_with_object({}).with_index { |(key, obj), index| obj[key.to_s] = index }
40
34
  @pg_types = @model_class.attribute_names.each_with_object({}) do |key, obj|
41
35
  obj[key.to_sym] = inventory_collection.model_class.columns_hash[key]
@@ -75,14 +69,8 @@ module InventoryRefresh::SaveCollection
75
69
 
76
70
  # Saves the InventoryCollection
77
71
  def save_inventory_collection!
78
- # If we have a targeted InventoryCollection that wouldn't do anything, quickly skip it
79
- return if inventory_collection.noop?
80
-
81
- # Delete_complement strategy using :all_manager_uuids attribute
82
- delete_complement unless inventory_collection.delete_complement_noop?
83
-
84
72
  # Create/Update/Archive/Delete records based on InventoryCollection data and scope
85
- save!(association) unless inventory_collection.saving_noop?
73
+ save!(association)
86
74
  end
87
75
 
88
76
  protected
@@ -101,6 +89,8 @@ module InventoryRefresh::SaveCollection
101
89
  # @param attributes [Hash] attributes hash
102
90
  # @return [Hash] modified hash from parameter attributes with casted values
103
91
  def values_for_database!(all_attribute_keys, attributes)
92
+ # TODO(lsmola) we'll need to fill default value from the DB to the NOT_NULL columns here, since sending NULL
93
+ # to column with NOT_NULL constraint always fails, even if there is a default value
104
94
  all_attribute_keys.each do |key|
105
95
  next unless attributes.key?(key)
106
96
 
@@ -112,11 +102,7 @@ module InventoryRefresh::SaveCollection
112
102
  end
113
103
 
114
104
  def transform_to_hash!(all_attribute_keys, hash)
115
- if inventory_collection.use_ar_object?
116
- record = inventory_collection.model_class.new(hash)
117
- values_for_database!(all_attribute_keys,
118
- record.attributes.slice(*record.changed_attributes.keys).symbolize_keys)
119
- elsif serializable_keys?
105
+ if serializable_keys?
120
106
  values_for_database!(all_attribute_keys,
121
107
  hash)
122
108
  else
@@ -127,101 +113,15 @@ module InventoryRefresh::SaveCollection
127
113
  private
128
114
 
129
115
  attr_reader :unique_index_keys, :unique_index_keys_to_s, :select_keys, :unique_db_primary_keys, :unique_db_indexes,
130
- :primary_key, :arel_primary_key, :record_key_method, :pure_sql_records_fetching, :select_keys_indexes,
116
+ :primary_key, :arel_primary_key, :record_key_method, :select_keys_indexes,
131
117
  :batch_size, :batch_size_for_persisting, :model_class, :serializable_keys, :deserializable_keys, :pg_types, :table_name,
132
118
  :q_table_name
133
119
 
134
120
  delegate :supports_column?, :to => :inventory_collection
135
121
 
136
- # Saves the InventoryCollection
137
- #
138
- # @param association [Symbol] An existing association on manager
139
- def save!(association)
140
- attributes_index = {}
141
- inventory_objects_index = {}
142
- inventory_collection.each do |inventory_object|
143
- attributes = inventory_object.attributes(inventory_collection)
144
- index = build_stringified_reference(attributes, unique_index_keys)
145
-
146
- attributes_index[index] = attributes
147
- inventory_objects_index[index] = inventory_object
148
- end
149
-
150
- logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
151
- # Records that are in the DB, we will be updating or deleting them.
152
- ActiveRecord::Base.transaction do
153
- association.find_each do |record|
154
- index = build_stringified_reference_for_record(record, unique_index_keys)
155
-
156
- next unless assert_distinct_relation(record.id)
157
- next unless assert_unique_record(record, index)
158
-
159
- inventory_object = inventory_objects_index.delete(index)
160
- hash = attributes_index.delete(index)
161
-
162
- if inventory_object.nil?
163
- # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
164
- # delete it from the DB.
165
- delete_record!(record) if inventory_collection.delete_allowed?
166
- else
167
- # Record was found in the DB and sent for saving, we will be updating the DB.
168
- update_record!(record, hash, inventory_object) if assert_referential_integrity(hash)
169
- end
170
- end
171
- end
172
-
173
- unless inventory_collection.custom_reconnect_block.nil?
174
- inventory_collection.custom_reconnect_block.call(inventory_collection, inventory_objects_index, attributes_index)
175
- end
176
-
177
- # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
178
- if inventory_collection.create_allowed?
179
- ActiveRecord::Base.transaction do
180
- inventory_objects_index.each do |index, inventory_object|
181
- hash = attributes_index.delete(index)
182
-
183
- create_record!(hash, inventory_object) if assert_referential_integrity(hash)
184
- end
185
- end
186
- end
187
- logger.debug("Processing #{inventory_collection}, "\
188
- "created=#{inventory_collection.created_records.count}, "\
189
- "updated=#{inventory_collection.updated_records.count}, "\
190
- "deleted=#{inventory_collection.deleted_records.count}...Complete")
191
- rescue => e
192
- logger.error("Error when saving #{inventory_collection} with #{inventory_collection_details}. Message: #{e.message}")
193
- raise e
194
- end
195
-
196
122
  # @return [String] a string for logging purposes
197
123
  def inventory_collection_details
198
- "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}, targeted: #{inventory_collection.targeted?}"
199
- end
200
-
201
- # @param record [ApplicationRecord] ApplicationRecord object
202
- # @param key [Symbol] A key that is an attribute of the AR object
203
- # @return [Object] Value of attribute name :key on the :record
204
- def record_key(record, key)
205
- record.public_send(key)
206
- end
207
-
208
- # Deletes a complement of referenced data
209
- def delete_complement
210
- raise(":delete_complement method is supported only for :saver_strategy => [:batch, :concurrent_safe_batch]")
211
- end
212
-
213
- # Deletes/soft-deletes a given record
214
- #
215
- # @param [ApplicationRecord] record we want to delete
216
- def delete_record!(record)
217
- record.public_send(inventory_collection.delete_method)
218
- inventory_collection.store_deleted_records(record)
219
- end
220
-
221
- # @return [TrueClass] always return true, this method is redefined in default saver
222
- def assert_unique_record(_record, _index)
223
- # TODO(lsmola) can go away once we indexed our DB with unique indexes
224
- true
124
+ "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}"
225
125
  end
226
126
 
227
127
  # Check if relation provided is distinct, i.e. the relation should not return the same primary key value twice.
@@ -46,40 +46,6 @@ module InventoryRefresh::SaveCollection
46
46
  record[select_keys_indexes[key]]
47
47
  end
48
48
 
49
- # Returns iterator or relation based on settings
50
- #
51
- # @param association [Symbol] An existing association on manager
52
- # @return [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or relation based on settings
53
- def batch_iterator(association)
54
- if pure_sql_records_fetching
55
- # Building fast iterator doing pure SQL query and therefore avoiding redundant creation of AR objects. The
56
- # iterator responds to find_in_batches, so it acts like the AR relation. For targeted refresh, the association
57
- # can already be ApplicationRecordIterator, so we will skip that.
58
- pure_sql_iterator = lambda do |&block|
59
- primary_key_offset = nil
60
- loop do
61
- relation = association.select(*select_keys)
62
- .reorder("#{primary_key} ASC")
63
- .limit(batch_size)
64
- # Using rails way of comparing primary key instead of offset
65
- relation = relation.where(arel_primary_key.gt(primary_key_offset)) if primary_key_offset
66
- records = get_connection.query(relation.to_sql)
67
- last_record = records.last
68
- block.call(records)
69
-
70
- break if records.size < batch_size
71
- primary_key_offset = record_key(last_record, primary_key)
72
- end
73
- end
74
-
75
- InventoryRefresh::ApplicationRecordIterator.new(:iterator => pure_sql_iterator)
76
- else
77
- # Normal Rails ActiveRecord::Relation where we can call find_in_batches or
78
- # InventoryRefresh::ApplicationRecordIterator passed from targeted refresh
79
- association
80
- end
81
- end
82
-
83
49
  # Saves the InventoryCollection
84
50
  #
85
51
  # @param association [Symbol] An existing association on manager
@@ -89,7 +55,7 @@ module InventoryRefresh::SaveCollection
89
55
  all_attribute_keys = Set.new + inventory_collection.batch_extra_attributes
90
56
 
91
57
  inventory_collection.each do |inventory_object|
92
- attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
58
+ attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
93
59
  index = build_stringified_reference(attributes, unique_index_keys)
94
60
 
95
61
  # Interesting fact: not building attributes_index and using only inventory_objects_index doesn't do much
@@ -103,7 +69,7 @@ module InventoryRefresh::SaveCollection
103
69
  logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
104
70
 
105
71
  unless inventory_collection.create_only?
106
- update_or_destroy_records!(batch_iterator(association), inventory_objects_index, attributes_index, all_attribute_keys)
72
+ load_and_update_records!(association, inventory_objects_index, attributes_index, all_attribute_keys)
107
73
  end
108
74
 
109
75
  unless inventory_collection.create_only?
@@ -112,15 +78,11 @@ module InventoryRefresh::SaveCollection
112
78
 
113
79
  # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
114
80
  if inventory_collection.create_allowed?
115
- on_conflict = inventory_collection.parallel_safe? ? :do_update : nil
116
-
117
81
  inventory_objects_index.each_slice(batch_size_for_persisting) do |batch|
118
- create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => on_conflict)
82
+ create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => :do_update)
119
83
  end
120
84
 
121
- if inventory_collection.parallel_safe?
122
- create_or_update_partial_records(all_attribute_keys)
123
- end
85
+ create_or_update_partial_records(all_attribute_keys)
124
86
  end
125
87
 
126
88
  logger.debug("Marking :last_seen_at of #{inventory_collection} of size #{inventory_collection.size}...")
@@ -149,7 +111,7 @@ module InventoryRefresh::SaveCollection
149
111
  end
150
112
 
151
113
  def mark_last_seen_at(attributes_index)
152
- return unless supports_column?(:last_seen_at) && inventory_collection.parallel_safe?
114
+ return unless supports_column?(:last_seen_at)
153
115
  return if attributes_index.blank?
154
116
 
155
117
  all_attribute_keys = [:last_seen_at]
@@ -162,8 +124,7 @@ module InventoryRefresh::SaveCollection
162
124
  get_connection.execute(query)
163
125
  end
164
126
 
165
- # Batch updates existing records that are in the DB using attributes_index. And delete the ones that were not
166
- # present in inventory_objects_index.
127
+ # Batch updates existing records that are in the DB using attributes_index.
167
128
  #
168
129
  # @param records_batch_iterator [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or
169
130
  # relation, both responding to :find_in_batches method
@@ -171,12 +132,11 @@ module InventoryRefresh::SaveCollection
171
132
  # @param attributes_index [Hash{String => Hash}] Hash of data hashes with only keys that are column names of the
172
133
  # models's table
173
134
  # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
174
- def update_or_destroy_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
175
- hashes_for_update = []
176
- records_for_destroy = []
135
+ def load_and_update_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
136
+ hashes_for_update = []
177
137
  indexed_inventory_objects = {}
178
138
 
179
- records_batch_iterator.find_in_batches(:batch_size => batch_size) do |batch|
139
+ records_batch_iterator.find_in_batches(:batch_size => batch_size, :attributes_index => attributes_index) do |batch|
180
140
  update_time = time_now
181
141
 
182
142
  batch.each do |record|
@@ -189,20 +149,14 @@ module InventoryRefresh::SaveCollection
189
149
  inventory_object = inventory_objects_index.delete(index)
190
150
  hash = attributes_index[index]
191
151
 
192
- if inventory_object.nil?
193
- # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
194
- # delete it from the DB.
195
- if inventory_collection.delete_allowed?
196
- records_for_destroy << record
197
- end
198
- else
152
+ if inventory_object
199
153
  # Record was found in the DB and sent for saving, we will be updating the DB.
200
154
  inventory_object.id = primary_key_value
201
155
  next unless assert_referential_integrity(hash)
202
- next unless changed?(record, hash, all_attribute_keys)
203
156
 
204
- if inventory_collection.parallel_safe? &&
205
- (supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys))
157
+ record_version = nil
158
+ record_version_max = nil
159
+ if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
206
160
 
207
161
  version_attr, max_version_attr = if supports_remote_data_timestamp?(all_attribute_keys)
208
162
  [:resource_timestamp, :resource_timestamps_max]
@@ -210,16 +164,16 @@ module InventoryRefresh::SaveCollection
210
164
  [:resource_counter, :resource_counters_max]
211
165
  end
212
166
 
213
- next if skeletonize_or_skip_record(record_key(record, version_attr),
214
- hash[version_attr],
215
- record_key(record, max_version_attr),
216
- inventory_object)
167
+ record_version = record_key(record, version_attr.to_s)
168
+ record_version_max = record_key(record, max_version_attr.to_s)
217
169
  end
218
170
 
219
171
  hash_for_update = if inventory_collection.use_ar_object?
220
172
  record.assign_attributes(hash.except(:id))
173
+ next unless changed?(record)
174
+
221
175
  values_for_database!(all_attribute_keys,
222
- record.attributes.symbolize_keys)
176
+ hash)
223
177
  elsif serializable_keys?
224
178
  # TODO(lsmola) hash data with current DB data to allow subset of data being sent,
225
179
  # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
@@ -230,6 +184,14 @@ module InventoryRefresh::SaveCollection
230
184
  # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
231
185
  hash
232
186
  end
187
+
188
+ if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
189
+ next if skeletonize_or_skip_record(record_version,
190
+ hash[version_attr],
191
+ record_version_max,
192
+ inventory_object)
193
+ end
194
+
233
195
  assign_attributes_for_update!(hash_for_update, update_time)
234
196
 
235
197
  hash_for_update[:id] = primary_key_value
@@ -245,39 +207,20 @@ module InventoryRefresh::SaveCollection
245
207
  hashes_for_update = []
246
208
  indexed_inventory_objects = {}
247
209
  end
248
-
249
- # Destroy in batches
250
- if records_for_destroy.size >= batch_size_for_persisting
251
- destroy_records!(records_for_destroy)
252
- records_for_destroy = []
253
- end
254
210
  end
255
211
 
256
212
  # Update the last batch
257
213
  update_records!(all_attribute_keys, hashes_for_update, indexed_inventory_objects)
258
214
  hashes_for_update = [] # Cleanup so GC can release it sooner
259
-
260
- # Destroy the last batch
261
- destroy_records!(records_for_destroy)
262
- records_for_destroy = [] # Cleanup so GC can release it sooner
263
215
  end
264
216
 
265
- def changed?(_record, _hash, _all_attribute_keys)
217
+ def changed?(record)
266
218
  return true unless inventory_collection.check_changed?
267
219
 
268
- # TODO(lsmola) this check needs to be disabled now, because it doesn't work with lazy_find having secondary
269
- # indexes. Examples: we save a pod before we save a project, that means the project lazy_find won't evaluate,
270
- # because we load it with secondary index and can't do skeletal precreate. Then when the object is being saved
271
- # again, the lazy_find is evaluated, but the resource version is not changed, so the row is not saved.
272
- #
273
- # To keep this quick .changed? check, we might need to extend this, so the resource_version doesn't save until
274
- # all lazy_links of the row are evaluated.
275
- #
276
- # if supports_resource_version?(all_attribute_keys) && supports_column?(resource_version_column)
277
- # record_resource_version = record_key(record, resource_version_column.to_s)
278
- #
279
- # return record_resource_version != hash[resource_version_column]
280
- # end
220
+ # Skip if nothing changed
221
+ return false if record.changed_attributes.empty?
222
+ # Skip if we only changed the resource_timestamp, but data stays the same
223
+ return false if record.changed_attributes.keys == ["resource_timestamp"]
281
224
 
282
225
  true
283
226
  end
@@ -285,10 +228,7 @@ module InventoryRefresh::SaveCollection
285
228
  def db_columns_index(record, pure_sql: false)
286
229
  # Incoming values are in SQL string form.
287
230
  # TODO(lsmola) unify this behavior with object_index_with_keys method in InventoryCollection
288
- # TODO(lsmola) maybe we can drop the whole pure sql fetching, since everything will be targeted refresh
289
231
  # with streaming refresh? Maybe just metrics and events will not be, but those should be upsert only
290
- # TODO(lsmola) taking ^ in account, we can't drop pure sql, since that is returned by batch insert and
291
- # update queries
292
232
  unique_index_keys_to_s.map do |attribute|
293
233
  value = if pure_sql
294
234
  record[attribute]
@@ -319,20 +259,13 @@ module InventoryRefresh::SaveCollection
319
259
  def update_records!(all_attribute_keys, hashes, indexed_inventory_objects)
320
260
  return if hashes.blank?
321
261
 
322
- unless inventory_collection.parallel_safe?
323
- # We need to update the stored records before we save it, since hashes are modified
324
- inventory_collection.store_updated_records(hashes)
325
- end
326
-
327
262
  query = build_update_query(all_attribute_keys, hashes)
328
263
  result = get_connection.execute(query)
329
264
 
330
- if inventory_collection.parallel_safe?
331
- # We will check for timestamp clashes of full row update and we will fallback to skeletal update
332
- inventory_collection.store_updated_records(result)
265
+ # We will check for timestamp clashes of full row update and we will fallback to skeletal update
266
+ inventory_collection.store_updated_records(result)
333
267
 
334
- skeletonize_ignored_records!(indexed_inventory_objects, result)
335
- end
268
+ skeletonize_ignored_records!(indexed_inventory_objects, result)
336
269
 
337
270
  result
338
271
  end
@@ -352,11 +285,7 @@ module InventoryRefresh::SaveCollection
352
285
  hashes = []
353
286
  create_time = time_now
354
287
  batch.each do |index, inventory_object|
355
- hash = if inventory_collection.use_ar_object?
356
- record = inventory_collection.model_class.new(attributes_index[index])
357
- values_for_database!(all_attribute_keys,
358
- record.attributes.symbolize_keys)
359
- elsif serializable_keys?
288
+ hash = if serializable_keys?
360
289
  values_for_database!(all_attribute_keys,
361
290
  attributes_index[index])
362
291
  else
@@ -378,24 +307,19 @@ module InventoryRefresh::SaveCollection
378
307
  build_insert_query(all_attribute_keys, hashes, :on_conflict => on_conflict, :mode => :full)
379
308
  )
380
309
 
381
- if inventory_collection.parallel_safe?
382
- # We've done upsert, so records were either created or updated. We can recognize that by checking if
383
- # created and updated timestamps are the same
384
- created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
385
- created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
386
- updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
387
- updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
388
-
389
- if created_attr && updated_attr
390
- created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
391
- inventory_collection.store_created_records(created)
392
- inventory_collection.store_updated_records(updated)
393
- else
394
- # The record doesn't have both created and updated attrs, so we'll take all as created
395
- inventory_collection.store_created_records(result)
396
- end
310
+ # We've done upsert, so records were either created or updated. We can recognize that by checking if
311
+ # created and updated timestamps are the same
312
+ created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
313
+ created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
314
+ updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
315
+ updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
316
+
317
+ if created_attr && updated_attr
318
+ created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
319
+ inventory_collection.store_created_records(created)
320
+ inventory_collection.store_updated_records(updated)
397
321
  else
398
- # We've done just insert, so all records were created
322
+ # The record doesn't have both created and updated attrs, so we'll take all as created
399
323
  inventory_collection.store_created_records(result)
400
324
  end
401
325
 
@@ -408,9 +332,7 @@ module InventoryRefresh::SaveCollection
408
332
  :on_conflict => on_conflict)
409
333
  end
410
334
 
411
- if inventory_collection.parallel_safe?
412
- skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
413
- end
335
+ skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
414
336
  end
415
337
 
416
338
  # Stores primary_key values of created records into associated InventoryObject objects.