inventory_refresh 0.2.2 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +0 -1
  3. data/.travis.yml +6 -8
  4. data/inventory_refresh.gemspec +2 -4
  5. data/lib/inventory_refresh.rb +0 -2
  6. data/lib/inventory_refresh/application_record_iterator.rb +9 -26
  7. data/lib/inventory_refresh/exception.rb +8 -0
  8. data/lib/inventory_refresh/inventory_collection.rb +36 -110
  9. data/lib/inventory_refresh/inventory_collection/builder.rb +6 -6
  10. data/lib/inventory_refresh/inventory_collection/data_storage.rb +0 -9
  11. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +34 -143
  12. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +1 -44
  13. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +6 -34
  14. data/lib/inventory_refresh/inventory_collection/index/type/base.rb +0 -8
  15. data/lib/inventory_refresh/inventory_collection/references_storage.rb +0 -17
  16. data/lib/inventory_refresh/inventory_collection/scanner.rb +1 -87
  17. data/lib/inventory_refresh/inventory_collection/serialization.rb +10 -16
  18. data/lib/inventory_refresh/inventory_object.rb +34 -68
  19. data/lib/inventory_refresh/inventory_object_lazy.rb +10 -17
  20. data/lib/inventory_refresh/persister.rb +63 -29
  21. data/lib/inventory_refresh/save_collection/base.rb +2 -4
  22. data/lib/inventory_refresh/save_collection/saver/base.rb +8 -108
  23. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +48 -126
  24. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +19 -1
  25. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +3 -68
  26. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +0 -125
  27. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +5 -9
  28. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +9 -17
  29. data/lib/inventory_refresh/save_collection/sweeper.rb +91 -18
  30. data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
  31. data/lib/inventory_refresh/save_inventory.rb +12 -5
  32. data/lib/inventory_refresh/version.rb +1 -1
  33. metadata +9 -45
  34. data/lib/inventory_refresh/save_collection/saver/batch.rb +0 -17
  35. data/lib/inventory_refresh/save_collection/saver/default.rb +0 -57
  36. data/lib/inventory_refresh/target.rb +0 -73
  37. data/lib/inventory_refresh/target_collection.rb +0 -92
@@ -30,7 +30,6 @@ module InventoryRefresh
30
30
 
31
31
  # @return [String] stringified reference
32
32
  def to_s
33
- # TODO(lsmola) do we need this method?
34
33
  stringified_reference
35
34
  end
36
35
 
@@ -71,10 +70,6 @@ module InventoryRefresh
71
70
 
72
71
  # @return [Boolean] true if the key is an association on inventory_collection_scope model class
73
72
  def association?(key)
74
- # TODO(lsmola) remove this if there will be better dependency scan, probably with transitive dependencies filled
75
- # in a second pass, then we can get rid of this hardcoded symbols. Right now we are not able to introspect these.
76
- return true if [:parent, :genealogy_parent].include?(key)
77
-
78
73
  inventory_collection.dependency_attributes.key?(key) ||
79
74
  !inventory_collection.association_to_foreign_key_mapping[key].nil?
80
75
  end
@@ -100,7 +95,7 @@ module InventoryRefresh
100
95
 
101
96
  private
102
97
 
103
- delegate :parallel_safe?, :saved?, :saver_strategy, :skeletal_primary_index, :targeted?, :to => :inventory_collection
98
+ delegate :saved?, :saver_strategy, :skeletal_primary_index, :to => :inventory_collection
104
99
  delegate :nested_secondary_index?, :primary?, :full_reference, :keys, :primary?, :to => :reference
105
100
 
106
101
  attr_writer :reference
@@ -112,20 +107,18 @@ module InventoryRefresh
112
107
  #
113
108
  # @return [InventoryRefresh::InventoryObject, NilClass] Returns pre-created InventoryObject or nil
114
109
  def skeletal_precreate!
115
- # We can do skeletal pre-create only for strategies using unique indexes. Since this can build records out of
116
- # the given :arel scope, we will always attempt to create the recod, so we need unique index to avoid duplication
117
- # of records.
118
- return unless parallel_safe?
119
110
  # Pre-create only for strategies that will be persisting data, i.e. are not saved already
120
111
  return if saved?
121
112
  # We can only do skeletal pre-create for primary index reference, since that is needed to create DB unique index
122
- return unless primary?
123
- # Full reference must be present
124
- return if full_reference.blank?
125
-
126
- # To avoid pre-creating invalid records all fields of a primary key must have value
127
- # TODO(lsmola) for composite keys, it's still valid to have one of the keys nil, figure out how to allow this
128
- return if keys.any? { |x| full_reference[x].blank? }
113
+ # and full reference must be present
114
+ return if !primary? || full_reference.blank?
115
+
116
+ # To avoid pre-creating invalid records all fields of a primary key must have non null value
117
+ # TODO(lsmola) for composite keys, it's still valid to have one of the keys nil, figure out how to allow this. We
118
+ # will need to scan the DB for NOT NULL constraint and allow it based on that. So we would move this check to
119
+ # saving code, but this will require bigger change, since having the column nil means we will have to batch it
120
+ # smartly, since having nil means we will need to use different unique index for the upsert/update query.
121
+ return if keys.any? { |x| full_reference[x].nil? }
129
122
 
130
123
  skeletal_primary_index.build(full_reference)
131
124
  end
@@ -3,15 +3,13 @@ module InventoryRefresh
3
3
  require 'json'
4
4
  require 'yaml'
5
5
 
6
- attr_reader :manager, :target, :collections
6
+ attr_reader :manager, :collections
7
7
 
8
8
  attr_accessor :refresh_state_uuid, :refresh_state_part_uuid, :total_parts, :sweep_scope, :retry_count, :retry_max
9
9
 
10
10
  # @param manager [ManageIQ::Providers::BaseManager] A manager object
11
- # @param target [Object] A refresh Target object
12
- def initialize(manager, target = nil)
11
+ def initialize(manager)
13
12
  @manager = manager
14
- @target = target
15
13
 
16
14
  @collections = {}
17
15
 
@@ -45,9 +43,6 @@ module InventoryRefresh
45
43
  &block)
46
44
 
47
45
  builder.add_properties(extra_properties) if extra_properties.present?
48
-
49
- builder.add_properties({:manager_uuids => target.try(:references, collection_name) || []}, :if_missing) if targeted?
50
-
51
46
  builder.evaluate_lambdas!(self)
52
47
 
53
48
  collections[collection_name] = builder.to_inventory_collection
@@ -104,8 +99,6 @@ module InventoryRefresh
104
99
  def to_hash
105
100
  collections_data = collections.map do |_, collection|
106
101
  next if collection.data.blank? &&
107
- collection.targeted_scope.primary_references.blank? &&
108
- collection.all_manager_uuids.nil? &&
109
102
  collection.skeletal_primary_index.index_data.blank?
110
103
 
111
104
  collection.to_hash
@@ -117,7 +110,7 @@ module InventoryRefresh
117
110
  :retry_count => retry_count,
118
111
  :retry_max => retry_max,
119
112
  :total_parts => total_parts,
120
- :sweep_scope => sweep_scope,
113
+ :sweep_scope => sweep_scope_to_hash(sweep_scope),
121
114
  :collections => collections_data,
122
115
  }
123
116
  end
@@ -127,22 +120,19 @@ module InventoryRefresh
127
120
  #
128
121
  # @param json_data [String] input JSON data
129
122
  # @return [ManageIQ::Providers::Inventory::Persister] Persister object loaded from a passed JSON
130
- def from_json(json_data, manager, target = nil)
131
- from_hash(JSON.parse(json_data), manager, target)
123
+ def from_json(json_data, manager)
124
+ from_hash(JSON.parse(json_data), manager)
132
125
  end
133
126
 
134
127
  # Returns Persister object built from serialized data
135
128
  #
136
129
  # @param persister_data [Hash] serialized Persister object in hash
137
130
  # @return [ManageIQ::Providers::Inventory::Persister] Persister object built from serialized data
138
- def from_hash(persister_data, manager, target = nil)
139
- # TODO(lsmola) we need to pass serialized targeted scope here
140
- target ||= InventoryRefresh::TargetCollection.new(:manager => manager)
141
-
142
- new(manager, target).tap do |persister|
131
+ def from_hash(persister_data, manager)
132
+ new(manager).tap do |persister|
143
133
  persister_data['collections'].each do |collection|
144
134
  inventory_collection = persister.collections[collection['name'].try(:to_sym)]
145
- raise "Unrecognized InventoryCollection name: #{inventory_collection}" if inventory_collection.blank?
135
+ raise "Unrecognized InventoryCollection name: #{collection['name']}" if inventory_collection.blank?
146
136
 
147
137
  inventory_collection.from_hash(collection, persister.collections)
148
138
  end
@@ -152,9 +142,44 @@ module InventoryRefresh
152
142
  persister.retry_count = persister_data['retry_count']
153
143
  persister.retry_max = persister_data['retry_max']
154
144
  persister.total_parts = persister_data['total_parts']
155
- persister.sweep_scope = persister_data['sweep_scope']
145
+ persister.sweep_scope = sweep_scope_from_hash(persister_data['sweep_scope'], persister.collections)
156
146
  end
157
147
  end
148
+
149
+ private
150
+
151
+ def assert_sweep_scope!(sweep_scope)
152
+ return unless sweep_scope
153
+
154
+ allowed_format_message = "Allowed format of sweep scope is Array<String> or Hash{String => Hash}, got #{sweep_scope}"
155
+
156
+ if sweep_scope.kind_of?(Array)
157
+ return if sweep_scope.all? { |x| x.kind_of?(String) || x.kind_of?(Symbol) }
158
+
159
+ raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
160
+ elsif sweep_scope.kind_of?(Hash)
161
+ return if sweep_scope.values.all? { |x| x.kind_of?(Array) }
162
+
163
+ raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
164
+ end
165
+
166
+ raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
167
+ end
168
+
169
+ def sweep_scope_from_hash(sweep_scope, available_inventory_collections)
170
+ assert_sweep_scope!(sweep_scope)
171
+
172
+ return sweep_scope unless sweep_scope.kind_of?(Hash)
173
+
174
+ sweep_scope.each_with_object({}) do |(k, v), obj|
175
+ inventory_collection = available_inventory_collections[k.try(:to_sym)]
176
+ raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
177
+
178
+ serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
179
+
180
+ obj[k] = serializer.sweep_scope_from_hash(v, available_inventory_collections)
181
+ end.symbolize_keys!
182
+ end
158
183
  end
159
184
 
160
185
  protected
@@ -185,28 +210,37 @@ module InventoryRefresh
185
210
  nil
186
211
  end
187
212
 
188
- def saver_strategy
189
- :default
190
- end
191
-
192
- # Persisters for targeted refresh can override to true
193
- def targeted?
213
+ def assert_graph_integrity?
194
214
  false
195
215
  end
196
216
 
197
- def assert_graph_integrity?
198
- false
217
+ def use_ar_object?
218
+ true
199
219
  end
200
220
 
201
221
  # @return [Hash] kwargs shared for all InventoryCollection objects
202
222
  def shared_options
203
223
  {
204
- :saver_strategy => saver_strategy,
205
224
  :strategy => strategy,
206
- :targeted => targeted?,
207
225
  :parent => manager.presence,
208
226
  :assert_graph_integrity => assert_graph_integrity?,
227
+ :use_ar_object => use_ar_object?,
209
228
  }
210
229
  end
230
+
231
+ private
232
+
233
+ def sweep_scope_to_hash(sweep_scope)
234
+ return sweep_scope unless sweep_scope.kind_of?(Hash)
235
+
236
+ sweep_scope.each_with_object({}) do |(k, v), obj|
237
+ inventory_collection = collections[k.try(:to_sym)]
238
+ raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
239
+
240
+ serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
241
+
242
+ obj[k] = serializer.sweep_scope_to_hash(v)
243
+ end
244
+ end
211
245
  end
212
246
  end
@@ -1,7 +1,5 @@
1
1
  require "inventory_refresh/logging"
2
- require "inventory_refresh/save_collection/saver/batch"
3
2
  require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
4
- require "inventory_refresh/save_collection/saver/default"
5
3
 
6
4
  module InventoryRefresh::SaveCollection
7
5
  class Base
@@ -16,7 +14,7 @@ module InventoryRefresh::SaveCollection
16
14
  return if skip?(inventory_collection)
17
15
 
18
16
  logger.debug("----- BEGIN ----- Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
19
- " the database, for the manager: '#{ems.name}'...")
17
+ " the database, for the manager: '#{ems.id}'...")
20
18
 
21
19
  if inventory_collection.custom_save_block.present?
22
20
  logger.debug("Saving collection #{inventory_collection} using a custom save block")
@@ -24,7 +22,7 @@ module InventoryRefresh::SaveCollection
24
22
  else
25
23
  save_inventory(inventory_collection)
26
24
  end
27
- logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.name}'...Complete")
25
+ logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.id}'...Complete")
28
26
  inventory_collection.saved = true
29
27
  end
30
28
 
@@ -12,8 +12,6 @@ module InventoryRefresh::SaveCollection
12
12
  # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we will be saving
13
13
  def initialize(inventory_collection)
14
14
  @inventory_collection = inventory_collection
15
- # TODO(lsmola) do I need to reload every time? Also it should be enough to clear the associations.
16
- inventory_collection.parent.reload if inventory_collection.parent
17
15
  @association = inventory_collection.db_collection_for_comparison
18
16
 
19
17
  # Private attrs
@@ -28,14 +26,10 @@ module InventoryRefresh::SaveCollection
28
26
  @unique_db_primary_keys = Set.new
29
27
  @unique_db_indexes = Set.new
30
28
 
31
- # Right now ApplicationRecordIterator in association is used for targeted refresh. Given the small amount of
32
- # records flowing through there, we probably don't need to optimize that association to fetch a pure SQL.
33
- @pure_sql_records_fetching = !inventory_collection.use_ar_object? && !@association.kind_of?(InventoryRefresh::ApplicationRecordIterator)
34
-
35
29
  @batch_size_for_persisting = inventory_collection.batch_size_pure_sql
30
+ @batch_size = inventory_collection.use_ar_object? ? @batch_size_for_persisting : inventory_collection.batch_size
36
31
 
37
- @batch_size = @pure_sql_records_fetching ? @batch_size_for_persisting : inventory_collection.batch_size
38
- @record_key_method = @pure_sql_records_fetching ? :pure_sql_record_key : :ar_record_key
32
+ @record_key_method = inventory_collection.pure_sql_record_fetching? ? :pure_sql_record_key : :ar_record_key
39
33
  @select_keys_indexes = @select_keys.each_with_object({}).with_index { |(key, obj), index| obj[key.to_s] = index }
40
34
  @pg_types = @model_class.attribute_names.each_with_object({}) do |key, obj|
41
35
  obj[key.to_sym] = inventory_collection.model_class.columns_hash[key]
@@ -75,14 +69,8 @@ module InventoryRefresh::SaveCollection
75
69
 
76
70
  # Saves the InventoryCollection
77
71
  def save_inventory_collection!
78
- # If we have a targeted InventoryCollection that wouldn't do anything, quickly skip it
79
- return if inventory_collection.noop?
80
-
81
- # Delete_complement strategy using :all_manager_uuids attribute
82
- delete_complement unless inventory_collection.delete_complement_noop?
83
-
84
72
  # Create/Update/Archive/Delete records based on InventoryCollection data and scope
85
- save!(association) unless inventory_collection.saving_noop?
73
+ save!(association)
86
74
  end
87
75
 
88
76
  protected
@@ -101,6 +89,8 @@ module InventoryRefresh::SaveCollection
101
89
  # @param attributes [Hash] attributes hash
102
90
  # @return [Hash] modified hash from parameter attributes with casted values
103
91
  def values_for_database!(all_attribute_keys, attributes)
92
+ # TODO(lsmola) we'll need to fill default value from the DB to the NOT_NULL columns here, since sending NULL
93
+ # to column with NOT_NULL constraint always fails, even if there is a default value
104
94
  all_attribute_keys.each do |key|
105
95
  next unless attributes.key?(key)
106
96
 
@@ -112,11 +102,7 @@ module InventoryRefresh::SaveCollection
112
102
  end
113
103
 
114
104
  def transform_to_hash!(all_attribute_keys, hash)
115
- if inventory_collection.use_ar_object?
116
- record = inventory_collection.model_class.new(hash)
117
- values_for_database!(all_attribute_keys,
118
- record.attributes.slice(*record.changed_attributes.keys).symbolize_keys)
119
- elsif serializable_keys?
105
+ if serializable_keys?
120
106
  values_for_database!(all_attribute_keys,
121
107
  hash)
122
108
  else
@@ -127,101 +113,15 @@ module InventoryRefresh::SaveCollection
127
113
  private
128
114
 
129
115
  attr_reader :unique_index_keys, :unique_index_keys_to_s, :select_keys, :unique_db_primary_keys, :unique_db_indexes,
130
- :primary_key, :arel_primary_key, :record_key_method, :pure_sql_records_fetching, :select_keys_indexes,
116
+ :primary_key, :arel_primary_key, :record_key_method, :select_keys_indexes,
131
117
  :batch_size, :batch_size_for_persisting, :model_class, :serializable_keys, :deserializable_keys, :pg_types, :table_name,
132
118
  :q_table_name
133
119
 
134
120
  delegate :supports_column?, :to => :inventory_collection
135
121
 
136
- # Saves the InventoryCollection
137
- #
138
- # @param association [Symbol] An existing association on manager
139
- def save!(association)
140
- attributes_index = {}
141
- inventory_objects_index = {}
142
- inventory_collection.each do |inventory_object|
143
- attributes = inventory_object.attributes(inventory_collection)
144
- index = build_stringified_reference(attributes, unique_index_keys)
145
-
146
- attributes_index[index] = attributes
147
- inventory_objects_index[index] = inventory_object
148
- end
149
-
150
- logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
151
- # Records that are in the DB, we will be updating or deleting them.
152
- ActiveRecord::Base.transaction do
153
- association.find_each do |record|
154
- index = build_stringified_reference_for_record(record, unique_index_keys)
155
-
156
- next unless assert_distinct_relation(record.id)
157
- next unless assert_unique_record(record, index)
158
-
159
- inventory_object = inventory_objects_index.delete(index)
160
- hash = attributes_index.delete(index)
161
-
162
- if inventory_object.nil?
163
- # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
164
- # delete it from the DB.
165
- delete_record!(record) if inventory_collection.delete_allowed?
166
- else
167
- # Record was found in the DB and sent for saving, we will be updating the DB.
168
- update_record!(record, hash, inventory_object) if assert_referential_integrity(hash)
169
- end
170
- end
171
- end
172
-
173
- unless inventory_collection.custom_reconnect_block.nil?
174
- inventory_collection.custom_reconnect_block.call(inventory_collection, inventory_objects_index, attributes_index)
175
- end
176
-
177
- # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
178
- if inventory_collection.create_allowed?
179
- ActiveRecord::Base.transaction do
180
- inventory_objects_index.each do |index, inventory_object|
181
- hash = attributes_index.delete(index)
182
-
183
- create_record!(hash, inventory_object) if assert_referential_integrity(hash)
184
- end
185
- end
186
- end
187
- logger.debug("Processing #{inventory_collection}, "\
188
- "created=#{inventory_collection.created_records.count}, "\
189
- "updated=#{inventory_collection.updated_records.count}, "\
190
- "deleted=#{inventory_collection.deleted_records.count}...Complete")
191
- rescue => e
192
- logger.error("Error when saving #{inventory_collection} with #{inventory_collection_details}. Message: #{e.message}")
193
- raise e
194
- end
195
-
196
122
  # @return [String] a string for logging purposes
197
123
  def inventory_collection_details
198
- "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}, targeted: #{inventory_collection.targeted?}"
199
- end
200
-
201
- # @param record [ApplicationRecord] ApplicationRecord object
202
- # @param key [Symbol] A key that is an attribute of the AR object
203
- # @return [Object] Value of attribute name :key on the :record
204
- def record_key(record, key)
205
- record.public_send(key)
206
- end
207
-
208
- # Deletes a complement of referenced data
209
- def delete_complement
210
- raise(":delete_complement method is supported only for :saver_strategy => [:batch, :concurrent_safe_batch]")
211
- end
212
-
213
- # Deletes/soft-deletes a given record
214
- #
215
- # @param [ApplicationRecord] record we want to delete
216
- def delete_record!(record)
217
- record.public_send(inventory_collection.delete_method)
218
- inventory_collection.store_deleted_records(record)
219
- end
220
-
221
- # @return [TrueClass] always return true, this method is redefined in default saver
222
- def assert_unique_record(_record, _index)
223
- # TODO(lsmola) can go away once we indexed our DB with unique indexes
224
- true
124
+ "strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}"
225
125
  end
226
126
 
227
127
  # Check if relation provided is distinct, i.e. the relation should not return the same primary key value twice.
@@ -46,40 +46,6 @@ module InventoryRefresh::SaveCollection
46
46
  record[select_keys_indexes[key]]
47
47
  end
48
48
 
49
- # Returns iterator or relation based on settings
50
- #
51
- # @param association [Symbol] An existing association on manager
52
- # @return [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or relation based on settings
53
- def batch_iterator(association)
54
- if pure_sql_records_fetching
55
- # Building fast iterator doing pure SQL query and therefore avoiding redundant creation of AR objects. The
56
- # iterator responds to find_in_batches, so it acts like the AR relation. For targeted refresh, the association
57
- # can already be ApplicationRecordIterator, so we will skip that.
58
- pure_sql_iterator = lambda do |&block|
59
- primary_key_offset = nil
60
- loop do
61
- relation = association.select(*select_keys)
62
- .reorder("#{primary_key} ASC")
63
- .limit(batch_size)
64
- # Using rails way of comparing primary key instead of offset
65
- relation = relation.where(arel_primary_key.gt(primary_key_offset)) if primary_key_offset
66
- records = get_connection.query(relation.to_sql)
67
- last_record = records.last
68
- block.call(records)
69
-
70
- break if records.size < batch_size
71
- primary_key_offset = record_key(last_record, primary_key)
72
- end
73
- end
74
-
75
- InventoryRefresh::ApplicationRecordIterator.new(:iterator => pure_sql_iterator)
76
- else
77
- # Normal Rails ActiveRecord::Relation where we can call find_in_batches or
78
- # InventoryRefresh::ApplicationRecordIterator passed from targeted refresh
79
- association
80
- end
81
- end
82
-
83
49
  # Saves the InventoryCollection
84
50
  #
85
51
  # @param association [Symbol] An existing association on manager
@@ -89,7 +55,7 @@ module InventoryRefresh::SaveCollection
89
55
  all_attribute_keys = Set.new + inventory_collection.batch_extra_attributes
90
56
 
91
57
  inventory_collection.each do |inventory_object|
92
- attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
58
+ attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
93
59
  index = build_stringified_reference(attributes, unique_index_keys)
94
60
 
95
61
  # Interesting fact: not building attributes_index and using only inventory_objects_index doesn't do much
@@ -103,7 +69,7 @@ module InventoryRefresh::SaveCollection
103
69
  logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
104
70
 
105
71
  unless inventory_collection.create_only?
106
- update_or_destroy_records!(batch_iterator(association), inventory_objects_index, attributes_index, all_attribute_keys)
72
+ load_and_update_records!(association, inventory_objects_index, attributes_index, all_attribute_keys)
107
73
  end
108
74
 
109
75
  unless inventory_collection.create_only?
@@ -112,15 +78,11 @@ module InventoryRefresh::SaveCollection
112
78
 
113
79
  # Records that were not found in the DB but sent for saving, we will be creating these in the DB.
114
80
  if inventory_collection.create_allowed?
115
- on_conflict = inventory_collection.parallel_safe? ? :do_update : nil
116
-
117
81
  inventory_objects_index.each_slice(batch_size_for_persisting) do |batch|
118
- create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => on_conflict)
82
+ create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => :do_update)
119
83
  end
120
84
 
121
- if inventory_collection.parallel_safe?
122
- create_or_update_partial_records(all_attribute_keys)
123
- end
85
+ create_or_update_partial_records(all_attribute_keys)
124
86
  end
125
87
 
126
88
  logger.debug("Marking :last_seen_at of #{inventory_collection} of size #{inventory_collection.size}...")
@@ -149,7 +111,7 @@ module InventoryRefresh::SaveCollection
149
111
  end
150
112
 
151
113
  def mark_last_seen_at(attributes_index)
152
- return unless supports_column?(:last_seen_at) && inventory_collection.parallel_safe?
114
+ return unless supports_column?(:last_seen_at)
153
115
  return if attributes_index.blank?
154
116
 
155
117
  all_attribute_keys = [:last_seen_at]
@@ -162,8 +124,7 @@ module InventoryRefresh::SaveCollection
162
124
  get_connection.execute(query)
163
125
  end
164
126
 
165
- # Batch updates existing records that are in the DB using attributes_index. And delete the ones that were not
166
- # present in inventory_objects_index.
127
+ # Batch updates existing records that are in the DB using attributes_index.
167
128
  #
168
129
  # @param records_batch_iterator [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or
169
130
  # relation, both responding to :find_in_batches method
@@ -171,12 +132,11 @@ module InventoryRefresh::SaveCollection
171
132
  # @param attributes_index [Hash{String => Hash}] Hash of data hashes with only keys that are column names of the
172
133
  # models's table
173
134
  # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
174
- def update_or_destroy_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
175
- hashes_for_update = []
176
- records_for_destroy = []
135
+ def load_and_update_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
136
+ hashes_for_update = []
177
137
  indexed_inventory_objects = {}
178
138
 
179
- records_batch_iterator.find_in_batches(:batch_size => batch_size) do |batch|
139
+ records_batch_iterator.find_in_batches(:batch_size => batch_size, :attributes_index => attributes_index) do |batch|
180
140
  update_time = time_now
181
141
 
182
142
  batch.each do |record|
@@ -189,20 +149,14 @@ module InventoryRefresh::SaveCollection
189
149
  inventory_object = inventory_objects_index.delete(index)
190
150
  hash = attributes_index[index]
191
151
 
192
- if inventory_object.nil?
193
- # Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
194
- # delete it from the DB.
195
- if inventory_collection.delete_allowed?
196
- records_for_destroy << record
197
- end
198
- else
152
+ if inventory_object
199
153
  # Record was found in the DB and sent for saving, we will be updating the DB.
200
154
  inventory_object.id = primary_key_value
201
155
  next unless assert_referential_integrity(hash)
202
- next unless changed?(record, hash, all_attribute_keys)
203
156
 
204
- if inventory_collection.parallel_safe? &&
205
- (supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys))
157
+ record_version = nil
158
+ record_version_max = nil
159
+ if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
206
160
 
207
161
  version_attr, max_version_attr = if supports_remote_data_timestamp?(all_attribute_keys)
208
162
  [:resource_timestamp, :resource_timestamps_max]
@@ -210,16 +164,16 @@ module InventoryRefresh::SaveCollection
210
164
  [:resource_counter, :resource_counters_max]
211
165
  end
212
166
 
213
- next if skeletonize_or_skip_record(record_key(record, version_attr),
214
- hash[version_attr],
215
- record_key(record, max_version_attr),
216
- inventory_object)
167
+ record_version = record_key(record, version_attr.to_s)
168
+ record_version_max = record_key(record, max_version_attr.to_s)
217
169
  end
218
170
 
219
171
  hash_for_update = if inventory_collection.use_ar_object?
220
172
  record.assign_attributes(hash.except(:id))
173
+ next unless changed?(record)
174
+
221
175
  values_for_database!(all_attribute_keys,
222
- record.attributes.symbolize_keys)
176
+ hash)
223
177
  elsif serializable_keys?
224
178
  # TODO(lsmola) hash data with current DB data to allow subset of data being sent,
225
179
  # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
@@ -230,6 +184,14 @@ module InventoryRefresh::SaveCollection
230
184
  # otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
231
185
  hash
232
186
  end
187
+
188
+ if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
189
+ next if skeletonize_or_skip_record(record_version,
190
+ hash[version_attr],
191
+ record_version_max,
192
+ inventory_object)
193
+ end
194
+
233
195
  assign_attributes_for_update!(hash_for_update, update_time)
234
196
 
235
197
  hash_for_update[:id] = primary_key_value
@@ -245,39 +207,20 @@ module InventoryRefresh::SaveCollection
245
207
  hashes_for_update = []
246
208
  indexed_inventory_objects = {}
247
209
  end
248
-
249
- # Destroy in batches
250
- if records_for_destroy.size >= batch_size_for_persisting
251
- destroy_records!(records_for_destroy)
252
- records_for_destroy = []
253
- end
254
210
  end
255
211
 
256
212
  # Update the last batch
257
213
  update_records!(all_attribute_keys, hashes_for_update, indexed_inventory_objects)
258
214
  hashes_for_update = [] # Cleanup so GC can release it sooner
259
-
260
- # Destroy the last batch
261
- destroy_records!(records_for_destroy)
262
- records_for_destroy = [] # Cleanup so GC can release it sooner
263
215
  end
264
216
 
265
- def changed?(_record, _hash, _all_attribute_keys)
217
+ def changed?(record)
266
218
  return true unless inventory_collection.check_changed?
267
219
 
268
- # TODO(lsmola) this check needs to be disabled now, because it doesn't work with lazy_find having secondary
269
- # indexes. Examples: we save a pod before we save a project, that means the project lazy_find won't evaluate,
270
- # because we load it with secondary index and can't do skeletal precreate. Then when the object is being saved
271
- # again, the lazy_find is evaluated, but the resource version is not changed, so the row is not saved.
272
- #
273
- # To keep this quick .changed? check, we might need to extend this, so the resource_version doesn't save until
274
- # all lazy_links of the row are evaluated.
275
- #
276
- # if supports_resource_version?(all_attribute_keys) && supports_column?(resource_version_column)
277
- # record_resource_version = record_key(record, resource_version_column.to_s)
278
- #
279
- # return record_resource_version != hash[resource_version_column]
280
- # end
220
+ # Skip if nothing changed
221
+ return false if record.changed_attributes.empty?
222
+ # Skip if we only changed the resource_timestamp, but data stays the same
223
+ return false if record.changed_attributes.keys == ["resource_timestamp"]
281
224
 
282
225
  true
283
226
  end
@@ -285,10 +228,7 @@ module InventoryRefresh::SaveCollection
285
228
  def db_columns_index(record, pure_sql: false)
286
229
  # Incoming values are in SQL string form.
287
230
  # TODO(lsmola) unify this behavior with object_index_with_keys method in InventoryCollection
288
- # TODO(lsmola) maybe we can drop the whole pure sql fetching, since everything will be targeted refresh
289
231
  # with streaming refresh? Maybe just metrics and events will not be, but those should be upsert only
290
- # TODO(lsmola) taking ^ in account, we can't drop pure sql, since that is returned by batch insert and
291
- # update queries
292
232
  unique_index_keys_to_s.map do |attribute|
293
233
  value = if pure_sql
294
234
  record[attribute]
@@ -319,20 +259,13 @@ module InventoryRefresh::SaveCollection
319
259
  def update_records!(all_attribute_keys, hashes, indexed_inventory_objects)
320
260
  return if hashes.blank?
321
261
 
322
- unless inventory_collection.parallel_safe?
323
- # We need to update the stored records before we save it, since hashes are modified
324
- inventory_collection.store_updated_records(hashes)
325
- end
326
-
327
262
  query = build_update_query(all_attribute_keys, hashes)
328
263
  result = get_connection.execute(query)
329
264
 
330
- if inventory_collection.parallel_safe?
331
- # We will check for timestamp clashes of full row update and we will fallback to skeletal update
332
- inventory_collection.store_updated_records(result)
265
+ # We will check for timestamp clashes of full row update and we will fallback to skeletal update
266
+ inventory_collection.store_updated_records(result)
333
267
 
334
- skeletonize_ignored_records!(indexed_inventory_objects, result)
335
- end
268
+ skeletonize_ignored_records!(indexed_inventory_objects, result)
336
269
 
337
270
  result
338
271
  end
@@ -352,11 +285,7 @@ module InventoryRefresh::SaveCollection
352
285
  hashes = []
353
286
  create_time = time_now
354
287
  batch.each do |index, inventory_object|
355
- hash = if inventory_collection.use_ar_object?
356
- record = inventory_collection.model_class.new(attributes_index[index])
357
- values_for_database!(all_attribute_keys,
358
- record.attributes.symbolize_keys)
359
- elsif serializable_keys?
288
+ hash = if serializable_keys?
360
289
  values_for_database!(all_attribute_keys,
361
290
  attributes_index[index])
362
291
  else
@@ -378,24 +307,19 @@ module InventoryRefresh::SaveCollection
378
307
  build_insert_query(all_attribute_keys, hashes, :on_conflict => on_conflict, :mode => :full)
379
308
  )
380
309
 
381
- if inventory_collection.parallel_safe?
382
- # We've done upsert, so records were either created or updated. We can recognize that by checking if
383
- # created and updated timestamps are the same
384
- created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
385
- created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
386
- updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
387
- updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
388
-
389
- if created_attr && updated_attr
390
- created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
391
- inventory_collection.store_created_records(created)
392
- inventory_collection.store_updated_records(updated)
393
- else
394
- # The record doesn't have both created and updated attrs, so we'll take all as created
395
- inventory_collection.store_created_records(result)
396
- end
310
+ # We've done upsert, so records were either created or updated. We can recognize that by checking if
311
+ # created and updated timestamps are the same
312
+ created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
313
+ created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
314
+ updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
315
+ updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
316
+
317
+ if created_attr && updated_attr
318
+ created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
319
+ inventory_collection.store_created_records(created)
320
+ inventory_collection.store_updated_records(updated)
397
321
  else
398
- # We've done just insert, so all records were created
322
+ # The record doesn't have both created and updated attrs, so we'll take all as created
399
323
  inventory_collection.store_created_records(result)
400
324
  end
401
325
 
@@ -408,9 +332,7 @@ module InventoryRefresh::SaveCollection
408
332
  :on_conflict => on_conflict)
409
333
  end
410
334
 
411
- if inventory_collection.parallel_safe?
412
- skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
413
- end
335
+ skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
414
336
  end
415
337
 
416
338
  # Stores primary_key values of created records into associated InventoryObject objects.