inventory_refresh 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +0 -1
- data/.travis.yml +6 -8
- data/inventory_refresh.gemspec +3 -5
- data/lib/inventory_refresh/application_record_iterator.rb +9 -26
- data/lib/inventory_refresh/exception.rb +8 -0
- data/lib/inventory_refresh/inventory_collection/builder.rb +6 -6
- data/lib/inventory_refresh/inventory_collection/data_storage.rb +0 -9
- data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +35 -144
- data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +1 -44
- data/lib/inventory_refresh/inventory_collection/index/proxy.rb +6 -34
- data/lib/inventory_refresh/inventory_collection/index/type/base.rb +0 -8
- data/lib/inventory_refresh/inventory_collection/references_storage.rb +0 -17
- data/lib/inventory_refresh/inventory_collection/scanner.rb +1 -87
- data/lib/inventory_refresh/inventory_collection/serialization.rb +10 -16
- data/lib/inventory_refresh/inventory_collection.rb +36 -110
- data/lib/inventory_refresh/inventory_object.rb +34 -68
- data/lib/inventory_refresh/inventory_object_lazy.rb +10 -17
- data/lib/inventory_refresh/persister.rb +63 -29
- data/lib/inventory_refresh/save_collection/base.rb +2 -4
- data/lib/inventory_refresh/save_collection/saver/base.rb +8 -108
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +48 -126
- data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +19 -1
- data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +3 -68
- data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +0 -125
- data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +5 -9
- data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +9 -17
- data/lib/inventory_refresh/save_collection/sweeper.rb +91 -18
- data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
- data/lib/inventory_refresh/save_inventory.rb +12 -5
- data/lib/inventory_refresh/version.rb +1 -1
- data/lib/inventory_refresh.rb +0 -2
- metadata +15 -57
- data/lib/inventory_refresh/save_collection/saver/batch.rb +0 -17
- data/lib/inventory_refresh/save_collection/saver/default.rb +0 -57
- data/lib/inventory_refresh/target.rb +0 -73
- data/lib/inventory_refresh/target_collection.rb +0 -92
@@ -3,15 +3,13 @@ module InventoryRefresh
|
|
3
3
|
require 'json'
|
4
4
|
require 'yaml'
|
5
5
|
|
6
|
-
attr_reader :manager, :
|
6
|
+
attr_reader :manager, :collections
|
7
7
|
|
8
8
|
attr_accessor :refresh_state_uuid, :refresh_state_part_uuid, :total_parts, :sweep_scope, :retry_count, :retry_max
|
9
9
|
|
10
10
|
# @param manager [ManageIQ::Providers::BaseManager] A manager object
|
11
|
-
|
12
|
-
def initialize(manager, target = nil)
|
11
|
+
def initialize(manager)
|
13
12
|
@manager = manager
|
14
|
-
@target = target
|
15
13
|
|
16
14
|
@collections = {}
|
17
15
|
|
@@ -45,9 +43,6 @@ module InventoryRefresh
|
|
45
43
|
&block)
|
46
44
|
|
47
45
|
builder.add_properties(extra_properties) if extra_properties.present?
|
48
|
-
|
49
|
-
builder.add_properties({:manager_uuids => target.try(:references, collection_name) || []}, :if_missing) if targeted?
|
50
|
-
|
51
46
|
builder.evaluate_lambdas!(self)
|
52
47
|
|
53
48
|
collections[collection_name] = builder.to_inventory_collection
|
@@ -104,8 +99,6 @@ module InventoryRefresh
|
|
104
99
|
def to_hash
|
105
100
|
collections_data = collections.map do |_, collection|
|
106
101
|
next if collection.data.blank? &&
|
107
|
-
collection.targeted_scope.primary_references.blank? &&
|
108
|
-
collection.all_manager_uuids.nil? &&
|
109
102
|
collection.skeletal_primary_index.index_data.blank?
|
110
103
|
|
111
104
|
collection.to_hash
|
@@ -117,7 +110,7 @@ module InventoryRefresh
|
|
117
110
|
:retry_count => retry_count,
|
118
111
|
:retry_max => retry_max,
|
119
112
|
:total_parts => total_parts,
|
120
|
-
:sweep_scope => sweep_scope,
|
113
|
+
:sweep_scope => sweep_scope_to_hash(sweep_scope),
|
121
114
|
:collections => collections_data,
|
122
115
|
}
|
123
116
|
end
|
@@ -127,22 +120,19 @@ module InventoryRefresh
|
|
127
120
|
#
|
128
121
|
# @param json_data [String] input JSON data
|
129
122
|
# @return [ManageIQ::Providers::Inventory::Persister] Persister object loaded from a passed JSON
|
130
|
-
def from_json(json_data, manager
|
131
|
-
from_hash(JSON.parse(json_data), manager
|
123
|
+
def from_json(json_data, manager)
|
124
|
+
from_hash(JSON.parse(json_data), manager)
|
132
125
|
end
|
133
126
|
|
134
127
|
# Returns Persister object built from serialized data
|
135
128
|
#
|
136
129
|
# @param persister_data [Hash] serialized Persister object in hash
|
137
130
|
# @return [ManageIQ::Providers::Inventory::Persister] Persister object built from serialized data
|
138
|
-
def from_hash(persister_data, manager
|
139
|
-
|
140
|
-
target ||= InventoryRefresh::TargetCollection.new(:manager => manager)
|
141
|
-
|
142
|
-
new(manager, target).tap do |persister|
|
131
|
+
def from_hash(persister_data, manager)
|
132
|
+
new(manager).tap do |persister|
|
143
133
|
persister_data['collections'].each do |collection|
|
144
134
|
inventory_collection = persister.collections[collection['name'].try(:to_sym)]
|
145
|
-
raise "Unrecognized InventoryCollection name: #{
|
135
|
+
raise "Unrecognized InventoryCollection name: #{collection['name']}" if inventory_collection.blank?
|
146
136
|
|
147
137
|
inventory_collection.from_hash(collection, persister.collections)
|
148
138
|
end
|
@@ -152,9 +142,44 @@ module InventoryRefresh
|
|
152
142
|
persister.retry_count = persister_data['retry_count']
|
153
143
|
persister.retry_max = persister_data['retry_max']
|
154
144
|
persister.total_parts = persister_data['total_parts']
|
155
|
-
persister.sweep_scope = persister_data['sweep_scope']
|
145
|
+
persister.sweep_scope = sweep_scope_from_hash(persister_data['sweep_scope'], persister.collections)
|
156
146
|
end
|
157
147
|
end
|
148
|
+
|
149
|
+
private
|
150
|
+
|
151
|
+
def assert_sweep_scope!(sweep_scope)
|
152
|
+
return unless sweep_scope
|
153
|
+
|
154
|
+
allowed_format_message = "Allowed format of sweep scope is Array<String> or Hash{String => Hash}, got #{sweep_scope}"
|
155
|
+
|
156
|
+
if sweep_scope.kind_of?(Array)
|
157
|
+
return if sweep_scope.all? { |x| x.kind_of?(String) || x.kind_of?(Symbol) }
|
158
|
+
|
159
|
+
raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
|
160
|
+
elsif sweep_scope.kind_of?(Hash)
|
161
|
+
return if sweep_scope.values.all? { |x| x.kind_of?(Array) }
|
162
|
+
|
163
|
+
raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
|
164
|
+
end
|
165
|
+
|
166
|
+
raise InventoryRefresh::Exception::SweeperScopeBadFormat, allowed_format_message
|
167
|
+
end
|
168
|
+
|
169
|
+
def sweep_scope_from_hash(sweep_scope, available_inventory_collections)
|
170
|
+
assert_sweep_scope!(sweep_scope)
|
171
|
+
|
172
|
+
return sweep_scope unless sweep_scope.kind_of?(Hash)
|
173
|
+
|
174
|
+
sweep_scope.each_with_object({}) do |(k, v), obj|
|
175
|
+
inventory_collection = available_inventory_collections[k.try(:to_sym)]
|
176
|
+
raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
|
177
|
+
|
178
|
+
serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
|
179
|
+
|
180
|
+
obj[k] = serializer.sweep_scope_from_hash(v, available_inventory_collections)
|
181
|
+
end.symbolize_keys!
|
182
|
+
end
|
158
183
|
end
|
159
184
|
|
160
185
|
protected
|
@@ -185,28 +210,37 @@ module InventoryRefresh
|
|
185
210
|
nil
|
186
211
|
end
|
187
212
|
|
188
|
-
def
|
189
|
-
:default
|
190
|
-
end
|
191
|
-
|
192
|
-
# Persisters for targeted refresh can override to true
|
193
|
-
def targeted?
|
213
|
+
def assert_graph_integrity?
|
194
214
|
false
|
195
215
|
end
|
196
216
|
|
197
|
-
def
|
198
|
-
|
217
|
+
def use_ar_object?
|
218
|
+
true
|
199
219
|
end
|
200
220
|
|
201
221
|
# @return [Hash] kwargs shared for all InventoryCollection objects
|
202
222
|
def shared_options
|
203
223
|
{
|
204
|
-
:saver_strategy => saver_strategy,
|
205
224
|
:strategy => strategy,
|
206
|
-
:targeted => targeted?,
|
207
225
|
:parent => manager.presence,
|
208
226
|
:assert_graph_integrity => assert_graph_integrity?,
|
227
|
+
:use_ar_object => use_ar_object?,
|
209
228
|
}
|
210
229
|
end
|
230
|
+
|
231
|
+
private
|
232
|
+
|
233
|
+
def sweep_scope_to_hash(sweep_scope)
|
234
|
+
return sweep_scope unless sweep_scope.kind_of?(Hash)
|
235
|
+
|
236
|
+
sweep_scope.each_with_object({}) do |(k, v), obj|
|
237
|
+
inventory_collection = collections[k.try(:to_sym)]
|
238
|
+
raise "Unrecognized InventoryCollection name: #{k}" if inventory_collection.blank?
|
239
|
+
|
240
|
+
serializer = InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection)
|
241
|
+
|
242
|
+
obj[k] = serializer.sweep_scope_to_hash(v)
|
243
|
+
end
|
244
|
+
end
|
211
245
|
end
|
212
246
|
end
|
@@ -1,7 +1,5 @@
|
|
1
1
|
require "inventory_refresh/logging"
|
2
|
-
require "inventory_refresh/save_collection/saver/batch"
|
3
2
|
require "inventory_refresh/save_collection/saver/concurrent_safe_batch"
|
4
|
-
require "inventory_refresh/save_collection/saver/default"
|
5
3
|
|
6
4
|
module InventoryRefresh::SaveCollection
|
7
5
|
class Base
|
@@ -16,7 +14,7 @@ module InventoryRefresh::SaveCollection
|
|
16
14
|
return if skip?(inventory_collection)
|
17
15
|
|
18
16
|
logger.debug("----- BEGIN ----- Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
|
19
|
-
" the database, for the manager: '#{ems.
|
17
|
+
" the database, for the manager: '#{ems.id}'...")
|
20
18
|
|
21
19
|
if inventory_collection.custom_save_block.present?
|
22
20
|
logger.debug("Saving collection #{inventory_collection} using a custom save block")
|
@@ -24,7 +22,7 @@ module InventoryRefresh::SaveCollection
|
|
24
22
|
else
|
25
23
|
save_inventory(inventory_collection)
|
26
24
|
end
|
27
|
-
logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.
|
25
|
+
logger.debug("----- END ----- Saving collection #{inventory_collection}, for the manager: '#{ems.id}'...Complete")
|
28
26
|
inventory_collection.saved = true
|
29
27
|
end
|
30
28
|
|
@@ -12,8 +12,6 @@ module InventoryRefresh::SaveCollection
|
|
12
12
|
# @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we will be saving
|
13
13
|
def initialize(inventory_collection)
|
14
14
|
@inventory_collection = inventory_collection
|
15
|
-
# TODO(lsmola) do I need to reload every time? Also it should be enough to clear the associations.
|
16
|
-
inventory_collection.parent.reload if inventory_collection.parent
|
17
15
|
@association = inventory_collection.db_collection_for_comparison
|
18
16
|
|
19
17
|
# Private attrs
|
@@ -28,14 +26,10 @@ module InventoryRefresh::SaveCollection
|
|
28
26
|
@unique_db_primary_keys = Set.new
|
29
27
|
@unique_db_indexes = Set.new
|
30
28
|
|
31
|
-
# Right now ApplicationRecordIterator in association is used for targeted refresh. Given the small amount of
|
32
|
-
# records flowing through there, we probably don't need to optimize that association to fetch a pure SQL.
|
33
|
-
@pure_sql_records_fetching = !inventory_collection.use_ar_object? && !@association.kind_of?(InventoryRefresh::ApplicationRecordIterator)
|
34
|
-
|
35
29
|
@batch_size_for_persisting = inventory_collection.batch_size_pure_sql
|
30
|
+
@batch_size = inventory_collection.use_ar_object? ? @batch_size_for_persisting : inventory_collection.batch_size
|
36
31
|
|
37
|
-
@
|
38
|
-
@record_key_method = @pure_sql_records_fetching ? :pure_sql_record_key : :ar_record_key
|
32
|
+
@record_key_method = inventory_collection.pure_sql_record_fetching? ? :pure_sql_record_key : :ar_record_key
|
39
33
|
@select_keys_indexes = @select_keys.each_with_object({}).with_index { |(key, obj), index| obj[key.to_s] = index }
|
40
34
|
@pg_types = @model_class.attribute_names.each_with_object({}) do |key, obj|
|
41
35
|
obj[key.to_sym] = inventory_collection.model_class.columns_hash[key]
|
@@ -75,14 +69,8 @@ module InventoryRefresh::SaveCollection
|
|
75
69
|
|
76
70
|
# Saves the InventoryCollection
|
77
71
|
def save_inventory_collection!
|
78
|
-
# If we have a targeted InventoryCollection that wouldn't do anything, quickly skip it
|
79
|
-
return if inventory_collection.noop?
|
80
|
-
|
81
|
-
# Delete_complement strategy using :all_manager_uuids attribute
|
82
|
-
delete_complement unless inventory_collection.delete_complement_noop?
|
83
|
-
|
84
72
|
# Create/Update/Archive/Delete records based on InventoryCollection data and scope
|
85
|
-
save!(association)
|
73
|
+
save!(association)
|
86
74
|
end
|
87
75
|
|
88
76
|
protected
|
@@ -101,6 +89,8 @@ module InventoryRefresh::SaveCollection
|
|
101
89
|
# @param attributes [Hash] attributes hash
|
102
90
|
# @return [Hash] modified hash from parameter attributes with casted values
|
103
91
|
def values_for_database!(all_attribute_keys, attributes)
|
92
|
+
# TODO(lsmola) we'll need to fill default value from the DB to the NOT_NULL columns here, since sending NULL
|
93
|
+
# to column with NOT_NULL constraint always fails, even if there is a default value
|
104
94
|
all_attribute_keys.each do |key|
|
105
95
|
next unless attributes.key?(key)
|
106
96
|
|
@@ -112,11 +102,7 @@ module InventoryRefresh::SaveCollection
|
|
112
102
|
end
|
113
103
|
|
114
104
|
def transform_to_hash!(all_attribute_keys, hash)
|
115
|
-
if
|
116
|
-
record = inventory_collection.model_class.new(hash)
|
117
|
-
values_for_database!(all_attribute_keys,
|
118
|
-
record.attributes.slice(*record.changed_attributes.keys).symbolize_keys)
|
119
|
-
elsif serializable_keys?
|
105
|
+
if serializable_keys?
|
120
106
|
values_for_database!(all_attribute_keys,
|
121
107
|
hash)
|
122
108
|
else
|
@@ -127,101 +113,15 @@ module InventoryRefresh::SaveCollection
|
|
127
113
|
private
|
128
114
|
|
129
115
|
attr_reader :unique_index_keys, :unique_index_keys_to_s, :select_keys, :unique_db_primary_keys, :unique_db_indexes,
|
130
|
-
:primary_key, :arel_primary_key, :record_key_method, :
|
116
|
+
:primary_key, :arel_primary_key, :record_key_method, :select_keys_indexes,
|
131
117
|
:batch_size, :batch_size_for_persisting, :model_class, :serializable_keys, :deserializable_keys, :pg_types, :table_name,
|
132
118
|
:q_table_name
|
133
119
|
|
134
120
|
delegate :supports_column?, :to => :inventory_collection
|
135
121
|
|
136
|
-
# Saves the InventoryCollection
|
137
|
-
#
|
138
|
-
# @param association [Symbol] An existing association on manager
|
139
|
-
def save!(association)
|
140
|
-
attributes_index = {}
|
141
|
-
inventory_objects_index = {}
|
142
|
-
inventory_collection.each do |inventory_object|
|
143
|
-
attributes = inventory_object.attributes(inventory_collection)
|
144
|
-
index = build_stringified_reference(attributes, unique_index_keys)
|
145
|
-
|
146
|
-
attributes_index[index] = attributes
|
147
|
-
inventory_objects_index[index] = inventory_object
|
148
|
-
end
|
149
|
-
|
150
|
-
logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
|
151
|
-
# Records that are in the DB, we will be updating or deleting them.
|
152
|
-
ActiveRecord::Base.transaction do
|
153
|
-
association.find_each do |record|
|
154
|
-
index = build_stringified_reference_for_record(record, unique_index_keys)
|
155
|
-
|
156
|
-
next unless assert_distinct_relation(record.id)
|
157
|
-
next unless assert_unique_record(record, index)
|
158
|
-
|
159
|
-
inventory_object = inventory_objects_index.delete(index)
|
160
|
-
hash = attributes_index.delete(index)
|
161
|
-
|
162
|
-
if inventory_object.nil?
|
163
|
-
# Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
|
164
|
-
# delete it from the DB.
|
165
|
-
delete_record!(record) if inventory_collection.delete_allowed?
|
166
|
-
else
|
167
|
-
# Record was found in the DB and sent for saving, we will be updating the DB.
|
168
|
-
update_record!(record, hash, inventory_object) if assert_referential_integrity(hash)
|
169
|
-
end
|
170
|
-
end
|
171
|
-
end
|
172
|
-
|
173
|
-
unless inventory_collection.custom_reconnect_block.nil?
|
174
|
-
inventory_collection.custom_reconnect_block.call(inventory_collection, inventory_objects_index, attributes_index)
|
175
|
-
end
|
176
|
-
|
177
|
-
# Records that were not found in the DB but sent for saving, we will be creating these in the DB.
|
178
|
-
if inventory_collection.create_allowed?
|
179
|
-
ActiveRecord::Base.transaction do
|
180
|
-
inventory_objects_index.each do |index, inventory_object|
|
181
|
-
hash = attributes_index.delete(index)
|
182
|
-
|
183
|
-
create_record!(hash, inventory_object) if assert_referential_integrity(hash)
|
184
|
-
end
|
185
|
-
end
|
186
|
-
end
|
187
|
-
logger.debug("Processing #{inventory_collection}, "\
|
188
|
-
"created=#{inventory_collection.created_records.count}, "\
|
189
|
-
"updated=#{inventory_collection.updated_records.count}, "\
|
190
|
-
"deleted=#{inventory_collection.deleted_records.count}...Complete")
|
191
|
-
rescue => e
|
192
|
-
logger.error("Error when saving #{inventory_collection} with #{inventory_collection_details}. Message: #{e.message}")
|
193
|
-
raise e
|
194
|
-
end
|
195
|
-
|
196
122
|
# @return [String] a string for logging purposes
|
197
123
|
def inventory_collection_details
|
198
|
-
"strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}
|
199
|
-
end
|
200
|
-
|
201
|
-
# @param record [ApplicationRecord] ApplicationRecord object
|
202
|
-
# @param key [Symbol] A key that is an attribute of the AR object
|
203
|
-
# @return [Object] Value of attribute name :key on the :record
|
204
|
-
def record_key(record, key)
|
205
|
-
record.public_send(key)
|
206
|
-
end
|
207
|
-
|
208
|
-
# Deletes a complement of referenced data
|
209
|
-
def delete_complement
|
210
|
-
raise(":delete_complement method is supported only for :saver_strategy => [:batch, :concurrent_safe_batch]")
|
211
|
-
end
|
212
|
-
|
213
|
-
# Deletes/soft-deletes a given record
|
214
|
-
#
|
215
|
-
# @param [ApplicationRecord] record we want to delete
|
216
|
-
def delete_record!(record)
|
217
|
-
record.public_send(inventory_collection.delete_method)
|
218
|
-
inventory_collection.store_deleted_records(record)
|
219
|
-
end
|
220
|
-
|
221
|
-
# @return [TrueClass] always return true, this method is redefined in default saver
|
222
|
-
def assert_unique_record(_record, _index)
|
223
|
-
# TODO(lsmola) can go away once we indexed our DB with unique indexes
|
224
|
-
true
|
124
|
+
"strategy: #{inventory_collection.strategy}, saver_strategy: #{inventory_collection.saver_strategy}"
|
225
125
|
end
|
226
126
|
|
227
127
|
# Check if relation provided is distinct, i.e. the relation should not return the same primary key value twice.
|
@@ -46,40 +46,6 @@ module InventoryRefresh::SaveCollection
|
|
46
46
|
record[select_keys_indexes[key]]
|
47
47
|
end
|
48
48
|
|
49
|
-
# Returns iterator or relation based on settings
|
50
|
-
#
|
51
|
-
# @param association [Symbol] An existing association on manager
|
52
|
-
# @return [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or relation based on settings
|
53
|
-
def batch_iterator(association)
|
54
|
-
if pure_sql_records_fetching
|
55
|
-
# Building fast iterator doing pure SQL query and therefore avoiding redundant creation of AR objects. The
|
56
|
-
# iterator responds to find_in_batches, so it acts like the AR relation. For targeted refresh, the association
|
57
|
-
# can already be ApplicationRecordIterator, so we will skip that.
|
58
|
-
pure_sql_iterator = lambda do |&block|
|
59
|
-
primary_key_offset = nil
|
60
|
-
loop do
|
61
|
-
relation = association.select(*select_keys)
|
62
|
-
.reorder("#{primary_key} ASC")
|
63
|
-
.limit(batch_size)
|
64
|
-
# Using rails way of comparing primary key instead of offset
|
65
|
-
relation = relation.where(arel_primary_key.gt(primary_key_offset)) if primary_key_offset
|
66
|
-
records = get_connection.query(relation.to_sql)
|
67
|
-
last_record = records.last
|
68
|
-
block.call(records)
|
69
|
-
|
70
|
-
break if records.size < batch_size
|
71
|
-
primary_key_offset = record_key(last_record, primary_key)
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
InventoryRefresh::ApplicationRecordIterator.new(:iterator => pure_sql_iterator)
|
76
|
-
else
|
77
|
-
# Normal Rails ActiveRecord::Relation where we can call find_in_batches or
|
78
|
-
# InventoryRefresh::ApplicationRecordIterator passed from targeted refresh
|
79
|
-
association
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
49
|
# Saves the InventoryCollection
|
84
50
|
#
|
85
51
|
# @param association [Symbol] An existing association on manager
|
@@ -89,7 +55,7 @@ module InventoryRefresh::SaveCollection
|
|
89
55
|
all_attribute_keys = Set.new + inventory_collection.batch_extra_attributes
|
90
56
|
|
91
57
|
inventory_collection.each do |inventory_object|
|
92
|
-
attributes = inventory_object.attributes_with_keys(inventory_collection, all_attribute_keys, inventory_object)
|
58
|
+
attributes = inventory_object.class.attributes_with_keys(inventory_object.data, inventory_collection, all_attribute_keys, inventory_object)
|
93
59
|
index = build_stringified_reference(attributes, unique_index_keys)
|
94
60
|
|
95
61
|
# Interesting fact: not building attributes_index and using only inventory_objects_index doesn't do much
|
@@ -103,7 +69,7 @@ module InventoryRefresh::SaveCollection
|
|
103
69
|
logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
|
104
70
|
|
105
71
|
unless inventory_collection.create_only?
|
106
|
-
|
72
|
+
load_and_update_records!(association, inventory_objects_index, attributes_index, all_attribute_keys)
|
107
73
|
end
|
108
74
|
|
109
75
|
unless inventory_collection.create_only?
|
@@ -112,15 +78,11 @@ module InventoryRefresh::SaveCollection
|
|
112
78
|
|
113
79
|
# Records that were not found in the DB but sent for saving, we will be creating these in the DB.
|
114
80
|
if inventory_collection.create_allowed?
|
115
|
-
on_conflict = inventory_collection.parallel_safe? ? :do_update : nil
|
116
|
-
|
117
81
|
inventory_objects_index.each_slice(batch_size_for_persisting) do |batch|
|
118
|
-
create_records!(all_attribute_keys, batch, attributes_index, :on_conflict =>
|
82
|
+
create_records!(all_attribute_keys, batch, attributes_index, :on_conflict => :do_update)
|
119
83
|
end
|
120
84
|
|
121
|
-
|
122
|
-
create_or_update_partial_records(all_attribute_keys)
|
123
|
-
end
|
85
|
+
create_or_update_partial_records(all_attribute_keys)
|
124
86
|
end
|
125
87
|
|
126
88
|
logger.debug("Marking :last_seen_at of #{inventory_collection} of size #{inventory_collection.size}...")
|
@@ -149,7 +111,7 @@ module InventoryRefresh::SaveCollection
|
|
149
111
|
end
|
150
112
|
|
151
113
|
def mark_last_seen_at(attributes_index)
|
152
|
-
return unless supports_column?(:last_seen_at)
|
114
|
+
return unless supports_column?(:last_seen_at)
|
153
115
|
return if attributes_index.blank?
|
154
116
|
|
155
117
|
all_attribute_keys = [:last_seen_at]
|
@@ -162,8 +124,7 @@ module InventoryRefresh::SaveCollection
|
|
162
124
|
get_connection.execute(query)
|
163
125
|
end
|
164
126
|
|
165
|
-
# Batch updates existing records that are in the DB using attributes_index.
|
166
|
-
# present in inventory_objects_index.
|
127
|
+
# Batch updates existing records that are in the DB using attributes_index.
|
167
128
|
#
|
168
129
|
# @param records_batch_iterator [ActiveRecord::Relation, InventoryRefresh::ApplicationRecordIterator] iterator or
|
169
130
|
# relation, both responding to :find_in_batches method
|
@@ -171,12 +132,11 @@ module InventoryRefresh::SaveCollection
|
|
171
132
|
# @param attributes_index [Hash{String => Hash}] Hash of data hashes with only keys that are column names of the
|
172
133
|
# models's table
|
173
134
|
# @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
|
174
|
-
def
|
175
|
-
hashes_for_update
|
176
|
-
records_for_destroy = []
|
135
|
+
def load_and_update_records!(records_batch_iterator, inventory_objects_index, attributes_index, all_attribute_keys)
|
136
|
+
hashes_for_update = []
|
177
137
|
indexed_inventory_objects = {}
|
178
138
|
|
179
|
-
records_batch_iterator.find_in_batches(:batch_size => batch_size) do |batch|
|
139
|
+
records_batch_iterator.find_in_batches(:batch_size => batch_size, :attributes_index => attributes_index) do |batch|
|
180
140
|
update_time = time_now
|
181
141
|
|
182
142
|
batch.each do |record|
|
@@ -189,20 +149,14 @@ module InventoryRefresh::SaveCollection
|
|
189
149
|
inventory_object = inventory_objects_index.delete(index)
|
190
150
|
hash = attributes_index[index]
|
191
151
|
|
192
|
-
if inventory_object
|
193
|
-
# Record was found in the DB but not sent for saving, that means it doesn't exist anymore and we should
|
194
|
-
# delete it from the DB.
|
195
|
-
if inventory_collection.delete_allowed?
|
196
|
-
records_for_destroy << record
|
197
|
-
end
|
198
|
-
else
|
152
|
+
if inventory_object
|
199
153
|
# Record was found in the DB and sent for saving, we will be updating the DB.
|
200
154
|
inventory_object.id = primary_key_value
|
201
155
|
next unless assert_referential_integrity(hash)
|
202
|
-
next unless changed?(record, hash, all_attribute_keys)
|
203
156
|
|
204
|
-
|
205
|
-
|
157
|
+
record_version = nil
|
158
|
+
record_version_max = nil
|
159
|
+
if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
|
206
160
|
|
207
161
|
version_attr, max_version_attr = if supports_remote_data_timestamp?(all_attribute_keys)
|
208
162
|
[:resource_timestamp, :resource_timestamps_max]
|
@@ -210,16 +164,16 @@ module InventoryRefresh::SaveCollection
|
|
210
164
|
[:resource_counter, :resource_counters_max]
|
211
165
|
end
|
212
166
|
|
213
|
-
|
214
|
-
|
215
|
-
record_key(record, max_version_attr),
|
216
|
-
inventory_object)
|
167
|
+
record_version = record_key(record, version_attr.to_s)
|
168
|
+
record_version_max = record_key(record, max_version_attr.to_s)
|
217
169
|
end
|
218
170
|
|
219
171
|
hash_for_update = if inventory_collection.use_ar_object?
|
220
172
|
record.assign_attributes(hash.except(:id))
|
173
|
+
next unless changed?(record)
|
174
|
+
|
221
175
|
values_for_database!(all_attribute_keys,
|
222
|
-
|
176
|
+
hash)
|
223
177
|
elsif serializable_keys?
|
224
178
|
# TODO(lsmola) hash data with current DB data to allow subset of data being sent,
|
225
179
|
# otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
|
@@ -230,6 +184,14 @@ module InventoryRefresh::SaveCollection
|
|
230
184
|
# otherwise we would nullify the not sent attributes. Test e.g. on disks in cloud
|
231
185
|
hash
|
232
186
|
end
|
187
|
+
|
188
|
+
if supports_remote_data_timestamp?(all_attribute_keys) || supports_remote_data_version?(all_attribute_keys)
|
189
|
+
next if skeletonize_or_skip_record(record_version,
|
190
|
+
hash[version_attr],
|
191
|
+
record_version_max,
|
192
|
+
inventory_object)
|
193
|
+
end
|
194
|
+
|
233
195
|
assign_attributes_for_update!(hash_for_update, update_time)
|
234
196
|
|
235
197
|
hash_for_update[:id] = primary_key_value
|
@@ -245,39 +207,20 @@ module InventoryRefresh::SaveCollection
|
|
245
207
|
hashes_for_update = []
|
246
208
|
indexed_inventory_objects = {}
|
247
209
|
end
|
248
|
-
|
249
|
-
# Destroy in batches
|
250
|
-
if records_for_destroy.size >= batch_size_for_persisting
|
251
|
-
destroy_records!(records_for_destroy)
|
252
|
-
records_for_destroy = []
|
253
|
-
end
|
254
210
|
end
|
255
211
|
|
256
212
|
# Update the last batch
|
257
213
|
update_records!(all_attribute_keys, hashes_for_update, indexed_inventory_objects)
|
258
214
|
hashes_for_update = [] # Cleanup so GC can release it sooner
|
259
|
-
|
260
|
-
# Destroy the last batch
|
261
|
-
destroy_records!(records_for_destroy)
|
262
|
-
records_for_destroy = [] # Cleanup so GC can release it sooner
|
263
215
|
end
|
264
216
|
|
265
|
-
def changed?(
|
217
|
+
def changed?(record)
|
266
218
|
return true unless inventory_collection.check_changed?
|
267
219
|
|
268
|
-
#
|
269
|
-
|
270
|
-
#
|
271
|
-
|
272
|
-
#
|
273
|
-
# To keep this quick .changed? check, we might need to extend this, so the resource_version doesn't save until
|
274
|
-
# all lazy_links of the row are evaluated.
|
275
|
-
#
|
276
|
-
# if supports_resource_version?(all_attribute_keys) && supports_column?(resource_version_column)
|
277
|
-
# record_resource_version = record_key(record, resource_version_column.to_s)
|
278
|
-
#
|
279
|
-
# return record_resource_version != hash[resource_version_column]
|
280
|
-
# end
|
220
|
+
# Skip if nothing changed
|
221
|
+
return false if record.changed_attributes.empty?
|
222
|
+
# Skip if we only changed the resource_timestamp, but data stays the same
|
223
|
+
return false if record.changed_attributes.keys == ["resource_timestamp"]
|
281
224
|
|
282
225
|
true
|
283
226
|
end
|
@@ -285,10 +228,7 @@ module InventoryRefresh::SaveCollection
|
|
285
228
|
def db_columns_index(record, pure_sql: false)
|
286
229
|
# Incoming values are in SQL string form.
|
287
230
|
# TODO(lsmola) unify this behavior with object_index_with_keys method in InventoryCollection
|
288
|
-
# TODO(lsmola) maybe we can drop the whole pure sql fetching, since everything will be targeted refresh
|
289
231
|
# with streaming refresh? Maybe just metrics and events will not be, but those should be upsert only
|
290
|
-
# TODO(lsmola) taking ^ in account, we can't drop pure sql, since that is returned by batch insert and
|
291
|
-
# update queries
|
292
232
|
unique_index_keys_to_s.map do |attribute|
|
293
233
|
value = if pure_sql
|
294
234
|
record[attribute]
|
@@ -319,20 +259,13 @@ module InventoryRefresh::SaveCollection
|
|
319
259
|
def update_records!(all_attribute_keys, hashes, indexed_inventory_objects)
|
320
260
|
return if hashes.blank?
|
321
261
|
|
322
|
-
unless inventory_collection.parallel_safe?
|
323
|
-
# We need to update the stored records before we save it, since hashes are modified
|
324
|
-
inventory_collection.store_updated_records(hashes)
|
325
|
-
end
|
326
|
-
|
327
262
|
query = build_update_query(all_attribute_keys, hashes)
|
328
263
|
result = get_connection.execute(query)
|
329
264
|
|
330
|
-
|
331
|
-
|
332
|
-
inventory_collection.store_updated_records(result)
|
265
|
+
# We will check for timestamp clashes of full row update and we will fallback to skeletal update
|
266
|
+
inventory_collection.store_updated_records(result)
|
333
267
|
|
334
|
-
|
335
|
-
end
|
268
|
+
skeletonize_ignored_records!(indexed_inventory_objects, result)
|
336
269
|
|
337
270
|
result
|
338
271
|
end
|
@@ -352,11 +285,7 @@ module InventoryRefresh::SaveCollection
|
|
352
285
|
hashes = []
|
353
286
|
create_time = time_now
|
354
287
|
batch.each do |index, inventory_object|
|
355
|
-
hash = if
|
356
|
-
record = inventory_collection.model_class.new(attributes_index[index])
|
357
|
-
values_for_database!(all_attribute_keys,
|
358
|
-
record.attributes.symbolize_keys)
|
359
|
-
elsif serializable_keys?
|
288
|
+
hash = if serializable_keys?
|
360
289
|
values_for_database!(all_attribute_keys,
|
361
290
|
attributes_index[index])
|
362
291
|
else
|
@@ -378,24 +307,19 @@ module InventoryRefresh::SaveCollection
|
|
378
307
|
build_insert_query(all_attribute_keys, hashes, :on_conflict => on_conflict, :mode => :full)
|
379
308
|
)
|
380
309
|
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
inventory_collection.store_updated_records(updated)
|
393
|
-
else
|
394
|
-
# The record doesn't have both created and updated attrs, so we'll take all as created
|
395
|
-
inventory_collection.store_created_records(result)
|
396
|
-
end
|
310
|
+
# We've done upsert, so records were either created or updated. We can recognize that by checking if
|
311
|
+
# created and updated timestamps are the same
|
312
|
+
created_attr = "created_on" if inventory_collection.supports_column?(:created_on)
|
313
|
+
created_attr ||= "created_at" if inventory_collection.supports_column?(:created_at)
|
314
|
+
updated_attr = "updated_on" if inventory_collection.supports_column?(:updated_on)
|
315
|
+
updated_attr ||= "updated_at" if inventory_collection.supports_column?(:updated_at)
|
316
|
+
|
317
|
+
if created_attr && updated_attr
|
318
|
+
created, updated = result.to_a.partition { |x| x[created_attr] == x[updated_attr] }
|
319
|
+
inventory_collection.store_created_records(created)
|
320
|
+
inventory_collection.store_updated_records(updated)
|
397
321
|
else
|
398
|
-
#
|
322
|
+
# The record doesn't have both created and updated attrs, so we'll take all as created
|
399
323
|
inventory_collection.store_created_records(result)
|
400
324
|
end
|
401
325
|
|
@@ -408,9 +332,7 @@ module InventoryRefresh::SaveCollection
|
|
408
332
|
:on_conflict => on_conflict)
|
409
333
|
end
|
410
334
|
|
411
|
-
|
412
|
-
skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
|
413
|
-
end
|
335
|
+
skeletonize_ignored_records!(indexed_inventory_objects, result, :all_unique_columns => true)
|
414
336
|
end
|
415
337
|
|
416
338
|
# Stores primary_key values of created records into associated InventoryObject objects.
|