deimos-ruby 1.24.2 → 1.24.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/docs/CONFIGURATION.md +1 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +4 -2
- data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
- data/lib/deimos/active_record_consumer.rb +5 -0
- data/lib/deimos/config/configuration.rb +7 -1
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/version.rb +1 -1
- data/spec/active_record_consume/mass_updater_spec.rb +137 -0
- data/spec/config/configuration_spec.rb +13 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: de8ca3bf7233f799c3bf9169fa647dabebd8e08cd2b05ec00511c132a5f3920d
|
4
|
+
data.tar.gz: 03177d9ac87184fe3ce6ef916bce0ad39e3b81a6576f60ba71c33b298f82dfd8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 40ecdddd9c0b8f5f9675f8058f4a8c8adf1eb69a293e04870a46b9d8ae1972cdc70088611c898f0ab91e072f6627c4760bca5ca5399b698316edda8f3212f5aa
|
7
|
+
data.tar.gz: 3f2fa463d5c777ce7e4dfc4cf610a7dd4ed3f6dba5b6dd459f1ca8fad037f0baa47278436c65c91dc90679f88e9d42a792b84ccbdc2cde4b58d1239d03b2fd4a
|
data/CHANGELOG.md
CHANGED
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
+
# 1.24.3 - 2024-05-13
|
11
|
+
|
12
|
+
- Feature: Enable `producers.persistent_connections` phobos setting
|
13
|
+
- Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
|
14
|
+
|
10
15
|
# 1.24.2 - 2024-05-01
|
11
16
|
- Fix: Deprecation notice with Rails 7.
|
12
17
|
|
data/docs/CONFIGURATION.md
CHANGED
@@ -102,6 +102,7 @@ heartbeat_interval|10|Interval between heartbeats; must be less than the session
|
|
102
102
|
backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
|
103
103
|
replace_associations|nil| Whether to delete existing associations for records during bulk consumption for this consumer. If no value is specified the provided/default value from the `consumers` configuration will be used.
|
104
104
|
bulk_import_id_generator|nil| Block to determine the `bulk_import_id` generated during bulk consumption. If no block is specified the provided/default block from the `consumers` configuration will be used.
|
105
|
+
save_associations_first|false|Whether to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
|
105
106
|
|
106
107
|
## Defining Database Pollers
|
107
108
|
|
@@ -83,7 +83,7 @@ module Deimos
|
|
83
83
|
def deleted_query(records)
|
84
84
|
keys = records.
|
85
85
|
map { |m| record_key(m.key)[@klass.primary_key] }.
|
86
|
-
|
86
|
+
compact
|
87
87
|
|
88
88
|
@klass.unscoped.where(@klass.primary_key => keys)
|
89
89
|
end
|
@@ -168,7 +168,9 @@ module Deimos
|
|
168
168
|
key_col_proc: key_col_proc,
|
169
169
|
col_proc: col_proc,
|
170
170
|
replace_associations: self.class.replace_associations,
|
171
|
-
bulk_import_id_generator: self.class.bulk_import_id_generator
|
171
|
+
bulk_import_id_generator: self.class.bulk_import_id_generator,
|
172
|
+
save_associations_first: self.class.save_associations_first,
|
173
|
+
bulk_import_id_column: self.class.bulk_import_id_column)
|
172
174
|
ActiveSupport::Notifications.instrument('batch_consumption.valid_records', {
|
173
175
|
records: updater.mass_update(record_list),
|
174
176
|
consumer: self.class
|
@@ -20,10 +20,13 @@ module Deimos
|
|
20
20
|
# @param col_proc [Proc<Class < ActiveRecord::Base>]
|
21
21
|
# @param replace_associations [Boolean]
|
22
22
|
def initialize(klass, key_col_proc: nil, col_proc: nil,
|
23
|
-
replace_associations: true, bulk_import_id_generator: nil
|
23
|
+
replace_associations: true, bulk_import_id_generator: nil, save_associations_first: false,
|
24
|
+
bulk_import_id_column: nil)
|
24
25
|
@klass = klass
|
25
26
|
@replace_associations = replace_associations
|
26
27
|
@bulk_import_id_generator = bulk_import_id_generator
|
28
|
+
@save_associations_first = save_associations_first
|
29
|
+
@bulk_import_id_column = bulk_import_id_column&.to_s
|
27
30
|
|
28
31
|
@key_cols = {}
|
29
32
|
@key_col_proc = key_col_proc
|
@@ -84,15 +87,67 @@ module Deimos
|
|
84
87
|
end
|
85
88
|
end
|
86
89
|
|
90
|
+
# Assign associated records to corresponding primary records
|
91
|
+
# @param record_list [BatchRecordList] RecordList of primary records for this consumer
|
92
|
+
# @return [Hash]
|
93
|
+
def assign_associations(record_list)
|
94
|
+
associations_info = {}
|
95
|
+
record_list.associations.each do |assoc|
|
96
|
+
col = @bulk_import_id_column if assoc.klass.column_names.include?(@bulk_import_id_column)
|
97
|
+
associations_info[[assoc, col]] = []
|
98
|
+
end
|
99
|
+
record_list.batch_records.each do |primary_batch_record|
|
100
|
+
associations_info.each_key do |assoc, col|
|
101
|
+
batch_record = BatchRecord.new(klass: assoc.klass,
|
102
|
+
attributes: primary_batch_record.associations[assoc.name],
|
103
|
+
bulk_import_column: col,
|
104
|
+
bulk_import_id_generator: @bulk_import_id_generator)
|
105
|
+
# Associate this associated batch record's record with the primary record to
|
106
|
+
# retrieve foreign_key after associated records have been saved and primary
|
107
|
+
# keys have been filled
|
108
|
+
primary_batch_record.record.assign_attributes({ assoc.name => batch_record.record })
|
109
|
+
associations_info[[assoc, col]] << batch_record
|
110
|
+
end
|
111
|
+
end
|
112
|
+
associations_info
|
113
|
+
end
|
114
|
+
|
115
|
+
# Save associated records and fill foreign keys on RecordList records
|
116
|
+
# @param record_list [BatchRecordList] RecordList of primary records for this consumer
|
117
|
+
# @param associations_info [Hash] Contains association info
|
118
|
+
def save_associations_first(record_list, associations_info)
|
119
|
+
associations_info.each_value do |records|
|
120
|
+
assoc_record_list = BatchRecordList.new(records)
|
121
|
+
Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
|
122
|
+
save_records_to_database(assoc_record_list)
|
123
|
+
end
|
124
|
+
import_associations(assoc_record_list)
|
125
|
+
end
|
126
|
+
record_list.records.each do |record|
|
127
|
+
associations_info.each_key do |assoc, _|
|
128
|
+
record.assign_attributes({ assoc.foreign_key => record.send(assoc.name).id })
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
87
133
|
# @param record_list [BatchRecordList]
|
88
134
|
# @return [Array<ActiveRecord::Base>]
|
89
135
|
def mass_update(record_list)
|
90
136
|
# The entire batch should be treated as one transaction so that if
|
91
137
|
# any message fails, the whole thing is rolled back or retried
|
92
138
|
# if there is deadlock
|
93
|
-
|
94
|
-
|
95
|
-
|
139
|
+
|
140
|
+
if @save_associations_first
|
141
|
+
associations_info = assign_associations(record_list)
|
142
|
+
save_associations_first(record_list, associations_info)
|
143
|
+
Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
|
144
|
+
save_records_to_database(record_list)
|
145
|
+
end
|
146
|
+
else
|
147
|
+
Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
|
148
|
+
save_records_to_database(record_list)
|
149
|
+
import_associations(record_list) if record_list.associations.any?
|
150
|
+
end
|
96
151
|
end
|
97
152
|
record_list.records
|
98
153
|
end
|
@@ -45,6 +45,11 @@ module Deimos
|
|
45
45
|
config[:replace_associations]
|
46
46
|
end
|
47
47
|
|
48
|
+
# @return [Boolean]
|
49
|
+
def save_associations_first
|
50
|
+
config[:save_associations_first]
|
51
|
+
end
|
52
|
+
|
48
53
|
# @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
|
49
54
|
# only the last message for each unique key in a batch is processed.
|
50
55
|
# @return [void]
|
@@ -97,7 +97,8 @@ module Deimos # rubocop:disable Metrics/ModuleLength
|
|
97
97
|
kafka_config.replace_associations
|
98
98
|
end,
|
99
99
|
bulk_import_id_generator: kafka_config.bulk_import_id_generator ||
|
100
|
-
Deimos.config.consumers.bulk_import_id_generator
|
100
|
+
Deimos.config.consumers.bulk_import_id_generator,
|
101
|
+
save_associations_first: kafka_config.save_associations_first
|
101
102
|
)
|
102
103
|
end
|
103
104
|
end
|
@@ -476,6 +477,11 @@ module Deimos # rubocop:disable Metrics/ModuleLength
|
|
476
477
|
# @return [Block]
|
477
478
|
setting :bulk_import_id_generator, nil
|
478
479
|
|
480
|
+
# If enabled save associated records prior to saving the main record class
|
481
|
+
# This will also set foreign keys for associated records
|
482
|
+
# @return [Boolean]
|
483
|
+
setting :save_associations_first, false
|
484
|
+
|
479
485
|
# These are the phobos "listener" configs. See CONFIGURATION.md for more
|
480
486
|
# info.
|
481
487
|
setting :group_id
|
@@ -52,7 +52,8 @@ module Deimos
|
|
52
52
|
compression_threshold: self.producers.compression_threshold,
|
53
53
|
max_queue_size: self.producers.max_queue_size,
|
54
54
|
delivery_threshold: self.producers.delivery_threshold,
|
55
|
-
delivery_interval: self.producers.delivery_interval
|
55
|
+
delivery_interval: self.producers.delivery_interval,
|
56
|
+
persistent_connections: self.producers.persistent_connections
|
56
57
|
},
|
57
58
|
consumer: {
|
58
59
|
session_timeout: self.consumers.session_timeout,
|
data/lib/deimos/version.rb
CHANGED
@@ -114,5 +114,142 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
114
114
|
|
115
115
|
end
|
116
116
|
|
117
|
+
context 'with save_associations_first' do
|
118
|
+
before(:all) do
|
119
|
+
ActiveRecord::Base.connection.create_table(:fidgets, force: true) do |t|
|
120
|
+
t.string(:test_id)
|
121
|
+
t.integer(:some_int)
|
122
|
+
t.string(:bulk_import_id)
|
123
|
+
t.timestamps
|
124
|
+
end
|
125
|
+
|
126
|
+
ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
|
127
|
+
t.string(:title)
|
128
|
+
t.string(:bulk_import_id)
|
129
|
+
t.belongs_to(:fidget)
|
130
|
+
|
131
|
+
t.index(%i(title), unique: true)
|
132
|
+
end
|
133
|
+
|
134
|
+
ActiveRecord::Base.connection.create_table(:widget_fidgets, force: true, id: false) do |t|
|
135
|
+
t.belongs_to(:fidget)
|
136
|
+
t.belongs_to(:widget)
|
137
|
+
t.string(:bulk_import_id)
|
138
|
+
t.string(:note)
|
139
|
+
t.index(%i(widget_id fidget_id), unique: true)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
after(:all) do
|
144
|
+
ActiveRecord::Base.connection.drop_table(:fidgets)
|
145
|
+
ActiveRecord::Base.connection.drop_table(:fidget_details)
|
146
|
+
ActiveRecord::Base.connection.drop_table(:widget_fidgets)
|
147
|
+
end
|
148
|
+
|
149
|
+
let(:fidget_detail_class) do
|
150
|
+
Class.new(ActiveRecord::Base) do
|
151
|
+
self.table_name = 'fidget_details'
|
152
|
+
belongs_to :fidget
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
let(:fidget_class) do
|
157
|
+
Class.new(ActiveRecord::Base) do
|
158
|
+
self.table_name = 'fidgets'
|
159
|
+
has_one :fidget_detail
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
let(:widget_fidget_class) do
|
164
|
+
Class.new(ActiveRecord::Base) do
|
165
|
+
self.table_name = 'widget_fidgets'
|
166
|
+
belongs_to :fidget
|
167
|
+
belongs_to :widget
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
let(:bulk_id_generator) { proc { SecureRandom.uuid } }
|
172
|
+
|
173
|
+
let(:key_proc) do
|
174
|
+
lambda do |klass|
|
175
|
+
case klass.to_s
|
176
|
+
when 'Widget', 'Fidget'
|
177
|
+
%w(id)
|
178
|
+
when 'WidgetFidget'
|
179
|
+
%w(widget_id fidget_id)
|
180
|
+
when 'FidgetDetail', 'Detail'
|
181
|
+
%w(title)
|
182
|
+
else
|
183
|
+
raise "Key Columns for #{klass} not defined"
|
184
|
+
end
|
185
|
+
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
before(:each) do
|
190
|
+
stub_const('Fidget', fidget_class)
|
191
|
+
stub_const('FidgetDetail', fidget_detail_class)
|
192
|
+
stub_const('WidgetFidget', widget_fidget_class)
|
193
|
+
Widget.reset_column_information
|
194
|
+
Fidget.reset_column_information
|
195
|
+
WidgetFidget.reset_column_information
|
196
|
+
end
|
197
|
+
|
198
|
+
# rubocop:disable RSpec/MultipleExpectations, RSpec/ExampleLength
|
199
|
+
it 'should backfill the associations when upserting primary records' do
|
200
|
+
batch = Deimos::ActiveRecordConsume::BatchRecordList.new(
|
201
|
+
[
|
202
|
+
Deimos::ActiveRecordConsume::BatchRecord.new(
|
203
|
+
klass: WidgetFidget,
|
204
|
+
attributes: {
|
205
|
+
widget: { test_id: 'id1', some_int: 10, detail: { title: 'Widget Title 1' } },
|
206
|
+
fidget: { test_id: 'id1', some_int: 10, fidget_detail: { title: 'Fidget Title 1' } },
|
207
|
+
note: 'Stuff 1'
|
208
|
+
},
|
209
|
+
bulk_import_column: 'bulk_import_id',
|
210
|
+
bulk_import_id_generator: bulk_id_generator
|
211
|
+
),
|
212
|
+
Deimos::ActiveRecordConsume::BatchRecord.new(
|
213
|
+
klass: WidgetFidget,
|
214
|
+
attributes: {
|
215
|
+
widget: { test_id: 'id2', some_int: 20, detail: { title: 'Widget Title 2' } },
|
216
|
+
fidget: { test_id: 'id2', some_int: 20, fidget_detail: { title: 'Fidget Title 2' } },
|
217
|
+
note: 'Stuff 2'
|
218
|
+
},
|
219
|
+
bulk_import_column: 'bulk_import_id',
|
220
|
+
bulk_import_id_generator: bulk_id_generator
|
221
|
+
)
|
222
|
+
]
|
223
|
+
)
|
224
|
+
|
225
|
+
results = described_class.new(WidgetFidget,
|
226
|
+
bulk_import_id_generator: bulk_id_generator,
|
227
|
+
bulk_import_id_column: 'bulk_import_id',
|
228
|
+
key_col_proc: key_proc,
|
229
|
+
save_associations_first: true).mass_update(batch)
|
230
|
+
expect(results.count).to eq(2)
|
231
|
+
expect(Widget.count).to eq(2)
|
232
|
+
expect(Detail.count).to eq(2)
|
233
|
+
expect(Fidget.count).to eq(2)
|
234
|
+
expect(FidgetDetail.count).to eq(2)
|
235
|
+
|
236
|
+
WidgetFidget.all.each_with_index do |widget_fidget, ind|
|
237
|
+
widget = Widget.find_by(id: widget_fidget.widget_id)
|
238
|
+
expect(widget.test_id).to eq("id#{ind + 1}")
|
239
|
+
expect(widget.some_int).to eq((ind + 1) * 10)
|
240
|
+
detail = Detail.find_by(widget_id: widget_fidget.widget_id)
|
241
|
+
expect(detail.title).to eq("Widget Title #{ind + 1}")
|
242
|
+
fidget = Fidget.find_by(id: widget_fidget.fidget_id)
|
243
|
+
expect(fidget.test_id).to eq("id#{ind + 1}")
|
244
|
+
expect(fidget.some_int).to eq((ind + 1) * 10)
|
245
|
+
fidget_detail = FidgetDetail.find_by(fidget_id: widget_fidget.fidget_id)
|
246
|
+
expect(fidget_detail.title).to eq("Fidget Title #{ind + 1}")
|
247
|
+
expect(widget_fidget.note).to eq("Stuff #{ind + 1}")
|
248
|
+
end
|
249
|
+
end
|
250
|
+
# rubocop:enable RSpec/MultipleExpectations, RSpec/ExampleLength
|
251
|
+
|
252
|
+
end
|
253
|
+
|
117
254
|
end
|
118
255
|
end
|
@@ -92,7 +92,8 @@ describe Deimos, 'configuration' do
|
|
92
92
|
handler: 'ConsumerTest::MyConsumer',
|
93
93
|
use_schema_classes: nil,
|
94
94
|
max_db_batch_size: nil,
|
95
|
-
bulk_import_id_generator: nil
|
95
|
+
bulk_import_id_generator: nil,
|
96
|
+
save_associations_first: false
|
96
97
|
}, {
|
97
98
|
topic: 'my_batch_consume_topic',
|
98
99
|
group_id: 'my_batch_group_id',
|
@@ -111,7 +112,8 @@ describe Deimos, 'configuration' do
|
|
111
112
|
handler: 'ConsumerTest::MyBatchConsumer',
|
112
113
|
use_schema_classes: nil,
|
113
114
|
max_db_batch_size: nil,
|
114
|
-
bulk_import_id_generator: nil
|
115
|
+
bulk_import_id_generator: nil,
|
116
|
+
save_associations_first: false
|
115
117
|
}
|
116
118
|
],
|
117
119
|
producer: {
|
@@ -125,7 +127,8 @@ describe Deimos, 'configuration' do
|
|
125
127
|
compression_threshold: 1,
|
126
128
|
max_queue_size: 10_000,
|
127
129
|
delivery_threshold: 0,
|
128
|
-
delivery_interval: 0
|
130
|
+
delivery_interval: 0,
|
131
|
+
persistent_connections: false
|
129
132
|
}
|
130
133
|
)
|
131
134
|
end
|
@@ -264,7 +267,8 @@ describe Deimos, 'configuration' do
|
|
264
267
|
handler: 'MyConfigConsumer',
|
265
268
|
use_schema_classes: false,
|
266
269
|
max_db_batch_size: nil,
|
267
|
-
bulk_import_id_generator: nil
|
270
|
+
bulk_import_id_generator: nil,
|
271
|
+
save_associations_first: false
|
268
272
|
}
|
269
273
|
],
|
270
274
|
producer: {
|
@@ -278,7 +282,8 @@ describe Deimos, 'configuration' do
|
|
278
282
|
compression_threshold: 2,
|
279
283
|
max_queue_size: 10,
|
280
284
|
delivery_threshold: 1,
|
281
|
-
delivery_interval: 1
|
285
|
+
delivery_interval: 1,
|
286
|
+
persistent_connections: true
|
282
287
|
}
|
283
288
|
)
|
284
289
|
end
|
@@ -295,6 +300,7 @@ describe Deimos, 'configuration' do
|
|
295
300
|
group_id 'myconsumerid'
|
296
301
|
bulk_import_id_generator(-> { 'consumer' })
|
297
302
|
replace_associations false
|
303
|
+
save_associations_first true
|
298
304
|
end
|
299
305
|
|
300
306
|
consumer do
|
@@ -312,10 +318,12 @@ describe Deimos, 'configuration' do
|
|
312
318
|
custom = MyConfigConsumer.config
|
313
319
|
expect(custom[:replace_associations]).to eq(false)
|
314
320
|
expect(custom[:bulk_import_id_generator].call).to eq('consumer')
|
321
|
+
expect(custom[:save_associations_first]).to eq(true)
|
315
322
|
|
316
323
|
default = MyConfigConsumer2.config
|
317
324
|
expect(default[:replace_associations]).to eq(true)
|
318
325
|
expect(default[:bulk_import_id_generator].call).to eq('global')
|
326
|
+
expect(default[:save_associations_first]).to eq(false)
|
319
327
|
|
320
328
|
end
|
321
329
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.24.
|
4
|
+
version: 1.24.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-05-
|
11
|
+
date: 2024-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|