deimos-ruby 1.22.5 → 1.23.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +18 -0
- data/README.md +6 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +43 -20
- data/lib/deimos/active_record_consume/batch_record.rb +5 -4
- data/lib/deimos/active_record_consume/batch_record_list.rb +11 -2
- data/lib/deimos/active_record_consume/mass_updater.rb +13 -4
- data/lib/deimos/active_record_consumer.rb +10 -0
- data/lib/deimos/config/configuration.rb +28 -2
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/instrumentation.rb +20 -8
- data/lib/deimos/test_helpers.rb +1 -0
- data/lib/deimos/tracing/datadog.rb +12 -6
- data/lib/deimos/tracing/mock.rb +31 -2
- data/lib/deimos/tracing/provider.rb +6 -0
- data/lib/deimos/utils/db_poller/base.rb +23 -0
- data/lib/deimos/utils/schema_class.rb +10 -2
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +14 -1
- data/spec/active_record_batch_consumer_association_spec.rb +32 -5
- data/spec/active_record_batch_consumer_spec.rb +376 -59
- data/spec/active_record_consume/mass_updater_spec.rb +46 -3
- data/spec/active_record_consumer_spec.rb +74 -1
- data/spec/active_record_producer_spec.rb +4 -1
- data/spec/batch_consumer_spec.rb +4 -1
- data/spec/config/configuration_spec.rb +42 -3
- data/spec/consumer_spec.rb +42 -1
- data/spec/schemas/my_namespace/my_updated_schema.rb +18 -0
- data/spec/utils/db_poller_spec.rb +42 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 42d51126a6bf86b0aa338798c9b9a4b479154eda722394940df0bbc8a23e3947
|
4
|
+
data.tar.gz: 8d6c27244277078ea3b5d2a23440071aba43db3938c43cbbfe4fef27db207556
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 86d98c4e4cc84a380b69196c4d04c51a6a8aaaa05601e2d7deac536495d7c689c987555e23e1bcafb895dc78e24492c9c0b174ee273c0859c9d76795a510d154
|
7
|
+
data.tar.gz: 8e7875b92bb043c95721a9a12747f6f6727bc225f1802f5d6a6ba34ee4757884347aaa42b3050d9b77f668b7ebd2654aa322cd46dea3c426d140a191c19f7fb0
|
data/.github/workflows/ci.yml
CHANGED
@@ -17,7 +17,7 @@ jobs:
|
|
17
17
|
BUNDLE_WITHOUT: development:test
|
18
18
|
|
19
19
|
steps:
|
20
|
-
- uses: actions/checkout@
|
20
|
+
- uses: actions/checkout@v3
|
21
21
|
|
22
22
|
- name: Set up Ruby 2.7
|
23
23
|
uses: ruby/setup-ruby@v1
|
@@ -39,7 +39,7 @@ jobs:
|
|
39
39
|
ruby: [ '2.6', '2.7', '3.0', '3.1' ]
|
40
40
|
|
41
41
|
steps:
|
42
|
-
- uses: actions/checkout@
|
42
|
+
- uses: actions/checkout@v3
|
43
43
|
- uses: ruby/setup-ruby@v1
|
44
44
|
with:
|
45
45
|
ruby-version: ${{ matrix.ruby }}
|
data/CHANGELOG.md
CHANGED
@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
+
# 1.23.3 - 2024-01-25
|
11
|
+
- Feature: Add configuration to skip messages that are too large to publish via DB poller.
|
12
|
+
|
13
|
+
# 1.23.2 - 2024-01-22
|
14
|
+
- Fix: Send a `publish_error` metric for errors other than `DeliveryFailed`.
|
15
|
+
|
16
|
+
# 1.23.0 - 2024-01-09
|
17
|
+
|
18
|
+
- Fix: Fixed handler metric for status:received, status:success in batch consumption
|
19
|
+
- Feature: Allow pre processing of messages prior to bulk consumption
|
20
|
+
- Feature: Add global configuration for custom `bulk_import_id_generator` proc for all consumers
|
21
|
+
- Feature: Add individual configuration for custom `bulk_import_id_generator` proc per consumer
|
22
|
+
- Feature: Add global `replace_assocations` value for for all consumers
|
23
|
+
- Feature: Add individual `replace_assocations` value for for individual consumers
|
24
|
+
- Feature: `should_consume?` method accepts BatchRecord associations
|
25
|
+
- Feature: Reintroduce `filter_records` for bulk filtering of records prior to insertion
|
26
|
+
- Feature: Return valid and invalid records saved during consumption for further processing in `batch_consumption.valid_records` and `batch_consumption.invalid_records` ActiveSupport Notifications
|
27
|
+
|
10
28
|
# 1.22.5 - 2023-07-18
|
11
29
|
- Fix: Fixed buffer overflow crash with DB producer.
|
12
30
|
|
data/README.md
CHANGED
@@ -189,6 +189,12 @@ produced by Phobos and RubyKafka):
|
|
189
189
|
* exception_object
|
190
190
|
* messages - the batch of messages (in the form of `Deimos::KafkaMessage`s)
|
191
191
|
that failed - this should have only a single message in the batch.
|
192
|
+
* `batch_consumption.valid_records` - sent when the consumer has successfully upserted records. Limited by `max_db_batch_size`.
|
193
|
+
* consumer: class of the consumer that upserted these records
|
194
|
+
* records: Records upserted into the DB (of type `ActiveRecord::Base`)
|
195
|
+
* `batch_consumption.invalid_records` - sent when the consumer has rejected records returned from `filtered_records`. Limited by `max_db_batch_size`.
|
196
|
+
* consumer: class of the consumer that rejected these records
|
197
|
+
* records: Rejected records (of type `Deimos::ActiveRecordConsume::BatchRecord`)
|
192
198
|
|
193
199
|
Similarly:
|
194
200
|
```ruby
|
data/docs/CONFIGURATION.md
CHANGED
@@ -100,6 +100,8 @@ offset_commit_threshold|0|Number of messages that can be processed before their
|
|
100
100
|
offset_retention_time|nil|The time period that committed offsets will be retained, in seconds. Defaults to the broker setting.
|
101
101
|
heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
|
102
102
|
backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
|
103
|
+
replace_associations|nil| Whether to delete existing associations for records during bulk consumption for this consumer. If no value is specified the provided/default value from the `consumers` configuration will be used.
|
104
|
+
bulk_import_id_generator|nil| Block to determine the `bulk_import_id` generated during bulk consumption. If no block is specified the provided/default block from the `consumers` configuration will be used.
|
103
105
|
|
104
106
|
## Defining Database Pollers
|
105
107
|
|
@@ -172,6 +174,8 @@ consumers.backoff|`(1000..60_000)`|Range representing the minimum and maximum nu
|
|
172
174
|
consumers.reraise_errors|false|Default behavior is to swallow uncaught exceptions and log to the metrics provider. Set this to true to instead raise all errors. Note that raising an error will ensure that the message cannot be processed - if there is a bad message which will always raise that error, your consumer will not be able to proceed past it and will be stuck forever until you fix your code. See also the `fatal_error` configuration. This is automatically set to true when using the `TestHelpers` module in RSpec.
|
173
175
|
consumers.report_lag|false|Whether to send the `consumer_lag` metric. This requires an extra thread per consumer.
|
174
176
|
consumers.fatal_error|`proc { false }`|Block taking an exception, payload and metadata and returning true if this should be considered a fatal error and false otherwise. E.g. you can use this to always fail if the database is available. Not needed if reraise_errors is set to true.
|
177
|
+
consumers.replace_associations|true|Whether to delete existing associations for records during bulk consumption prior to inserting new associated records
|
178
|
+
consumers.bulk_import_id_generator|`proc { SecureRandom.uuid }`| Block to determine the `bulk_import_id` generated during bulk consumption. Block will be used for all bulk consumers unless explicitly set for individual consumers
|
175
179
|
|
176
180
|
## Producer Configuration
|
177
181
|
|
@@ -28,18 +28,14 @@ module Deimos
|
|
28
28
|
zip(metadata[:keys]).
|
29
29
|
map { |p, k| Deimos::Message.new(p, nil, key: k) }
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
update_database(compact_messages(messages))
|
40
|
-
else
|
41
|
-
uncompacted_update(messages)
|
42
|
-
end
|
31
|
+
tag = metadata[:topic]
|
32
|
+
Deimos.config.tracer.active_span.set_tag('topic', tag)
|
33
|
+
|
34
|
+
Deimos.instrument('ar_consumer.consume_batch', tag) do
|
35
|
+
if @compacted || self.class.config[:no_keys]
|
36
|
+
update_database(compact_messages(messages))
|
37
|
+
else
|
38
|
+
uncompacted_update(messages)
|
43
39
|
end
|
44
40
|
end
|
45
41
|
end
|
@@ -93,8 +89,9 @@ module Deimos
|
|
93
89
|
end
|
94
90
|
|
95
91
|
# @param _record [ActiveRecord::Base]
|
92
|
+
# @param _associations [Hash]
|
96
93
|
# @return [Boolean]
|
97
|
-
def should_consume?(_record)
|
94
|
+
def should_consume?(_record, _associations=nil)
|
98
95
|
true
|
99
96
|
end
|
100
97
|
|
@@ -155,8 +152,13 @@ module Deimos
|
|
155
152
|
# @return [void]
|
156
153
|
def upsert_records(messages)
|
157
154
|
record_list = build_records(messages)
|
158
|
-
record_list
|
159
|
-
|
155
|
+
invalid = filter_records(record_list)
|
156
|
+
if invalid.any?
|
157
|
+
ActiveSupport::Notifications.instrument('batch_consumption.invalid_records', {
|
158
|
+
records: invalid,
|
159
|
+
consumer: self.class
|
160
|
+
})
|
161
|
+
end
|
160
162
|
return if record_list.empty?
|
161
163
|
|
162
164
|
key_col_proc = self.method(:key_columns).to_proc
|
@@ -165,13 +167,31 @@ module Deimos
|
|
165
167
|
updater = MassUpdater.new(@klass,
|
166
168
|
key_col_proc: key_col_proc,
|
167
169
|
col_proc: col_proc,
|
168
|
-
replace_associations: self.class.
|
169
|
-
|
170
|
+
replace_associations: self.class.replace_associations,
|
171
|
+
bulk_import_id_generator: self.class.bulk_import_id_generator)
|
172
|
+
ActiveSupport::Notifications.instrument('batch_consumption.valid_records', {
|
173
|
+
records: updater.mass_update(record_list),
|
174
|
+
consumer: self.class
|
175
|
+
})
|
176
|
+
end
|
177
|
+
|
178
|
+
# @param record_list [BatchRecordList]
|
179
|
+
# @return [Array<BatchRecord>]
|
180
|
+
def filter_records(record_list)
|
181
|
+
record_list.filter!(self.method(:should_consume?).to_proc)
|
182
|
+
end
|
183
|
+
|
184
|
+
# Process messages prior to saving to database
|
185
|
+
# @param _messages [Array<Deimos::Message>]
|
186
|
+
# @return [Void]
|
187
|
+
def pre_process(_messages)
|
188
|
+
nil
|
170
189
|
end
|
171
190
|
|
172
191
|
# @param messages [Array<Deimos::Message>]
|
173
192
|
# @return [BatchRecordList]
|
174
193
|
def build_records(messages)
|
194
|
+
pre_process(messages)
|
175
195
|
records = messages.map do |m|
|
176
196
|
attrs = if self.method(:record_attributes).parameters.size == 2
|
177
197
|
record_attributes(m.payload, m.key)
|
@@ -189,7 +209,8 @@ module Deimos
|
|
189
209
|
|
190
210
|
BatchRecord.new(klass: @klass,
|
191
211
|
attributes: attrs,
|
192
|
-
bulk_import_column: col
|
212
|
+
bulk_import_column: col,
|
213
|
+
bulk_import_id_generator: self.class.bulk_import_id_generator)
|
193
214
|
end
|
194
215
|
BatchRecordList.new(records.compact)
|
195
216
|
end
|
@@ -199,9 +220,11 @@ module Deimos
|
|
199
220
|
# deleted records.
|
200
221
|
# @return [void]
|
201
222
|
def remove_records(messages)
|
202
|
-
|
223
|
+
Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
|
224
|
+
clause = deleted_query(messages)
|
203
225
|
|
204
|
-
|
226
|
+
clause.delete_all
|
227
|
+
end
|
205
228
|
end
|
206
229
|
end
|
207
230
|
end
|
@@ -17,16 +17,17 @@ module Deimos
|
|
17
17
|
# @return [String] The column name to use for bulk IDs - defaults to `bulk_import_id`.
|
18
18
|
attr_accessor :bulk_import_column
|
19
19
|
|
20
|
-
delegate :valid?, to: :record
|
20
|
+
delegate :valid?, :errors, :send, :attributes, to: :record
|
21
21
|
|
22
22
|
# @param klass [Class < ActiveRecord::Base]
|
23
23
|
# @param attributes [Hash] the full attribute list, including associations.
|
24
24
|
# @param bulk_import_column [String]
|
25
|
-
|
25
|
+
# @param bulk_import_id_generator [Proc]
|
26
|
+
def initialize(klass:, attributes:, bulk_import_column: nil, bulk_import_id_generator: nil)
|
26
27
|
@klass = klass
|
27
28
|
if bulk_import_column
|
28
29
|
self.bulk_import_column = bulk_import_column
|
29
|
-
self.bulk_import_id =
|
30
|
+
self.bulk_import_id = bulk_import_id_generator&.call
|
30
31
|
attributes[bulk_import_column] = bulk_import_id
|
31
32
|
end
|
32
33
|
attributes = attributes.with_indifferent_access
|
@@ -43,7 +44,7 @@ module Deimos
|
|
43
44
|
return if @klass.column_names.include?(self.bulk_import_column.to_s)
|
44
45
|
|
45
46
|
raise "Create bulk_import_id on the #{@klass.table_name} table." \
|
46
|
-
|
47
|
+
' Run rails g deimos:bulk_import_id {table} to create the migration.'
|
47
48
|
end
|
48
49
|
|
49
50
|
# @return [Class < ActiveRecord::Base]
|
@@ -17,10 +17,19 @@ module Deimos
|
|
17
17
|
self.bulk_import_column = records.first&.bulk_import_column&.to_sym
|
18
18
|
end
|
19
19
|
|
20
|
-
# Filter
|
20
|
+
# Filter and return removed invalid batch records by the specified method
|
21
21
|
# @param method [Proc]
|
22
|
+
# @return [Array<BatchRecord>]
|
22
23
|
def filter!(method)
|
23
|
-
self.batch_records
|
24
|
+
self.batch_records, invalid = self.batch_records.partition do |batch_record|
|
25
|
+
case method.parameters.size
|
26
|
+
when 2
|
27
|
+
method.call(batch_record.record, batch_record.associations)
|
28
|
+
else
|
29
|
+
method.call(batch_record.record)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
invalid
|
24
33
|
end
|
25
34
|
|
26
35
|
# Get the original ActiveRecord objects.
|
@@ -19,9 +19,11 @@ module Deimos
|
|
19
19
|
# @param key_col_proc [Proc<Class < ActiveRecord::Base>]
|
20
20
|
# @param col_proc [Proc<Class < ActiveRecord::Base>]
|
21
21
|
# @param replace_associations [Boolean]
|
22
|
-
def initialize(klass, key_col_proc: nil, col_proc: nil,
|
22
|
+
def initialize(klass, key_col_proc: nil, col_proc: nil,
|
23
|
+
replace_associations: true, bulk_import_id_generator: nil)
|
23
24
|
@klass = klass
|
24
25
|
@replace_associations = replace_associations
|
26
|
+
@bulk_import_id_generator = bulk_import_id_generator
|
25
27
|
|
26
28
|
@key_cols = {}
|
27
29
|
@key_col_proc = key_col_proc
|
@@ -69,7 +71,7 @@ module Deimos
|
|
69
71
|
def import_associations(record_list)
|
70
72
|
record_list.fill_primary_keys!
|
71
73
|
|
72
|
-
import_id = @replace_associations ?
|
74
|
+
import_id = @replace_associations ? @bulk_import_id_generator&.call : nil
|
73
75
|
record_list.associations.each do |assoc|
|
74
76
|
sub_records = record_list.map { |r| r.sub_records(assoc.name, import_id) }.flatten
|
75
77
|
next unless sub_records.any?
|
@@ -82,9 +84,16 @@ module Deimos
|
|
82
84
|
end
|
83
85
|
|
84
86
|
# @param record_list [BatchRecordList]
|
87
|
+
# @return [Array<ActiveRecord::Base>]
|
85
88
|
def mass_update(record_list)
|
86
|
-
|
87
|
-
|
89
|
+
# The entire batch should be treated as one transaction so that if
|
90
|
+
# any message fails, the whole thing is rolled back or retried
|
91
|
+
# if there is deadlock
|
92
|
+
Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
|
93
|
+
save_records_to_database(record_list)
|
94
|
+
import_associations(record_list) if record_list.associations.any?
|
95
|
+
end
|
96
|
+
record_list.records
|
88
97
|
end
|
89
98
|
|
90
99
|
end
|
@@ -35,6 +35,16 @@ module Deimos
|
|
35
35
|
config[:bulk_import_id_column]
|
36
36
|
end
|
37
37
|
|
38
|
+
# @return [Proc]
|
39
|
+
def bulk_import_id_generator
|
40
|
+
config[:bulk_import_id_generator]
|
41
|
+
end
|
42
|
+
|
43
|
+
# @return [Boolean]
|
44
|
+
def replace_associations
|
45
|
+
config[:replace_associations]
|
46
|
+
end
|
47
|
+
|
38
48
|
# @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
|
39
49
|
# only the last message for each unique key in a batch is processed.
|
40
50
|
# @return [void]
|
@@ -79,6 +79,7 @@ module Deimos # rubocop:disable Metrics/ModuleLength
|
|
79
79
|
|
80
80
|
# @!visibility private
|
81
81
|
# @param kafka_config [FigTree::ConfigStruct]
|
82
|
+
# rubocop:disable Metrics/PerceivedComplexity, Metrics/AbcSize
|
82
83
|
def self.configure_producer_or_consumer(kafka_config)
|
83
84
|
klass = kafka_config.class_name.constantize
|
84
85
|
klass.class_eval do
|
@@ -90,11 +91,18 @@ module Deimos # rubocop:disable Metrics/ModuleLength
|
|
90
91
|
if kafka_config.respond_to?(:bulk_import_id_column) # consumer
|
91
92
|
klass.config.merge!(
|
92
93
|
bulk_import_id_column: kafka_config.bulk_import_id_column,
|
93
|
-
replace_associations: kafka_config.replace_associations
|
94
|
+
replace_associations: if kafka_config.replace_associations.nil?
|
95
|
+
Deimos.config.consumers.replace_associations
|
96
|
+
else
|
97
|
+
kafka_config.replace_associations
|
98
|
+
end,
|
99
|
+
bulk_import_id_generator: kafka_config.bulk_import_id_generator ||
|
100
|
+
Deimos.config.consumers.bulk_import_id_generator
|
94
101
|
)
|
95
102
|
end
|
96
103
|
end
|
97
104
|
end
|
105
|
+
# rubocop:enable Metrics/PerceivedComplexity, Metrics/AbcSize
|
98
106
|
|
99
107
|
define_settings do
|
100
108
|
|
@@ -242,6 +250,15 @@ module Deimos # rubocop:disable Metrics/ModuleLength
|
|
242
250
|
# Not needed if reraise_errors is set to true.
|
243
251
|
# @return [Block]
|
244
252
|
setting(:fatal_error, proc { false })
|
253
|
+
|
254
|
+
# The default function to generate a bulk ID for bulk consumers
|
255
|
+
# @return [Block]
|
256
|
+
setting(:bulk_import_id_generator, proc { SecureRandom.uuid })
|
257
|
+
|
258
|
+
# If true, multi-table consumers will blow away associations rather than appending to them.
|
259
|
+
# Applies to all consumers unless specified otherwise
|
260
|
+
# @return [Boolean]
|
261
|
+
setting :replace_associations, true
|
245
262
|
end
|
246
263
|
|
247
264
|
setting :producers do
|
@@ -445,7 +462,13 @@ module Deimos # rubocop:disable Metrics/ModuleLength
|
|
445
462
|
setting :bulk_import_id_column, :bulk_import_id
|
446
463
|
# If true, multi-table consumers will blow away associations rather than appending to them.
|
447
464
|
# @return [Boolean]
|
448
|
-
setting :replace_associations,
|
465
|
+
setting :replace_associations, nil
|
466
|
+
|
467
|
+
# The default function to generate a bulk ID for this consumer
|
468
|
+
# Uses the consumers proc defined in the consumers config by default unless
|
469
|
+
# specified for individual consumers
|
470
|
+
# @return [Block]
|
471
|
+
setting :bulk_import_id_generator, nil
|
449
472
|
|
450
473
|
# These are the phobos "listener" configs. See CONFIGURATION.md for more
|
451
474
|
# info.
|
@@ -477,6 +500,9 @@ module Deimos # rubocop:disable Metrics/ModuleLength
|
|
477
500
|
# The number of times to retry production when encountering a *non-Kafka* error. Set to nil
|
478
501
|
# for infinite retries.
|
479
502
|
setting :retries, 1
|
503
|
+
# If true, rather than shutting down when finding a message that is too large, log an
|
504
|
+
# error and skip it.
|
505
|
+
setting :skip_too_large_messages, false
|
480
506
|
# Amount of time, in seconds, to wait before catching updates, to allow transactions
|
481
507
|
# to complete but still pick up the right records. Should only be set for time-based mode.
|
482
508
|
setting :delay_time, 2
|
@@ -64,7 +64,7 @@ module Deimos
|
|
64
64
|
))
|
65
65
|
Deimos.config.metrics&.increment(
|
66
66
|
'handler',
|
67
|
-
by: metadata[
|
67
|
+
by: metadata[:batch_size],
|
68
68
|
tags: %W(
|
69
69
|
status:received
|
70
70
|
topic:#{metadata[:topic]}
|
@@ -115,7 +115,7 @@ module Deimos
|
|
115
115
|
))
|
116
116
|
Deimos.config.metrics&.increment(
|
117
117
|
'handler',
|
118
|
-
by: metadata[
|
118
|
+
by: metadata[:batch_size],
|
119
119
|
tags: %W(
|
120
120
|
status:success
|
121
121
|
topic:#{metadata[:topic]}
|
@@ -43,14 +43,8 @@ module Deimos
|
|
43
43
|
|
44
44
|
# This module listens to events published by RubyKafka.
|
45
45
|
module KafkaListener
|
46
|
-
#
|
47
|
-
|
48
|
-
# @param event [ActiveSupport::Notifications::Event]
|
49
|
-
# @return [void]
|
50
|
-
def self.send_produce_error(event)
|
51
|
-
exception = event.payload[:exception_object]
|
52
|
-
return if !exception || !exception.respond_to?(:failed_messages)
|
53
|
-
|
46
|
+
# @param exception [Exception]
|
47
|
+
def self.handle_exception_with_messages(exception)
|
54
48
|
messages = exception.failed_messages
|
55
49
|
messages.group_by(&:topic).each do |topic, batch|
|
56
50
|
producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
|
@@ -74,6 +68,24 @@ module Deimos
|
|
74
68
|
)
|
75
69
|
end
|
76
70
|
end
|
71
|
+
|
72
|
+
# Listens for any exceptions that happen during publishing and re-publishes
|
73
|
+
# as a Deimos event.
|
74
|
+
# @param event [ActiveSupport::Notifications::Event]
|
75
|
+
# @return [void]
|
76
|
+
def self.send_produce_error(event)
|
77
|
+
exception = event.payload[:exception_object]
|
78
|
+
return unless exception
|
79
|
+
|
80
|
+
if exception.respond_to?(:failed_messages)
|
81
|
+
handle_exception_with_messages(exception)
|
82
|
+
else
|
83
|
+
Deimos.config.metrics&.increment(
|
84
|
+
'publish_error',
|
85
|
+
by: event.payload[:message_count] || 1
|
86
|
+
)
|
87
|
+
end
|
88
|
+
end
|
77
89
|
end
|
78
90
|
|
79
91
|
ActiveSupport::Notifications.subscribe('deliver_messages.producer.kafka') do |*args|
|
data/lib/deimos/test_helpers.rb
CHANGED
@@ -15,11 +15,7 @@ module Deimos
|
|
15
15
|
|
16
16
|
# :nodoc:
|
17
17
|
def start(span_name, options={})
|
18
|
-
span =
|
19
|
-
::Datadog.tracer.trace(span_name)
|
20
|
-
else
|
21
|
-
::Datadog::Tracing.trace(span_name)
|
22
|
-
end
|
18
|
+
span = tracer.trace(span_name)
|
23
19
|
span.service = @service
|
24
20
|
span.resource = options[:resource]
|
25
21
|
span
|
@@ -30,9 +26,14 @@ module Deimos
|
|
30
26
|
span.finish
|
31
27
|
end
|
32
28
|
|
29
|
+
# :nodoc:
|
30
|
+
def tracer
|
31
|
+
@tracer ||= ::Datadog.respond_to?(:tracer) ? ::Datadog.tracer : ::Datadog::Tracing
|
32
|
+
end
|
33
|
+
|
33
34
|
# :nodoc:
|
34
35
|
def active_span
|
35
|
-
|
36
|
+
tracer.active_span
|
36
37
|
end
|
37
38
|
|
38
39
|
# :nodoc:
|
@@ -45,6 +46,11 @@ module Deimos
|
|
45
46
|
(span || active_span).set_tag(tag, value)
|
46
47
|
end
|
47
48
|
|
49
|
+
# :nodoc:
|
50
|
+
def get_tag(tag)
|
51
|
+
active_span.get_tag(tag)
|
52
|
+
end
|
53
|
+
|
48
54
|
end
|
49
55
|
end
|
50
56
|
end
|
data/lib/deimos/tracing/mock.rb
CHANGED
@@ -10,6 +10,7 @@ module Deimos
|
|
10
10
|
def initialize(logger=nil)
|
11
11
|
@logger = logger || Logger.new(STDOUT)
|
12
12
|
@logger.info('MockTracingProvider initialized')
|
13
|
+
@active_span = MockSpan.new
|
13
14
|
end
|
14
15
|
|
15
16
|
# @param span_name [String]
|
@@ -32,12 +33,22 @@ module Deimos
|
|
32
33
|
|
33
34
|
# :nodoc:
|
34
35
|
def active_span
|
35
|
-
|
36
|
+
@active_span ||= MockSpan.new
|
36
37
|
end
|
37
38
|
|
38
39
|
# :nodoc:
|
39
40
|
def set_tag(tag, value, span=nil)
|
40
|
-
|
41
|
+
if span
|
42
|
+
span.set_tag(tag, value)
|
43
|
+
else
|
44
|
+
active_span.set_tag(tag, value)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Get a tag from a span with the specified tag.
|
49
|
+
# @param tag [String]
|
50
|
+
def get_tag(tag)
|
51
|
+
@span.get_tag(tag)
|
41
52
|
end
|
42
53
|
|
43
54
|
# :nodoc:
|
@@ -47,5 +58,23 @@ module Deimos
|
|
47
58
|
@logger.info("Mock span '#{name}' set an error: #{exception}")
|
48
59
|
end
|
49
60
|
end
|
61
|
+
|
62
|
+
# Mock Span class
|
63
|
+
class MockSpan
|
64
|
+
# :nodoc:
|
65
|
+
def initialize
|
66
|
+
@span = {}
|
67
|
+
end
|
68
|
+
|
69
|
+
# :nodoc:
|
70
|
+
def set_tag(tag, value)
|
71
|
+
@span[tag] = value
|
72
|
+
end
|
73
|
+
|
74
|
+
# :nodoc:
|
75
|
+
def get_tag(tag)
|
76
|
+
@span[tag]
|
77
|
+
end
|
78
|
+
end
|
50
79
|
end
|
51
80
|
end
|
@@ -105,6 +105,25 @@ module Deimos
|
|
105
105
|
raise Deimos::MissingImplementationError
|
106
106
|
end
|
107
107
|
|
108
|
+
# @param exception [Exception]
|
109
|
+
# @param batch [Array<ActiveRecord::Base>]
|
110
|
+
# @param status [PollStatus]
|
111
|
+
# @param span [Object]
|
112
|
+
# @return [Boolean]
|
113
|
+
def handle_message_too_large(exception, batch, status, span)
|
114
|
+
Deimos.config.logger.error("Error publishing through DB Poller: #{exception.message}")
|
115
|
+
if @config.skip_too_large_messages
|
116
|
+
Deimos.config.logger.error("Skipping messages #{batch.map(&:id).join(', ')} since they are too large")
|
117
|
+
Deimos.config.tracer&.set_error(span, exception)
|
118
|
+
status.batches_errored += 1
|
119
|
+
true
|
120
|
+
else # do the same thing as regular Kafka::Error
|
121
|
+
sleep(0.5)
|
122
|
+
false
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
# rubocop:disable Metrics/AbcSize
|
108
127
|
# @param batch [Array<ActiveRecord::Base>]
|
109
128
|
# @param status [PollStatus]
|
110
129
|
# @return [Boolean]
|
@@ -118,6 +137,9 @@ module Deimos
|
|
118
137
|
process_batch(batch)
|
119
138
|
Deimos.config.tracer&.finish(span)
|
120
139
|
status.batches_processed += 1
|
140
|
+
rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,
|
141
|
+
Kafka::RecordListTooLarge => e
|
142
|
+
retry unless handle_message_too_large(e, batch, status, span)
|
121
143
|
rescue Kafka::Error => e # keep trying till it fixes itself
|
122
144
|
Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
|
123
145
|
sleep(0.5)
|
@@ -139,6 +161,7 @@ module Deimos
|
|
139
161
|
end
|
140
162
|
true
|
141
163
|
end
|
164
|
+
# rubocop:enable Metrics/AbcSize
|
142
165
|
|
143
166
|
# Publish batch using the configured producers
|
144
167
|
# @param batch [Array<ActiveRecord::Base>]
|
@@ -25,13 +25,21 @@ module Deimos
|
|
25
25
|
def instance(payload, schema, namespace='')
|
26
26
|
return payload if payload.is_a?(Deimos::SchemaClass::Base)
|
27
27
|
|
28
|
-
|
29
|
-
klass = constants.join('::').safe_constantize
|
28
|
+
klass = klass(schema, namespace)
|
30
29
|
return payload if klass.nil? || payload.nil?
|
31
30
|
|
32
31
|
klass.new(**payload.symbolize_keys)
|
33
32
|
end
|
34
33
|
|
34
|
+
# Determine and return the SchemaClass with the provided schema and namespace
|
35
|
+
# @param schema [String]
|
36
|
+
# @param namespace [String]
|
37
|
+
# @return [Deimos::SchemaClass]
|
38
|
+
def klass(schema, namespace)
|
39
|
+
constants = modules_for(namespace) + [schema.underscore.camelize.singularize]
|
40
|
+
constants.join('::').safe_constantize
|
41
|
+
end
|
42
|
+
|
35
43
|
# @param config [Hash] Producer or Consumer config
|
36
44
|
# @return [Boolean]
|
37
45
|
def use?(config)
|
data/lib/deimos/version.rb
CHANGED
data/lib/deimos.rb
CHANGED
@@ -57,7 +57,20 @@ module Deimos
|
|
57
57
|
# @param namespace [String]
|
58
58
|
# @return [Deimos::SchemaBackends::Base]
|
59
59
|
def schema_backend(schema:, namespace:)
|
60
|
-
|
60
|
+
if Utils::SchemaClass.use?(config.to_h)
|
61
|
+
# Initialize an instance of the provided schema
|
62
|
+
# in the event the schema class is an override, the inherited
|
63
|
+
# schema and namespace will be applied
|
64
|
+
schema_class = Utils::SchemaClass.klass(schema, namespace)
|
65
|
+
if schema_class.nil?
|
66
|
+
schema_backend_class.new(schema: schema, namespace: namespace)
|
67
|
+
else
|
68
|
+
schema_instance = schema_class.new
|
69
|
+
schema_backend_class.new(schema: schema_instance.schema, namespace: schema_instance.namespace)
|
70
|
+
end
|
71
|
+
else
|
72
|
+
schema_backend_class.new(schema: schema, namespace: namespace)
|
73
|
+
end
|
61
74
|
end
|
62
75
|
|
63
76
|
# @param schema [String]
|