deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -4
- data/CHANGELOG.md +50 -0
- data/Gemfile.lock +109 -75
- data/README.md +147 -16
- data/deimos-ruby.gemspec +4 -2
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos.rb +8 -7
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/batch_consumer.rb +2 -142
- data/lib/deimos/config/configuration.rb +8 -10
- data/lib/deimos/consume/batch_consumption.rb +150 -0
- data/lib/deimos/consume/message_consumption.rb +94 -0
- data/lib/deimos/consumer.rb +79 -72
- data/lib/deimos/instrumentation.rb +10 -5
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_topic_info.rb +21 -2
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/schema_backends/avro_base.rb +33 -1
- data/lib/deimos/schema_backends/avro_schema_coercer.rb +30 -11
- data/lib/deimos/schema_backends/base.rb +21 -2
- data/lib/deimos/utils/db_poller.rb +6 -6
- data/lib/deimos/utils/db_producer.rb +57 -15
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/utils/schema_controller_mixin.rb +111 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
- data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
- data/lib/generators/deimos/active_record_generator.rb +79 -0
- data/lib/generators/deimos/db_backend/templates/migration +1 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +1 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +3 -11
- data/spec/batch_consumer_spec.rb +24 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +6 -6
- data/spec/deimos_spec.rb +57 -49
- data/spec/generators/active_record_generator_spec.rb +56 -0
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/kafka_listener_spec.rb +54 -0
- data/spec/kafka_topic_info_spec.rb +39 -16
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +34 -0
- data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
- data/spec/schemas/com/my-namespace/MyNestedSchema.avsc +55 -0
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/schemas/com/my-namespace/request/Index.avsc +11 -0
- data/spec/schemas/com/my-namespace/request/UpdateRequest.avsc +11 -0
- data/spec/schemas/com/my-namespace/response/Index.avsc +11 -0
- data/spec/schemas/com/my-namespace/response/UpdateResponse.avsc +11 -0
- data/spec/spec_helper.rb +24 -0
- data/spec/utils/db_poller_spec.rb +2 -2
- data/spec/utils/db_producer_spec.rb +84 -10
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- data/spec/utils/schema_controller_mixin_spec.rb +68 -0
- metadata +87 -30
- data/lib/deimos/base_consumer.rb +0 -100
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Deimos
|
4
|
+
module ActiveRecordConsume
|
5
|
+
# Convert a message with a schema to an ActiveRecord model
|
6
|
+
class SchemaModelConverter
|
7
|
+
# Create new converter
|
8
|
+
# @param decoder [SchemaBackends::Base] Incoming message schema.
|
9
|
+
# @param klass [ActiveRecord::Base] Model to map to.
|
10
|
+
def initialize(decoder, klass)
|
11
|
+
@decoder = decoder
|
12
|
+
@klass = klass
|
13
|
+
end
|
14
|
+
|
15
|
+
# Convert a message from a decoded hash to a set of ActiveRecord
|
16
|
+
# attributes. Attributes that don't exist in the model will be ignored.
|
17
|
+
# @param payload [Hash] Decoded message payload.
|
18
|
+
# @return [Hash] Model attributes.
|
19
|
+
def convert(payload)
|
20
|
+
attributes = {}
|
21
|
+
@decoder.schema_fields.each do |field|
|
22
|
+
column = @klass.columns.find { |c| c.name == field.name }
|
23
|
+
next if column.nil?
|
24
|
+
next if %w(updated_at created_at).include?(field.name)
|
25
|
+
|
26
|
+
attributes[field.name] = _coerce_field(column, payload[field.name])
|
27
|
+
end
|
28
|
+
attributes
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
# @param column [ActiveRecord::ConnectionAdapters::Column]
|
34
|
+
# @param val [Object]
|
35
|
+
def _coerce_field(column, val)
|
36
|
+
return nil if val.nil?
|
37
|
+
|
38
|
+
if column.type == :datetime
|
39
|
+
int_val = begin
|
40
|
+
val.is_a?(Integer) ? val : (val.is_a?(String) && Integer(val))
|
41
|
+
rescue StandardError
|
42
|
+
nil
|
43
|
+
end
|
44
|
+
|
45
|
+
return Time.zone.at(int_val) if int_val
|
46
|
+
end
|
47
|
+
|
48
|
+
val
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -1,101 +1,59 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'deimos/active_record_consume/batch_consumption'
|
4
|
+
require 'deimos/active_record_consume/message_consumption'
|
5
|
+
require 'deimos/active_record_consume/schema_model_converter'
|
3
6
|
require 'deimos/consumer'
|
4
7
|
|
5
8
|
module Deimos
|
6
|
-
#
|
9
|
+
# Basic ActiveRecord consumer class. Consumes messages and upserts them to
|
10
|
+
# the database. For tombstones (null payloads), deletes corresponding
|
11
|
+
# records from the database. Can operate in either message-by-message mode
|
12
|
+
# or in batch mode.
|
13
|
+
#
|
14
|
+
# In batch mode, ActiveRecord callbacks will be skipped and messages will
|
15
|
+
# be batched to minimize database calls.
|
16
|
+
|
17
|
+
# To configure batch vs. message mode, change the delivery mode of your
|
18
|
+
# Phobos listener.
|
19
|
+
# Message-by-message -> use `delivery: message` or `delivery: batch`
|
20
|
+
# Batch -> use `delivery: inline_batch`
|
7
21
|
class ActiveRecordConsumer < Consumer
|
22
|
+
include ActiveRecordConsume::MessageConsumption
|
23
|
+
include ActiveRecordConsume::BatchConsumption
|
24
|
+
|
8
25
|
class << self
|
9
26
|
# param klass [Class < ActiveRecord::Base] the class used to save to the
|
10
27
|
# database.
|
11
28
|
def record_class(klass)
|
12
29
|
config[:record_class] = klass
|
13
30
|
end
|
14
|
-
end
|
15
31
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
# @param key [Object]
|
22
|
-
# @return [ActiveRecord::Base]
|
23
|
-
def fetch_record(klass, _payload, key)
|
24
|
-
klass.unscoped.where(klass.primary_key => key).first
|
32
|
+
# param val [Boolean] Turn pre-compaction of the batch on or off. If true,
|
33
|
+
# only the last message for each unique key in a batch is processed.
|
34
|
+
def compacted(val)
|
35
|
+
config[:compacted] = val
|
36
|
+
end
|
25
37
|
end
|
26
38
|
|
27
|
-
#
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
def assign_key(record, _payload, key)
|
32
|
-
record[record.class.primary_key] = key
|
33
|
-
end
|
39
|
+
# Setup
|
40
|
+
def initialize
|
41
|
+
@klass = self.class.config[:record_class]
|
42
|
+
@converter = ActiveRecordConsume::SchemaModelConverter.new(self.class.decoder, @klass)
|
34
43
|
|
35
|
-
|
36
|
-
|
37
|
-
key = metadata.with_indifferent_access[:key]
|
38
|
-
klass = self.class.config[:record_class]
|
39
|
-
record = fetch_record(klass, (payload || {}).with_indifferent_access, key)
|
40
|
-
if payload.nil?
|
41
|
-
destroy_record(record)
|
42
|
-
return
|
44
|
+
if self.class.config[:key_schema]
|
45
|
+
@key_converter = ActiveRecordConsume::SchemaModelConverter.new(self.class.key_decoder, @klass)
|
43
46
|
end
|
44
|
-
if record.blank?
|
45
|
-
record = klass.new
|
46
|
-
assign_key(record, payload, key)
|
47
|
-
end
|
48
|
-
attrs = record_attributes(payload.with_indifferent_access)
|
49
|
-
# don't use attributes= - bypass Rails < 5 attr_protected
|
50
|
-
attrs.each do |k, v|
|
51
|
-
record.send("#{k}=", v)
|
52
|
-
end
|
53
|
-
record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
|
54
|
-
record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
|
55
|
-
record.save!
|
56
|
-
end
|
57
47
|
|
58
|
-
|
59
|
-
# to do something other than a straight destroy (e.g. mark as archived).
|
60
|
-
# @param record [ActiveRecord::Base]
|
61
|
-
def destroy_record(record)
|
62
|
-
record&.destroy
|
48
|
+
@compacted = self.class.config[:compacted] != false
|
63
49
|
end
|
64
50
|
|
65
51
|
# Override this method (with `super`) if you want to add/change the default
|
66
52
|
# attributes set to the new/existing record.
|
67
53
|
# @param payload [Hash]
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
self.class.decoder.schema_fields.each do |field|
|
72
|
-
column = klass.columns.find { |c| c.name == field.name }
|
73
|
-
next if column.nil?
|
74
|
-
next if %w(updated_at created_at).include?(field.name)
|
75
|
-
|
76
|
-
attributes[field.name] = _coerce_field(column, payload[field.name])
|
77
|
-
end
|
78
|
-
attributes
|
79
|
-
end
|
80
|
-
|
81
|
-
private
|
82
|
-
|
83
|
-
# @param column [ActiveRecord::ConnectionAdapters::Column]
|
84
|
-
# @param val [Object]
|
85
|
-
def _coerce_field(column, val)
|
86
|
-
return nil if val.nil?
|
87
|
-
|
88
|
-
if column.type == :datetime
|
89
|
-
int_val = begin
|
90
|
-
val.is_a?(Integer) ? val : (val.is_a?(String) && Integer(val))
|
91
|
-
rescue StandardError
|
92
|
-
nil
|
93
|
-
end
|
94
|
-
|
95
|
-
return Time.zone.at(int_val) if int_val
|
96
|
-
end
|
97
|
-
|
98
|
-
val
|
54
|
+
# @param _key [String]
|
55
|
+
def record_attributes(payload, _key=nil)
|
56
|
+
@converter.convert(payload)
|
99
57
|
end
|
100
58
|
end
|
101
59
|
end
|
@@ -1,147 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'deimos/base_consumer'
|
4
|
-
require 'phobos/batch_handler'
|
5
|
-
|
6
3
|
module Deimos
|
7
|
-
#
|
8
|
-
|
9
|
-
# for every incoming batch of messages. This class should be lightweight.
|
10
|
-
class BatchConsumer < BaseConsumer
|
11
|
-
include Phobos::BatchHandler
|
12
|
-
|
13
|
-
# :nodoc:
|
14
|
-
def around_consume_batch(batch, metadata)
|
15
|
-
payloads = []
|
16
|
-
benchmark = Benchmark.measure do
|
17
|
-
if self.class.config[:key_configured]
|
18
|
-
metadata[:keys] = batch.map do |message|
|
19
|
-
decode_key(message.key)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
payloads = batch.map do |message|
|
24
|
-
message.payload ? self.class.decoder.decode(message.payload) : nil
|
25
|
-
end
|
26
|
-
_received_batch(payloads, metadata)
|
27
|
-
_with_span do
|
28
|
-
yield payloads, metadata
|
29
|
-
end
|
30
|
-
end
|
31
|
-
_handle_success(benchmark.real, payloads, metadata)
|
32
|
-
rescue StandardError => e
|
33
|
-
_handle_error(e, payloads, metadata)
|
34
|
-
end
|
35
|
-
|
36
|
-
# Consume a batch of incoming messages.
|
37
|
-
# @param _payloads [Array<Phobos::BatchMessage>]
|
38
|
-
# @param _metadata [Hash]
|
39
|
-
def consume_batch(_payloads, _metadata)
|
40
|
-
raise NotImplementedError
|
41
|
-
end
|
42
|
-
|
43
|
-
protected
|
44
|
-
|
45
|
-
def _received_batch(payloads, metadata)
|
46
|
-
Deimos.config.logger.info(
|
47
|
-
message: 'Got Kafka batch event',
|
48
|
-
message_ids: _payload_identifiers(payloads, metadata),
|
49
|
-
metadata: metadata.except(:keys)
|
50
|
-
)
|
51
|
-
Deimos.config.logger.debug(
|
52
|
-
message: 'Kafka batch event payloads',
|
53
|
-
payloads: payloads
|
54
|
-
)
|
55
|
-
Deimos.config.metrics&.increment(
|
56
|
-
'handler',
|
57
|
-
tags: %W(
|
58
|
-
status:batch_received
|
59
|
-
topic:#{metadata[:topic]}
|
60
|
-
))
|
61
|
-
Deimos.config.metrics&.increment(
|
62
|
-
'handler',
|
63
|
-
by: metadata['batch_size'],
|
64
|
-
tags: %W(
|
65
|
-
status:received
|
66
|
-
topic:#{metadata[:topic]}
|
67
|
-
))
|
68
|
-
if payloads.present?
|
69
|
-
payloads.each { |payload| _report_time_delayed(payload, metadata) }
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
# @param exception [Throwable]
|
74
|
-
# @param payloads [Array<Hash>]
|
75
|
-
# @param metadata [Hash]
|
76
|
-
def _handle_error(exception, payloads, metadata)
|
77
|
-
Deimos.config.metrics&.increment(
|
78
|
-
'handler',
|
79
|
-
tags: %W(
|
80
|
-
status:batch_error
|
81
|
-
topic:#{metadata[:topic]}
|
82
|
-
))
|
83
|
-
Deimos.config.logger.warn(
|
84
|
-
message: 'Error consuming message batch',
|
85
|
-
handler: self.class.name,
|
86
|
-
metadata: metadata.except(:keys),
|
87
|
-
message_ids: _payload_identifiers(payloads, metadata),
|
88
|
-
error_message: exception.message,
|
89
|
-
error: exception.backtrace
|
90
|
-
)
|
91
|
-
super
|
92
|
-
end
|
93
|
-
|
94
|
-
# @param time_taken [Float]
|
95
|
-
# @param payloads [Array<Hash>]
|
96
|
-
# @param metadata [Hash]
|
97
|
-
def _handle_success(time_taken, payloads, metadata)
|
98
|
-
Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
|
99
|
-
time:consume_batch
|
100
|
-
topic:#{metadata[:topic]}
|
101
|
-
))
|
102
|
-
Deimos.config.metrics&.increment(
|
103
|
-
'handler',
|
104
|
-
tags: %W(
|
105
|
-
status:batch_success
|
106
|
-
topic:#{metadata[:topic]}
|
107
|
-
))
|
108
|
-
Deimos.config.metrics&.increment(
|
109
|
-
'handler',
|
110
|
-
by: metadata['batch_size'],
|
111
|
-
tags: %W(
|
112
|
-
status:success
|
113
|
-
topic:#{metadata[:topic]}
|
114
|
-
))
|
115
|
-
Deimos.config.logger.info(
|
116
|
-
message: 'Finished processing Kafka batch event',
|
117
|
-
message_ids: _payload_identifiers(payloads, metadata),
|
118
|
-
time_elapsed: time_taken,
|
119
|
-
metadata: metadata.except(:keys)
|
120
|
-
)
|
121
|
-
end
|
122
|
-
|
123
|
-
# Get payload identifiers (key and message_id if present) for logging.
|
124
|
-
# @param payloads [Array<Hash>]
|
125
|
-
# @param metadata [Hash]
|
126
|
-
# @return [Hash] the identifiers.
|
127
|
-
def _payload_identifiers(payloads, metadata)
|
128
|
-
message_ids = payloads&.map do |payload|
|
129
|
-
if payload.is_a?(Hash) && payload.key?('message_id')
|
130
|
-
payload['message_id']
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
# Payloads may be nil if preprocessing failed
|
135
|
-
messages = payloads || metadata[:keys] || []
|
136
|
-
|
137
|
-
messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
|
138
|
-
ids = {}
|
139
|
-
|
140
|
-
ids[:key] = k if k.present?
|
141
|
-
ids[:message_id] = m_id if m_id.present?
|
142
|
-
|
143
|
-
ids
|
144
|
-
end
|
145
|
-
end
|
4
|
+
# @deprecated Use Deimos::Consumer with `delivery: inline_batch` configured instead
|
5
|
+
class BatchConsumer < Consumer
|
146
6
|
end
|
147
7
|
end
|
@@ -47,17 +47,15 @@ module Deimos
|
|
47
47
|
handler_class = listener.handler.constantize
|
48
48
|
delivery = listener.delivery
|
49
49
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
elsif handler_class < Deimos::Consumer
|
57
|
-
if delivery.present? && !%w(message batch).include?(delivery)
|
58
|
-
raise "Non-batch Consumer #{listener.handler} must have delivery"\
|
59
|
-
' set to `message` or `batch`'
|
50
|
+
next unless handler_class < Deimos::Consumer
|
51
|
+
|
52
|
+
# Validate that each consumer implements the correct method for its type
|
53
|
+
if delivery == 'inline_batch'
|
54
|
+
if handler_class.instance_method(:consume_batch).owner == Deimos::Consume::BatchConsumption
|
55
|
+
raise "BatchConsumer #{listener.handler} does not implement `consume_batch`"
|
60
56
|
end
|
57
|
+
elsif handler_class.instance_method(:consume).owner == Deimos::Consume::MessageConsumption
|
58
|
+
raise "Non-batch Consumer #{listener.handler} does not implement `consume`"
|
61
59
|
end
|
62
60
|
end
|
63
61
|
end
|
@@ -0,0 +1,150 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Deimos
|
4
|
+
module Consume
|
5
|
+
# Helper methods used by batch consumers, i.e. those with "inline_batch"
|
6
|
+
# delivery. Payloads are decoded then consumers are invoked with arrays
|
7
|
+
# of messages to be handled at once
|
8
|
+
module BatchConsumption
|
9
|
+
extend ActiveSupport::Concern
|
10
|
+
include Phobos::BatchHandler
|
11
|
+
|
12
|
+
# :nodoc:
|
13
|
+
def around_consume_batch(batch, metadata)
|
14
|
+
payloads = []
|
15
|
+
benchmark = Benchmark.measure do
|
16
|
+
if self.class.config[:key_configured]
|
17
|
+
metadata[:keys] = batch.map do |message|
|
18
|
+
decode_key(message.key)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
metadata[:first_offset] = batch.first&.offset
|
22
|
+
|
23
|
+
payloads = batch.map do |message|
|
24
|
+
message.payload ? self.class.decoder.decode(message.payload) : nil
|
25
|
+
end
|
26
|
+
_received_batch(payloads, metadata)
|
27
|
+
_with_span do
|
28
|
+
yield payloads, metadata
|
29
|
+
end
|
30
|
+
end
|
31
|
+
_handle_batch_success(benchmark.real, payloads, metadata)
|
32
|
+
rescue StandardError => e
|
33
|
+
_handle_batch_error(e, payloads, metadata)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Consume a batch of incoming messages.
|
37
|
+
# @param _payloads [Array<Phobos::BatchMessage>]
|
38
|
+
# @param _metadata [Hash]
|
39
|
+
def consume_batch(_payloads, _metadata)
|
40
|
+
raise NotImplementedError
|
41
|
+
end
|
42
|
+
|
43
|
+
protected
|
44
|
+
|
45
|
+
def _received_batch(payloads, metadata)
|
46
|
+
Deimos.config.logger.info(
|
47
|
+
message: 'Got Kafka batch event',
|
48
|
+
message_ids: _payload_identifiers(payloads, metadata),
|
49
|
+
metadata: metadata.except(:keys)
|
50
|
+
)
|
51
|
+
Deimos.config.logger.debug(
|
52
|
+
message: 'Kafka batch event payloads',
|
53
|
+
payloads: payloads
|
54
|
+
)
|
55
|
+
Deimos.config.metrics&.increment(
|
56
|
+
'handler',
|
57
|
+
tags: %W(
|
58
|
+
status:batch_received
|
59
|
+
topic:#{metadata[:topic]}
|
60
|
+
))
|
61
|
+
Deimos.config.metrics&.increment(
|
62
|
+
'handler',
|
63
|
+
by: metadata['batch_size'],
|
64
|
+
tags: %W(
|
65
|
+
status:received
|
66
|
+
topic:#{metadata[:topic]}
|
67
|
+
))
|
68
|
+
if payloads.present?
|
69
|
+
payloads.each { |payload| _report_time_delayed(payload, metadata) }
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# @param exception [Throwable]
|
74
|
+
# @param payloads [Array<Hash>]
|
75
|
+
# @param metadata [Hash]
|
76
|
+
def _handle_batch_error(exception, payloads, metadata)
|
77
|
+
Deimos.config.metrics&.increment(
|
78
|
+
'handler',
|
79
|
+
tags: %W(
|
80
|
+
status:batch_error
|
81
|
+
topic:#{metadata[:topic]}
|
82
|
+
))
|
83
|
+
Deimos.config.logger.warn(
|
84
|
+
message: 'Error consuming message batch',
|
85
|
+
handler: self.class.name,
|
86
|
+
metadata: metadata.except(:keys),
|
87
|
+
message_ids: _payload_identifiers(payloads, metadata),
|
88
|
+
error_message: exception.message,
|
89
|
+
error: exception.backtrace
|
90
|
+
)
|
91
|
+
_error(exception, payloads, metadata)
|
92
|
+
end
|
93
|
+
|
94
|
+
# @param time_taken [Float]
|
95
|
+
# @param payloads [Array<Hash>]
|
96
|
+
# @param metadata [Hash]
|
97
|
+
def _handle_batch_success(time_taken, payloads, metadata)
|
98
|
+
Deimos.config.metrics&.histogram('handler',
|
99
|
+
time_taken,
|
100
|
+
tags: %W(
|
101
|
+
time:consume_batch
|
102
|
+
topic:#{metadata[:topic]}
|
103
|
+
))
|
104
|
+
Deimos.config.metrics&.increment(
|
105
|
+
'handler',
|
106
|
+
tags: %W(
|
107
|
+
status:batch_success
|
108
|
+
topic:#{metadata[:topic]}
|
109
|
+
))
|
110
|
+
Deimos.config.metrics&.increment(
|
111
|
+
'handler',
|
112
|
+
by: metadata['batch_size'],
|
113
|
+
tags: %W(
|
114
|
+
status:success
|
115
|
+
topic:#{metadata[:topic]}
|
116
|
+
))
|
117
|
+
Deimos.config.logger.info(
|
118
|
+
message: 'Finished processing Kafka batch event',
|
119
|
+
message_ids: _payload_identifiers(payloads, metadata),
|
120
|
+
time_elapsed: time_taken,
|
121
|
+
metadata: metadata.except(:keys)
|
122
|
+
)
|
123
|
+
end
|
124
|
+
|
125
|
+
# Get payload identifiers (key and message_id if present) for logging.
|
126
|
+
# @param payloads [Array<Hash>]
|
127
|
+
# @param metadata [Hash]
|
128
|
+
# @return [Array<Array>] the identifiers.
|
129
|
+
def _payload_identifiers(payloads, metadata)
|
130
|
+
message_ids = payloads&.map do |payload|
|
131
|
+
if payload.is_a?(Hash) && payload.key?('message_id')
|
132
|
+
payload['message_id']
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
# Payloads may be nil if preprocessing failed
|
137
|
+
messages = payloads || metadata[:keys] || []
|
138
|
+
|
139
|
+
messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
|
140
|
+
ids = {}
|
141
|
+
|
142
|
+
ids[:key] = k if k.present?
|
143
|
+
ids[:message_id] = m_id if m_id.present?
|
144
|
+
|
145
|
+
ids
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|