deimos-ruby 1.0.0.pre.beta22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +74 -0
- data/.gitignore +41 -0
- data/.gitmodules +0 -0
- data/.rspec +1 -0
- data/.rubocop.yml +321 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +32 -0
- data/CODE_OF_CONDUCT.md +77 -0
- data/Dockerfile +23 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +165 -0
- data/Guardfile +22 -0
- data/LICENSE.md +195 -0
- data/README.md +752 -0
- data/Rakefile +13 -0
- data/bin/deimos +4 -0
- data/deimos-kafka.gemspec +42 -0
- data/docker-compose.yml +71 -0
- data/docs/DATABASE_BACKEND.md +147 -0
- data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
- data/lib/deimos/active_record_consumer.rb +81 -0
- data/lib/deimos/active_record_producer.rb +64 -0
- data/lib/deimos/avro_data_coder.rb +89 -0
- data/lib/deimos/avro_data_decoder.rb +36 -0
- data/lib/deimos/avro_data_encoder.rb +51 -0
- data/lib/deimos/backends/db.rb +27 -0
- data/lib/deimos/backends/kafka.rb +27 -0
- data/lib/deimos/backends/kafka_async.rb +27 -0
- data/lib/deimos/configuration.rb +90 -0
- data/lib/deimos/consumer.rb +164 -0
- data/lib/deimos/instrumentation.rb +71 -0
- data/lib/deimos/kafka_message.rb +27 -0
- data/lib/deimos/kafka_source.rb +126 -0
- data/lib/deimos/kafka_topic_info.rb +86 -0
- data/lib/deimos/message.rb +74 -0
- data/lib/deimos/metrics/datadog.rb +47 -0
- data/lib/deimos/metrics/mock.rb +39 -0
- data/lib/deimos/metrics/provider.rb +38 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
- data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
- data/lib/deimos/monkey_patches/schema_store.rb +19 -0
- data/lib/deimos/producer.rb +218 -0
- data/lib/deimos/publish_backend.rb +30 -0
- data/lib/deimos/railtie.rb +8 -0
- data/lib/deimos/schema_coercer.rb +108 -0
- data/lib/deimos/shared_config.rb +59 -0
- data/lib/deimos/test_helpers.rb +356 -0
- data/lib/deimos/tracing/datadog.rb +35 -0
- data/lib/deimos/tracing/mock.rb +40 -0
- data/lib/deimos/tracing/provider.rb +31 -0
- data/lib/deimos/utils/db_producer.rb +122 -0
- data/lib/deimos/utils/executor.rb +117 -0
- data/lib/deimos/utils/inline_consumer.rb +144 -0
- data/lib/deimos/utils/lag_reporter.rb +182 -0
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +68 -0
- data/lib/deimos/version.rb +5 -0
- data/lib/deimos.rb +133 -0
- data/lib/generators/deimos/db_backend/templates/migration +24 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
- data/lib/generators/deimos/db_backend_generator.rb +48 -0
- data/lib/tasks/deimos.rake +27 -0
- data/spec/active_record_consumer_spec.rb +81 -0
- data/spec/active_record_producer_spec.rb +107 -0
- data/spec/avro_data_decoder_spec.rb +18 -0
- data/spec/avro_data_encoder_spec.rb +37 -0
- data/spec/backends/db_spec.rb +35 -0
- data/spec/backends/kafka_async_spec.rb +11 -0
- data/spec/backends/kafka_spec.rb +11 -0
- data/spec/consumer_spec.rb +169 -0
- data/spec/deimos_spec.rb +120 -0
- data/spec/kafka_source_spec.rb +168 -0
- data/spec/kafka_topic_info_spec.rb +88 -0
- data/spec/phobos.bad_db.yml +73 -0
- data/spec/phobos.yml +73 -0
- data/spec/producer_spec.rb +397 -0
- data/spec/publish_backend_spec.rb +10 -0
- data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
- data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
- data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
- data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
- data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
- data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
- data/spec/spec_helper.rb +207 -0
- data/spec/updateable_schema_store_spec.rb +36 -0
- data/spec/utils/db_producer_spec.rb +259 -0
- data/spec/utils/executor_spec.rb +42 -0
- data/spec/utils/lag_reporter_spec.rb +69 -0
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/signal_handler_spec.rb +16 -0
- data/support/deimos-solo.png +0 -0
- data/support/deimos-with-name-next.png +0 -0
- data/support/deimos-with-name.png +0 -0
- data/support/flipp-logo.png +0 -0
- metadata +452 -0
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/concern'
|
4
|
+
|
5
|
+
module Deimos
|
6
|
+
# Module that producers and consumers can share which sets up configuration.
|
7
|
+
module SharedConfig
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
# need to use this instead of class_methods to be backwards-compatible
|
11
|
+
# with Rails 3
|
12
|
+
module ClassMethods
|
13
|
+
# @return [Hash]
|
14
|
+
def config
|
15
|
+
return @config if @config
|
16
|
+
|
17
|
+
@config = {
|
18
|
+
encode_key: true
|
19
|
+
}
|
20
|
+
klass = self.superclass
|
21
|
+
while klass.respond_to?(:config)
|
22
|
+
klass_config = klass.config
|
23
|
+
if klass_config
|
24
|
+
# default is true for this so don't include it in the merge
|
25
|
+
klass_config.delete(:encode_key) if klass_config[:encode_key]
|
26
|
+
@config.merge!(klass_config) if klass.config
|
27
|
+
end
|
28
|
+
klass = klass.superclass
|
29
|
+
end
|
30
|
+
@config
|
31
|
+
end
|
32
|
+
|
33
|
+
# Set the schema.
|
34
|
+
# @param schema [String]
|
35
|
+
def schema(schema)
|
36
|
+
config[:schema] = schema
|
37
|
+
end
|
38
|
+
|
39
|
+
# Set the namespace.
|
40
|
+
# @param namespace [String]
|
41
|
+
def namespace(namespace)
|
42
|
+
config[:namespace] = namespace
|
43
|
+
end
|
44
|
+
|
45
|
+
# Set key configuration.
|
46
|
+
# @param field [Symbol] the name of a field to use in the value schema as
|
47
|
+
# a generated key schema
|
48
|
+
# @param schema [String|Symbol] the name of a schema to use for the key
|
49
|
+
# @param plain [Boolean] if true, do not encode keys at all
|
50
|
+
# @param none [Boolean] if true, do not use keys at all
|
51
|
+
def key_config(plain: nil, field: nil, schema: nil, none: nil)
|
52
|
+
config[:no_keys] = none
|
53
|
+
config[:encode_key] = !plain && !none
|
54
|
+
config[:key_field] = field&.to_s
|
55
|
+
config[:key_schema] = schema
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,356 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/concern'
|
4
|
+
require 'active_support/core_ext'
|
5
|
+
require 'avro_turf'
|
6
|
+
require 'deimos/tracing/mock'
|
7
|
+
require 'deimos/metrics/mock'
|
8
|
+
|
9
|
+
module Deimos
|
10
|
+
# Include this module in your RSpec spec_helper
|
11
|
+
# to stub out external dependencies
|
12
|
+
# and add methods to use to test encoding/decoding.
|
13
|
+
module TestHelpers
|
14
|
+
extend ActiveSupport::Concern
|
15
|
+
|
16
|
+
class << self
|
17
|
+
# @return [Array<Hash>]
|
18
|
+
def sent_messages
|
19
|
+
@sent_messages ||= []
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
included do
|
24
|
+
# @param encoder_schema [String]
|
25
|
+
# @param namespace [String]
|
26
|
+
# @return [Deimos::AvroDataEncoder]
|
27
|
+
def create_encoder(encoder_schema, namespace)
|
28
|
+
encoder = Deimos::AvroDataEncoder.new(schema: encoder_schema,
|
29
|
+
namespace: namespace)
|
30
|
+
|
31
|
+
# we added and_wrap_original to RSpec 2 but the regular block
|
32
|
+
# syntax wasn't working for some reason - block wasn't being passed
|
33
|
+
# to the method
|
34
|
+
block = proc do |m, *args|
|
35
|
+
m.call(*args)
|
36
|
+
args[0]
|
37
|
+
end
|
38
|
+
allow(encoder).to receive(:encode_local).and_wrap_original(&block)
|
39
|
+
allow(encoder).to receive(:encode) do |payload, schema: nil, topic: nil|
|
40
|
+
encoder.encode_local(payload, schema: schema)
|
41
|
+
end
|
42
|
+
|
43
|
+
block = proc do |m, *args|
|
44
|
+
m.call(*args)&.values&.first
|
45
|
+
end
|
46
|
+
allow(encoder).to receive(:encode_key).and_wrap_original(&block)
|
47
|
+
encoder
|
48
|
+
end
|
49
|
+
|
50
|
+
# @param decoder_schema [String]
|
51
|
+
# @param namespace [String]
|
52
|
+
# @return [Deimos::AvroDataDecoder]
|
53
|
+
def create_decoder(decoder_schema, namespace)
|
54
|
+
decoder = Deimos::AvroDataDecoder.new(schema: decoder_schema,
|
55
|
+
namespace: namespace)
|
56
|
+
allow(decoder).to receive(:decode_local) { |payload| payload }
|
57
|
+
allow(decoder).to receive(:decode) do |payload, schema: nil|
|
58
|
+
schema ||= decoder.schema
|
59
|
+
if schema && decoder.namespace
|
60
|
+
# Validate against local schema.
|
61
|
+
encoder = Deimos::AvroDataEncoder.new(schema: schema,
|
62
|
+
namespace: decoder.namespace)
|
63
|
+
encoder.schema_store = decoder.schema_store
|
64
|
+
encoder.encode_local(payload)
|
65
|
+
end
|
66
|
+
payload
|
67
|
+
end
|
68
|
+
allow(decoder).to receive(:decode_key) do |payload, _key_id|
|
69
|
+
payload.values.first
|
70
|
+
end
|
71
|
+
decoder
|
72
|
+
end
|
73
|
+
|
74
|
+
RSpec.configure do |config|
|
75
|
+
|
76
|
+
config.before(:suite) do
|
77
|
+
Deimos.configure do |fr_config|
|
78
|
+
fr_config.logger = Logger.new(STDOUT)
|
79
|
+
fr_config.seed_broker ||= 'test_broker'
|
80
|
+
fr_config.tracer = Deimos::Tracing::Mock.new
|
81
|
+
fr_config.metrics = Deimos::Metrics::Mock.new
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
before(:each) do
|
87
|
+
client = double('client').as_null_object
|
88
|
+
allow(client).to receive(:time) do |*_args, &block|
|
89
|
+
block.call
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# Stub all already-loaded producers and consumers fir unit testing purposes.
|
95
|
+
def stub_producers_and_consumers!
|
96
|
+
Deimos::TestHelpers.sent_messages.clear
|
97
|
+
|
98
|
+
allow(Deimos::Producer).to receive(:produce_batch) do |_, batch|
|
99
|
+
Deimos::TestHelpers.sent_messages.concat(batch.map(&:to_h))
|
100
|
+
end
|
101
|
+
|
102
|
+
Deimos::Producer.descendants.each do |klass|
|
103
|
+
next if klass == Deimos::ActiveRecordProducer # "abstract" class
|
104
|
+
|
105
|
+
stub_producer(klass)
|
106
|
+
end
|
107
|
+
|
108
|
+
Deimos::Consumer.descendants.each do |klass|
|
109
|
+
next if klass == Deimos::ActiveRecordConsumer # "abstract" class
|
110
|
+
|
111
|
+
stub_consumer(klass)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
# Stub a given producer class.
|
116
|
+
# @param klass [Class < Deimos::Producer]
|
117
|
+
def stub_producer(klass)
|
118
|
+
allow(klass).to receive(:encoder) do
|
119
|
+
create_encoder(klass.config[:schema], klass.config[:namespace])
|
120
|
+
end
|
121
|
+
allow(klass).to receive(:key_encoder) do
|
122
|
+
create_encoder(klass.config[:key_schema], klass.config[:namespace])
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
# Stub a given consumer class.
|
127
|
+
# @param klass [Class < Deimos::Consumer]
|
128
|
+
def stub_consumer(klass)
|
129
|
+
allow(klass).to receive(:decoder) do
|
130
|
+
create_decoder(klass.config[:schema], klass.config[:namespace])
|
131
|
+
end
|
132
|
+
klass.class_eval do
|
133
|
+
alias_method(:old_consume, :consume) unless self.instance_methods.include?(:old_consume)
|
134
|
+
end
|
135
|
+
allow_any_instance_of(klass).to receive(:consume) do |instance, payload, metadata|
|
136
|
+
metadata[:key] = klass.new.decode_key(metadata[:key])
|
137
|
+
instance.old_consume(payload, metadata)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
# get the difference of 2 hashes.
|
142
|
+
# @param hash1 [Hash]
|
143
|
+
# @param hash2 [Hash]
|
144
|
+
def _hash_diff(hash1, hash2)
|
145
|
+
if hash1.nil? || !hash1.is_a?(Hash)
|
146
|
+
hash2
|
147
|
+
elsif hash2.nil? || !hash2.is_a?(Hash)
|
148
|
+
hash1
|
149
|
+
else
|
150
|
+
hash1.dup.
|
151
|
+
delete_if { |k, v| hash2[k] == v }.
|
152
|
+
merge!(hash2.dup.delete_if { |k, _v| hash1.key?(k) })
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
# :nodoc:
|
157
|
+
def _frk_failure_message(topic, message, key=nil, partition_key=nil, was_negated=false)
|
158
|
+
messages = Deimos::TestHelpers.sent_messages.
|
159
|
+
select { |m| m[:topic] == topic }.
|
160
|
+
map { |m| m.except(:topic) }
|
161
|
+
message_string = ''
|
162
|
+
diff = nil
|
163
|
+
min_hash_diff = nil
|
164
|
+
if messages.any?
|
165
|
+
message_string = messages.map(&:inspect).join("\n")
|
166
|
+
min_hash_diff = messages.min_by { |m| _hash_diff(m, message).keys.size }
|
167
|
+
diff = RSpec::Expectations.differ.
|
168
|
+
diff_as_object(message, min_hash_diff[:payload])
|
169
|
+
end
|
170
|
+
description = if message.respond_to?(:description)
|
171
|
+
message.description
|
172
|
+
elsif message.nil?
|
173
|
+
'nil'
|
174
|
+
else
|
175
|
+
message
|
176
|
+
end
|
177
|
+
str = "Expected #{topic} #{'not ' if was_negated}to have sent #{description}"
|
178
|
+
str += " with key #{key}" if key
|
179
|
+
str += " with partition key #{partition_key}" if partition_key
|
180
|
+
str += "\nClosest message received: #{min_hash_diff}" if min_hash_diff
|
181
|
+
str += "\nDiff: #{diff}" if diff
|
182
|
+
str + "\nAll Messages received:\n#{message_string}"
|
183
|
+
end
|
184
|
+
|
185
|
+
RSpec::Matchers.define :have_sent do |msg, key=nil, partition_key=nil|
|
186
|
+
message = if msg.respond_to?(:with_indifferent_access)
|
187
|
+
msg.with_indifferent_access
|
188
|
+
else
|
189
|
+
msg
|
190
|
+
end
|
191
|
+
match do |topic|
|
192
|
+
Deimos::TestHelpers.sent_messages.any? do |m|
|
193
|
+
hash_matcher = RSpec::Matchers::BuiltIn::Match.new(message)
|
194
|
+
hash_matcher.send(:match,
|
195
|
+
message,
|
196
|
+
m[:payload]&.with_indifferent_access) &&
|
197
|
+
topic == m[:topic] &&
|
198
|
+
(key.present? ? key == m[:key] : true) &&
|
199
|
+
(partition_key.present? ? partition_key == m[:partition_key] : true)
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
if respond_to?(:failure_message)
|
204
|
+
failure_message do |topic|
|
205
|
+
_frk_failure_message(topic, message, key, partition_key)
|
206
|
+
end
|
207
|
+
failure_message_when_negated do |topic|
|
208
|
+
_frk_failure_message(topic, message, key, partition_key, true)
|
209
|
+
end
|
210
|
+
else
|
211
|
+
failure_message_for_should do |topic|
|
212
|
+
_frk_failure_message(topic, message, key, partition_key)
|
213
|
+
end
|
214
|
+
failure_message_for_should_not do |topic|
|
215
|
+
_frk_failure_message(topic, message, key, partition_key, true)
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
# Clear all sent messages - e.g. if we want to check that
|
221
|
+
# particular messages were sent or not sent after a point in time.
|
222
|
+
def clear_kafka_messages!
|
223
|
+
Deimos::TestHelpers.sent_messages.clear
|
224
|
+
end
|
225
|
+
|
226
|
+
# test that a message was sent on the given topic.
|
227
|
+
# DEPRECATED - use the "have_sent" matcher instead.
|
228
|
+
# @param message [Hash]
|
229
|
+
# @param topic [String]
|
230
|
+
# @param key [String|Integer]
|
231
|
+
# @return [Boolean]
|
232
|
+
def was_message_sent?(message, topic, key=nil)
|
233
|
+
Deimos::TestHelpers.sent_messages.any? do |m|
|
234
|
+
message == m[:payload] && m[:topic] == topic &&
|
235
|
+
(key.present? ? m[:key] == key : true)
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
# Test that a given handler will consume a given payload correctly, i.e.
|
240
|
+
# that the Avro schema is correct. If
|
241
|
+
# a block is given, that block will be executed when `consume` is called.
|
242
|
+
# Otherwise it will just confirm that `consume` is called at all.
|
243
|
+
# @param handler_class_or_topic [Class|String] Class which inherits from
|
244
|
+
# Deimos::Consumer or the topic as a string
|
245
|
+
# @param payload [Hash] the payload to consume
|
246
|
+
# @param call_original [Boolean] if true, allow the consume handler
|
247
|
+
# to continue as normal. Not compatible with a block.
|
248
|
+
# @param ignore_expectation [Boolean] Set to true to not place any
|
249
|
+
# expectations on the consumer. Primarily used internally to Deimos.
|
250
|
+
# @param key [Object] the key to use.
|
251
|
+
# @param partition_key [Object] the partition key to use.
|
252
|
+
def test_consume_message(handler_class_or_topic,
|
253
|
+
payload,
|
254
|
+
call_original: false,
|
255
|
+
key: nil,
|
256
|
+
partition_key: nil,
|
257
|
+
skip_expectation: false,
|
258
|
+
&block)
|
259
|
+
raise 'Cannot have both call_original and be given a block!' if call_original && block_given?
|
260
|
+
|
261
|
+
payload&.stringify_keys!
|
262
|
+
handler_class = if handler_class_or_topic.is_a?(String)
|
263
|
+
_get_handler_class_from_topic(handler_class_or_topic)
|
264
|
+
else
|
265
|
+
handler_class_or_topic
|
266
|
+
end
|
267
|
+
handler = handler_class.new
|
268
|
+
allow(handler_class).to receive(:new).and_return(handler)
|
269
|
+
listener = double('listener',
|
270
|
+
handler_class: handler_class,
|
271
|
+
encoding: nil)
|
272
|
+
key ||= _key_from_consumer(handler_class)
|
273
|
+
message = double('message',
|
274
|
+
'key' => key,
|
275
|
+
'partition_key' => partition_key,
|
276
|
+
'partition' => 1,
|
277
|
+
'offset' => 1,
|
278
|
+
'value' => payload)
|
279
|
+
|
280
|
+
unless skip_expectation
|
281
|
+
expectation = expect(handler).to receive(:consume).
|
282
|
+
with(payload, anything, &block)
|
283
|
+
expectation.and_call_original if call_original
|
284
|
+
end
|
285
|
+
Phobos::Actions::ProcessMessage.new(
|
286
|
+
listener: listener,
|
287
|
+
message: message,
|
288
|
+
listener_metadata: { topic: 'my-topic' }
|
289
|
+
).send(:process_message, payload)
|
290
|
+
end
|
291
|
+
|
292
|
+
# Check to see that a given message will fail due to Avro errors.
|
293
|
+
# @param handler_class [Class]
|
294
|
+
# @param payload [Hash]
|
295
|
+
def test_consume_invalid_message(handler_class, payload)
|
296
|
+
handler = handler_class.new
|
297
|
+
allow(handler_class).to receive(:new).and_return(handler)
|
298
|
+
listener = double('listener',
|
299
|
+
handler_class: handler_class,
|
300
|
+
encoding: nil)
|
301
|
+
message = double('message',
|
302
|
+
key: _key_from_consumer(handler_class),
|
303
|
+
partition_key: nil,
|
304
|
+
partition: 1,
|
305
|
+
offset: 1,
|
306
|
+
value: payload)
|
307
|
+
|
308
|
+
expect {
|
309
|
+
Phobos::Actions::ProcessMessage.new(
|
310
|
+
listener: listener,
|
311
|
+
message: message,
|
312
|
+
listener_metadata: { topic: 'my-topic' }
|
313
|
+
).send(:process_message, payload)
|
314
|
+
}.to raise_error(Avro::SchemaValidator::ValidationError)
|
315
|
+
end
|
316
|
+
|
317
|
+
# @param schema1 [String|Hash] a file path, JSON string, or
|
318
|
+
# hash representing a schema.
|
319
|
+
# @param schema2 [String|Hash] a file path, JSON string, or
|
320
|
+
# hash representing a schema.
|
321
|
+
# @return [Boolean] true if the schemas are compatible, false otherwise.
|
322
|
+
def self.schemas_compatible?(schema1, schema2)
|
323
|
+
json1, json2 = [schema1, schema2].map do |schema|
|
324
|
+
if schema.is_a?(String)
|
325
|
+
schema = File.read(schema) unless schema.strip.starts_with?('{') # file path
|
326
|
+
MultiJson.load(schema)
|
327
|
+
else
|
328
|
+
schema
|
329
|
+
end
|
330
|
+
end
|
331
|
+
avro_schema1 = Avro::Schema.real_parse(json1, {})
|
332
|
+
avro_schema2 = Avro::Schema.real_parse(json2, {})
|
333
|
+
Avro::SchemaCompatibility.mutual_read?(avro_schema1, avro_schema2)
|
334
|
+
end
|
335
|
+
|
336
|
+
private
|
337
|
+
|
338
|
+
def _key_from_consumer(consumer)
|
339
|
+
if consumer.config[:key_field] || consumer.config[:key_schema]
|
340
|
+
{ 'test' => 1 }
|
341
|
+
else
|
342
|
+
1
|
343
|
+
end
|
344
|
+
end
|
345
|
+
|
346
|
+
# @param topic [String]
|
347
|
+
# @return [Class]
|
348
|
+
def _get_handler_class_from_topic(topic)
|
349
|
+
listeners = Phobos.config['listeners']
|
350
|
+
handler = listeners.find { |l| l.topic == topic }
|
351
|
+
raise "No consumer found in Phobos configuration for topic #{topic}!" if handler.nil?
|
352
|
+
|
353
|
+
handler.handler.constantize
|
354
|
+
end
|
355
|
+
end
|
356
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'deimos/tracing/provider'
|
4
|
+
|
5
|
+
module Deimos
|
6
|
+
module Tracing
|
7
|
+
# Tracing wrapper class for Datadog.
|
8
|
+
class Datadog < Tracing::Provider
|
9
|
+
# :nodoc:
|
10
|
+
def initialize(config)
|
11
|
+
raise 'Tracing config must specify service_name' if config[:service_name].nil?
|
12
|
+
|
13
|
+
@service = config[:service_name]
|
14
|
+
end
|
15
|
+
|
16
|
+
# :nodoc:
|
17
|
+
def start(span_name, options={})
|
18
|
+
span = ::Datadog.tracer.trace(span_name)
|
19
|
+
span.service = @service
|
20
|
+
span.resource = options[:resource]
|
21
|
+
span
|
22
|
+
end
|
23
|
+
|
24
|
+
# :nodoc:
|
25
|
+
def finish(span)
|
26
|
+
span.finish
|
27
|
+
end
|
28
|
+
|
29
|
+
# :nodoc:
|
30
|
+
def set_error(span, exception)
|
31
|
+
span.set_error(exception)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'deimos/tracing/provider'
|
4
|
+
|
5
|
+
module Deimos
|
6
|
+
module Tracing
|
7
|
+
# Class that mocks out tracing functionality
|
8
|
+
class Mock < Tracing::Provider
|
9
|
+
# :nodoc:
|
10
|
+
def initialize(logger=nil)
|
11
|
+
@logger = logger || Logger.new(STDOUT)
|
12
|
+
@logger.info('MockTracingProvider initialized')
|
13
|
+
end
|
14
|
+
|
15
|
+
# :nodoc:
|
16
|
+
def start(span_name, _options={})
|
17
|
+
@logger.info("Mock span '#{span_name}' started")
|
18
|
+
{
|
19
|
+
name: span_name,
|
20
|
+
started_at: Time.zone.now
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
# :nodoc:
|
25
|
+
def finish(span)
|
26
|
+
name = span[:name]
|
27
|
+
start = span[:started_at]
|
28
|
+
finish = Time.zone.now
|
29
|
+
@logger.info("Mock span '#{name}' finished: #{start} to #{finish}")
|
30
|
+
end
|
31
|
+
|
32
|
+
# :nodoc:
|
33
|
+
def set_error(span, exception)
|
34
|
+
span[:exception] = exception
|
35
|
+
name = span[:name]
|
36
|
+
@logger.info("Mock span '#{name}' set an error: #{exception}")
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# rubocop:disable Lint/UnusedMethodArgument
|
4
|
+
module Deimos
|
5
|
+
module Tracing
|
6
|
+
# Base class for all tracing providers.
|
7
|
+
class Provider
|
8
|
+
# Returns a span object and starts the trace.
|
9
|
+
# @param span_name [String] The name of the span/trace
|
10
|
+
# @param options [Hash] Options for the span
|
11
|
+
# @return [Object] The span object
|
12
|
+
def start(span_name, options={})
|
13
|
+
raise NotImplementedError
|
14
|
+
end
|
15
|
+
|
16
|
+
# Finishes the trace on the span object.
|
17
|
+
# @param span [Object] The span to finish trace on
|
18
|
+
def finish(span)
|
19
|
+
raise NotImplementedError
|
20
|
+
end
|
21
|
+
|
22
|
+
# Set an error on the span.
|
23
|
+
# @param span [Object] The span to set error on
|
24
|
+
# @param exception [Exception] The exception that occurred
|
25
|
+
def set_error(span, exception)
|
26
|
+
raise NotImplementedError
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
# rubocop:enable Lint/UnusedMethodArgument
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Deimos
|
4
|
+
module Utils
|
5
|
+
# Class which continually polls the database and sends Kafka messages.
|
6
|
+
class DbProducer
|
7
|
+
include Phobos::Producer
|
8
|
+
attr_accessor :id, :current_topic
|
9
|
+
|
10
|
+
BATCH_SIZE = 1000
|
11
|
+
|
12
|
+
# @param logger [Logger]
|
13
|
+
def initialize(logger=Logger.new(STDOUT))
|
14
|
+
@id = SecureRandom.uuid
|
15
|
+
@logger = logger
|
16
|
+
@logger.push_tags("DbProducer #{@id}") if @logger.respond_to?(:push_tags)
|
17
|
+
end
|
18
|
+
|
19
|
+
# Start the poll.
|
20
|
+
def start
|
21
|
+
@logger.info('Starting...')
|
22
|
+
@signal_to_stop = false
|
23
|
+
loop do
|
24
|
+
if @signal_to_stop
|
25
|
+
@logger.info('Shutting down')
|
26
|
+
break
|
27
|
+
end
|
28
|
+
send_pending_metrics
|
29
|
+
process_next_messages
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Stop the poll.
|
34
|
+
def stop
|
35
|
+
@logger.info('Received signal to stop')
|
36
|
+
@signal_to_stop = true
|
37
|
+
end
|
38
|
+
|
39
|
+
# Complete one loop of processing all messages in the DB.
|
40
|
+
def process_next_messages
|
41
|
+
topics = retrieve_topics
|
42
|
+
@logger.info("Found topics: #{topics}")
|
43
|
+
topics.each(&method(:process_topic))
|
44
|
+
sleep(0.5)
|
45
|
+
end
|
46
|
+
|
47
|
+
# @return [Array<String>]
|
48
|
+
def retrieve_topics
|
49
|
+
KafkaMessage.select('distinct topic').map(&:topic).uniq
|
50
|
+
end
|
51
|
+
|
52
|
+
# @param topic [String]
|
53
|
+
# @return [String] the topic that was locked, or nil if none were.
|
54
|
+
def process_topic(topic)
|
55
|
+
# If the topic is already locked, another producer is currently
|
56
|
+
# working on it. Move on to the next one.
|
57
|
+
unless KafkaTopicInfo.lock(topic, @id)
|
58
|
+
@logger.debug("Could not lock topic #{topic} - continuing")
|
59
|
+
return
|
60
|
+
end
|
61
|
+
@current_topic = topic
|
62
|
+
messages = retrieve_messages
|
63
|
+
|
64
|
+
while messages.any?
|
65
|
+
produce_messages(messages.map(&:phobos_message))
|
66
|
+
messages.first.class.where(id: messages.map(&:id)).delete_all
|
67
|
+
break if messages.size < BATCH_SIZE
|
68
|
+
|
69
|
+
KafkaTopicInfo.heartbeat(@current_topic, @id) # keep alive
|
70
|
+
send_pending_metrics
|
71
|
+
messages = retrieve_messages
|
72
|
+
end
|
73
|
+
KafkaTopicInfo.clear_lock(@current_topic, @id)
|
74
|
+
rescue StandardError => e
|
75
|
+
@logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
|
76
|
+
KafkaTopicInfo.register_error(@current_topic, @id)
|
77
|
+
end
|
78
|
+
|
79
|
+
# @return [Array<Deimos::KafkaMessage>]
|
80
|
+
def retrieve_messages
|
81
|
+
KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
|
82
|
+
end
|
83
|
+
|
84
|
+
# Send metrics to Datadog.
|
85
|
+
def send_pending_metrics
|
86
|
+
first_message = KafkaMessage.first
|
87
|
+
time_diff = first_message ? Time.zone.now - KafkaMessage.first.created_at : 0
|
88
|
+
Deimos.config.metrics&.gauge('pending_db_messages_max_wait', time_diff)
|
89
|
+
end
|
90
|
+
|
91
|
+
# @param batch [Array<Hash>]
|
92
|
+
def produce_messages(batch)
|
93
|
+
batch_size = batch.size
|
94
|
+
begin
|
95
|
+
batch.in_groups_of(batch_size, false).each do |group|
|
96
|
+
@logger.debug("Publishing #{group.size} messages to #{@current_topic}")
|
97
|
+
producer.publish_list(group)
|
98
|
+
Deimos.config.metrics&.increment(
|
99
|
+
'publish',
|
100
|
+
tags: %W(status:success topic:#{@current_topic}),
|
101
|
+
by: group.size
|
102
|
+
)
|
103
|
+
@logger.info("Sent #{group.size} messages to #{@current_topic}")
|
104
|
+
end
|
105
|
+
rescue Kafka::BufferOverflow
|
106
|
+
raise if batch_size == 1
|
107
|
+
|
108
|
+
@logger.error("Buffer overflow when publishing #{batch.size} in groups of #{batch_size}, retrying...")
|
109
|
+
if batch_size < 10
|
110
|
+
batch_size = 1
|
111
|
+
else
|
112
|
+
batch_size /= 10
|
113
|
+
end
|
114
|
+
if self.class.producer.respond_to?(:sync_producer_shutdown) # Phobos 1.8.3
|
115
|
+
self.class.producer.sync_producer_shutdown
|
116
|
+
end
|
117
|
+
retry
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|