deimos-ruby 1.0.0.pre.beta22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +74 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +321 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +32 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +165 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +752 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-kafka.gemspec +42 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/DATABASE_BACKEND.md +147 -0
  22. data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
  23. data/lib/deimos/active_record_consumer.rb +81 -0
  24. data/lib/deimos/active_record_producer.rb +64 -0
  25. data/lib/deimos/avro_data_coder.rb +89 -0
  26. data/lib/deimos/avro_data_decoder.rb +36 -0
  27. data/lib/deimos/avro_data_encoder.rb +51 -0
  28. data/lib/deimos/backends/db.rb +27 -0
  29. data/lib/deimos/backends/kafka.rb +27 -0
  30. data/lib/deimos/backends/kafka_async.rb +27 -0
  31. data/lib/deimos/configuration.rb +90 -0
  32. data/lib/deimos/consumer.rb +164 -0
  33. data/lib/deimos/instrumentation.rb +71 -0
  34. data/lib/deimos/kafka_message.rb +27 -0
  35. data/lib/deimos/kafka_source.rb +126 -0
  36. data/lib/deimos/kafka_topic_info.rb +86 -0
  37. data/lib/deimos/message.rb +74 -0
  38. data/lib/deimos/metrics/datadog.rb +47 -0
  39. data/lib/deimos/metrics/mock.rb +39 -0
  40. data/lib/deimos/metrics/provider.rb +38 -0
  41. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  42. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  43. data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
  44. data/lib/deimos/monkey_patches/schema_store.rb +19 -0
  45. data/lib/deimos/producer.rb +218 -0
  46. data/lib/deimos/publish_backend.rb +30 -0
  47. data/lib/deimos/railtie.rb +8 -0
  48. data/lib/deimos/schema_coercer.rb +108 -0
  49. data/lib/deimos/shared_config.rb +59 -0
  50. data/lib/deimos/test_helpers.rb +356 -0
  51. data/lib/deimos/tracing/datadog.rb +35 -0
  52. data/lib/deimos/tracing/mock.rb +40 -0
  53. data/lib/deimos/tracing/provider.rb +31 -0
  54. data/lib/deimos/utils/db_producer.rb +122 -0
  55. data/lib/deimos/utils/executor.rb +117 -0
  56. data/lib/deimos/utils/inline_consumer.rb +144 -0
  57. data/lib/deimos/utils/lag_reporter.rb +182 -0
  58. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  59. data/lib/deimos/utils/signal_handler.rb +68 -0
  60. data/lib/deimos/version.rb +5 -0
  61. data/lib/deimos.rb +133 -0
  62. data/lib/generators/deimos/db_backend/templates/migration +24 -0
  63. data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
  64. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  65. data/lib/tasks/deimos.rake +27 -0
  66. data/spec/active_record_consumer_spec.rb +81 -0
  67. data/spec/active_record_producer_spec.rb +107 -0
  68. data/spec/avro_data_decoder_spec.rb +18 -0
  69. data/spec/avro_data_encoder_spec.rb +37 -0
  70. data/spec/backends/db_spec.rb +35 -0
  71. data/spec/backends/kafka_async_spec.rb +11 -0
  72. data/spec/backends/kafka_spec.rb +11 -0
  73. data/spec/consumer_spec.rb +169 -0
  74. data/spec/deimos_spec.rb +120 -0
  75. data/spec/kafka_source_spec.rb +168 -0
  76. data/spec/kafka_topic_info_spec.rb +88 -0
  77. data/spec/phobos.bad_db.yml +73 -0
  78. data/spec/phobos.yml +73 -0
  79. data/spec/producer_spec.rb +397 -0
  80. data/spec/publish_backend_spec.rb +10 -0
  81. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  82. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  83. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  84. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  85. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  86. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  87. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  88. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  89. data/spec/spec_helper.rb +207 -0
  90. data/spec/updateable_schema_store_spec.rb +36 -0
  91. data/spec/utils/db_producer_spec.rb +259 -0
  92. data/spec/utils/executor_spec.rb +42 -0
  93. data/spec/utils/lag_reporter_spec.rb +69 -0
  94. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  95. data/spec/utils/signal_handler_spec.rb +16 -0
  96. data/support/deimos-solo.png +0 -0
  97. data/support/deimos-with-name-next.png +0 -0
  98. data/support/deimos-with-name.png +0 -0
  99. data/support/flipp-logo.png +0 -0
  100. metadata +452 -0
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support/concern'
4
+
5
+ module Deimos
6
+ # Module that producers and consumers can share which sets up configuration.
7
+ module SharedConfig
8
+ extend ActiveSupport::Concern
9
+
10
+ # need to use this instead of class_methods to be backwards-compatible
11
+ # with Rails 3
12
+ module ClassMethods
13
+ # @return [Hash]
14
+ def config
15
+ return @config if @config
16
+
17
+ @config = {
18
+ encode_key: true
19
+ }
20
+ klass = self.superclass
21
+ while klass.respond_to?(:config)
22
+ klass_config = klass.config
23
+ if klass_config
24
+ # default is true for this so don't include it in the merge
25
+ klass_config.delete(:encode_key) if klass_config[:encode_key]
26
+ @config.merge!(klass_config) if klass.config
27
+ end
28
+ klass = klass.superclass
29
+ end
30
+ @config
31
+ end
32
+
33
+ # Set the schema.
34
+ # @param schema [String]
35
+ def schema(schema)
36
+ config[:schema] = schema
37
+ end
38
+
39
+ # Set the namespace.
40
+ # @param namespace [String]
41
+ def namespace(namespace)
42
+ config[:namespace] = namespace
43
+ end
44
+
45
+ # Set key configuration.
46
+ # @param field [Symbol] the name of a field to use in the value schema as
47
+ # a generated key schema
48
+ # @param schema [String|Symbol] the name of a schema to use for the key
49
+ # @param plain [Boolean] if true, do not encode keys at all
50
+ # @param none [Boolean] if true, do not use keys at all
51
+ def key_config(plain: nil, field: nil, schema: nil, none: nil)
52
+ config[:no_keys] = none
53
+ config[:encode_key] = !plain && !none
54
+ config[:key_field] = field&.to_s
55
+ config[:key_schema] = schema
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,356 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support/concern'
4
+ require 'active_support/core_ext'
5
+ require 'avro_turf'
6
+ require 'deimos/tracing/mock'
7
+ require 'deimos/metrics/mock'
8
+
9
+ module Deimos
10
+ # Include this module in your RSpec spec_helper
11
+ # to stub out external dependencies
12
+ # and add methods to use to test encoding/decoding.
13
+ module TestHelpers
14
+ extend ActiveSupport::Concern
15
+
16
+ class << self
17
+ # @return [Array<Hash>]
18
+ def sent_messages
19
+ @sent_messages ||= []
20
+ end
21
+ end
22
+
23
+ included do
24
+ # @param encoder_schema [String]
25
+ # @param namespace [String]
26
+ # @return [Deimos::AvroDataEncoder]
27
+ def create_encoder(encoder_schema, namespace)
28
+ encoder = Deimos::AvroDataEncoder.new(schema: encoder_schema,
29
+ namespace: namespace)
30
+
31
+ # we added and_wrap_original to RSpec 2 but the regular block
32
+ # syntax wasn't working for some reason - block wasn't being passed
33
+ # to the method
34
+ block = proc do |m, *args|
35
+ m.call(*args)
36
+ args[0]
37
+ end
38
+ allow(encoder).to receive(:encode_local).and_wrap_original(&block)
39
+ allow(encoder).to receive(:encode) do |payload, schema: nil, topic: nil|
40
+ encoder.encode_local(payload, schema: schema)
41
+ end
42
+
43
+ block = proc do |m, *args|
44
+ m.call(*args)&.values&.first
45
+ end
46
+ allow(encoder).to receive(:encode_key).and_wrap_original(&block)
47
+ encoder
48
+ end
49
+
50
+ # @param decoder_schema [String]
51
+ # @param namespace [String]
52
+ # @return [Deimos::AvroDataDecoder]
53
+ def create_decoder(decoder_schema, namespace)
54
+ decoder = Deimos::AvroDataDecoder.new(schema: decoder_schema,
55
+ namespace: namespace)
56
+ allow(decoder).to receive(:decode_local) { |payload| payload }
57
+ allow(decoder).to receive(:decode) do |payload, schema: nil|
58
+ schema ||= decoder.schema
59
+ if schema && decoder.namespace
60
+ # Validate against local schema.
61
+ encoder = Deimos::AvroDataEncoder.new(schema: schema,
62
+ namespace: decoder.namespace)
63
+ encoder.schema_store = decoder.schema_store
64
+ encoder.encode_local(payload)
65
+ end
66
+ payload
67
+ end
68
+ allow(decoder).to receive(:decode_key) do |payload, _key_id|
69
+ payload.values.first
70
+ end
71
+ decoder
72
+ end
73
+
74
+ RSpec.configure do |config|
75
+
76
+ config.before(:suite) do
77
+ Deimos.configure do |fr_config|
78
+ fr_config.logger = Logger.new(STDOUT)
79
+ fr_config.seed_broker ||= 'test_broker'
80
+ fr_config.tracer = Deimos::Tracing::Mock.new
81
+ fr_config.metrics = Deimos::Metrics::Mock.new
82
+ end
83
+ end
84
+
85
+ end
86
+ before(:each) do
87
+ client = double('client').as_null_object
88
+ allow(client).to receive(:time) do |*_args, &block|
89
+ block.call
90
+ end
91
+ end
92
+ end
93
+
94
+ # Stub all already-loaded producers and consumers fir unit testing purposes.
95
+ def stub_producers_and_consumers!
96
+ Deimos::TestHelpers.sent_messages.clear
97
+
98
+ allow(Deimos::Producer).to receive(:produce_batch) do |_, batch|
99
+ Deimos::TestHelpers.sent_messages.concat(batch.map(&:to_h))
100
+ end
101
+
102
+ Deimos::Producer.descendants.each do |klass|
103
+ next if klass == Deimos::ActiveRecordProducer # "abstract" class
104
+
105
+ stub_producer(klass)
106
+ end
107
+
108
+ Deimos::Consumer.descendants.each do |klass|
109
+ next if klass == Deimos::ActiveRecordConsumer # "abstract" class
110
+
111
+ stub_consumer(klass)
112
+ end
113
+ end
114
+
115
+ # Stub a given producer class.
116
+ # @param klass [Class < Deimos::Producer]
117
+ def stub_producer(klass)
118
+ allow(klass).to receive(:encoder) do
119
+ create_encoder(klass.config[:schema], klass.config[:namespace])
120
+ end
121
+ allow(klass).to receive(:key_encoder) do
122
+ create_encoder(klass.config[:key_schema], klass.config[:namespace])
123
+ end
124
+ end
125
+
126
+ # Stub a given consumer class.
127
+ # @param klass [Class < Deimos::Consumer]
128
+ def stub_consumer(klass)
129
+ allow(klass).to receive(:decoder) do
130
+ create_decoder(klass.config[:schema], klass.config[:namespace])
131
+ end
132
+ klass.class_eval do
133
+ alias_method(:old_consume, :consume) unless self.instance_methods.include?(:old_consume)
134
+ end
135
+ allow_any_instance_of(klass).to receive(:consume) do |instance, payload, metadata|
136
+ metadata[:key] = klass.new.decode_key(metadata[:key])
137
+ instance.old_consume(payload, metadata)
138
+ end
139
+ end
140
+
141
+ # get the difference of 2 hashes.
142
+ # @param hash1 [Hash]
143
+ # @param hash2 [Hash]
144
+ def _hash_diff(hash1, hash2)
145
+ if hash1.nil? || !hash1.is_a?(Hash)
146
+ hash2
147
+ elsif hash2.nil? || !hash2.is_a?(Hash)
148
+ hash1
149
+ else
150
+ hash1.dup.
151
+ delete_if { |k, v| hash2[k] == v }.
152
+ merge!(hash2.dup.delete_if { |k, _v| hash1.key?(k) })
153
+ end
154
+ end
155
+
156
+ # :nodoc:
157
+ def _frk_failure_message(topic, message, key=nil, partition_key=nil, was_negated=false)
158
+ messages = Deimos::TestHelpers.sent_messages.
159
+ select { |m| m[:topic] == topic }.
160
+ map { |m| m.except(:topic) }
161
+ message_string = ''
162
+ diff = nil
163
+ min_hash_diff = nil
164
+ if messages.any?
165
+ message_string = messages.map(&:inspect).join("\n")
166
+ min_hash_diff = messages.min_by { |m| _hash_diff(m, message).keys.size }
167
+ diff = RSpec::Expectations.differ.
168
+ diff_as_object(message, min_hash_diff[:payload])
169
+ end
170
+ description = if message.respond_to?(:description)
171
+ message.description
172
+ elsif message.nil?
173
+ 'nil'
174
+ else
175
+ message
176
+ end
177
+ str = "Expected #{topic} #{'not ' if was_negated}to have sent #{description}"
178
+ str += " with key #{key}" if key
179
+ str += " with partition key #{partition_key}" if partition_key
180
+ str += "\nClosest message received: #{min_hash_diff}" if min_hash_diff
181
+ str += "\nDiff: #{diff}" if diff
182
+ str + "\nAll Messages received:\n#{message_string}"
183
+ end
184
+
185
+ RSpec::Matchers.define :have_sent do |msg, key=nil, partition_key=nil|
186
+ message = if msg.respond_to?(:with_indifferent_access)
187
+ msg.with_indifferent_access
188
+ else
189
+ msg
190
+ end
191
+ match do |topic|
192
+ Deimos::TestHelpers.sent_messages.any? do |m|
193
+ hash_matcher = RSpec::Matchers::BuiltIn::Match.new(message)
194
+ hash_matcher.send(:match,
195
+ message,
196
+ m[:payload]&.with_indifferent_access) &&
197
+ topic == m[:topic] &&
198
+ (key.present? ? key == m[:key] : true) &&
199
+ (partition_key.present? ? partition_key == m[:partition_key] : true)
200
+ end
201
+ end
202
+
203
+ if respond_to?(:failure_message)
204
+ failure_message do |topic|
205
+ _frk_failure_message(topic, message, key, partition_key)
206
+ end
207
+ failure_message_when_negated do |topic|
208
+ _frk_failure_message(topic, message, key, partition_key, true)
209
+ end
210
+ else
211
+ failure_message_for_should do |topic|
212
+ _frk_failure_message(topic, message, key, partition_key)
213
+ end
214
+ failure_message_for_should_not do |topic|
215
+ _frk_failure_message(topic, message, key, partition_key, true)
216
+ end
217
+ end
218
+ end
219
+
220
+ # Clear all sent messages - e.g. if we want to check that
221
+ # particular messages were sent or not sent after a point in time.
222
+ def clear_kafka_messages!
223
+ Deimos::TestHelpers.sent_messages.clear
224
+ end
225
+
226
+ # test that a message was sent on the given topic.
227
+ # DEPRECATED - use the "have_sent" matcher instead.
228
+ # @param message [Hash]
229
+ # @param topic [String]
230
+ # @param key [String|Integer]
231
+ # @return [Boolean]
232
+ def was_message_sent?(message, topic, key=nil)
233
+ Deimos::TestHelpers.sent_messages.any? do |m|
234
+ message == m[:payload] && m[:topic] == topic &&
235
+ (key.present? ? m[:key] == key : true)
236
+ end
237
+ end
238
+
239
+ # Test that a given handler will consume a given payload correctly, i.e.
240
+ # that the Avro schema is correct. If
241
+ # a block is given, that block will be executed when `consume` is called.
242
+ # Otherwise it will just confirm that `consume` is called at all.
243
+ # @param handler_class_or_topic [Class|String] Class which inherits from
244
+ # Deimos::Consumer or the topic as a string
245
+ # @param payload [Hash] the payload to consume
246
+ # @param call_original [Boolean] if true, allow the consume handler
247
+ # to continue as normal. Not compatible with a block.
248
+ # @param ignore_expectation [Boolean] Set to true to not place any
249
+ # expectations on the consumer. Primarily used internally to Deimos.
250
+ # @param key [Object] the key to use.
251
+ # @param partition_key [Object] the partition key to use.
252
+ def test_consume_message(handler_class_or_topic,
253
+ payload,
254
+ call_original: false,
255
+ key: nil,
256
+ partition_key: nil,
257
+ skip_expectation: false,
258
+ &block)
259
+ raise 'Cannot have both call_original and be given a block!' if call_original && block_given?
260
+
261
+ payload&.stringify_keys!
262
+ handler_class = if handler_class_or_topic.is_a?(String)
263
+ _get_handler_class_from_topic(handler_class_or_topic)
264
+ else
265
+ handler_class_or_topic
266
+ end
267
+ handler = handler_class.new
268
+ allow(handler_class).to receive(:new).and_return(handler)
269
+ listener = double('listener',
270
+ handler_class: handler_class,
271
+ encoding: nil)
272
+ key ||= _key_from_consumer(handler_class)
273
+ message = double('message',
274
+ 'key' => key,
275
+ 'partition_key' => partition_key,
276
+ 'partition' => 1,
277
+ 'offset' => 1,
278
+ 'value' => payload)
279
+
280
+ unless skip_expectation
281
+ expectation = expect(handler).to receive(:consume).
282
+ with(payload, anything, &block)
283
+ expectation.and_call_original if call_original
284
+ end
285
+ Phobos::Actions::ProcessMessage.new(
286
+ listener: listener,
287
+ message: message,
288
+ listener_metadata: { topic: 'my-topic' }
289
+ ).send(:process_message, payload)
290
+ end
291
+
292
+ # Check to see that a given message will fail due to Avro errors.
293
+ # @param handler_class [Class]
294
+ # @param payload [Hash]
295
+ def test_consume_invalid_message(handler_class, payload)
296
+ handler = handler_class.new
297
+ allow(handler_class).to receive(:new).and_return(handler)
298
+ listener = double('listener',
299
+ handler_class: handler_class,
300
+ encoding: nil)
301
+ message = double('message',
302
+ key: _key_from_consumer(handler_class),
303
+ partition_key: nil,
304
+ partition: 1,
305
+ offset: 1,
306
+ value: payload)
307
+
308
+ expect {
309
+ Phobos::Actions::ProcessMessage.new(
310
+ listener: listener,
311
+ message: message,
312
+ listener_metadata: { topic: 'my-topic' }
313
+ ).send(:process_message, payload)
314
+ }.to raise_error(Avro::SchemaValidator::ValidationError)
315
+ end
316
+
317
+ # @param schema1 [String|Hash] a file path, JSON string, or
318
+ # hash representing a schema.
319
+ # @param schema2 [String|Hash] a file path, JSON string, or
320
+ # hash representing a schema.
321
+ # @return [Boolean] true if the schemas are compatible, false otherwise.
322
+ def self.schemas_compatible?(schema1, schema2)
323
+ json1, json2 = [schema1, schema2].map do |schema|
324
+ if schema.is_a?(String)
325
+ schema = File.read(schema) unless schema.strip.starts_with?('{') # file path
326
+ MultiJson.load(schema)
327
+ else
328
+ schema
329
+ end
330
+ end
331
+ avro_schema1 = Avro::Schema.real_parse(json1, {})
332
+ avro_schema2 = Avro::Schema.real_parse(json2, {})
333
+ Avro::SchemaCompatibility.mutual_read?(avro_schema1, avro_schema2)
334
+ end
335
+
336
+ private
337
+
338
+ def _key_from_consumer(consumer)
339
+ if consumer.config[:key_field] || consumer.config[:key_schema]
340
+ { 'test' => 1 }
341
+ else
342
+ 1
343
+ end
344
+ end
345
+
346
+ # @param topic [String]
347
+ # @return [Class]
348
+ def _get_handler_class_from_topic(topic)
349
+ listeners = Phobos.config['listeners']
350
+ handler = listeners.find { |l| l.topic == topic }
351
+ raise "No consumer found in Phobos configuration for topic #{topic}!" if handler.nil?
352
+
353
+ handler.handler.constantize
354
+ end
355
+ end
356
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'deimos/tracing/provider'
4
+
5
+ module Deimos
6
+ module Tracing
7
+ # Tracing wrapper class for Datadog.
8
+ class Datadog < Tracing::Provider
9
+ # :nodoc:
10
+ def initialize(config)
11
+ raise 'Tracing config must specify service_name' if config[:service_name].nil?
12
+
13
+ @service = config[:service_name]
14
+ end
15
+
16
+ # :nodoc:
17
+ def start(span_name, options={})
18
+ span = ::Datadog.tracer.trace(span_name)
19
+ span.service = @service
20
+ span.resource = options[:resource]
21
+ span
22
+ end
23
+
24
+ # :nodoc:
25
+ def finish(span)
26
+ span.finish
27
+ end
28
+
29
+ # :nodoc:
30
+ def set_error(span, exception)
31
+ span.set_error(exception)
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'deimos/tracing/provider'
4
+
5
+ module Deimos
6
+ module Tracing
7
+ # Class that mocks out tracing functionality
8
+ class Mock < Tracing::Provider
9
+ # :nodoc:
10
+ def initialize(logger=nil)
11
+ @logger = logger || Logger.new(STDOUT)
12
+ @logger.info('MockTracingProvider initialized')
13
+ end
14
+
15
+ # :nodoc:
16
+ def start(span_name, _options={})
17
+ @logger.info("Mock span '#{span_name}' started")
18
+ {
19
+ name: span_name,
20
+ started_at: Time.zone.now
21
+ }
22
+ end
23
+
24
+ # :nodoc:
25
+ def finish(span)
26
+ name = span[:name]
27
+ start = span[:started_at]
28
+ finish = Time.zone.now
29
+ @logger.info("Mock span '#{name}' finished: #{start} to #{finish}")
30
+ end
31
+
32
+ # :nodoc:
33
+ def set_error(span, exception)
34
+ span[:exception] = exception
35
+ name = span[:name]
36
+ @logger.info("Mock span '#{name}' set an error: #{exception}")
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ # rubocop:disable Lint/UnusedMethodArgument
4
+ module Deimos
5
+ module Tracing
6
+ # Base class for all tracing providers.
7
+ class Provider
8
+ # Returns a span object and starts the trace.
9
+ # @param span_name [String] The name of the span/trace
10
+ # @param options [Hash] Options for the span
11
+ # @return [Object] The span object
12
+ def start(span_name, options={})
13
+ raise NotImplementedError
14
+ end
15
+
16
+ # Finishes the trace on the span object.
17
+ # @param span [Object] The span to finish trace on
18
+ def finish(span)
19
+ raise NotImplementedError
20
+ end
21
+
22
+ # Set an error on the span.
23
+ # @param span [Object] The span to set error on
24
+ # @param exception [Exception] The exception that occurred
25
+ def set_error(span, exception)
26
+ raise NotImplementedError
27
+ end
28
+ end
29
+ end
30
+ end
31
+ # rubocop:enable Lint/UnusedMethodArgument
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Deimos
4
+ module Utils
5
+ # Class which continually polls the database and sends Kafka messages.
6
+ class DbProducer
7
+ include Phobos::Producer
8
+ attr_accessor :id, :current_topic
9
+
10
+ BATCH_SIZE = 1000
11
+
12
+ # @param logger [Logger]
13
+ def initialize(logger=Logger.new(STDOUT))
14
+ @id = SecureRandom.uuid
15
+ @logger = logger
16
+ @logger.push_tags("DbProducer #{@id}") if @logger.respond_to?(:push_tags)
17
+ end
18
+
19
+ # Start the poll.
20
+ def start
21
+ @logger.info('Starting...')
22
+ @signal_to_stop = false
23
+ loop do
24
+ if @signal_to_stop
25
+ @logger.info('Shutting down')
26
+ break
27
+ end
28
+ send_pending_metrics
29
+ process_next_messages
30
+ end
31
+ end
32
+
33
+ # Stop the poll.
34
+ def stop
35
+ @logger.info('Received signal to stop')
36
+ @signal_to_stop = true
37
+ end
38
+
39
+ # Complete one loop of processing all messages in the DB.
40
+ def process_next_messages
41
+ topics = retrieve_topics
42
+ @logger.info("Found topics: #{topics}")
43
+ topics.each(&method(:process_topic))
44
+ sleep(0.5)
45
+ end
46
+
47
+ # @return [Array<String>]
48
+ def retrieve_topics
49
+ KafkaMessage.select('distinct topic').map(&:topic).uniq
50
+ end
51
+
52
+ # @param topic [String]
53
+ # @return [String] the topic that was locked, or nil if none were.
54
+ def process_topic(topic)
55
+ # If the topic is already locked, another producer is currently
56
+ # working on it. Move on to the next one.
57
+ unless KafkaTopicInfo.lock(topic, @id)
58
+ @logger.debug("Could not lock topic #{topic} - continuing")
59
+ return
60
+ end
61
+ @current_topic = topic
62
+ messages = retrieve_messages
63
+
64
+ while messages.any?
65
+ produce_messages(messages.map(&:phobos_message))
66
+ messages.first.class.where(id: messages.map(&:id)).delete_all
67
+ break if messages.size < BATCH_SIZE
68
+
69
+ KafkaTopicInfo.heartbeat(@current_topic, @id) # keep alive
70
+ send_pending_metrics
71
+ messages = retrieve_messages
72
+ end
73
+ KafkaTopicInfo.clear_lock(@current_topic, @id)
74
+ rescue StandardError => e
75
+ @logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
76
+ KafkaTopicInfo.register_error(@current_topic, @id)
77
+ end
78
+
79
+ # @return [Array<Deimos::KafkaMessage>]
80
+ def retrieve_messages
81
+ KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
82
+ end
83
+
84
+ # Send metrics to Datadog.
85
+ def send_pending_metrics
86
+ first_message = KafkaMessage.first
87
+ time_diff = first_message ? Time.zone.now - KafkaMessage.first.created_at : 0
88
+ Deimos.config.metrics&.gauge('pending_db_messages_max_wait', time_diff)
89
+ end
90
+
91
+ # @param batch [Array<Hash>]
92
+ def produce_messages(batch)
93
+ batch_size = batch.size
94
+ begin
95
+ batch.in_groups_of(batch_size, false).each do |group|
96
+ @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
97
+ producer.publish_list(group)
98
+ Deimos.config.metrics&.increment(
99
+ 'publish',
100
+ tags: %W(status:success topic:#{@current_topic}),
101
+ by: group.size
102
+ )
103
+ @logger.info("Sent #{group.size} messages to #{@current_topic}")
104
+ end
105
+ rescue Kafka::BufferOverflow
106
+ raise if batch_size == 1
107
+
108
+ @logger.error("Buffer overflow when publishing #{batch.size} in groups of #{batch_size}, retrying...")
109
+ if batch_size < 10
110
+ batch_size = 1
111
+ else
112
+ batch_size /= 10
113
+ end
114
+ if self.class.producer.respond_to?(:sync_producer_shutdown) # Phobos 1.8.3
115
+ self.class.producer.sync_producer_shutdown
116
+ end
117
+ retry
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end