deimos-ruby 1.0.0.pre.beta22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +74 -0
- data/.gitignore +41 -0
- data/.gitmodules +0 -0
- data/.rspec +1 -0
- data/.rubocop.yml +321 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +32 -0
- data/CODE_OF_CONDUCT.md +77 -0
- data/Dockerfile +23 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +165 -0
- data/Guardfile +22 -0
- data/LICENSE.md +195 -0
- data/README.md +752 -0
- data/Rakefile +13 -0
- data/bin/deimos +4 -0
- data/deimos-kafka.gemspec +42 -0
- data/docker-compose.yml +71 -0
- data/docs/DATABASE_BACKEND.md +147 -0
- data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
- data/lib/deimos/active_record_consumer.rb +81 -0
- data/lib/deimos/active_record_producer.rb +64 -0
- data/lib/deimos/avro_data_coder.rb +89 -0
- data/lib/deimos/avro_data_decoder.rb +36 -0
- data/lib/deimos/avro_data_encoder.rb +51 -0
- data/lib/deimos/backends/db.rb +27 -0
- data/lib/deimos/backends/kafka.rb +27 -0
- data/lib/deimos/backends/kafka_async.rb +27 -0
- data/lib/deimos/configuration.rb +90 -0
- data/lib/deimos/consumer.rb +164 -0
- data/lib/deimos/instrumentation.rb +71 -0
- data/lib/deimos/kafka_message.rb +27 -0
- data/lib/deimos/kafka_source.rb +126 -0
- data/lib/deimos/kafka_topic_info.rb +86 -0
- data/lib/deimos/message.rb +74 -0
- data/lib/deimos/metrics/datadog.rb +47 -0
- data/lib/deimos/metrics/mock.rb +39 -0
- data/lib/deimos/metrics/provider.rb +38 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
- data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
- data/lib/deimos/monkey_patches/schema_store.rb +19 -0
- data/lib/deimos/producer.rb +218 -0
- data/lib/deimos/publish_backend.rb +30 -0
- data/lib/deimos/railtie.rb +8 -0
- data/lib/deimos/schema_coercer.rb +108 -0
- data/lib/deimos/shared_config.rb +59 -0
- data/lib/deimos/test_helpers.rb +356 -0
- data/lib/deimos/tracing/datadog.rb +35 -0
- data/lib/deimos/tracing/mock.rb +40 -0
- data/lib/deimos/tracing/provider.rb +31 -0
- data/lib/deimos/utils/db_producer.rb +122 -0
- data/lib/deimos/utils/executor.rb +117 -0
- data/lib/deimos/utils/inline_consumer.rb +144 -0
- data/lib/deimos/utils/lag_reporter.rb +182 -0
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +68 -0
- data/lib/deimos/version.rb +5 -0
- data/lib/deimos.rb +133 -0
- data/lib/generators/deimos/db_backend/templates/migration +24 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
- data/lib/generators/deimos/db_backend_generator.rb +48 -0
- data/lib/tasks/deimos.rake +27 -0
- data/spec/active_record_consumer_spec.rb +81 -0
- data/spec/active_record_producer_spec.rb +107 -0
- data/spec/avro_data_decoder_spec.rb +18 -0
- data/spec/avro_data_encoder_spec.rb +37 -0
- data/spec/backends/db_spec.rb +35 -0
- data/spec/backends/kafka_async_spec.rb +11 -0
- data/spec/backends/kafka_spec.rb +11 -0
- data/spec/consumer_spec.rb +169 -0
- data/spec/deimos_spec.rb +120 -0
- data/spec/kafka_source_spec.rb +168 -0
- data/spec/kafka_topic_info_spec.rb +88 -0
- data/spec/phobos.bad_db.yml +73 -0
- data/spec/phobos.yml +73 -0
- data/spec/producer_spec.rb +397 -0
- data/spec/publish_backend_spec.rb +10 -0
- data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
- data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
- data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
- data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
- data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
- data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
- data/spec/spec_helper.rb +207 -0
- data/spec/updateable_schema_store_spec.rb +36 -0
- data/spec/utils/db_producer_spec.rb +259 -0
- data/spec/utils/executor_spec.rb +42 -0
- data/spec/utils/lag_reporter_spec.rb +69 -0
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/signal_handler_spec.rb +16 -0
- data/support/deimos-solo.png +0 -0
- data/support/deimos-with-name-next.png +0 -0
- data/support/deimos-with-name.png +0 -0
- data/support/flipp-logo.png +0 -0
- metadata +452 -0
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# rubocop:disable Lint/RescueException
|
4
|
+
module Deimos
|
5
|
+
module Utils
|
6
|
+
# Mostly copied from Phobos::Executor. We should DRY this up by putting in a
|
7
|
+
# PR to make it more generic. Might even make sense to move to a separate
|
8
|
+
# gem.
|
9
|
+
class Executor
|
10
|
+
# @return [Array<#start, #stop, #id>]
|
11
|
+
attr_accessor :runners
|
12
|
+
|
13
|
+
# @param runners [Array<#start, #stop, #id>] A list of objects that can be
|
14
|
+
# started or stopped.
|
15
|
+
# @param logger [Logger]
|
16
|
+
def initialize(runners, logger=Logger.new(STDOUT))
|
17
|
+
@threads = Concurrent::Array.new
|
18
|
+
@runners = runners
|
19
|
+
@logger = logger
|
20
|
+
end
|
21
|
+
|
22
|
+
# Start the executor.
|
23
|
+
def start
|
24
|
+
@logger.info('Starting executor')
|
25
|
+
@signal_to_stop = false
|
26
|
+
@threads.clear
|
27
|
+
@thread_pool = Concurrent::FixedThreadPool.new(@runners.size)
|
28
|
+
|
29
|
+
@runners.each do |runner|
|
30
|
+
@thread_pool.post do
|
31
|
+
thread = Thread.current
|
32
|
+
thread.abort_on_exception = true
|
33
|
+
@threads << thread
|
34
|
+
run_object(runner)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
true
|
39
|
+
end
|
40
|
+
|
41
|
+
# Stop the executor.
|
42
|
+
def stop
|
43
|
+
return if @signal_to_stop
|
44
|
+
|
45
|
+
@logger.info('Stopping executor')
|
46
|
+
@signal_to_stop = true
|
47
|
+
@runners.each(&:stop)
|
48
|
+
@threads.select(&:alive?).each do |thread|
|
49
|
+
begin
|
50
|
+
thread.wakeup
|
51
|
+
rescue StandardError
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
end
|
55
|
+
@thread_pool&.shutdown
|
56
|
+
@thread_pool&.wait_for_termination
|
57
|
+
@logger.info('Executor stopped')
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
# @param exception [Throwable]
|
63
|
+
# @return [Hash]
|
64
|
+
def error_metadata(exception)
|
65
|
+
{
|
66
|
+
exception_class: exception.class.name,
|
67
|
+
exception_message: exception.message,
|
68
|
+
backtrace: exception.backtrace
|
69
|
+
}
|
70
|
+
end
|
71
|
+
|
72
|
+
def run_object(runner)
|
73
|
+
retry_count = 0
|
74
|
+
|
75
|
+
begin
|
76
|
+
@logger.info("Running #{runner.id}")
|
77
|
+
runner.start
|
78
|
+
retry_count = 0 # success - reset retry count
|
79
|
+
rescue Exception => e
|
80
|
+
handle_crashed_runner(runner, e, retry_count)
|
81
|
+
retry_count += 1
|
82
|
+
retry unless @signal_to_stop
|
83
|
+
end
|
84
|
+
rescue Exception => e
|
85
|
+
@logger.error("Failed to run listener (#{e.message}) #{error_metadata(e)}")
|
86
|
+
raise e
|
87
|
+
end
|
88
|
+
|
89
|
+
# @return [ExponentialBackoff]
|
90
|
+
def create_exponential_backoff
|
91
|
+
min = 1
|
92
|
+
max = 60
|
93
|
+
ExponentialBackoff.new(min, max).tap do |backoff|
|
94
|
+
backoff.randomize_factor = rand
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# When "runner#start" is interrupted / crashes we assume it's
|
99
|
+
# safe to be called again
|
100
|
+
def handle_crashed_runner(runner, error, retry_count)
|
101
|
+
backoff = create_exponential_backoff
|
102
|
+
interval = backoff.interval_at(retry_count).round(2)
|
103
|
+
|
104
|
+
metadata = {
|
105
|
+
listener_id: runner.id,
|
106
|
+
retry_count: retry_count,
|
107
|
+
waiting_time: interval
|
108
|
+
}.merge(error_metadata(error))
|
109
|
+
|
110
|
+
@logger.error("Runner crashed, waiting #{interval}s (#{error.message}) #{metadata}")
|
111
|
+
sleep(interval)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
# rubocop:enable Lint/RescueException
|
@@ -0,0 +1,144 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Class to consume messages. Can be used with integration testing frameworks.
|
4
|
+
# Assumes that you have a topic with only one partition.
|
5
|
+
module Deimos
|
6
|
+
module Utils
|
7
|
+
# Listener that can seek to get the last X messages in a topic.
|
8
|
+
class SeekListener < Phobos::Listener
|
9
|
+
attr_accessor :num_messages
|
10
|
+
|
11
|
+
# :nodoc:
|
12
|
+
def start_listener
|
13
|
+
@num_messages ||= 10
|
14
|
+
@consumer = create_kafka_consumer
|
15
|
+
@consumer.subscribe(topic, @subscribe_opts)
|
16
|
+
|
17
|
+
begin
|
18
|
+
last_offset = @kafka_client.last_offset_for(topic, 0)
|
19
|
+
offset = last_offset - num_messages
|
20
|
+
if offset.positive?
|
21
|
+
Deimos.config.logger.info("Seeking to #{offset}")
|
22
|
+
@consumer.seek(topic, 0, offset)
|
23
|
+
end
|
24
|
+
rescue StandardError => e
|
25
|
+
"Could not seek to offset: #{e.message}"
|
26
|
+
end
|
27
|
+
|
28
|
+
instrument('listener.start_handler', listener_metadata) do
|
29
|
+
@handler_class.start(@kafka_client)
|
30
|
+
end
|
31
|
+
log_info('Listener started', listener_metadata)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# Class to return the messages consumed.
|
36
|
+
class MessageBankHandler < Deimos::Consumer
|
37
|
+
include Phobos::Handler
|
38
|
+
|
39
|
+
cattr_accessor :total_messages
|
40
|
+
|
41
|
+
# @param klass [Class < Deimos::Consumer]
|
42
|
+
def self.config_class=(klass)
|
43
|
+
self.config.merge!(klass.config)
|
44
|
+
end
|
45
|
+
|
46
|
+
# :nodoc:
|
47
|
+
def self.start(_kafka_client)
|
48
|
+
self.total_messages = []
|
49
|
+
end
|
50
|
+
|
51
|
+
# :nodoc:
|
52
|
+
def consume(payload, metadata)
|
53
|
+
puts "Got #{payload}"
|
54
|
+
self.class.total_messages << {
|
55
|
+
key: metadata[:key],
|
56
|
+
payload: payload
|
57
|
+
}
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
# Class which can process/consume messages inline.
|
62
|
+
class InlineConsumer
|
63
|
+
MAX_MESSAGE_WAIT_TIME = 1.second
|
64
|
+
MAX_TOPIC_WAIT_TIME = 10.seconds
|
65
|
+
|
66
|
+
# Get the last X messages from a topic. You can specify a subclass of
|
67
|
+
# Deimos::Consumer or Deimos::Producer, or provide the
|
68
|
+
# schema, namespace and key_config directly.
|
69
|
+
# @param topic [String]
|
70
|
+
# @param config_class [Class < Deimos::Consumer|Deimos::Producer>]
|
71
|
+
# @param schema [String]
|
72
|
+
# @param namespace [String]
|
73
|
+
# @param key_config [Hash]
|
74
|
+
# @param num_messages [Number]
|
75
|
+
# @return [Array<Hash>]
|
76
|
+
def self.get_messages_for(topic:, schema: nil, namespace: nil, key_config: nil,
|
77
|
+
config_class: nil, num_messages: 10)
|
78
|
+
if config_class
|
79
|
+
MessageBankHandler.config_class = config_class
|
80
|
+
elsif schema.nil? || key_config.nil?
|
81
|
+
raise 'You must specify either a config_class or a schema, namespace and key_config!'
|
82
|
+
else
|
83
|
+
MessageBankHandler.class_eval do
|
84
|
+
schema schema
|
85
|
+
namespace namespace
|
86
|
+
key_config key_config
|
87
|
+
@decoder = nil
|
88
|
+
@key_decoder = nil
|
89
|
+
end
|
90
|
+
end
|
91
|
+
self.consume(topic: topic,
|
92
|
+
frk_consumer: MessageBankHandler,
|
93
|
+
num_messages: num_messages)
|
94
|
+
messages = MessageBankHandler.total_messages
|
95
|
+
messages.size <= num_messages ? messages : messages[-num_messages..-1]
|
96
|
+
end
|
97
|
+
|
98
|
+
# Consume the last X messages from a topic.
|
99
|
+
# @param topic [String]
|
100
|
+
# @param frk_consumer [Class]
|
101
|
+
# @param num_messages [Integer] If this number is >= the number
|
102
|
+
# of messages in the topic, all messages will be consumed.
|
103
|
+
def self.consume(topic:, frk_consumer:, num_messages: 10)
|
104
|
+
listener = SeekListener.new(
|
105
|
+
handler: frk_consumer,
|
106
|
+
group_id: SecureRandom.hex,
|
107
|
+
topic: topic,
|
108
|
+
heartbeat_interval: 1
|
109
|
+
)
|
110
|
+
listener.num_messages = num_messages
|
111
|
+
|
112
|
+
# Add the start_time and last_message_time attributes to the
|
113
|
+
# consumer class so we can kill it if it's gone on too long
|
114
|
+
class << frk_consumer
|
115
|
+
attr_accessor :start_time, :last_message_time
|
116
|
+
end
|
117
|
+
|
118
|
+
subscribers = []
|
119
|
+
subscribers << ActiveSupport::Notifications.
|
120
|
+
subscribe('phobos.listener.process_message') do
|
121
|
+
frk_consumer.last_message_time = Time.zone.now
|
122
|
+
end
|
123
|
+
subscribers << ActiveSupport::Notifications.
|
124
|
+
subscribe('phobos.listener.start_handler') do
|
125
|
+
frk_consumer.start_time = Time.zone.now
|
126
|
+
frk_consumer.last_message_time = nil
|
127
|
+
end
|
128
|
+
subscribers << ActiveSupport::Notifications.
|
129
|
+
subscribe('heartbeat.consumer.kafka') do
|
130
|
+
if frk_consumer.last_message_time
|
131
|
+
if Time.zone.now - frk_consumer.last_message_time > MAX_MESSAGE_WAIT_TIME
|
132
|
+
raise Phobos::AbortError
|
133
|
+
end
|
134
|
+
elsif Time.zone.now - frk_consumer.start_time > MAX_TOPIC_WAIT_TIME
|
135
|
+
Deimos.config.logger.error('Aborting - initial wait too long')
|
136
|
+
raise Phobos::AbortError
|
137
|
+
end
|
138
|
+
end
|
139
|
+
listener.start
|
140
|
+
subscribers.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,182 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'mutex_m'
|
4
|
+
|
5
|
+
# :nodoc:
|
6
|
+
module Deimos
|
7
|
+
module Utils
|
8
|
+
# Class that manages reporting lag.
|
9
|
+
class LagReporter
|
10
|
+
extend Mutex_m
|
11
|
+
|
12
|
+
# Class that has a list of topics
|
13
|
+
class ConsumerGroup
|
14
|
+
# @return [Hash<String, Topic>]
|
15
|
+
attr_accessor :topics
|
16
|
+
# @return [String]
|
17
|
+
attr_accessor :id
|
18
|
+
|
19
|
+
# @param id [String]
|
20
|
+
def initialize(id)
|
21
|
+
self.id = id
|
22
|
+
self.topics = {}
|
23
|
+
end
|
24
|
+
|
25
|
+
# @param topic [String]
|
26
|
+
# @param partition [Integer]
|
27
|
+
def report_lag(topic, partition)
|
28
|
+
self.topics[topic.to_s] ||= Topic.new(topic, self)
|
29
|
+
self.topics[topic.to_s].report_lag(partition)
|
30
|
+
end
|
31
|
+
|
32
|
+
# @param topic [String]
|
33
|
+
# @param partition [Integer]
|
34
|
+
# @param lag [Integer]
|
35
|
+
def assign_lag(topic, partition, lag)
|
36
|
+
self.topics[topic.to_s] ||= Topic.new(topic, self)
|
37
|
+
self.topics[topic.to_s].assign_lag(partition, lag)
|
38
|
+
end
|
39
|
+
|
40
|
+
# Figure out the current lag by asking Kafka based on the current offset.
|
41
|
+
# @param topic [String]
|
42
|
+
# @param partition [Integer]
|
43
|
+
# @param offset [Integer]
|
44
|
+
def compute_lag(topic, partition, offset)
|
45
|
+
self.topics[topic.to_s] ||= Topic.new(topic, self)
|
46
|
+
self.topics[topic.to_s].compute_lag(partition, offset)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# Topic which has a hash of partition => last known offset lag
|
51
|
+
class Topic
|
52
|
+
# @return [String]
|
53
|
+
attr_accessor :topic_name
|
54
|
+
# @return [Hash<Integer, Integer>]
|
55
|
+
attr_accessor :partition_offset_lags
|
56
|
+
# @return [ConsumerGroup]
|
57
|
+
attr_accessor :consumer_group
|
58
|
+
|
59
|
+
# @param topic_name [String]
|
60
|
+
# @param group [ConsumerGroup]
|
61
|
+
def initialize(topic_name, group)
|
62
|
+
self.topic_name = topic_name
|
63
|
+
self.consumer_group = group
|
64
|
+
self.partition_offset_lags = {}
|
65
|
+
end
|
66
|
+
|
67
|
+
# @param partition [Integer]
|
68
|
+
# @param lag [Integer]
|
69
|
+
def assign_lag(partition, lag)
|
70
|
+
self.partition_offset_lags[partition.to_i] = lag
|
71
|
+
end
|
72
|
+
|
73
|
+
# @param partition [Integer]
|
74
|
+
# @param offset [Integer]
|
75
|
+
def compute_lag(partition, offset)
|
76
|
+
return if self.partition_offset_lags[partition.to_i]
|
77
|
+
|
78
|
+
begin
|
79
|
+
client = Phobos.create_kafka_client
|
80
|
+
last_offset = client.last_offset_for(self.topic_name, partition)
|
81
|
+
assign_lag(partition, [last_offset - offset, 0].max)
|
82
|
+
rescue StandardError # don't do anything, just wait
|
83
|
+
Deimos.config.logger.
|
84
|
+
debug("Error computing lag for #{self.topic_name}, will retry")
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# @param partition [Integer]
|
89
|
+
def report_lag(partition)
|
90
|
+
lag = self.partition_offset_lags[partition.to_i]
|
91
|
+
return unless lag
|
92
|
+
|
93
|
+
group = self.consumer_group.id
|
94
|
+
Deimos.config.logger.
|
95
|
+
debug("Sending lag: #{group}/#{partition}: #{lag}")
|
96
|
+
Deimos.config.metrics&.gauge('consumer_lag', lag, tags: %W(
|
97
|
+
consumer_group:#{group}
|
98
|
+
partition:#{partition}
|
99
|
+
topic:#{self.topic_name}
|
100
|
+
))
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
@groups = {}
|
105
|
+
|
106
|
+
class << self
|
107
|
+
# Reset all group information.
|
108
|
+
def reset
|
109
|
+
@groups = {}
|
110
|
+
end
|
111
|
+
|
112
|
+
# @param payload [Hash]
|
113
|
+
def message_processed(payload)
|
114
|
+
lag = payload[:offset_lag]
|
115
|
+
topic = payload[:topic]
|
116
|
+
group = payload[:group_id]
|
117
|
+
partition = payload[:partition]
|
118
|
+
|
119
|
+
synchronize do
|
120
|
+
@groups[group.to_s] ||= ConsumerGroup.new(group)
|
121
|
+
@groups[group.to_s].assign_lag(topic, partition, lag)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
# @param payload [Hash]
|
126
|
+
def offset_seek(payload)
|
127
|
+
offset = payload[:offset]
|
128
|
+
topic = payload[:topic]
|
129
|
+
group = payload[:group_id]
|
130
|
+
partition = payload[:partition]
|
131
|
+
|
132
|
+
synchronize do
|
133
|
+
@groups[group.to_s] ||= ConsumerGroup.new(group)
|
134
|
+
@groups[group.to_s].compute_lag(topic, partition, offset)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
# @param payload [Hash]
|
139
|
+
def heartbeat(payload)
|
140
|
+
group = payload[:group_id]
|
141
|
+
synchronize do
|
142
|
+
@groups[group.to_s] ||= ConsumerGroup.new(group)
|
143
|
+
consumer_group = @groups[group.to_s]
|
144
|
+
payload[:topic_partitions].each do |topic, partitions|
|
145
|
+
partitions.each do |partition|
|
146
|
+
consumer_group.report_lag(topic, partition)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
ActiveSupport::Notifications.subscribe('start_process_message.consumer.kafka') do |*args|
|
156
|
+
next unless Deimos.config.report_lag
|
157
|
+
|
158
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
159
|
+
Deimos::Utils::LagReporter.message_processed(event.payload)
|
160
|
+
end
|
161
|
+
|
162
|
+
ActiveSupport::Notifications.subscribe('start_process_batch.consumer.kafka') do |*args|
|
163
|
+
next unless Deimos.config.report_lag
|
164
|
+
|
165
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
166
|
+
Deimos::Utils::LagReporter.message_processed(event.payload)
|
167
|
+
end
|
168
|
+
|
169
|
+
ActiveSupport::Notifications.subscribe('seek.consumer.kafka') do |*args|
|
170
|
+
next unless Deimos.config.report_lag
|
171
|
+
|
172
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
173
|
+
Deimos::Utils::LagReporter.offset_seek(event.payload)
|
174
|
+
end
|
175
|
+
|
176
|
+
ActiveSupport::Notifications.subscribe('heartbeat.consumer.kafka') do |*args|
|
177
|
+
next unless Deimos.config.report_lag
|
178
|
+
|
179
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
180
|
+
Deimos::Utils::LagReporter.heartbeat(event.payload)
|
181
|
+
end
|
182
|
+
end
|
File without changes
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Deimos
|
4
|
+
module Utils
|
5
|
+
# Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
|
6
|
+
# basically replace that implementation with this one to make it more generic.
|
7
|
+
class SignalHandler
|
8
|
+
SIGNALS = %i(INT TERM QUIT).freeze
|
9
|
+
|
10
|
+
# Takes any object that responds to the `start` and `stop` methods.
|
11
|
+
# @param runner[#start, #stop]
|
12
|
+
def initialize(runner)
|
13
|
+
@signal_queue = []
|
14
|
+
@reader, @writer = IO.pipe
|
15
|
+
@runner = runner
|
16
|
+
end
|
17
|
+
|
18
|
+
# Run the runner.
|
19
|
+
def run!
|
20
|
+
setup_signals
|
21
|
+
@runner.start
|
22
|
+
|
23
|
+
loop do
|
24
|
+
case signal_queue.pop
|
25
|
+
when *SIGNALS
|
26
|
+
@runner.stop
|
27
|
+
break
|
28
|
+
else
|
29
|
+
ready = IO.select([reader, writer])
|
30
|
+
|
31
|
+
# drain the self-pipe so it won't be returned again next time
|
32
|
+
reader.read_nonblock(1) if ready[0].include?(reader)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
attr_reader :reader, :writer, :signal_queue, :executor
|
40
|
+
|
41
|
+
# https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
|
42
|
+
def prepend_handler(signal)
|
43
|
+
previous = Signal.trap(signal) do
|
44
|
+
previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
|
45
|
+
yield
|
46
|
+
previous.call
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# Trap signals using the self-pipe trick.
|
51
|
+
def setup_signals
|
52
|
+
at_exit { @runner&.stop }
|
53
|
+
SIGNALS.each do |signal|
|
54
|
+
prepend_handler(signal) do
|
55
|
+
unblock(signal)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# Save the signal to the queue and continue on.
|
61
|
+
# @param signal [Symbol]
|
62
|
+
def unblock(signal)
|
63
|
+
writer.write_nonblock('.')
|
64
|
+
signal_queue << signal
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
data/lib/deimos.rb
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'avro-patches'
|
4
|
+
require 'avro_turf'
|
5
|
+
require 'phobos'
|
6
|
+
require 'deimos/version'
|
7
|
+
require 'deimos/avro_data_encoder'
|
8
|
+
require 'deimos/avro_data_decoder'
|
9
|
+
require 'deimos/producer'
|
10
|
+
require 'deimos/active_record_producer'
|
11
|
+
require 'deimos/active_record_consumer'
|
12
|
+
require 'deimos/consumer'
|
13
|
+
require 'deimos/configuration'
|
14
|
+
require 'deimos/instrumentation'
|
15
|
+
require 'deimos/utils/lag_reporter'
|
16
|
+
|
17
|
+
require 'deimos/publish_backend'
|
18
|
+
require 'deimos/backends/kafka'
|
19
|
+
require 'deimos/backends/kafka_async'
|
20
|
+
|
21
|
+
require 'deimos/monkey_patches/ruby_kafka_heartbeat'
|
22
|
+
require 'deimos/monkey_patches/schema_store'
|
23
|
+
require 'deimos/monkey_patches/phobos_producer'
|
24
|
+
require 'deimos/monkey_patches/phobos_cli'
|
25
|
+
|
26
|
+
require 'deimos/railtie' if defined?(Rails)
|
27
|
+
if defined?(ActiveRecord)
|
28
|
+
require 'deimos/kafka_source'
|
29
|
+
require 'deimos/kafka_topic_info'
|
30
|
+
require 'deimos/backends/db'
|
31
|
+
require 'deimos/utils/signal_handler.rb'
|
32
|
+
require 'deimos/utils/executor.rb'
|
33
|
+
require 'deimos/utils/db_producer.rb'
|
34
|
+
end
|
35
|
+
require 'deimos/utils/inline_consumer'
|
36
|
+
require 'yaml'
|
37
|
+
require 'erb'
|
38
|
+
|
39
|
+
# Parent module.
|
40
|
+
module Deimos
|
41
|
+
class << self
|
42
|
+
attr_accessor :config
|
43
|
+
|
44
|
+
# Configure Deimos.
|
45
|
+
def configure
|
46
|
+
first_time_config = self.config.nil?
|
47
|
+
self.config ||= Configuration.new
|
48
|
+
old_config = self.config.dup
|
49
|
+
yield(config)
|
50
|
+
|
51
|
+
# Don't re-configure Phobos every time
|
52
|
+
if first_time_config || config.phobos_config_changed?(old_config)
|
53
|
+
|
54
|
+
file = config.phobos_config_file
|
55
|
+
phobos_config = YAML.load(ERB.new(File.read(File.expand_path(file))).result)
|
56
|
+
|
57
|
+
configure_kafka_for_phobos(phobos_config)
|
58
|
+
configure_loggers(phobos_config)
|
59
|
+
|
60
|
+
Phobos.configure(phobos_config)
|
61
|
+
end
|
62
|
+
|
63
|
+
validate_db_backend if self.config.publish_backend == :db
|
64
|
+
end
|
65
|
+
|
66
|
+
# Ensure everything is set up correctly for the DB backend.
|
67
|
+
def validate_db_backend
|
68
|
+
begin
|
69
|
+
require 'activerecord-import'
|
70
|
+
rescue LoadError
|
71
|
+
raise 'Cannot set publish_backend to :db without activerecord-import! Please add it to your Gemfile.'
|
72
|
+
end
|
73
|
+
if Phobos.config.producer_hash[:required_acks] != :all
|
74
|
+
raise 'Cannot set publish_backend to :db unless required_acks is set to ":all" in phobos.yml!'
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# Start the DB producers to send Kafka messages.
|
79
|
+
# @param thread_count [Integer] the number of threads to start.
|
80
|
+
def start_db_backend!(thread_count: 1)
|
81
|
+
if self.config.publish_backend != :db # rubocop:disable Style/IfUnlessModifier
|
82
|
+
raise('Publish backend is not set to :db, exiting')
|
83
|
+
end
|
84
|
+
|
85
|
+
if thread_count.nil? || thread_count.zero?
|
86
|
+
raise('Thread count is not given or set to zero, exiting')
|
87
|
+
end
|
88
|
+
|
89
|
+
producers = (1..thread_count).map do
|
90
|
+
Deimos::Utils::DbProducer.new(self.config.logger)
|
91
|
+
end
|
92
|
+
executor = Deimos::Utils::Executor.new(producers,
|
93
|
+
self.config.logger)
|
94
|
+
signal_handler = Deimos::Utils::SignalHandler.new(executor)
|
95
|
+
signal_handler.run!
|
96
|
+
end
|
97
|
+
|
98
|
+
# @param phobos_config [Hash]
|
99
|
+
def configure_kafka_for_phobos(phobos_config)
|
100
|
+
if config.ssl_enabled
|
101
|
+
%w(ssl_ca_cert ssl_client_cert ssl_client_cert_key).each do |key|
|
102
|
+
next if config.send(key).blank?
|
103
|
+
|
104
|
+
phobos_config['kafka'][key] = ssl_var_contents(config.send(key))
|
105
|
+
end
|
106
|
+
end
|
107
|
+
phobos_config['kafka']['seed_brokers'] = config.seed_broker if config.seed_broker
|
108
|
+
end
|
109
|
+
|
110
|
+
# @param phobos_config [Hash]
|
111
|
+
def configure_loggers(phobos_config)
|
112
|
+
phobos_config['custom_logger'] = config.phobos_logger
|
113
|
+
phobos_config['custom_kafka_logger'] = config.kafka_logger
|
114
|
+
end
|
115
|
+
|
116
|
+
# @param filename [String] a file to read, or the contents of the SSL var
|
117
|
+
# @return [String] the contents of the file
|
118
|
+
def ssl_var_contents(filename)
|
119
|
+
File.exist?(filename) ? File.read(filename) : filename
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
at_exit do
|
125
|
+
begin
|
126
|
+
Deimos::Backends::KafkaAsync.producer.async_producer_shutdown
|
127
|
+
Deimos::Backends::KafkaAsync.producer.kafka_client&.close
|
128
|
+
rescue StandardError => e
|
129
|
+
Deimos.config.logger.error(
|
130
|
+
"Error closing async producer on shutdown: #{e.message} #{e.backtrace.join("\n")}"
|
131
|
+
)
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
+
def change
|
3
|
+
create_table :kafka_messages, force: true do |t|
|
4
|
+
t.string :topic, null: false
|
5
|
+
t.binary :message, limit: 10.megabytes
|
6
|
+
t.binary :key
|
7
|
+
t.string :partition_key
|
8
|
+
t.timestamps
|
9
|
+
end
|
10
|
+
|
11
|
+
add_index :kafka_messages, [:topic, :id]
|
12
|
+
|
13
|
+
create_table :kafka_topic_info, force: true do |t| # rubocop:disable Rails/CreateTableWithTimestamps
|
14
|
+
t.string :topic, null: false
|
15
|
+
t.string :locked_by
|
16
|
+
t.datetime :locked_at
|
17
|
+
t.boolean :error, null: false, default: false
|
18
|
+
t.integer :retries, null: false, default: 0
|
19
|
+
end
|
20
|
+
add_index :kafka_topic_info, :topic, unique: true
|
21
|
+
add_index :kafka_topic_info, [:locked_by, :error]
|
22
|
+
add_index :kafka_topic_info, :locked_at
|
23
|
+
end
|
24
|
+
end
|