deimos-ruby 1.0.0.pre.beta22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +74 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +321 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +32 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +165 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +752 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-kafka.gemspec +42 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/DATABASE_BACKEND.md +147 -0
  22. data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
  23. data/lib/deimos/active_record_consumer.rb +81 -0
  24. data/lib/deimos/active_record_producer.rb +64 -0
  25. data/lib/deimos/avro_data_coder.rb +89 -0
  26. data/lib/deimos/avro_data_decoder.rb +36 -0
  27. data/lib/deimos/avro_data_encoder.rb +51 -0
  28. data/lib/deimos/backends/db.rb +27 -0
  29. data/lib/deimos/backends/kafka.rb +27 -0
  30. data/lib/deimos/backends/kafka_async.rb +27 -0
  31. data/lib/deimos/configuration.rb +90 -0
  32. data/lib/deimos/consumer.rb +164 -0
  33. data/lib/deimos/instrumentation.rb +71 -0
  34. data/lib/deimos/kafka_message.rb +27 -0
  35. data/lib/deimos/kafka_source.rb +126 -0
  36. data/lib/deimos/kafka_topic_info.rb +86 -0
  37. data/lib/deimos/message.rb +74 -0
  38. data/lib/deimos/metrics/datadog.rb +47 -0
  39. data/lib/deimos/metrics/mock.rb +39 -0
  40. data/lib/deimos/metrics/provider.rb +38 -0
  41. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  42. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  43. data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
  44. data/lib/deimos/monkey_patches/schema_store.rb +19 -0
  45. data/lib/deimos/producer.rb +218 -0
  46. data/lib/deimos/publish_backend.rb +30 -0
  47. data/lib/deimos/railtie.rb +8 -0
  48. data/lib/deimos/schema_coercer.rb +108 -0
  49. data/lib/deimos/shared_config.rb +59 -0
  50. data/lib/deimos/test_helpers.rb +356 -0
  51. data/lib/deimos/tracing/datadog.rb +35 -0
  52. data/lib/deimos/tracing/mock.rb +40 -0
  53. data/lib/deimos/tracing/provider.rb +31 -0
  54. data/lib/deimos/utils/db_producer.rb +122 -0
  55. data/lib/deimos/utils/executor.rb +117 -0
  56. data/lib/deimos/utils/inline_consumer.rb +144 -0
  57. data/lib/deimos/utils/lag_reporter.rb +182 -0
  58. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  59. data/lib/deimos/utils/signal_handler.rb +68 -0
  60. data/lib/deimos/version.rb +5 -0
  61. data/lib/deimos.rb +133 -0
  62. data/lib/generators/deimos/db_backend/templates/migration +24 -0
  63. data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
  64. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  65. data/lib/tasks/deimos.rake +27 -0
  66. data/spec/active_record_consumer_spec.rb +81 -0
  67. data/spec/active_record_producer_spec.rb +107 -0
  68. data/spec/avro_data_decoder_spec.rb +18 -0
  69. data/spec/avro_data_encoder_spec.rb +37 -0
  70. data/spec/backends/db_spec.rb +35 -0
  71. data/spec/backends/kafka_async_spec.rb +11 -0
  72. data/spec/backends/kafka_spec.rb +11 -0
  73. data/spec/consumer_spec.rb +169 -0
  74. data/spec/deimos_spec.rb +120 -0
  75. data/spec/kafka_source_spec.rb +168 -0
  76. data/spec/kafka_topic_info_spec.rb +88 -0
  77. data/spec/phobos.bad_db.yml +73 -0
  78. data/spec/phobos.yml +73 -0
  79. data/spec/producer_spec.rb +397 -0
  80. data/spec/publish_backend_spec.rb +10 -0
  81. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  82. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  83. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  84. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  85. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  86. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  87. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  88. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  89. data/spec/spec_helper.rb +207 -0
  90. data/spec/updateable_schema_store_spec.rb +36 -0
  91. data/spec/utils/db_producer_spec.rb +259 -0
  92. data/spec/utils/executor_spec.rb +42 -0
  93. data/spec/utils/lag_reporter_spec.rb +69 -0
  94. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  95. data/spec/utils/signal_handler_spec.rb +16 -0
  96. data/support/deimos-solo.png +0 -0
  97. data/support/deimos-with-name-next.png +0 -0
  98. data/support/deimos-with-name.png +0 -0
  99. data/support/flipp-logo.png +0 -0
  100. metadata +452 -0
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ # rubocop:disable Lint/RescueException
4
+ module Deimos
5
+ module Utils
6
+ # Mostly copied from Phobos::Executor. We should DRY this up by putting in a
7
+ # PR to make it more generic. Might even make sense to move to a separate
8
+ # gem.
9
+ class Executor
10
+ # @return [Array<#start, #stop, #id>]
11
+ attr_accessor :runners
12
+
13
+ # @param runners [Array<#start, #stop, #id>] A list of objects that can be
14
+ # started or stopped.
15
+ # @param logger [Logger]
16
+ def initialize(runners, logger=Logger.new(STDOUT))
17
+ @threads = Concurrent::Array.new
18
+ @runners = runners
19
+ @logger = logger
20
+ end
21
+
22
+ # Start the executor.
23
+ def start
24
+ @logger.info('Starting executor')
25
+ @signal_to_stop = false
26
+ @threads.clear
27
+ @thread_pool = Concurrent::FixedThreadPool.new(@runners.size)
28
+
29
+ @runners.each do |runner|
30
+ @thread_pool.post do
31
+ thread = Thread.current
32
+ thread.abort_on_exception = true
33
+ @threads << thread
34
+ run_object(runner)
35
+ end
36
+ end
37
+
38
+ true
39
+ end
40
+
41
+ # Stop the executor.
42
+ def stop
43
+ return if @signal_to_stop
44
+
45
+ @logger.info('Stopping executor')
46
+ @signal_to_stop = true
47
+ @runners.each(&:stop)
48
+ @threads.select(&:alive?).each do |thread|
49
+ begin
50
+ thread.wakeup
51
+ rescue StandardError
52
+ nil
53
+ end
54
+ end
55
+ @thread_pool&.shutdown
56
+ @thread_pool&.wait_for_termination
57
+ @logger.info('Executor stopped')
58
+ end
59
+
60
+ private
61
+
62
+ # @param exception [Throwable]
63
+ # @return [Hash]
64
+ def error_metadata(exception)
65
+ {
66
+ exception_class: exception.class.name,
67
+ exception_message: exception.message,
68
+ backtrace: exception.backtrace
69
+ }
70
+ end
71
+
72
+ def run_object(runner)
73
+ retry_count = 0
74
+
75
+ begin
76
+ @logger.info("Running #{runner.id}")
77
+ runner.start
78
+ retry_count = 0 # success - reset retry count
79
+ rescue Exception => e
80
+ handle_crashed_runner(runner, e, retry_count)
81
+ retry_count += 1
82
+ retry unless @signal_to_stop
83
+ end
84
+ rescue Exception => e
85
+ @logger.error("Failed to run listener (#{e.message}) #{error_metadata(e)}")
86
+ raise e
87
+ end
88
+
89
+ # @return [ExponentialBackoff]
90
+ def create_exponential_backoff
91
+ min = 1
92
+ max = 60
93
+ ExponentialBackoff.new(min, max).tap do |backoff|
94
+ backoff.randomize_factor = rand
95
+ end
96
+ end
97
+
98
+ # When "runner#start" is interrupted / crashes we assume it's
99
+ # safe to be called again
100
+ def handle_crashed_runner(runner, error, retry_count)
101
+ backoff = create_exponential_backoff
102
+ interval = backoff.interval_at(retry_count).round(2)
103
+
104
+ metadata = {
105
+ listener_id: runner.id,
106
+ retry_count: retry_count,
107
+ waiting_time: interval
108
+ }.merge(error_metadata(error))
109
+
110
+ @logger.error("Runner crashed, waiting #{interval}s (#{error.message}) #{metadata}")
111
+ sleep(interval)
112
+ end
113
+ end
114
+ end
115
+ end
116
+
117
+ # rubocop:enable Lint/RescueException
@@ -0,0 +1,144 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Class to consume messages. Can be used with integration testing frameworks.
4
+ # Assumes that you have a topic with only one partition.
5
+ module Deimos
6
+ module Utils
7
+ # Listener that can seek to get the last X messages in a topic.
8
+ class SeekListener < Phobos::Listener
9
+ attr_accessor :num_messages
10
+
11
+ # :nodoc:
12
+ def start_listener
13
+ @num_messages ||= 10
14
+ @consumer = create_kafka_consumer
15
+ @consumer.subscribe(topic, @subscribe_opts)
16
+
17
+ begin
18
+ last_offset = @kafka_client.last_offset_for(topic, 0)
19
+ offset = last_offset - num_messages
20
+ if offset.positive?
21
+ Deimos.config.logger.info("Seeking to #{offset}")
22
+ @consumer.seek(topic, 0, offset)
23
+ end
24
+ rescue StandardError => e
25
+ "Could not seek to offset: #{e.message}"
26
+ end
27
+
28
+ instrument('listener.start_handler', listener_metadata) do
29
+ @handler_class.start(@kafka_client)
30
+ end
31
+ log_info('Listener started', listener_metadata)
32
+ end
33
+ end
34
+
35
+ # Class to return the messages consumed.
36
+ class MessageBankHandler < Deimos::Consumer
37
+ include Phobos::Handler
38
+
39
+ cattr_accessor :total_messages
40
+
41
+ # @param klass [Class < Deimos::Consumer]
42
+ def self.config_class=(klass)
43
+ self.config.merge!(klass.config)
44
+ end
45
+
46
+ # :nodoc:
47
+ def self.start(_kafka_client)
48
+ self.total_messages = []
49
+ end
50
+
51
+ # :nodoc:
52
+ def consume(payload, metadata)
53
+ puts "Got #{payload}"
54
+ self.class.total_messages << {
55
+ key: metadata[:key],
56
+ payload: payload
57
+ }
58
+ end
59
+ end
60
+
61
+ # Class which can process/consume messages inline.
62
+ class InlineConsumer
63
+ MAX_MESSAGE_WAIT_TIME = 1.second
64
+ MAX_TOPIC_WAIT_TIME = 10.seconds
65
+
66
+ # Get the last X messages from a topic. You can specify a subclass of
67
+ # Deimos::Consumer or Deimos::Producer, or provide the
68
+ # schema, namespace and key_config directly.
69
+ # @param topic [String]
70
+ # @param config_class [Class < Deimos::Consumer|Deimos::Producer>]
71
+ # @param schema [String]
72
+ # @param namespace [String]
73
+ # @param key_config [Hash]
74
+ # @param num_messages [Number]
75
+ # @return [Array<Hash>]
76
+ def self.get_messages_for(topic:, schema: nil, namespace: nil, key_config: nil,
77
+ config_class: nil, num_messages: 10)
78
+ if config_class
79
+ MessageBankHandler.config_class = config_class
80
+ elsif schema.nil? || key_config.nil?
81
+ raise 'You must specify either a config_class or a schema, namespace and key_config!'
82
+ else
83
+ MessageBankHandler.class_eval do
84
+ schema schema
85
+ namespace namespace
86
+ key_config key_config
87
+ @decoder = nil
88
+ @key_decoder = nil
89
+ end
90
+ end
91
+ self.consume(topic: topic,
92
+ frk_consumer: MessageBankHandler,
93
+ num_messages: num_messages)
94
+ messages = MessageBankHandler.total_messages
95
+ messages.size <= num_messages ? messages : messages[-num_messages..-1]
96
+ end
97
+
98
+ # Consume the last X messages from a topic.
99
+ # @param topic [String]
100
+ # @param frk_consumer [Class]
101
+ # @param num_messages [Integer] If this number is >= the number
102
+ # of messages in the topic, all messages will be consumed.
103
+ def self.consume(topic:, frk_consumer:, num_messages: 10)
104
+ listener = SeekListener.new(
105
+ handler: frk_consumer,
106
+ group_id: SecureRandom.hex,
107
+ topic: topic,
108
+ heartbeat_interval: 1
109
+ )
110
+ listener.num_messages = num_messages
111
+
112
+ # Add the start_time and last_message_time attributes to the
113
+ # consumer class so we can kill it if it's gone on too long
114
+ class << frk_consumer
115
+ attr_accessor :start_time, :last_message_time
116
+ end
117
+
118
+ subscribers = []
119
+ subscribers << ActiveSupport::Notifications.
120
+ subscribe('phobos.listener.process_message') do
121
+ frk_consumer.last_message_time = Time.zone.now
122
+ end
123
+ subscribers << ActiveSupport::Notifications.
124
+ subscribe('phobos.listener.start_handler') do
125
+ frk_consumer.start_time = Time.zone.now
126
+ frk_consumer.last_message_time = nil
127
+ end
128
+ subscribers << ActiveSupport::Notifications.
129
+ subscribe('heartbeat.consumer.kafka') do
130
+ if frk_consumer.last_message_time
131
+ if Time.zone.now - frk_consumer.last_message_time > MAX_MESSAGE_WAIT_TIME
132
+ raise Phobos::AbortError
133
+ end
134
+ elsif Time.zone.now - frk_consumer.start_time > MAX_TOPIC_WAIT_TIME
135
+ Deimos.config.logger.error('Aborting - initial wait too long')
136
+ raise Phobos::AbortError
137
+ end
138
+ end
139
+ listener.start
140
+ subscribers.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,182 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'mutex_m'
4
+
5
+ # :nodoc:
6
+ module Deimos
7
+ module Utils
8
+ # Class that manages reporting lag.
9
+ class LagReporter
10
+ extend Mutex_m
11
+
12
+ # Class that has a list of topics
13
+ class ConsumerGroup
14
+ # @return [Hash<String, Topic>]
15
+ attr_accessor :topics
16
+ # @return [String]
17
+ attr_accessor :id
18
+
19
+ # @param id [String]
20
+ def initialize(id)
21
+ self.id = id
22
+ self.topics = {}
23
+ end
24
+
25
+ # @param topic [String]
26
+ # @param partition [Integer]
27
+ def report_lag(topic, partition)
28
+ self.topics[topic.to_s] ||= Topic.new(topic, self)
29
+ self.topics[topic.to_s].report_lag(partition)
30
+ end
31
+
32
+ # @param topic [String]
33
+ # @param partition [Integer]
34
+ # @param lag [Integer]
35
+ def assign_lag(topic, partition, lag)
36
+ self.topics[topic.to_s] ||= Topic.new(topic, self)
37
+ self.topics[topic.to_s].assign_lag(partition, lag)
38
+ end
39
+
40
+ # Figure out the current lag by asking Kafka based on the current offset.
41
+ # @param topic [String]
42
+ # @param partition [Integer]
43
+ # @param offset [Integer]
44
+ def compute_lag(topic, partition, offset)
45
+ self.topics[topic.to_s] ||= Topic.new(topic, self)
46
+ self.topics[topic.to_s].compute_lag(partition, offset)
47
+ end
48
+ end
49
+
50
+ # Topic which has a hash of partition => last known offset lag
51
+ class Topic
52
+ # @return [String]
53
+ attr_accessor :topic_name
54
+ # @return [Hash<Integer, Integer>]
55
+ attr_accessor :partition_offset_lags
56
+ # @return [ConsumerGroup]
57
+ attr_accessor :consumer_group
58
+
59
+ # @param topic_name [String]
60
+ # @param group [ConsumerGroup]
61
+ def initialize(topic_name, group)
62
+ self.topic_name = topic_name
63
+ self.consumer_group = group
64
+ self.partition_offset_lags = {}
65
+ end
66
+
67
+ # @param partition [Integer]
68
+ # @param lag [Integer]
69
+ def assign_lag(partition, lag)
70
+ self.partition_offset_lags[partition.to_i] = lag
71
+ end
72
+
73
+ # @param partition [Integer]
74
+ # @param offset [Integer]
75
+ def compute_lag(partition, offset)
76
+ return if self.partition_offset_lags[partition.to_i]
77
+
78
+ begin
79
+ client = Phobos.create_kafka_client
80
+ last_offset = client.last_offset_for(self.topic_name, partition)
81
+ assign_lag(partition, [last_offset - offset, 0].max)
82
+ rescue StandardError # don't do anything, just wait
83
+ Deimos.config.logger.
84
+ debug("Error computing lag for #{self.topic_name}, will retry")
85
+ end
86
+ end
87
+
88
+ # @param partition [Integer]
89
+ def report_lag(partition)
90
+ lag = self.partition_offset_lags[partition.to_i]
91
+ return unless lag
92
+
93
+ group = self.consumer_group.id
94
+ Deimos.config.logger.
95
+ debug("Sending lag: #{group}/#{partition}: #{lag}")
96
+ Deimos.config.metrics&.gauge('consumer_lag', lag, tags: %W(
97
+ consumer_group:#{group}
98
+ partition:#{partition}
99
+ topic:#{self.topic_name}
100
+ ))
101
+ end
102
+ end
103
+
104
+ @groups = {}
105
+
106
+ class << self
107
+ # Reset all group information.
108
+ def reset
109
+ @groups = {}
110
+ end
111
+
112
+ # @param payload [Hash]
113
+ def message_processed(payload)
114
+ lag = payload[:offset_lag]
115
+ topic = payload[:topic]
116
+ group = payload[:group_id]
117
+ partition = payload[:partition]
118
+
119
+ synchronize do
120
+ @groups[group.to_s] ||= ConsumerGroup.new(group)
121
+ @groups[group.to_s].assign_lag(topic, partition, lag)
122
+ end
123
+ end
124
+
125
+ # @param payload [Hash]
126
+ def offset_seek(payload)
127
+ offset = payload[:offset]
128
+ topic = payload[:topic]
129
+ group = payload[:group_id]
130
+ partition = payload[:partition]
131
+
132
+ synchronize do
133
+ @groups[group.to_s] ||= ConsumerGroup.new(group)
134
+ @groups[group.to_s].compute_lag(topic, partition, offset)
135
+ end
136
+ end
137
+
138
+ # @param payload [Hash]
139
+ def heartbeat(payload)
140
+ group = payload[:group_id]
141
+ synchronize do
142
+ @groups[group.to_s] ||= ConsumerGroup.new(group)
143
+ consumer_group = @groups[group.to_s]
144
+ payload[:topic_partitions].each do |topic, partitions|
145
+ partitions.each do |partition|
146
+ consumer_group.report_lag(topic, partition)
147
+ end
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end
154
+
155
+ ActiveSupport::Notifications.subscribe('start_process_message.consumer.kafka') do |*args|
156
+ next unless Deimos.config.report_lag
157
+
158
+ event = ActiveSupport::Notifications::Event.new(*args)
159
+ Deimos::Utils::LagReporter.message_processed(event.payload)
160
+ end
161
+
162
+ ActiveSupport::Notifications.subscribe('start_process_batch.consumer.kafka') do |*args|
163
+ next unless Deimos.config.report_lag
164
+
165
+ event = ActiveSupport::Notifications::Event.new(*args)
166
+ Deimos::Utils::LagReporter.message_processed(event.payload)
167
+ end
168
+
169
+ ActiveSupport::Notifications.subscribe('seek.consumer.kafka') do |*args|
170
+ next unless Deimos.config.report_lag
171
+
172
+ event = ActiveSupport::Notifications::Event.new(*args)
173
+ Deimos::Utils::LagReporter.offset_seek(event.payload)
174
+ end
175
+
176
+ ActiveSupport::Notifications.subscribe('heartbeat.consumer.kafka') do |*args|
177
+ next unless Deimos.config.report_lag
178
+
179
+ event = ActiveSupport::Notifications::Event.new(*args)
180
+ Deimos::Utils::LagReporter.heartbeat(event.payload)
181
+ end
182
+ end
File without changes
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Deimos
4
+ module Utils
5
+ # Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
6
+ # basically replace that implementation with this one to make it more generic.
7
+ class SignalHandler
8
+ SIGNALS = %i(INT TERM QUIT).freeze
9
+
10
+ # Takes any object that responds to the `start` and `stop` methods.
11
+ # @param runner[#start, #stop]
12
+ def initialize(runner)
13
+ @signal_queue = []
14
+ @reader, @writer = IO.pipe
15
+ @runner = runner
16
+ end
17
+
18
+ # Run the runner.
19
+ def run!
20
+ setup_signals
21
+ @runner.start
22
+
23
+ loop do
24
+ case signal_queue.pop
25
+ when *SIGNALS
26
+ @runner.stop
27
+ break
28
+ else
29
+ ready = IO.select([reader, writer])
30
+
31
+ # drain the self-pipe so it won't be returned again next time
32
+ reader.read_nonblock(1) if ready[0].include?(reader)
33
+ end
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ attr_reader :reader, :writer, :signal_queue, :executor
40
+
41
+ # https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
42
+ def prepend_handler(signal)
43
+ previous = Signal.trap(signal) do
44
+ previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
45
+ yield
46
+ previous.call
47
+ end
48
+ end
49
+
50
+ # Trap signals using the self-pipe trick.
51
+ def setup_signals
52
+ at_exit { @runner&.stop }
53
+ SIGNALS.each do |signal|
54
+ prepend_handler(signal) do
55
+ unblock(signal)
56
+ end
57
+ end
58
+ end
59
+
60
+ # Save the signal to the queue and continue on.
61
+ # @param signal [Symbol]
62
+ def unblock(signal)
63
+ writer.write_nonblock('.')
64
+ signal_queue << signal
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Deimos
4
+ VERSION = '1.0.0-beta22'
5
+ end
data/lib/deimos.rb ADDED
@@ -0,0 +1,133 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'avro-patches'
4
+ require 'avro_turf'
5
+ require 'phobos'
6
+ require 'deimos/version'
7
+ require 'deimos/avro_data_encoder'
8
+ require 'deimos/avro_data_decoder'
9
+ require 'deimos/producer'
10
+ require 'deimos/active_record_producer'
11
+ require 'deimos/active_record_consumer'
12
+ require 'deimos/consumer'
13
+ require 'deimos/configuration'
14
+ require 'deimos/instrumentation'
15
+ require 'deimos/utils/lag_reporter'
16
+
17
+ require 'deimos/publish_backend'
18
+ require 'deimos/backends/kafka'
19
+ require 'deimos/backends/kafka_async'
20
+
21
+ require 'deimos/monkey_patches/ruby_kafka_heartbeat'
22
+ require 'deimos/monkey_patches/schema_store'
23
+ require 'deimos/monkey_patches/phobos_producer'
24
+ require 'deimos/monkey_patches/phobos_cli'
25
+
26
+ require 'deimos/railtie' if defined?(Rails)
27
+ if defined?(ActiveRecord)
28
+ require 'deimos/kafka_source'
29
+ require 'deimos/kafka_topic_info'
30
+ require 'deimos/backends/db'
31
+ require 'deimos/utils/signal_handler.rb'
32
+ require 'deimos/utils/executor.rb'
33
+ require 'deimos/utils/db_producer.rb'
34
+ end
35
+ require 'deimos/utils/inline_consumer'
36
+ require 'yaml'
37
+ require 'erb'
38
+
39
+ # Parent module.
40
+ module Deimos
41
+ class << self
42
+ attr_accessor :config
43
+
44
+ # Configure Deimos.
45
+ def configure
46
+ first_time_config = self.config.nil?
47
+ self.config ||= Configuration.new
48
+ old_config = self.config.dup
49
+ yield(config)
50
+
51
+ # Don't re-configure Phobos every time
52
+ if first_time_config || config.phobos_config_changed?(old_config)
53
+
54
+ file = config.phobos_config_file
55
+ phobos_config = YAML.load(ERB.new(File.read(File.expand_path(file))).result)
56
+
57
+ configure_kafka_for_phobos(phobos_config)
58
+ configure_loggers(phobos_config)
59
+
60
+ Phobos.configure(phobos_config)
61
+ end
62
+
63
+ validate_db_backend if self.config.publish_backend == :db
64
+ end
65
+
66
+ # Ensure everything is set up correctly for the DB backend.
67
+ def validate_db_backend
68
+ begin
69
+ require 'activerecord-import'
70
+ rescue LoadError
71
+ raise 'Cannot set publish_backend to :db without activerecord-import! Please add it to your Gemfile.'
72
+ end
73
+ if Phobos.config.producer_hash[:required_acks] != :all
74
+ raise 'Cannot set publish_backend to :db unless required_acks is set to ":all" in phobos.yml!'
75
+ end
76
+ end
77
+
78
+ # Start the DB producers to send Kafka messages.
79
+ # @param thread_count [Integer] the number of threads to start.
80
+ def start_db_backend!(thread_count: 1)
81
+ if self.config.publish_backend != :db # rubocop:disable Style/IfUnlessModifier
82
+ raise('Publish backend is not set to :db, exiting')
83
+ end
84
+
85
+ if thread_count.nil? || thread_count.zero?
86
+ raise('Thread count is not given or set to zero, exiting')
87
+ end
88
+
89
+ producers = (1..thread_count).map do
90
+ Deimos::Utils::DbProducer.new(self.config.logger)
91
+ end
92
+ executor = Deimos::Utils::Executor.new(producers,
93
+ self.config.logger)
94
+ signal_handler = Deimos::Utils::SignalHandler.new(executor)
95
+ signal_handler.run!
96
+ end
97
+
98
+ # @param phobos_config [Hash]
99
+ def configure_kafka_for_phobos(phobos_config)
100
+ if config.ssl_enabled
101
+ %w(ssl_ca_cert ssl_client_cert ssl_client_cert_key).each do |key|
102
+ next if config.send(key).blank?
103
+
104
+ phobos_config['kafka'][key] = ssl_var_contents(config.send(key))
105
+ end
106
+ end
107
+ phobos_config['kafka']['seed_brokers'] = config.seed_broker if config.seed_broker
108
+ end
109
+
110
+ # @param phobos_config [Hash]
111
+ def configure_loggers(phobos_config)
112
+ phobos_config['custom_logger'] = config.phobos_logger
113
+ phobos_config['custom_kafka_logger'] = config.kafka_logger
114
+ end
115
+
116
+ # @param filename [String] a file to read, or the contents of the SSL var
117
+ # @return [String] the contents of the file
118
+ def ssl_var_contents(filename)
119
+ File.exist?(filename) ? File.read(filename) : filename
120
+ end
121
+ end
122
+ end
123
+
124
+ at_exit do
125
+ begin
126
+ Deimos::Backends::KafkaAsync.producer.async_producer_shutdown
127
+ Deimos::Backends::KafkaAsync.producer.kafka_client&.close
128
+ rescue StandardError => e
129
+ Deimos.config.logger.error(
130
+ "Error closing async producer on shutdown: #{e.message} #{e.backtrace.join("\n")}"
131
+ )
132
+ end
133
+ end
@@ -0,0 +1,24 @@
1
+ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
2
+ def change
3
+ create_table :kafka_messages, force: true do |t|
4
+ t.string :topic, null: false
5
+ t.binary :message, limit: 10.megabytes
6
+ t.binary :key
7
+ t.string :partition_key
8
+ t.timestamps
9
+ end
10
+
11
+ add_index :kafka_messages, [:topic, :id]
12
+
13
+ create_table :kafka_topic_info, force: true do |t| # rubocop:disable Rails/CreateTableWithTimestamps
14
+ t.string :topic, null: false
15
+ t.string :locked_by
16
+ t.datetime :locked_at
17
+ t.boolean :error, null: false, default: false
18
+ t.integer :retries, null: false, default: 0
19
+ end
20
+ add_index :kafka_topic_info, :topic, unique: true
21
+ add_index :kafka_topic_info, [:locked_by, :error]
22
+ add_index :kafka_topic_info, :locked_at
23
+ end
24
+ end