deimos-ruby 1.6.3 → 1.8.1.pre.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +9 -0
  3. data/.rubocop.yml +22 -16
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +42 -0
  6. data/Gemfile.lock +125 -98
  7. data/README.md +164 -16
  8. data/Rakefile +1 -1
  9. data/deimos-ruby.gemspec +4 -3
  10. data/docs/ARCHITECTURE.md +144 -0
  11. data/docs/CONFIGURATION.md +27 -0
  12. data/lib/deimos.rb +8 -7
  13. data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
  14. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  15. data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
  16. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  17. data/lib/deimos/active_record_consumer.rb +33 -75
  18. data/lib/deimos/active_record_producer.rb +23 -0
  19. data/lib/deimos/batch_consumer.rb +2 -140
  20. data/lib/deimos/config/configuration.rb +28 -10
  21. data/lib/deimos/consume/batch_consumption.rb +150 -0
  22. data/lib/deimos/consume/message_consumption.rb +94 -0
  23. data/lib/deimos/consumer.rb +79 -70
  24. data/lib/deimos/kafka_message.rb +1 -1
  25. data/lib/deimos/kafka_topic_info.rb +22 -3
  26. data/lib/deimos/message.rb +6 -1
  27. data/lib/deimos/metrics/provider.rb +0 -2
  28. data/lib/deimos/poll_info.rb +9 -0
  29. data/lib/deimos/schema_backends/avro_base.rb +28 -1
  30. data/lib/deimos/schema_backends/base.rb +15 -2
  31. data/lib/deimos/tracing/provider.rb +0 -2
  32. data/lib/deimos/utils/db_poller.rb +149 -0
  33. data/lib/deimos/utils/db_producer.rb +59 -16
  34. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  35. data/lib/deimos/utils/lag_reporter.rb +19 -26
  36. data/lib/deimos/version.rb +1 -1
  37. data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
  38. data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
  39. data/lib/generators/deimos/active_record_generator.rb +79 -0
  40. data/lib/generators/deimos/db_backend/templates/migration +1 -0
  41. data/lib/generators/deimos/db_backend/templates/rails3_migration +1 -0
  42. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  43. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  44. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  45. data/lib/tasks/deimos.rake +7 -0
  46. data/spec/active_record_batch_consumer_spec.rb +481 -0
  47. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  48. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  49. data/spec/active_record_consumer_spec.rb +3 -11
  50. data/spec/active_record_producer_spec.rb +66 -88
  51. data/spec/batch_consumer_spec.rb +24 -7
  52. data/spec/config/configuration_spec.rb +4 -0
  53. data/spec/consumer_spec.rb +8 -8
  54. data/spec/deimos_spec.rb +57 -49
  55. data/spec/generators/active_record_generator_spec.rb +56 -0
  56. data/spec/handlers/my_batch_consumer.rb +6 -1
  57. data/spec/handlers/my_consumer.rb +6 -1
  58. data/spec/kafka_topic_info_spec.rb +39 -16
  59. data/spec/message_spec.rb +19 -0
  60. data/spec/producer_spec.rb +3 -3
  61. data/spec/rake_spec.rb +1 -1
  62. data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
  63. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  64. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  65. data/spec/spec_helper.rb +62 -6
  66. data/spec/utils/db_poller_spec.rb +320 -0
  67. data/spec/utils/db_producer_spec.rb +84 -10
  68. data/spec/utils/deadlock_retry_spec.rb +74 -0
  69. data/spec/utils/lag_reporter_spec.rb +29 -22
  70. metadata +66 -30
  71. data/lib/deimos/base_consumer.rb +0 -104
  72. data/lib/deimos/utils/executor.rb +0 -124
  73. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  74. data/lib/deimos/utils/signal_handler.rb +0 -68
  75. data/spec/utils/executor_spec.rb +0 -53
  76. data/spec/utils/signal_handler_spec.rb +0 -16
@@ -2,6 +2,7 @@
2
2
 
3
3
  $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
4
4
  require 'active_record'
5
+ require 'database_cleaner'
5
6
  require 'deimos'
6
7
  require 'deimos/metrics/mock'
7
8
  require 'deimos/tracing/mock'
@@ -31,6 +32,7 @@ module TestRunners
31
32
  # Test runner
32
33
  class TestRunner
33
34
  attr_accessor :id, :started, :stopped, :should_error
35
+
34
36
  # :nodoc:
35
37
  def initialize(id=nil)
36
38
  @id = id
@@ -100,9 +102,8 @@ module DbConfigs
100
102
  end
101
103
  end
102
104
 
103
- # Set up the given database.
104
- def setup_db(options)
105
- ActiveRecord::Base.establish_connection(options)
105
+ # :nodoc:
106
+ def run_db_backend_migration
106
107
  migration_class_name = 'DbBackendMigration'
107
108
  migration_version = '[5.2]'
108
109
  migration = ERB.new(
@@ -110,6 +111,24 @@ module DbConfigs
110
111
  ).result(binding)
111
112
  eval(migration) # rubocop:disable Security/Eval
112
113
  ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
114
+ end
115
+
116
+ # :nodoc:
117
+ def run_db_poller_migration
118
+ migration_class_name = 'DbPollerMigration'
119
+ migration_version = '[5.2]'
120
+ migration = ERB.new(
121
+ File.read('lib/generators/deimos/db_poller/templates/migration')
122
+ ).result(binding)
123
+ eval(migration) # rubocop:disable Security/Eval
124
+ ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
125
+ end
126
+
127
+ # Set up the given database.
128
+ def setup_db(options)
129
+ ActiveRecord::Base.establish_connection(options)
130
+ run_db_backend_migration
131
+ run_db_poller_migration
113
132
 
114
133
  ActiveRecord::Base.descendants.each do |klass|
115
134
  klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
@@ -130,8 +149,11 @@ RSpec.configure do |config|
130
149
  # true by default for RSpec 4.0
131
150
  config.shared_context_metadata_behavior = :apply_to_host_groups
132
151
 
152
+ config.filter_run(focus: true)
153
+ config.run_all_when_everything_filtered = true
154
+
133
155
  config.before(:all) do
134
- Time.zone = 'EST'
156
+ Time.zone = 'Eastern Time (US & Canada)'
135
157
  ActiveRecord::Base.logger = Logger.new('/dev/null')
136
158
  ActiveRecord::Base.establish_connection(
137
159
  'adapter' => 'sqlite3',
@@ -141,9 +163,10 @@ RSpec.configure do |config|
141
163
  config.include Deimos::TestHelpers
142
164
  config.include ActiveSupport::Testing::TimeHelpers
143
165
  config.before(:suite) do
144
- Time.zone = 'EST'
145
- ActiveRecord::Base.logger = Logger.new('/dev/null')
146
166
  setup_db(DbConfigs::DB_OPTIONS.last)
167
+
168
+ DatabaseCleaner.strategy = :transaction
169
+ DatabaseCleaner.clean_with(:truncation)
147
170
  end
148
171
 
149
172
  config.mock_with(:rspec) do |mocks|
@@ -164,6 +187,39 @@ RSpec.configure do |config|
164
187
  deimos_config.schema.backend = :avro_validation
165
188
  end
166
189
  end
190
+
191
+ config.around(:each) do |example|
192
+ use_cleaner = !example.metadata[:integration]
193
+
194
+ DatabaseCleaner.start if use_cleaner
195
+
196
+ example.run
197
+
198
+ DatabaseCleaner.clean if use_cleaner
199
+ end
200
+ end
201
+
202
+ RSpec.shared_context('with widgets') do
203
+ before(:all) do
204
+ ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
205
+ t.string(:test_id)
206
+ t.integer(:some_int)
207
+ t.boolean(:some_bool)
208
+ t.timestamps
209
+ end
210
+
211
+ # :nodoc:
212
+ class Widget < ActiveRecord::Base
213
+ # @return [String]
214
+ def generated_id
215
+ 'generated_id'
216
+ end
217
+ end
218
+ end
219
+
220
+ after(:all) do
221
+ ActiveRecord::Base.connection.drop_table(:widgets)
222
+ end
167
223
  end
168
224
 
169
225
  RSpec.shared_context('with DB') do
@@ -0,0 +1,320 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @param seconds [Integer]
4
+ # @return [Time]
5
+ def time_value(secs: 0, mins: 0)
6
+ Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
7
+ end
8
+
9
+ each_db_config(Deimos::Utils::DbPoller) do
10
+
11
+ before(:each) do
12
+ Deimos::PollInfo.delete_all
13
+ end
14
+
15
+ describe '#start!' do
16
+
17
+ before(:each) do
18
+ producer_class = Class.new(Deimos::Producer) do
19
+ schema 'MySchema'
20
+ namespace 'com.my-namespace'
21
+ topic 'my-topic'
22
+ key_config field: 'test_id'
23
+ end
24
+ stub_const('MyProducer', producer_class)
25
+
26
+ producer_class = Class.new(Deimos::Producer) do
27
+ schema 'MySchemaWithId'
28
+ namespace 'com.my-namespace'
29
+ topic 'my-topic'
30
+ key_config plain: true
31
+ end
32
+ stub_const('MyProducerWithID', producer_class)
33
+ end
34
+
35
+ it 'should raise an error if no pollers configured' do
36
+ Deimos.configure {}
37
+ expect { described_class.start! }.to raise_error('No pollers configured!')
38
+ end
39
+
40
+ it 'should start pollers as configured' do
41
+ Deimos.configure do
42
+ db_poller do
43
+ producer_class 'MyProducer'
44
+ end
45
+ db_poller do
46
+ producer_class 'MyProducerWithID'
47
+ end
48
+ end
49
+
50
+ allow(Deimos::Utils::DbPoller).to receive(:new)
51
+ signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
52
+ allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
53
+ described_class.start!
54
+ expect(Deimos::Utils::DbPoller).to have_received(:new).twice
55
+ expect(Deimos::Utils::DbPoller).to have_received(:new).
56
+ with(Deimos.config.db_poller_objects[0])
57
+ expect(Deimos::Utils::DbPoller).to have_received(:new).
58
+ with(Deimos.config.db_poller_objects[1])
59
+ end
60
+ end
61
+
62
+ describe 'pollers' do
63
+ include_context 'with widgets'
64
+
65
+ let(:poller) do
66
+ poller = described_class.new(config)
67
+ allow(poller).to receive(:sleep)
68
+ poller
69
+ end
70
+
71
+ let(:config) { Deimos.config.db_poller_objects.first.dup }
72
+
73
+ before(:each) do
74
+ Widget.delete_all
75
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
76
+ schema 'MySchemaWithId'
77
+ namespace 'com.my-namespace'
78
+ topic 'my-topic-with-id'
79
+ key_config none: true
80
+ record_class Widget
81
+
82
+ # :nodoc:
83
+ def self.generate_payload(attrs, widget)
84
+ super.merge(message_id: widget.generated_id)
85
+ end
86
+ end
87
+ stub_const('MyProducer', producer_class)
88
+
89
+ Deimos.configure do
90
+ db_poller do
91
+ producer_class 'MyProducer'
92
+ run_every 1.minute
93
+ end
94
+ end
95
+ end
96
+
97
+ after(:each) do
98
+ travel_back
99
+ end
100
+
101
+ it 'should crash if initialized with an invalid producer' do
102
+ config.producer_class = 'NoProducer'
103
+ expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
104
+ end
105
+
106
+ describe '#retrieve_poll_info' do
107
+
108
+ it 'should start from beginning when configured' do
109
+ poller.retrieve_poll_info
110
+ expect(Deimos::PollInfo.count).to eq(1)
111
+ info = Deimos::PollInfo.last
112
+ expect(info.producer).to eq('MyProducer')
113
+ expect(info.last_sent).to eq(Time.new(0))
114
+ expect(info.last_sent_id).to eq(0)
115
+ end
116
+
117
+ it 'should start from now when configured' do
118
+ travel_to time_value
119
+ config.start_from_beginning = false
120
+ poller.retrieve_poll_info
121
+ expect(Deimos::PollInfo.count).to eq(1)
122
+ info = Deimos::PollInfo.last
123
+ expect(info.producer).to eq('MyProducer')
124
+ expect(info.last_sent).to eq(time_value)
125
+ expect(info.last_sent_id).to eq(0)
126
+ end
127
+
128
+ end
129
+
130
+ specify '#start' do
131
+ i = 0
132
+ expect(poller).to receive(:process_updates).twice do
133
+ i += 1
134
+ poller.stop if i == 2
135
+ end
136
+ poller.start
137
+ end
138
+
139
+ specify '#should_run?' do
140
+ Deimos::PollInfo.create!(producer: 'MyProducer',
141
+ last_sent: time_value)
142
+ poller.retrieve_poll_info
143
+
144
+ # run_every is set to 1 minute
145
+ travel_to time_value(secs: 62)
146
+ expect(poller.should_run?).to eq(true)
147
+
148
+ travel_to time_value(secs: 30)
149
+ expect(poller.should_run?).to eq(false)
150
+
151
+ travel_to time_value(mins: -1) # this shouldn't be possible but meh
152
+ expect(poller.should_run?).to eq(false)
153
+
154
+ # take the 2 seconds of delay_time into account
155
+ travel_to time_value(secs: 60)
156
+ expect(poller.should_run?).to eq(false)
157
+ end
158
+
159
+ specify '#process_batch' do
160
+ travel_to time_value
161
+ widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
162
+ widgets.last.update_attribute(:updated_at, time_value(mins: -30))
163
+ expect(MyProducer).to receive(:send_events).with(widgets)
164
+ poller.retrieve_poll_info
165
+ poller.process_batch(widgets)
166
+ info = Deimos::PollInfo.last
167
+ expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
168
+ expect(info.last_sent_id).to eq(widgets.last.id)
169
+ end
170
+
171
+ describe '#process_updates' do
172
+ before(:each) do
173
+ Deimos::PollInfo.create!(producer: 'MyProducer',
174
+ last_sent: time_value(mins: -61),
175
+ last_sent_id: 0)
176
+ poller.retrieve_poll_info
177
+ travel_to time_value
178
+ stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
179
+ end
180
+
181
+ let!(:old_widget) do
182
+ # old widget, earlier than window
183
+ Widget.create!(test_id: 'some_id', some_int: 40,
184
+ updated_at: time_value(mins: -200))
185
+ end
186
+
187
+ let!(:last_widget) do
188
+ # new widget, before delay
189
+ Widget.create!(test_id: 'some_id', some_int: 10,
190
+ updated_at: time_value(secs: -1))
191
+ end
192
+
193
+ let!(:widgets) do
194
+ (1..7).map do |i|
195
+ Widget.create!(test_id: 'some_id', some_int: i,
196
+ updated_at: time_value(mins: -61, secs: 30 + i))
197
+ end
198
+ end
199
+
200
+ it 'should update the full table' do
201
+ info = Deimos::PollInfo.last
202
+ config.full_table = true
203
+ expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
204
+ expect(poller).to receive(:process_batch).ordered.
205
+ with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
206
+ m.call(*args)
207
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
208
+ expect(info.last_sent_id).to eq(widgets[1].id)
209
+ end
210
+ expect(poller).to receive(:process_batch).ordered.
211
+ with([widgets[2], widgets[3], widgets[4]]).and_call_original
212
+ expect(poller).to receive(:process_batch).ordered.
213
+ with([widgets[5], widgets[6]]).and_call_original
214
+ poller.process_updates
215
+
216
+ # this is the updated_at of widgets[6]
217
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
218
+ expect(info.last_sent_id).to eq(widgets[6].id)
219
+
220
+ last_widget.update_attribute(:updated_at, time_value(mins: -250))
221
+
222
+ travel 61.seconds
223
+ # should reprocess the table
224
+ expect(poller).to receive(:process_batch).ordered.
225
+ with([last_widget, old_widget, widgets[0]]).and_call_original
226
+ expect(poller).to receive(:process_batch).ordered.
227
+ with([widgets[1], widgets[2], widgets[3]]).and_call_original
228
+ expect(poller).to receive(:process_batch).ordered.
229
+ with([widgets[4], widgets[5], widgets[6]]).and_call_original
230
+ poller.process_updates
231
+
232
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
233
+ expect(info.last_sent_id).to eq(widgets[6].id)
234
+ end
235
+
236
+ it 'should send events across multiple batches' do
237
+ allow(MyProducer).to receive(:poll_query).and_call_original
238
+ expect(poller).to receive(:process_batch).ordered.
239
+ with([widgets[0], widgets[1], widgets[2]]).and_call_original
240
+ expect(poller).to receive(:process_batch).ordered.
241
+ with([widgets[3], widgets[4], widgets[5]]).and_call_original
242
+ expect(poller).to receive(:process_batch).ordered.
243
+ with([widgets[6]]).and_call_original
244
+ poller.process_updates
245
+
246
+ expect(MyProducer).to have_received(:poll_query).
247
+ with(time_from: time_value(mins: -61),
248
+ time_to: time_value(secs: -2),
249
+ column_name: :updated_at,
250
+ min_id: 0)
251
+
252
+ travel 61.seconds
253
+ # process the last widget which came in during the delay
254
+ expect(poller).to receive(:process_batch).with([last_widget]).
255
+ and_call_original
256
+ poller.process_updates
257
+
258
+ # widgets[6] updated_at value
259
+ expect(MyProducer).to have_received(:poll_query).
260
+ with(time_from: time_value(mins: -61, secs: 37),
261
+ time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
262
+ column_name: :updated_at,
263
+ min_id: widgets[6].id)
264
+
265
+ travel 61.seconds
266
+ # nothing else to process
267
+ expect(poller).not_to receive(:process_batch)
268
+ poller.process_updates
269
+ poller.process_updates
270
+
271
+ expect(MyProducer).to have_received(:poll_query).twice.
272
+ with(time_from: time_value(secs: -1),
273
+ time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
274
+ column_name: :updated_at,
275
+ min_id: last_widget.id)
276
+ end
277
+
278
+ it 'should recover correctly with errors and save the right ID' do
279
+ widgets.each do |w|
280
+ w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
281
+ end
282
+ allow(MyProducer).to receive(:poll_query).and_call_original
283
+ expect(poller).to receive(:process_batch).ordered.
284
+ with([widgets[0], widgets[1], widgets[2]]).and_call_original
285
+ expect(poller).to receive(:process_batch).ordered.
286
+ with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
287
+
288
+ expect { poller.process_updates }.to raise_exception('OH NOES')
289
+
290
+ expect(MyProducer).to have_received(:poll_query).
291
+ with(time_from: time_value(mins: -61),
292
+ time_to: time_value(secs: -2),
293
+ column_name: :updated_at,
294
+ min_id: 0)
295
+
296
+ info = Deimos::PollInfo.last
297
+ expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
298
+ expect(info.last_sent_id).to eq(widgets[2].id)
299
+
300
+ travel 61.seconds
301
+ # process the last widget which came in during the delay
302
+ expect(poller).to receive(:process_batch).ordered.
303
+ with([widgets[3], widgets[4], widgets[5]]).and_call_original
304
+ expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
305
+ and_call_original
306
+ poller.process_updates
307
+ expect(MyProducer).to have_received(:poll_query).
308
+ with(time_from: time_value(mins: -61, secs: 30),
309
+ time_to: time_value(secs: 59),
310
+ column_name: :updated_at,
311
+ min_id: widgets[2].id)
312
+
313
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
314
+ expect(info.last_sent_id).to eq(last_widget.id)
315
+ end
316
+
317
+ end
318
+
319
+ end
320
+ end
@@ -16,11 +16,13 @@ each_db_config(Deimos::Utils::DbProducer) do
16
16
 
17
17
  before(:each) do
18
18
  stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
19
+ stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
19
20
  end
20
21
 
21
22
  specify '#process_next_messages' do
22
23
  expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
23
24
  expect(producer).to receive(:process_topic).twice
25
+ expect(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics).with(%w(topic1 topic2))
24
26
  expect(producer).to receive(:sleep).with(0.5)
25
27
  producer.process_next_messages
26
28
  end
@@ -40,6 +42,9 @@ each_db_config(Deimos::Utils::DbProducer) do
40
42
  Deimos::KafkaMessage.create!(topic: 'topic1',
41
43
  message: 'blah',
42
44
  key: "key#{i}")
45
+ Deimos::KafkaMessage.create!(topic: 'topic2',
46
+ message: 'blah',
47
+ key: "key#{i}")
43
48
  end
44
49
  stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
45
50
  producer.current_topic = 'topic1'
@@ -280,6 +285,12 @@ each_db_config(Deimos::Utils::DbProducer) do
280
285
  message: "mess#{i}",
281
286
  partition_key: "key#{i}"
282
287
  )
288
+ Deimos::KafkaMessage.create!(
289
+ id: i,
290
+ topic: 'my-topic2',
291
+ message: "mess#{i}",
292
+ partition_key: "key#{i}"
293
+ )
283
294
  end
284
295
 
285
296
  expect(Deimos::KafkaTopicInfo).to receive(:lock).
@@ -288,9 +299,60 @@ each_db_config(Deimos::Utils::DbProducer) do
288
299
  expect(producer).to receive(:retrieve_messages).and_return(messages)
289
300
  expect(Deimos::KafkaTopicInfo).to receive(:register_error)
290
301
 
302
+ expect(Deimos::KafkaMessage.count).to eq(8)
303
+ producer.process_topic('my-topic')
291
304
  expect(Deimos::KafkaMessage.count).to eq(4)
305
+ end
306
+
307
+ it 'should retry deletes and not re-publish' do
308
+ messages = (1..4).map do |i|
309
+ Deimos::KafkaMessage.create!(
310
+ id: i,
311
+ topic: 'my-topic',
312
+ message: "mess#{i}",
313
+ partition_key: "key#{i}"
314
+ )
315
+ end
316
+ (5..8).each do |i|
317
+ Deimos::KafkaMessage.create!(
318
+ id: i,
319
+ topic: 'my-topic2',
320
+ message: "mess#{i}",
321
+ partition_key: "key#{i}"
322
+ )
323
+ end
324
+
325
+ raise_error = true
326
+ expect(Deimos::KafkaMessage).to receive(:where).exactly(5).times.and_wrap_original do |m, *args|
327
+ if raise_error
328
+ raise_error = false
329
+ raise 'Lock wait timeout'
330
+ end
331
+ m.call(*args)
332
+ end
333
+
334
+ expect(Deimos::KafkaTopicInfo).to receive(:lock).
335
+ with('my-topic', 'abc').and_return(true)
336
+ expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
337
+ expect(producer).to receive(:retrieve_messages).ordered.and_return([])
338
+ expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
339
+
340
+ expect(Deimos::KafkaMessage.count).to eq(8)
292
341
  producer.process_topic('my-topic')
293
- expect(Deimos::KafkaMessage.count).to eq(0)
342
+ expect(Deimos::KafkaMessage.count).to eq(4)
343
+ end
344
+
345
+ it 'should re-raise misc errors on delete' do
346
+ messages = (1..3).map do |i|
347
+ Deimos::KafkaMessage.create!(
348
+ id: i,
349
+ topic: 'my-topic',
350
+ message: "mess#{i}",
351
+ partition_key: "key#{i}"
352
+ )
353
+ end
354
+ expect(Deimos::KafkaMessage).to receive(:where).once.and_raise('OH NOES')
355
+ expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
294
356
  end
295
357
 
296
358
  end
@@ -309,21 +371,34 @@ each_db_config(Deimos::Utils::DbProducer) do
309
371
  Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
310
372
  created_at: (1 + i).minute.ago)
311
373
  end
374
+ Deimos::KafkaTopicInfo.create!(topic: 'topic1',
375
+ last_processed_at: 6.minutes.ago)
376
+ Deimos::KafkaTopicInfo.create!(topic: 'topic2',
377
+ last_processed_at: 3.minutes.ago)
378
+ Deimos::KafkaTopicInfo.create!(topic: 'topic3',
379
+ last_processed_at: 5.minutes.ago)
312
380
  allow(Deimos.config.metrics).to receive(:gauge)
313
381
  producer.send_pending_metrics
314
- expect(Deimos.config.metrics).to have_received(:gauge).twice
382
+ expect(Deimos.config.metrics).to have_received(:gauge).exactly(6).times
383
+ # topic1 has the earliest message 4 minutes ago and last processed 6
384
+ # minutes ago, so the most amount of time we've seen nothing is 4 minutes
315
385
  expect(Deimos.config.metrics).to have_received(:gauge).
316
386
  with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
387
+ # topic2 has earliest message 5 minutes ago and last processed 3 minutes
388
+ # ago, so we should give it 3 minutes
389
+ expect(Deimos.config.metrics).to have_received(:gauge).
390
+ with('pending_db_messages_max_wait', 3.minutes.to_i, tags: ['topic:topic2'])
391
+ # topic3 has no messages, so should get 0
317
392
  expect(Deimos.config.metrics).to have_received(:gauge).
318
- with('pending_db_messages_max_wait', 5.minutes.to_i, tags: ['topic:topic2'])
393
+ with('pending_db_messages_max_wait', 0, tags: ['topic:topic3'])
394
+ expect(Deimos.config.metrics).to have_received(:gauge).
395
+ with('pending_db_messages_count', 3, tags: ['topic:topic1'])
396
+ expect(Deimos.config.metrics).to have_received(:gauge).
397
+ with('pending_db_messages_count', 3, tags: ['topic:topic2'])
398
+ expect(Deimos.config.metrics).to have_received(:gauge).
399
+ with('pending_db_messages_count', 0, tags: ['topic:topic3'])
319
400
  end
320
401
  end
321
-
322
- it 'should send 0 if no messages' do
323
- expect(Deimos.config.metrics).to receive(:gauge).
324
- with('pending_db_messages_max_wait', 0)
325
- producer.send_pending_metrics
326
- end
327
402
  end
328
403
 
329
404
  example 'Full integration test' do
@@ -403,5 +478,4 @@ each_db_config(Deimos::Utils::DbProducer) do
403
478
  }
404
479
  ])
405
480
  end
406
-
407
481
  end