deimos-ruby 1.6.3 → 1.8.1.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +22 -16
- data/.ruby-version +1 -1
- data/CHANGELOG.md +42 -0
- data/Gemfile.lock +125 -98
- data/README.md +164 -16
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +4 -3
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +27 -0
- data/lib/deimos.rb +8 -7
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/batch_consumer.rb +2 -140
- data/lib/deimos/config/configuration.rb +28 -10
- data/lib/deimos/consume/batch_consumption.rb +150 -0
- data/lib/deimos/consume/message_consumption.rb +94 -0
- data/lib/deimos/consumer.rb +79 -70
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_topic_info.rb +22 -3
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/schema_backends/avro_base.rb +28 -1
- data/lib/deimos/schema_backends/base.rb +15 -2
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +59 -16
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
- data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
- data/lib/generators/deimos/active_record_generator.rb +79 -0
- data/lib/generators/deimos/db_backend/templates/migration +1 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +1 -0
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +3 -11
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/batch_consumer_spec.rb +24 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +8 -8
- data/spec/deimos_spec.rb +57 -49
- data/spec/generators/active_record_generator_spec.rb +56 -0
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/kafka_topic_info_spec.rb +39 -16
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/spec_helper.rb +62 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- data/spec/utils/db_producer_spec.rb +84 -10
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- metadata +66 -30
- data/lib/deimos/base_consumer.rb +0 -104
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
data/spec/spec_helper.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
4
4
|
require 'active_record'
|
5
|
+
require 'database_cleaner'
|
5
6
|
require 'deimos'
|
6
7
|
require 'deimos/metrics/mock'
|
7
8
|
require 'deimos/tracing/mock'
|
@@ -31,6 +32,7 @@ module TestRunners
|
|
31
32
|
# Test runner
|
32
33
|
class TestRunner
|
33
34
|
attr_accessor :id, :started, :stopped, :should_error
|
35
|
+
|
34
36
|
# :nodoc:
|
35
37
|
def initialize(id=nil)
|
36
38
|
@id = id
|
@@ -100,9 +102,8 @@ module DbConfigs
|
|
100
102
|
end
|
101
103
|
end
|
102
104
|
|
103
|
-
#
|
104
|
-
def
|
105
|
-
ActiveRecord::Base.establish_connection(options)
|
105
|
+
# :nodoc:
|
106
|
+
def run_db_backend_migration
|
106
107
|
migration_class_name = 'DbBackendMigration'
|
107
108
|
migration_version = '[5.2]'
|
108
109
|
migration = ERB.new(
|
@@ -110,6 +111,24 @@ module DbConfigs
|
|
110
111
|
).result(binding)
|
111
112
|
eval(migration) # rubocop:disable Security/Eval
|
112
113
|
ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
|
114
|
+
end
|
115
|
+
|
116
|
+
# :nodoc:
|
117
|
+
def run_db_poller_migration
|
118
|
+
migration_class_name = 'DbPollerMigration'
|
119
|
+
migration_version = '[5.2]'
|
120
|
+
migration = ERB.new(
|
121
|
+
File.read('lib/generators/deimos/db_poller/templates/migration')
|
122
|
+
).result(binding)
|
123
|
+
eval(migration) # rubocop:disable Security/Eval
|
124
|
+
ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
|
125
|
+
end
|
126
|
+
|
127
|
+
# Set up the given database.
|
128
|
+
def setup_db(options)
|
129
|
+
ActiveRecord::Base.establish_connection(options)
|
130
|
+
run_db_backend_migration
|
131
|
+
run_db_poller_migration
|
113
132
|
|
114
133
|
ActiveRecord::Base.descendants.each do |klass|
|
115
134
|
klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
|
@@ -130,8 +149,11 @@ RSpec.configure do |config|
|
|
130
149
|
# true by default for RSpec 4.0
|
131
150
|
config.shared_context_metadata_behavior = :apply_to_host_groups
|
132
151
|
|
152
|
+
config.filter_run(focus: true)
|
153
|
+
config.run_all_when_everything_filtered = true
|
154
|
+
|
133
155
|
config.before(:all) do
|
134
|
-
Time.zone = '
|
156
|
+
Time.zone = 'Eastern Time (US & Canada)'
|
135
157
|
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
136
158
|
ActiveRecord::Base.establish_connection(
|
137
159
|
'adapter' => 'sqlite3',
|
@@ -141,9 +163,10 @@ RSpec.configure do |config|
|
|
141
163
|
config.include Deimos::TestHelpers
|
142
164
|
config.include ActiveSupport::Testing::TimeHelpers
|
143
165
|
config.before(:suite) do
|
144
|
-
Time.zone = 'EST'
|
145
|
-
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
146
166
|
setup_db(DbConfigs::DB_OPTIONS.last)
|
167
|
+
|
168
|
+
DatabaseCleaner.strategy = :transaction
|
169
|
+
DatabaseCleaner.clean_with(:truncation)
|
147
170
|
end
|
148
171
|
|
149
172
|
config.mock_with(:rspec) do |mocks|
|
@@ -164,6 +187,39 @@ RSpec.configure do |config|
|
|
164
187
|
deimos_config.schema.backend = :avro_validation
|
165
188
|
end
|
166
189
|
end
|
190
|
+
|
191
|
+
config.around(:each) do |example|
|
192
|
+
use_cleaner = !example.metadata[:integration]
|
193
|
+
|
194
|
+
DatabaseCleaner.start if use_cleaner
|
195
|
+
|
196
|
+
example.run
|
197
|
+
|
198
|
+
DatabaseCleaner.clean if use_cleaner
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
RSpec.shared_context('with widgets') do
|
203
|
+
before(:all) do
|
204
|
+
ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
|
205
|
+
t.string(:test_id)
|
206
|
+
t.integer(:some_int)
|
207
|
+
t.boolean(:some_bool)
|
208
|
+
t.timestamps
|
209
|
+
end
|
210
|
+
|
211
|
+
# :nodoc:
|
212
|
+
class Widget < ActiveRecord::Base
|
213
|
+
# @return [String]
|
214
|
+
def generated_id
|
215
|
+
'generated_id'
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
after(:all) do
|
221
|
+
ActiveRecord::Base.connection.drop_table(:widgets)
|
222
|
+
end
|
167
223
|
end
|
168
224
|
|
169
225
|
RSpec.shared_context('with DB') do
|
@@ -0,0 +1,320 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# @param seconds [Integer]
|
4
|
+
# @return [Time]
|
5
|
+
def time_value(secs: 0, mins: 0)
|
6
|
+
Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
|
7
|
+
end
|
8
|
+
|
9
|
+
each_db_config(Deimos::Utils::DbPoller) do
|
10
|
+
|
11
|
+
before(:each) do
|
12
|
+
Deimos::PollInfo.delete_all
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '#start!' do
|
16
|
+
|
17
|
+
before(:each) do
|
18
|
+
producer_class = Class.new(Deimos::Producer) do
|
19
|
+
schema 'MySchema'
|
20
|
+
namespace 'com.my-namespace'
|
21
|
+
topic 'my-topic'
|
22
|
+
key_config field: 'test_id'
|
23
|
+
end
|
24
|
+
stub_const('MyProducer', producer_class)
|
25
|
+
|
26
|
+
producer_class = Class.new(Deimos::Producer) do
|
27
|
+
schema 'MySchemaWithId'
|
28
|
+
namespace 'com.my-namespace'
|
29
|
+
topic 'my-topic'
|
30
|
+
key_config plain: true
|
31
|
+
end
|
32
|
+
stub_const('MyProducerWithID', producer_class)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should raise an error if no pollers configured' do
|
36
|
+
Deimos.configure {}
|
37
|
+
expect { described_class.start! }.to raise_error('No pollers configured!')
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'should start pollers as configured' do
|
41
|
+
Deimos.configure do
|
42
|
+
db_poller do
|
43
|
+
producer_class 'MyProducer'
|
44
|
+
end
|
45
|
+
db_poller do
|
46
|
+
producer_class 'MyProducerWithID'
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
allow(Deimos::Utils::DbPoller).to receive(:new)
|
51
|
+
signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
|
52
|
+
allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
|
53
|
+
described_class.start!
|
54
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).twice
|
55
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
56
|
+
with(Deimos.config.db_poller_objects[0])
|
57
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
58
|
+
with(Deimos.config.db_poller_objects[1])
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
describe 'pollers' do
|
63
|
+
include_context 'with widgets'
|
64
|
+
|
65
|
+
let(:poller) do
|
66
|
+
poller = described_class.new(config)
|
67
|
+
allow(poller).to receive(:sleep)
|
68
|
+
poller
|
69
|
+
end
|
70
|
+
|
71
|
+
let(:config) { Deimos.config.db_poller_objects.first.dup }
|
72
|
+
|
73
|
+
before(:each) do
|
74
|
+
Widget.delete_all
|
75
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
76
|
+
schema 'MySchemaWithId'
|
77
|
+
namespace 'com.my-namespace'
|
78
|
+
topic 'my-topic-with-id'
|
79
|
+
key_config none: true
|
80
|
+
record_class Widget
|
81
|
+
|
82
|
+
# :nodoc:
|
83
|
+
def self.generate_payload(attrs, widget)
|
84
|
+
super.merge(message_id: widget.generated_id)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
stub_const('MyProducer', producer_class)
|
88
|
+
|
89
|
+
Deimos.configure do
|
90
|
+
db_poller do
|
91
|
+
producer_class 'MyProducer'
|
92
|
+
run_every 1.minute
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
after(:each) do
|
98
|
+
travel_back
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should crash if initialized with an invalid producer' do
|
102
|
+
config.producer_class = 'NoProducer'
|
103
|
+
expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
|
104
|
+
end
|
105
|
+
|
106
|
+
describe '#retrieve_poll_info' do
|
107
|
+
|
108
|
+
it 'should start from beginning when configured' do
|
109
|
+
poller.retrieve_poll_info
|
110
|
+
expect(Deimos::PollInfo.count).to eq(1)
|
111
|
+
info = Deimos::PollInfo.last
|
112
|
+
expect(info.producer).to eq('MyProducer')
|
113
|
+
expect(info.last_sent).to eq(Time.new(0))
|
114
|
+
expect(info.last_sent_id).to eq(0)
|
115
|
+
end
|
116
|
+
|
117
|
+
it 'should start from now when configured' do
|
118
|
+
travel_to time_value
|
119
|
+
config.start_from_beginning = false
|
120
|
+
poller.retrieve_poll_info
|
121
|
+
expect(Deimos::PollInfo.count).to eq(1)
|
122
|
+
info = Deimos::PollInfo.last
|
123
|
+
expect(info.producer).to eq('MyProducer')
|
124
|
+
expect(info.last_sent).to eq(time_value)
|
125
|
+
expect(info.last_sent_id).to eq(0)
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|
129
|
+
|
130
|
+
specify '#start' do
|
131
|
+
i = 0
|
132
|
+
expect(poller).to receive(:process_updates).twice do
|
133
|
+
i += 1
|
134
|
+
poller.stop if i == 2
|
135
|
+
end
|
136
|
+
poller.start
|
137
|
+
end
|
138
|
+
|
139
|
+
specify '#should_run?' do
|
140
|
+
Deimos::PollInfo.create!(producer: 'MyProducer',
|
141
|
+
last_sent: time_value)
|
142
|
+
poller.retrieve_poll_info
|
143
|
+
|
144
|
+
# run_every is set to 1 minute
|
145
|
+
travel_to time_value(secs: 62)
|
146
|
+
expect(poller.should_run?).to eq(true)
|
147
|
+
|
148
|
+
travel_to time_value(secs: 30)
|
149
|
+
expect(poller.should_run?).to eq(false)
|
150
|
+
|
151
|
+
travel_to time_value(mins: -1) # this shouldn't be possible but meh
|
152
|
+
expect(poller.should_run?).to eq(false)
|
153
|
+
|
154
|
+
# take the 2 seconds of delay_time into account
|
155
|
+
travel_to time_value(secs: 60)
|
156
|
+
expect(poller.should_run?).to eq(false)
|
157
|
+
end
|
158
|
+
|
159
|
+
specify '#process_batch' do
|
160
|
+
travel_to time_value
|
161
|
+
widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
|
162
|
+
widgets.last.update_attribute(:updated_at, time_value(mins: -30))
|
163
|
+
expect(MyProducer).to receive(:send_events).with(widgets)
|
164
|
+
poller.retrieve_poll_info
|
165
|
+
poller.process_batch(widgets)
|
166
|
+
info = Deimos::PollInfo.last
|
167
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
|
168
|
+
expect(info.last_sent_id).to eq(widgets.last.id)
|
169
|
+
end
|
170
|
+
|
171
|
+
describe '#process_updates' do
|
172
|
+
before(:each) do
|
173
|
+
Deimos::PollInfo.create!(producer: 'MyProducer',
|
174
|
+
last_sent: time_value(mins: -61),
|
175
|
+
last_sent_id: 0)
|
176
|
+
poller.retrieve_poll_info
|
177
|
+
travel_to time_value
|
178
|
+
stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
|
179
|
+
end
|
180
|
+
|
181
|
+
let!(:old_widget) do
|
182
|
+
# old widget, earlier than window
|
183
|
+
Widget.create!(test_id: 'some_id', some_int: 40,
|
184
|
+
updated_at: time_value(mins: -200))
|
185
|
+
end
|
186
|
+
|
187
|
+
let!(:last_widget) do
|
188
|
+
# new widget, before delay
|
189
|
+
Widget.create!(test_id: 'some_id', some_int: 10,
|
190
|
+
updated_at: time_value(secs: -1))
|
191
|
+
end
|
192
|
+
|
193
|
+
let!(:widgets) do
|
194
|
+
(1..7).map do |i|
|
195
|
+
Widget.create!(test_id: 'some_id', some_int: i,
|
196
|
+
updated_at: time_value(mins: -61, secs: 30 + i))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
it 'should update the full table' do
|
201
|
+
info = Deimos::PollInfo.last
|
202
|
+
config.full_table = true
|
203
|
+
expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
|
204
|
+
expect(poller).to receive(:process_batch).ordered.
|
205
|
+
with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
|
206
|
+
m.call(*args)
|
207
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
|
208
|
+
expect(info.last_sent_id).to eq(widgets[1].id)
|
209
|
+
end
|
210
|
+
expect(poller).to receive(:process_batch).ordered.
|
211
|
+
with([widgets[2], widgets[3], widgets[4]]).and_call_original
|
212
|
+
expect(poller).to receive(:process_batch).ordered.
|
213
|
+
with([widgets[5], widgets[6]]).and_call_original
|
214
|
+
poller.process_updates
|
215
|
+
|
216
|
+
# this is the updated_at of widgets[6]
|
217
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
|
218
|
+
expect(info.last_sent_id).to eq(widgets[6].id)
|
219
|
+
|
220
|
+
last_widget.update_attribute(:updated_at, time_value(mins: -250))
|
221
|
+
|
222
|
+
travel 61.seconds
|
223
|
+
# should reprocess the table
|
224
|
+
expect(poller).to receive(:process_batch).ordered.
|
225
|
+
with([last_widget, old_widget, widgets[0]]).and_call_original
|
226
|
+
expect(poller).to receive(:process_batch).ordered.
|
227
|
+
with([widgets[1], widgets[2], widgets[3]]).and_call_original
|
228
|
+
expect(poller).to receive(:process_batch).ordered.
|
229
|
+
with([widgets[4], widgets[5], widgets[6]]).and_call_original
|
230
|
+
poller.process_updates
|
231
|
+
|
232
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
|
233
|
+
expect(info.last_sent_id).to eq(widgets[6].id)
|
234
|
+
end
|
235
|
+
|
236
|
+
it 'should send events across multiple batches' do
|
237
|
+
allow(MyProducer).to receive(:poll_query).and_call_original
|
238
|
+
expect(poller).to receive(:process_batch).ordered.
|
239
|
+
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
240
|
+
expect(poller).to receive(:process_batch).ordered.
|
241
|
+
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
242
|
+
expect(poller).to receive(:process_batch).ordered.
|
243
|
+
with([widgets[6]]).and_call_original
|
244
|
+
poller.process_updates
|
245
|
+
|
246
|
+
expect(MyProducer).to have_received(:poll_query).
|
247
|
+
with(time_from: time_value(mins: -61),
|
248
|
+
time_to: time_value(secs: -2),
|
249
|
+
column_name: :updated_at,
|
250
|
+
min_id: 0)
|
251
|
+
|
252
|
+
travel 61.seconds
|
253
|
+
# process the last widget which came in during the delay
|
254
|
+
expect(poller).to receive(:process_batch).with([last_widget]).
|
255
|
+
and_call_original
|
256
|
+
poller.process_updates
|
257
|
+
|
258
|
+
# widgets[6] updated_at value
|
259
|
+
expect(MyProducer).to have_received(:poll_query).
|
260
|
+
with(time_from: time_value(mins: -61, secs: 37),
|
261
|
+
time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
|
262
|
+
column_name: :updated_at,
|
263
|
+
min_id: widgets[6].id)
|
264
|
+
|
265
|
+
travel 61.seconds
|
266
|
+
# nothing else to process
|
267
|
+
expect(poller).not_to receive(:process_batch)
|
268
|
+
poller.process_updates
|
269
|
+
poller.process_updates
|
270
|
+
|
271
|
+
expect(MyProducer).to have_received(:poll_query).twice.
|
272
|
+
with(time_from: time_value(secs: -1),
|
273
|
+
time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
|
274
|
+
column_name: :updated_at,
|
275
|
+
min_id: last_widget.id)
|
276
|
+
end
|
277
|
+
|
278
|
+
it 'should recover correctly with errors and save the right ID' do
|
279
|
+
widgets.each do |w|
|
280
|
+
w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
|
281
|
+
end
|
282
|
+
allow(MyProducer).to receive(:poll_query).and_call_original
|
283
|
+
expect(poller).to receive(:process_batch).ordered.
|
284
|
+
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
285
|
+
expect(poller).to receive(:process_batch).ordered.
|
286
|
+
with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
|
287
|
+
|
288
|
+
expect { poller.process_updates }.to raise_exception('OH NOES')
|
289
|
+
|
290
|
+
expect(MyProducer).to have_received(:poll_query).
|
291
|
+
with(time_from: time_value(mins: -61),
|
292
|
+
time_to: time_value(secs: -2),
|
293
|
+
column_name: :updated_at,
|
294
|
+
min_id: 0)
|
295
|
+
|
296
|
+
info = Deimos::PollInfo.last
|
297
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
|
298
|
+
expect(info.last_sent_id).to eq(widgets[2].id)
|
299
|
+
|
300
|
+
travel 61.seconds
|
301
|
+
# process the last widget which came in during the delay
|
302
|
+
expect(poller).to receive(:process_batch).ordered.
|
303
|
+
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
304
|
+
expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
|
305
|
+
and_call_original
|
306
|
+
poller.process_updates
|
307
|
+
expect(MyProducer).to have_received(:poll_query).
|
308
|
+
with(time_from: time_value(mins: -61, secs: 30),
|
309
|
+
time_to: time_value(secs: 59),
|
310
|
+
column_name: :updated_at,
|
311
|
+
min_id: widgets[2].id)
|
312
|
+
|
313
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
|
314
|
+
expect(info.last_sent_id).to eq(last_widget.id)
|
315
|
+
end
|
316
|
+
|
317
|
+
end
|
318
|
+
|
319
|
+
end
|
320
|
+
end
|
@@ -16,11 +16,13 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
16
16
|
|
17
17
|
before(:each) do
|
18
18
|
stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
|
19
|
+
stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
|
19
20
|
end
|
20
21
|
|
21
22
|
specify '#process_next_messages' do
|
22
23
|
expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
|
23
24
|
expect(producer).to receive(:process_topic).twice
|
25
|
+
expect(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics).with(%w(topic1 topic2))
|
24
26
|
expect(producer).to receive(:sleep).with(0.5)
|
25
27
|
producer.process_next_messages
|
26
28
|
end
|
@@ -40,6 +42,9 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
40
42
|
Deimos::KafkaMessage.create!(topic: 'topic1',
|
41
43
|
message: 'blah',
|
42
44
|
key: "key#{i}")
|
45
|
+
Deimos::KafkaMessage.create!(topic: 'topic2',
|
46
|
+
message: 'blah',
|
47
|
+
key: "key#{i}")
|
43
48
|
end
|
44
49
|
stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
|
45
50
|
producer.current_topic = 'topic1'
|
@@ -280,6 +285,12 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
280
285
|
message: "mess#{i}",
|
281
286
|
partition_key: "key#{i}"
|
282
287
|
)
|
288
|
+
Deimos::KafkaMessage.create!(
|
289
|
+
id: i,
|
290
|
+
topic: 'my-topic2',
|
291
|
+
message: "mess#{i}",
|
292
|
+
partition_key: "key#{i}"
|
293
|
+
)
|
283
294
|
end
|
284
295
|
|
285
296
|
expect(Deimos::KafkaTopicInfo).to receive(:lock).
|
@@ -288,9 +299,60 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
288
299
|
expect(producer).to receive(:retrieve_messages).and_return(messages)
|
289
300
|
expect(Deimos::KafkaTopicInfo).to receive(:register_error)
|
290
301
|
|
302
|
+
expect(Deimos::KafkaMessage.count).to eq(8)
|
303
|
+
producer.process_topic('my-topic')
|
291
304
|
expect(Deimos::KafkaMessage.count).to eq(4)
|
305
|
+
end
|
306
|
+
|
307
|
+
it 'should retry deletes and not re-publish' do
|
308
|
+
messages = (1..4).map do |i|
|
309
|
+
Deimos::KafkaMessage.create!(
|
310
|
+
id: i,
|
311
|
+
topic: 'my-topic',
|
312
|
+
message: "mess#{i}",
|
313
|
+
partition_key: "key#{i}"
|
314
|
+
)
|
315
|
+
end
|
316
|
+
(5..8).each do |i|
|
317
|
+
Deimos::KafkaMessage.create!(
|
318
|
+
id: i,
|
319
|
+
topic: 'my-topic2',
|
320
|
+
message: "mess#{i}",
|
321
|
+
partition_key: "key#{i}"
|
322
|
+
)
|
323
|
+
end
|
324
|
+
|
325
|
+
raise_error = true
|
326
|
+
expect(Deimos::KafkaMessage).to receive(:where).exactly(5).times.and_wrap_original do |m, *args|
|
327
|
+
if raise_error
|
328
|
+
raise_error = false
|
329
|
+
raise 'Lock wait timeout'
|
330
|
+
end
|
331
|
+
m.call(*args)
|
332
|
+
end
|
333
|
+
|
334
|
+
expect(Deimos::KafkaTopicInfo).to receive(:lock).
|
335
|
+
with('my-topic', 'abc').and_return(true)
|
336
|
+
expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
|
337
|
+
expect(producer).to receive(:retrieve_messages).ordered.and_return([])
|
338
|
+
expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
|
339
|
+
|
340
|
+
expect(Deimos::KafkaMessage.count).to eq(8)
|
292
341
|
producer.process_topic('my-topic')
|
293
|
-
expect(Deimos::KafkaMessage.count).to eq(
|
342
|
+
expect(Deimos::KafkaMessage.count).to eq(4)
|
343
|
+
end
|
344
|
+
|
345
|
+
it 'should re-raise misc errors on delete' do
|
346
|
+
messages = (1..3).map do |i|
|
347
|
+
Deimos::KafkaMessage.create!(
|
348
|
+
id: i,
|
349
|
+
topic: 'my-topic',
|
350
|
+
message: "mess#{i}",
|
351
|
+
partition_key: "key#{i}"
|
352
|
+
)
|
353
|
+
end
|
354
|
+
expect(Deimos::KafkaMessage).to receive(:where).once.and_raise('OH NOES')
|
355
|
+
expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
|
294
356
|
end
|
295
357
|
|
296
358
|
end
|
@@ -309,21 +371,34 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
309
371
|
Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
|
310
372
|
created_at: (1 + i).minute.ago)
|
311
373
|
end
|
374
|
+
Deimos::KafkaTopicInfo.create!(topic: 'topic1',
|
375
|
+
last_processed_at: 6.minutes.ago)
|
376
|
+
Deimos::KafkaTopicInfo.create!(topic: 'topic2',
|
377
|
+
last_processed_at: 3.minutes.ago)
|
378
|
+
Deimos::KafkaTopicInfo.create!(topic: 'topic3',
|
379
|
+
last_processed_at: 5.minutes.ago)
|
312
380
|
allow(Deimos.config.metrics).to receive(:gauge)
|
313
381
|
producer.send_pending_metrics
|
314
|
-
expect(Deimos.config.metrics).to have_received(:gauge).
|
382
|
+
expect(Deimos.config.metrics).to have_received(:gauge).exactly(6).times
|
383
|
+
# topic1 has the earliest message 4 minutes ago and last processed 6
|
384
|
+
# minutes ago, so the most amount of time we've seen nothing is 4 minutes
|
315
385
|
expect(Deimos.config.metrics).to have_received(:gauge).
|
316
386
|
with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
|
387
|
+
# topic2 has earliest message 5 minutes ago and last processed 3 minutes
|
388
|
+
# ago, so we should give it 3 minutes
|
389
|
+
expect(Deimos.config.metrics).to have_received(:gauge).
|
390
|
+
with('pending_db_messages_max_wait', 3.minutes.to_i, tags: ['topic:topic2'])
|
391
|
+
# topic3 has no messages, so should get 0
|
317
392
|
expect(Deimos.config.metrics).to have_received(:gauge).
|
318
|
-
with('pending_db_messages_max_wait',
|
393
|
+
with('pending_db_messages_max_wait', 0, tags: ['topic:topic3'])
|
394
|
+
expect(Deimos.config.metrics).to have_received(:gauge).
|
395
|
+
with('pending_db_messages_count', 3, tags: ['topic:topic1'])
|
396
|
+
expect(Deimos.config.metrics).to have_received(:gauge).
|
397
|
+
with('pending_db_messages_count', 3, tags: ['topic:topic2'])
|
398
|
+
expect(Deimos.config.metrics).to have_received(:gauge).
|
399
|
+
with('pending_db_messages_count', 0, tags: ['topic:topic3'])
|
319
400
|
end
|
320
401
|
end
|
321
|
-
|
322
|
-
it 'should send 0 if no messages' do
|
323
|
-
expect(Deimos.config.metrics).to receive(:gauge).
|
324
|
-
with('pending_db_messages_max_wait', 0)
|
325
|
-
producer.send_pending_metrics
|
326
|
-
end
|
327
402
|
end
|
328
403
|
|
329
404
|
example 'Full integration test' do
|
@@ -403,5 +478,4 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
403
478
|
}
|
404
479
|
])
|
405
480
|
end
|
406
|
-
|
407
481
|
end
|