deimos-ruby 1.6.2 → 1.8.0.pre.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +15 -13
- data/.ruby-version +1 -1
- data/CHANGELOG.md +31 -0
- data/Gemfile.lock +43 -36
- data/README.md +141 -16
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +2 -1
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +27 -0
- data/lib/deimos.rb +7 -6
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/batch_consumer.rb +2 -140
- data/lib/deimos/config/configuration.rb +28 -10
- data/lib/deimos/consume/batch_consumption.rb +150 -0
- data/lib/deimos/consume/message_consumption.rb +94 -0
- data/lib/deimos/consumer.rb +79 -69
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +8 -3
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +3 -11
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/batch_consumer_spec.rb +24 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +8 -8
- data/spec/deimos_spec.rb +57 -49
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/spec_helper.rb +61 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- metadata +55 -20
- data/lib/deimos/base_consumer.rb +0 -104
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
data/spec/deimos_spec.rb
CHANGED
@@ -68,11 +68,11 @@ describe Deimos do
|
|
68
68
|
|
69
69
|
describe '#start_db_backend!' do
|
70
70
|
it 'should start if backend is db and thread_count is > 0' do
|
71
|
-
signal_handler = instance_double(
|
71
|
+
signal_handler = instance_double(Sigurd::SignalHandler)
|
72
72
|
allow(signal_handler).to receive(:run!)
|
73
|
-
expect(
|
73
|
+
expect(Sigurd::Executor).to receive(:new).
|
74
74
|
with(anything, sleep_seconds: 5, logger: anything).and_call_original
|
75
|
-
expect(
|
75
|
+
expect(Sigurd::SignalHandler).to receive(:new) do |executor|
|
76
76
|
expect(executor.runners.size).to eq(2)
|
77
77
|
signal_handler
|
78
78
|
end
|
@@ -83,7 +83,7 @@ describe Deimos do
|
|
83
83
|
end
|
84
84
|
|
85
85
|
it 'should not start if backend is not db' do
|
86
|
-
expect(
|
86
|
+
expect(Sigurd::SignalHandler).not_to receive(:new)
|
87
87
|
described_class.configure do |config|
|
88
88
|
config.producers.backend = :kafka
|
89
89
|
end
|
@@ -92,7 +92,7 @@ describe Deimos do
|
|
92
92
|
end
|
93
93
|
|
94
94
|
it 'should not start if thread_count is nil' do
|
95
|
-
expect(
|
95
|
+
expect(Sigurd::SignalHandler).not_to receive(:new)
|
96
96
|
described_class.configure do |config|
|
97
97
|
config.producers.backend = :db
|
98
98
|
end
|
@@ -101,61 +101,69 @@ describe Deimos do
|
|
101
101
|
end
|
102
102
|
|
103
103
|
it 'should not start if thread_count is 0' do
|
104
|
-
expect(
|
104
|
+
expect(Sigurd::SignalHandler).not_to receive(:new)
|
105
105
|
described_class.configure do |config|
|
106
106
|
config.producers.backend = :db
|
107
107
|
end
|
108
108
|
expect { described_class.start_db_backend!(thread_count: 0) }.
|
109
109
|
to raise_error('Thread count is not given or set to zero, exiting')
|
110
110
|
end
|
111
|
+
end
|
111
112
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
it 'should not raise an error with properly configured handlers' do
|
118
|
-
path = config_path # for scope issues in the block below
|
119
|
-
# Add explicit consumers
|
120
|
-
phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
|
121
|
-
'delivery' => 'message' }
|
122
|
-
phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
|
123
|
-
'delivery' => 'batch' }
|
124
|
-
|
125
|
-
expect {
|
126
|
-
described_class.configure { |c| c.phobos_config_file = path }
|
127
|
-
}.not_to raise_error
|
128
|
-
end
|
129
|
-
|
130
|
-
it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
|
131
|
-
path = config_path # for scope issues in the block below
|
132
|
-
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
|
133
|
-
'delivery' => 'message' }]
|
134
|
-
|
135
|
-
expect {
|
136
|
-
described_class.configure { |c| c.phobos_config_file = path }
|
137
|
-
}.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
|
138
|
-
end
|
139
|
-
|
140
|
-
it 'should raise an error if Consumers do not have message or batch delivery' do
|
141
|
-
path = config_path # for scope issues in the block below
|
142
|
-
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
|
143
|
-
'delivery' => 'inline_batch' }]
|
113
|
+
describe 'delivery configuration' do
|
114
|
+
before(:each) do
|
115
|
+
allow(YAML).to receive(:load).and_return(phobos_configuration)
|
116
|
+
end
|
144
117
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
118
|
+
it 'should not raise an error with properly configured handlers' do
|
119
|
+
expect {
|
120
|
+
described_class.configure do
|
121
|
+
consumer do
|
122
|
+
class_name 'ConsumerTest::MyConsumer'
|
123
|
+
delivery :message
|
124
|
+
end
|
125
|
+
consumer do
|
126
|
+
class_name 'ConsumerTest::MyConsumer'
|
127
|
+
delivery :batch
|
128
|
+
end
|
129
|
+
consumer do
|
130
|
+
class_name 'ConsumerTest::MyBatchConsumer'
|
131
|
+
delivery :inline_batch
|
132
|
+
end
|
133
|
+
end
|
134
|
+
}.not_to raise_error
|
135
|
+
end
|
149
136
|
|
150
|
-
|
151
|
-
|
152
|
-
|
137
|
+
it 'should raise an error if inline_batch listeners do not implement consume_batch' do
|
138
|
+
expect {
|
139
|
+
described_class.configure do
|
140
|
+
consumer do
|
141
|
+
class_name 'ConsumerTest::MyConsumer'
|
142
|
+
delivery :inline_batch
|
143
|
+
end
|
144
|
+
end
|
145
|
+
}.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
|
146
|
+
end
|
153
147
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
148
|
+
it 'should raise an error if Consumers do not have message or batch delivery' do
|
149
|
+
expect {
|
150
|
+
described_class.configure do
|
151
|
+
consumer do
|
152
|
+
class_name 'ConsumerTest::MyBatchConsumer'
|
153
|
+
delivery :message
|
154
|
+
end
|
155
|
+
end
|
156
|
+
}.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
|
157
|
+
end
|
158
158
|
|
159
|
+
it 'should treat nil as `batch`' do
|
160
|
+
expect {
|
161
|
+
described_class.configure do
|
162
|
+
consumer do
|
163
|
+
class_name 'ConsumerTest::MyConsumer'
|
164
|
+
end
|
165
|
+
end
|
166
|
+
}.not_to raise_error
|
159
167
|
end
|
160
168
|
end
|
161
169
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.describe(Deimos::Message) do
|
4
|
+
it 'should detect tombstones' do
|
5
|
+
expect(described_class.new(nil, nil, key: 'key1')).
|
6
|
+
to be_tombstone
|
7
|
+
expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
|
8
|
+
not_to be_tombstone
|
9
|
+
expect(described_class.new({ v: '' }, nil, key: 'key1')).
|
10
|
+
not_to be_tombstone
|
11
|
+
expect(described_class.new({ v: 'val1' }, nil, key: nil)).
|
12
|
+
not_to be_tombstone
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'can support complex keys/values' do
|
16
|
+
expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
|
17
|
+
not_to raise_exception
|
18
|
+
end
|
19
|
+
end
|
data/spec/producer_spec.rb
CHANGED
@@ -148,7 +148,7 @@ module ProducerTest
|
|
148
148
|
Deimos.disable_producers do
|
149
149
|
raise 'OH NOES'
|
150
150
|
end
|
151
|
-
}
|
151
|
+
}.to raise_error('OH NOES')
|
152
152
|
expect(Deimos).not_to be_producers_disabled
|
153
153
|
end
|
154
154
|
|
@@ -246,7 +246,7 @@ module ProducerTest
|
|
246
246
|
MyNonEncodedProducer.publish_list(
|
247
247
|
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
248
248
|
)
|
249
|
-
}
|
249
|
+
}.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
|
250
250
|
end
|
251
251
|
|
252
252
|
it 'should allow nil keys if none: true is configured' do
|
@@ -254,7 +254,7 @@ module ProducerTest
|
|
254
254
|
MyNoKeyProducer.publish_list(
|
255
255
|
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
256
256
|
)
|
257
|
-
}
|
257
|
+
}.not_to raise_error
|
258
258
|
end
|
259
259
|
|
260
260
|
it 'should use a partition key' do
|
data/spec/rake_spec.rb
CHANGED
@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
|
|
9
9
|
Rake::Task.define_task(:environment)
|
10
10
|
end
|
11
11
|
|
12
|
-
describe 'Rakefile' do
|
12
|
+
describe 'Rakefile' do
|
13
13
|
it 'should start listeners' do
|
14
14
|
runner = instance_double(Phobos::CLI::Runner)
|
15
15
|
expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
|
@@ -0,0 +1,18 @@
|
|
1
|
+
{
|
2
|
+
"namespace": "com.my-namespace",
|
3
|
+
"name": "MySchemaCompound-key",
|
4
|
+
"type": "record",
|
5
|
+
"doc": "Test schema",
|
6
|
+
"fields": [
|
7
|
+
{
|
8
|
+
"name": "part_one",
|
9
|
+
"type": "string",
|
10
|
+
"doc": "test string one"
|
11
|
+
},
|
12
|
+
{
|
13
|
+
"name": "part_two",
|
14
|
+
"type": "string",
|
15
|
+
"doc": "test string two"
|
16
|
+
}
|
17
|
+
]
|
18
|
+
}
|
@@ -0,0 +1,43 @@
|
|
1
|
+
{
|
2
|
+
"namespace": "com.my-namespace",
|
3
|
+
"name": "Wibble",
|
4
|
+
"type": "record",
|
5
|
+
"fields": [
|
6
|
+
{
|
7
|
+
"name": "id",
|
8
|
+
"type": "long"
|
9
|
+
},
|
10
|
+
{
|
11
|
+
"name": "wibble_id",
|
12
|
+
"type": "long"
|
13
|
+
},
|
14
|
+
{
|
15
|
+
"name": "name",
|
16
|
+
"type": "string"
|
17
|
+
},
|
18
|
+
{
|
19
|
+
"name": "floop",
|
20
|
+
"type": "string"
|
21
|
+
},
|
22
|
+
{
|
23
|
+
"name": "birthday_int",
|
24
|
+
"type": "int"
|
25
|
+
},
|
26
|
+
{
|
27
|
+
"name": "birthday_long",
|
28
|
+
"type": "long"
|
29
|
+
},
|
30
|
+
{
|
31
|
+
"name": "birthday_optional",
|
32
|
+
"type": ["null", "int"]
|
33
|
+
},
|
34
|
+
{
|
35
|
+
"name": "updated_at",
|
36
|
+
"type": "long"
|
37
|
+
},
|
38
|
+
{
|
39
|
+
"name": "created_at",
|
40
|
+
"type": "long"
|
41
|
+
}
|
42
|
+
]
|
43
|
+
}
|
data/spec/spec_helper.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
4
4
|
require 'active_record'
|
5
|
+
require 'database_cleaner'
|
5
6
|
require 'deimos'
|
6
7
|
require 'deimos/metrics/mock'
|
7
8
|
require 'deimos/tracing/mock'
|
@@ -100,9 +101,8 @@ module DbConfigs
|
|
100
101
|
end
|
101
102
|
end
|
102
103
|
|
103
|
-
#
|
104
|
-
def
|
105
|
-
ActiveRecord::Base.establish_connection(options)
|
104
|
+
# :nodoc:
|
105
|
+
def run_db_backend_migration
|
106
106
|
migration_class_name = 'DbBackendMigration'
|
107
107
|
migration_version = '[5.2]'
|
108
108
|
migration = ERB.new(
|
@@ -110,6 +110,24 @@ module DbConfigs
|
|
110
110
|
).result(binding)
|
111
111
|
eval(migration) # rubocop:disable Security/Eval
|
112
112
|
ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
|
113
|
+
end
|
114
|
+
|
115
|
+
# :nodoc:
|
116
|
+
def run_db_poller_migration
|
117
|
+
migration_class_name = 'DbPollerMigration'
|
118
|
+
migration_version = '[5.2]'
|
119
|
+
migration = ERB.new(
|
120
|
+
File.read('lib/generators/deimos/db_poller/templates/migration')
|
121
|
+
).result(binding)
|
122
|
+
eval(migration) # rubocop:disable Security/Eval
|
123
|
+
ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
|
124
|
+
end
|
125
|
+
|
126
|
+
# Set up the given database.
|
127
|
+
def setup_db(options)
|
128
|
+
ActiveRecord::Base.establish_connection(options)
|
129
|
+
run_db_backend_migration
|
130
|
+
run_db_poller_migration
|
113
131
|
|
114
132
|
ActiveRecord::Base.descendants.each do |klass|
|
115
133
|
klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
|
@@ -130,8 +148,11 @@ RSpec.configure do |config|
|
|
130
148
|
# true by default for RSpec 4.0
|
131
149
|
config.shared_context_metadata_behavior = :apply_to_host_groups
|
132
150
|
|
151
|
+
config.filter_run(focus: true)
|
152
|
+
config.run_all_when_everything_filtered = true
|
153
|
+
|
133
154
|
config.before(:all) do
|
134
|
-
Time.zone = '
|
155
|
+
Time.zone = 'Eastern Time (US & Canada)'
|
135
156
|
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
136
157
|
ActiveRecord::Base.establish_connection(
|
137
158
|
'adapter' => 'sqlite3',
|
@@ -141,9 +162,10 @@ RSpec.configure do |config|
|
|
141
162
|
config.include Deimos::TestHelpers
|
142
163
|
config.include ActiveSupport::Testing::TimeHelpers
|
143
164
|
config.before(:suite) do
|
144
|
-
Time.zone = 'EST'
|
145
|
-
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
146
165
|
setup_db(DbConfigs::DB_OPTIONS.last)
|
166
|
+
|
167
|
+
DatabaseCleaner.strategy = :transaction
|
168
|
+
DatabaseCleaner.clean_with(:truncation)
|
147
169
|
end
|
148
170
|
|
149
171
|
config.mock_with(:rspec) do |mocks|
|
@@ -164,6 +186,39 @@ RSpec.configure do |config|
|
|
164
186
|
deimos_config.schema.backend = :avro_validation
|
165
187
|
end
|
166
188
|
end
|
189
|
+
|
190
|
+
config.around(:each) do |example|
|
191
|
+
use_cleaner = !example.metadata[:integration]
|
192
|
+
|
193
|
+
DatabaseCleaner.start if use_cleaner
|
194
|
+
|
195
|
+
example.run
|
196
|
+
|
197
|
+
DatabaseCleaner.clean if use_cleaner
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
RSpec.shared_context('with widgets') do
|
202
|
+
before(:all) do
|
203
|
+
ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
|
204
|
+
t.string(:test_id)
|
205
|
+
t.integer(:some_int)
|
206
|
+
t.boolean(:some_bool)
|
207
|
+
t.timestamps
|
208
|
+
end
|
209
|
+
|
210
|
+
# :nodoc:
|
211
|
+
class Widget < ActiveRecord::Base
|
212
|
+
# @return [String]
|
213
|
+
def generated_id
|
214
|
+
'generated_id'
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
after(:all) do
|
220
|
+
ActiveRecord::Base.connection.drop_table(:widgets)
|
221
|
+
end
|
167
222
|
end
|
168
223
|
|
169
224
|
RSpec.shared_context('with DB') do
|
@@ -0,0 +1,320 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# @param seconds [Integer]
|
4
|
+
# @return [Time]
|
5
|
+
def time_value(secs: 0, mins: 0)
|
6
|
+
Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
|
7
|
+
end
|
8
|
+
|
9
|
+
each_db_config(Deimos::Utils::DbPoller) do
|
10
|
+
|
11
|
+
before(:each) do
|
12
|
+
Deimos::PollInfo.delete_all
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '#start!' do
|
16
|
+
|
17
|
+
before(:each) do
|
18
|
+
producer_class = Class.new(Deimos::Producer) do
|
19
|
+
schema 'MySchema'
|
20
|
+
namespace 'com.my-namespace'
|
21
|
+
topic 'my-topic'
|
22
|
+
key_config field: 'test_id'
|
23
|
+
end
|
24
|
+
stub_const('MyProducer', producer_class)
|
25
|
+
|
26
|
+
producer_class = Class.new(Deimos::Producer) do
|
27
|
+
schema 'MySchemaWithId'
|
28
|
+
namespace 'com.my-namespace'
|
29
|
+
topic 'my-topic'
|
30
|
+
key_config plain: true
|
31
|
+
end
|
32
|
+
stub_const('MyProducerWithID', producer_class)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should raise an error if no pollers configured' do
|
36
|
+
Deimos.configure {}
|
37
|
+
expect { described_class.start! }.to raise_error('No pollers configured!')
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'should start pollers as configured' do
|
41
|
+
Deimos.configure do
|
42
|
+
db_poller do
|
43
|
+
producer_class 'MyProducer'
|
44
|
+
end
|
45
|
+
db_poller do
|
46
|
+
producer_class 'MyProducerWithID'
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
allow(Deimos::Utils::DbPoller).to receive(:new)
|
51
|
+
signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
|
52
|
+
allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
|
53
|
+
described_class.start!
|
54
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).twice
|
55
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
56
|
+
with(Deimos.config.db_poller_objects[0])
|
57
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
58
|
+
with(Deimos.config.db_poller_objects[1])
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
describe 'pollers' do
|
63
|
+
include_context 'with widgets'
|
64
|
+
|
65
|
+
let(:poller) do
|
66
|
+
poller = described_class.new(config)
|
67
|
+
allow(poller).to receive(:sleep)
|
68
|
+
poller
|
69
|
+
end
|
70
|
+
|
71
|
+
let(:config) { Deimos.config.db_poller_objects.first.dup }
|
72
|
+
|
73
|
+
before(:each) do
|
74
|
+
Widget.delete_all
|
75
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
76
|
+
schema 'MySchemaWithId'
|
77
|
+
namespace 'com.my-namespace'
|
78
|
+
topic 'my-topic-with-id'
|
79
|
+
key_config none: true
|
80
|
+
record_class Widget
|
81
|
+
|
82
|
+
# :nodoc:
|
83
|
+
def self.generate_payload(attrs, widget)
|
84
|
+
super.merge(message_id: widget.generated_id)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
stub_const('MyProducer', producer_class)
|
88
|
+
|
89
|
+
Deimos.configure do
|
90
|
+
db_poller do
|
91
|
+
producer_class 'MyProducer'
|
92
|
+
run_every 1.minute
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
after(:each) do
|
98
|
+
travel_back
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should crash if initialized with an invalid producer' do
|
102
|
+
config.producer_class = 'NoProducer'
|
103
|
+
expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
|
104
|
+
end
|
105
|
+
|
106
|
+
describe '#retrieve_poll_info' do
|
107
|
+
|
108
|
+
it 'should start from beginning when configured' do
|
109
|
+
poller.retrieve_poll_info
|
110
|
+
expect(Deimos::PollInfo.count).to eq(1)
|
111
|
+
info = Deimos::PollInfo.last
|
112
|
+
expect(info.producer).to eq('MyProducer')
|
113
|
+
expect(info.last_sent).to eq(Time.new(0))
|
114
|
+
expect(info.last_sent_id).to eq(0)
|
115
|
+
end
|
116
|
+
|
117
|
+
it 'should start from now when configured' do
|
118
|
+
travel_to time_value
|
119
|
+
config.start_from_beginning = false
|
120
|
+
poller.retrieve_poll_info
|
121
|
+
expect(Deimos::PollInfo.count).to eq(1)
|
122
|
+
info = Deimos::PollInfo.last
|
123
|
+
expect(info.producer).to eq('MyProducer')
|
124
|
+
expect(info.last_sent).to eq(time_value)
|
125
|
+
expect(info.last_sent_id).to eq(0)
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|
129
|
+
|
130
|
+
specify '#start' do
|
131
|
+
i = 0
|
132
|
+
expect(poller).to receive(:process_updates).twice do
|
133
|
+
i += 1
|
134
|
+
poller.stop if i == 2
|
135
|
+
end
|
136
|
+
poller.start
|
137
|
+
end
|
138
|
+
|
139
|
+
specify '#should_run?' do
|
140
|
+
Deimos::PollInfo.create!(producer: 'MyProducer',
|
141
|
+
last_sent: time_value)
|
142
|
+
poller.retrieve_poll_info
|
143
|
+
|
144
|
+
# run_every is set to 1 minute
|
145
|
+
travel_to time_value(secs: 62)
|
146
|
+
expect(poller.should_run?).to eq(true)
|
147
|
+
|
148
|
+
travel_to time_value(secs: 30)
|
149
|
+
expect(poller.should_run?).to eq(false)
|
150
|
+
|
151
|
+
travel_to time_value(mins: -1) # this shouldn't be possible but meh
|
152
|
+
expect(poller.should_run?).to eq(false)
|
153
|
+
|
154
|
+
# take the 2 seconds of delay_time into account
|
155
|
+
travel_to time_value(secs: 60)
|
156
|
+
expect(poller.should_run?).to eq(false)
|
157
|
+
end
|
158
|
+
|
159
|
+
specify '#process_batch' do
|
160
|
+
travel_to time_value
|
161
|
+
widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
|
162
|
+
widgets.last.update_attribute(:updated_at, time_value(mins: -30))
|
163
|
+
expect(MyProducer).to receive(:send_events).with(widgets)
|
164
|
+
poller.retrieve_poll_info
|
165
|
+
poller.process_batch(widgets)
|
166
|
+
info = Deimos::PollInfo.last
|
167
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
|
168
|
+
expect(info.last_sent_id).to eq(widgets.last.id)
|
169
|
+
end
|
170
|
+
|
171
|
+
describe '#process_updates' do
|
172
|
+
before(:each) do
|
173
|
+
Deimos::PollInfo.create!(producer: 'MyProducer',
|
174
|
+
last_sent: time_value(mins: -61),
|
175
|
+
last_sent_id: 0)
|
176
|
+
poller.retrieve_poll_info
|
177
|
+
travel_to time_value
|
178
|
+
stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
|
179
|
+
end
|
180
|
+
|
181
|
+
let!(:old_widget) do
|
182
|
+
# old widget, earlier than window
|
183
|
+
Widget.create!(test_id: 'some_id', some_int: 40,
|
184
|
+
updated_at: time_value(mins: -200))
|
185
|
+
end
|
186
|
+
|
187
|
+
let!(:last_widget) do
|
188
|
+
# new widget, before delay
|
189
|
+
Widget.create!(test_id: 'some_id', some_int: 10,
|
190
|
+
updated_at: time_value(secs: -1))
|
191
|
+
end
|
192
|
+
|
193
|
+
let!(:widgets) do
|
194
|
+
(1..7).map do |i|
|
195
|
+
Widget.create!(test_id: 'some_id', some_int: i,
|
196
|
+
updated_at: time_value(mins: -61, secs: 30 + i))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
it 'should update the full table' do
|
201
|
+
info = Deimos::PollInfo.last
|
202
|
+
config.full_table = true
|
203
|
+
expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
|
204
|
+
expect(poller).to receive(:process_batch).ordered.
|
205
|
+
with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
|
206
|
+
m.call(*args)
|
207
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
|
208
|
+
expect(info.last_sent_id).to eq(widgets[1].id)
|
209
|
+
end
|
210
|
+
expect(poller).to receive(:process_batch).ordered.
|
211
|
+
with([widgets[2], widgets[3], widgets[4]]).and_call_original
|
212
|
+
expect(poller).to receive(:process_batch).ordered.
|
213
|
+
with([widgets[5], widgets[6]]).and_call_original
|
214
|
+
poller.process_updates
|
215
|
+
|
216
|
+
# this is the updated_at of widgets[6]
|
217
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
|
218
|
+
expect(info.last_sent_id).to eq(widgets[6].id)
|
219
|
+
|
220
|
+
last_widget.update_attribute(:updated_at, time_value(mins: -250))
|
221
|
+
|
222
|
+
travel 61.seconds
|
223
|
+
# should reprocess the table
|
224
|
+
expect(poller).to receive(:process_batch).ordered.
|
225
|
+
with([last_widget, old_widget, widgets[0]]).and_call_original
|
226
|
+
expect(poller).to receive(:process_batch).ordered.
|
227
|
+
with([widgets[1], widgets[2], widgets[3]]).and_call_original
|
228
|
+
expect(poller).to receive(:process_batch).ordered.
|
229
|
+
with([widgets[4], widgets[5], widgets[6]]).and_call_original
|
230
|
+
poller.process_updates
|
231
|
+
|
232
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
|
233
|
+
expect(info.last_sent_id).to eq(widgets[6].id)
|
234
|
+
end
|
235
|
+
|
236
|
+
it 'should send events across multiple batches' do
|
237
|
+
allow(MyProducer).to receive(:poll_query).and_call_original
|
238
|
+
expect(poller).to receive(:process_batch).ordered.
|
239
|
+
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
240
|
+
expect(poller).to receive(:process_batch).ordered.
|
241
|
+
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
242
|
+
expect(poller).to receive(:process_batch).ordered.
|
243
|
+
with([widgets[6]]).and_call_original
|
244
|
+
poller.process_updates
|
245
|
+
|
246
|
+
expect(MyProducer).to have_received(:poll_query).
|
247
|
+
with(time_from: time_value(mins: -61),
|
248
|
+
time_to: time_value(secs: -2),
|
249
|
+
column_name: :updated_at,
|
250
|
+
min_id: 0)
|
251
|
+
|
252
|
+
travel 61.seconds
|
253
|
+
# process the last widget which came in during the delay
|
254
|
+
expect(poller).to receive(:process_batch).with([last_widget]).
|
255
|
+
and_call_original
|
256
|
+
poller.process_updates
|
257
|
+
|
258
|
+
# widgets[6] updated_at value
|
259
|
+
expect(MyProducer).to have_received(:poll_query).
|
260
|
+
with(time_from: time_value(mins: -61, secs: 37),
|
261
|
+
time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
|
262
|
+
column_name: :updated_at,
|
263
|
+
min_id: widgets[6].id)
|
264
|
+
|
265
|
+
travel 61.seconds
|
266
|
+
# nothing else to process
|
267
|
+
expect(poller).not_to receive(:process_batch)
|
268
|
+
poller.process_updates
|
269
|
+
poller.process_updates
|
270
|
+
|
271
|
+
expect(MyProducer).to have_received(:poll_query).twice.
|
272
|
+
with(time_from: time_value(secs: -1),
|
273
|
+
time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
|
274
|
+
column_name: :updated_at,
|
275
|
+
min_id: last_widget.id)
|
276
|
+
end
|
277
|
+
|
278
|
+
it 'should recover correctly with errors and save the right ID' do
|
279
|
+
widgets.each do |w|
|
280
|
+
w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
|
281
|
+
end
|
282
|
+
allow(MyProducer).to receive(:poll_query).and_call_original
|
283
|
+
expect(poller).to receive(:process_batch).ordered.
|
284
|
+
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
285
|
+
expect(poller).to receive(:process_batch).ordered.
|
286
|
+
with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
|
287
|
+
|
288
|
+
expect { poller.process_updates }.to raise_exception('OH NOES')
|
289
|
+
|
290
|
+
expect(MyProducer).to have_received(:poll_query).
|
291
|
+
with(time_from: time_value(mins: -61),
|
292
|
+
time_to: time_value(secs: -2),
|
293
|
+
column_name: :updated_at,
|
294
|
+
min_id: 0)
|
295
|
+
|
296
|
+
info = Deimos::PollInfo.last
|
297
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
|
298
|
+
expect(info.last_sent_id).to eq(widgets[2].id)
|
299
|
+
|
300
|
+
travel 61.seconds
|
301
|
+
# process the last widget which came in during the delay
|
302
|
+
expect(poller).to receive(:process_batch).ordered.
|
303
|
+
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
304
|
+
expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
|
305
|
+
and_call_original
|
306
|
+
poller.process_updates
|
307
|
+
expect(MyProducer).to have_received(:poll_query).
|
308
|
+
with(time_from: time_value(mins: -61, secs: 30),
|
309
|
+
time_to: time_value(secs: 59),
|
310
|
+
column_name: :updated_at,
|
311
|
+
min_id: widgets[2].id)
|
312
|
+
|
313
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
|
314
|
+
expect(info.last_sent_id).to eq(last_widget.id)
|
315
|
+
end
|
316
|
+
|
317
|
+
end
|
318
|
+
|
319
|
+
end
|
320
|
+
end
|