deimos-ruby 1.6.2 → 1.8.0.pre.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +15 -13
- data/.ruby-version +1 -1
- data/CHANGELOG.md +31 -0
- data/Gemfile.lock +43 -36
- data/README.md +141 -16
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +2 -1
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +27 -0
- data/lib/deimos.rb +7 -6
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/batch_consumer.rb +2 -140
- data/lib/deimos/config/configuration.rb +28 -10
- data/lib/deimos/consume/batch_consumption.rb +150 -0
- data/lib/deimos/consume/message_consumption.rb +94 -0
- data/lib/deimos/consumer.rb +79 -69
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +8 -3
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +3 -11
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/batch_consumer_spec.rb +24 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +8 -8
- data/spec/deimos_spec.rb +57 -49
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/spec_helper.rb +61 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- metadata +55 -20
- data/lib/deimos/base_consumer.rb +0 -104
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
data/spec/deimos_spec.rb
CHANGED
@@ -68,11 +68,11 @@ describe Deimos do
|
|
68
68
|
|
69
69
|
describe '#start_db_backend!' do
|
70
70
|
it 'should start if backend is db and thread_count is > 0' do
|
71
|
-
signal_handler = instance_double(
|
71
|
+
signal_handler = instance_double(Sigurd::SignalHandler)
|
72
72
|
allow(signal_handler).to receive(:run!)
|
73
|
-
expect(
|
73
|
+
expect(Sigurd::Executor).to receive(:new).
|
74
74
|
with(anything, sleep_seconds: 5, logger: anything).and_call_original
|
75
|
-
expect(
|
75
|
+
expect(Sigurd::SignalHandler).to receive(:new) do |executor|
|
76
76
|
expect(executor.runners.size).to eq(2)
|
77
77
|
signal_handler
|
78
78
|
end
|
@@ -83,7 +83,7 @@ describe Deimos do
|
|
83
83
|
end
|
84
84
|
|
85
85
|
it 'should not start if backend is not db' do
|
86
|
-
expect(
|
86
|
+
expect(Sigurd::SignalHandler).not_to receive(:new)
|
87
87
|
described_class.configure do |config|
|
88
88
|
config.producers.backend = :kafka
|
89
89
|
end
|
@@ -92,7 +92,7 @@ describe Deimos do
|
|
92
92
|
end
|
93
93
|
|
94
94
|
it 'should not start if thread_count is nil' do
|
95
|
-
expect(
|
95
|
+
expect(Sigurd::SignalHandler).not_to receive(:new)
|
96
96
|
described_class.configure do |config|
|
97
97
|
config.producers.backend = :db
|
98
98
|
end
|
@@ -101,61 +101,69 @@ describe Deimos do
|
|
101
101
|
end
|
102
102
|
|
103
103
|
it 'should not start if thread_count is 0' do
|
104
|
-
expect(
|
104
|
+
expect(Sigurd::SignalHandler).not_to receive(:new)
|
105
105
|
described_class.configure do |config|
|
106
106
|
config.producers.backend = :db
|
107
107
|
end
|
108
108
|
expect { described_class.start_db_backend!(thread_count: 0) }.
|
109
109
|
to raise_error('Thread count is not given or set to zero, exiting')
|
110
110
|
end
|
111
|
+
end
|
111
112
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
it 'should not raise an error with properly configured handlers' do
|
118
|
-
path = config_path # for scope issues in the block below
|
119
|
-
# Add explicit consumers
|
120
|
-
phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
|
121
|
-
'delivery' => 'message' }
|
122
|
-
phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
|
123
|
-
'delivery' => 'batch' }
|
124
|
-
|
125
|
-
expect {
|
126
|
-
described_class.configure { |c| c.phobos_config_file = path }
|
127
|
-
}.not_to raise_error
|
128
|
-
end
|
129
|
-
|
130
|
-
it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
|
131
|
-
path = config_path # for scope issues in the block below
|
132
|
-
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
|
133
|
-
'delivery' => 'message' }]
|
134
|
-
|
135
|
-
expect {
|
136
|
-
described_class.configure { |c| c.phobos_config_file = path }
|
137
|
-
}.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
|
138
|
-
end
|
139
|
-
|
140
|
-
it 'should raise an error if Consumers do not have message or batch delivery' do
|
141
|
-
path = config_path # for scope issues in the block below
|
142
|
-
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
|
143
|
-
'delivery' => 'inline_batch' }]
|
113
|
+
describe 'delivery configuration' do
|
114
|
+
before(:each) do
|
115
|
+
allow(YAML).to receive(:load).and_return(phobos_configuration)
|
116
|
+
end
|
144
117
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
118
|
+
it 'should not raise an error with properly configured handlers' do
|
119
|
+
expect {
|
120
|
+
described_class.configure do
|
121
|
+
consumer do
|
122
|
+
class_name 'ConsumerTest::MyConsumer'
|
123
|
+
delivery :message
|
124
|
+
end
|
125
|
+
consumer do
|
126
|
+
class_name 'ConsumerTest::MyConsumer'
|
127
|
+
delivery :batch
|
128
|
+
end
|
129
|
+
consumer do
|
130
|
+
class_name 'ConsumerTest::MyBatchConsumer'
|
131
|
+
delivery :inline_batch
|
132
|
+
end
|
133
|
+
end
|
134
|
+
}.not_to raise_error
|
135
|
+
end
|
149
136
|
|
150
|
-
|
151
|
-
|
152
|
-
|
137
|
+
it 'should raise an error if inline_batch listeners do not implement consume_batch' do
|
138
|
+
expect {
|
139
|
+
described_class.configure do
|
140
|
+
consumer do
|
141
|
+
class_name 'ConsumerTest::MyConsumer'
|
142
|
+
delivery :inline_batch
|
143
|
+
end
|
144
|
+
end
|
145
|
+
}.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
|
146
|
+
end
|
153
147
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
148
|
+
it 'should raise an error if Consumers do not have message or batch delivery' do
|
149
|
+
expect {
|
150
|
+
described_class.configure do
|
151
|
+
consumer do
|
152
|
+
class_name 'ConsumerTest::MyBatchConsumer'
|
153
|
+
delivery :message
|
154
|
+
end
|
155
|
+
end
|
156
|
+
}.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
|
157
|
+
end
|
158
158
|
|
159
|
+
it 'should treat nil as `batch`' do
|
160
|
+
expect {
|
161
|
+
described_class.configure do
|
162
|
+
consumer do
|
163
|
+
class_name 'ConsumerTest::MyConsumer'
|
164
|
+
end
|
165
|
+
end
|
166
|
+
}.not_to raise_error
|
159
167
|
end
|
160
168
|
end
|
161
169
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.describe(Deimos::Message) do
|
4
|
+
it 'should detect tombstones' do
|
5
|
+
expect(described_class.new(nil, nil, key: 'key1')).
|
6
|
+
to be_tombstone
|
7
|
+
expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
|
8
|
+
not_to be_tombstone
|
9
|
+
expect(described_class.new({ v: '' }, nil, key: 'key1')).
|
10
|
+
not_to be_tombstone
|
11
|
+
expect(described_class.new({ v: 'val1' }, nil, key: nil)).
|
12
|
+
not_to be_tombstone
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'can support complex keys/values' do
|
16
|
+
expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
|
17
|
+
not_to raise_exception
|
18
|
+
end
|
19
|
+
end
|
data/spec/producer_spec.rb
CHANGED
@@ -148,7 +148,7 @@ module ProducerTest
|
|
148
148
|
Deimos.disable_producers do
|
149
149
|
raise 'OH NOES'
|
150
150
|
end
|
151
|
-
}
|
151
|
+
}.to raise_error('OH NOES')
|
152
152
|
expect(Deimos).not_to be_producers_disabled
|
153
153
|
end
|
154
154
|
|
@@ -246,7 +246,7 @@ module ProducerTest
|
|
246
246
|
MyNonEncodedProducer.publish_list(
|
247
247
|
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
248
248
|
)
|
249
|
-
}
|
249
|
+
}.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
|
250
250
|
end
|
251
251
|
|
252
252
|
it 'should allow nil keys if none: true is configured' do
|
@@ -254,7 +254,7 @@ module ProducerTest
|
|
254
254
|
MyNoKeyProducer.publish_list(
|
255
255
|
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
256
256
|
)
|
257
|
-
}
|
257
|
+
}.not_to raise_error
|
258
258
|
end
|
259
259
|
|
260
260
|
it 'should use a partition key' do
|
data/spec/rake_spec.rb
CHANGED
@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
|
|
9
9
|
Rake::Task.define_task(:environment)
|
10
10
|
end
|
11
11
|
|
12
|
-
describe 'Rakefile' do
|
12
|
+
describe 'Rakefile' do
|
13
13
|
it 'should start listeners' do
|
14
14
|
runner = instance_double(Phobos::CLI::Runner)
|
15
15
|
expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
|
@@ -0,0 +1,18 @@
|
|
1
|
+
{
|
2
|
+
"namespace": "com.my-namespace",
|
3
|
+
"name": "MySchemaCompound-key",
|
4
|
+
"type": "record",
|
5
|
+
"doc": "Test schema",
|
6
|
+
"fields": [
|
7
|
+
{
|
8
|
+
"name": "part_one",
|
9
|
+
"type": "string",
|
10
|
+
"doc": "test string one"
|
11
|
+
},
|
12
|
+
{
|
13
|
+
"name": "part_two",
|
14
|
+
"type": "string",
|
15
|
+
"doc": "test string two"
|
16
|
+
}
|
17
|
+
]
|
18
|
+
}
|
@@ -0,0 +1,43 @@
|
|
1
|
+
{
|
2
|
+
"namespace": "com.my-namespace",
|
3
|
+
"name": "Wibble",
|
4
|
+
"type": "record",
|
5
|
+
"fields": [
|
6
|
+
{
|
7
|
+
"name": "id",
|
8
|
+
"type": "long"
|
9
|
+
},
|
10
|
+
{
|
11
|
+
"name": "wibble_id",
|
12
|
+
"type": "long"
|
13
|
+
},
|
14
|
+
{
|
15
|
+
"name": "name",
|
16
|
+
"type": "string"
|
17
|
+
},
|
18
|
+
{
|
19
|
+
"name": "floop",
|
20
|
+
"type": "string"
|
21
|
+
},
|
22
|
+
{
|
23
|
+
"name": "birthday_int",
|
24
|
+
"type": "int"
|
25
|
+
},
|
26
|
+
{
|
27
|
+
"name": "birthday_long",
|
28
|
+
"type": "long"
|
29
|
+
},
|
30
|
+
{
|
31
|
+
"name": "birthday_optional",
|
32
|
+
"type": ["null", "int"]
|
33
|
+
},
|
34
|
+
{
|
35
|
+
"name": "updated_at",
|
36
|
+
"type": "long"
|
37
|
+
},
|
38
|
+
{
|
39
|
+
"name": "created_at",
|
40
|
+
"type": "long"
|
41
|
+
}
|
42
|
+
]
|
43
|
+
}
|
data/spec/spec_helper.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
4
4
|
require 'active_record'
|
5
|
+
require 'database_cleaner'
|
5
6
|
require 'deimos'
|
6
7
|
require 'deimos/metrics/mock'
|
7
8
|
require 'deimos/tracing/mock'
|
@@ -100,9 +101,8 @@ module DbConfigs
|
|
100
101
|
end
|
101
102
|
end
|
102
103
|
|
103
|
-
#
|
104
|
-
def
|
105
|
-
ActiveRecord::Base.establish_connection(options)
|
104
|
+
# :nodoc:
|
105
|
+
def run_db_backend_migration
|
106
106
|
migration_class_name = 'DbBackendMigration'
|
107
107
|
migration_version = '[5.2]'
|
108
108
|
migration = ERB.new(
|
@@ -110,6 +110,24 @@ module DbConfigs
|
|
110
110
|
).result(binding)
|
111
111
|
eval(migration) # rubocop:disable Security/Eval
|
112
112
|
ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
|
113
|
+
end
|
114
|
+
|
115
|
+
# :nodoc:
|
116
|
+
def run_db_poller_migration
|
117
|
+
migration_class_name = 'DbPollerMigration'
|
118
|
+
migration_version = '[5.2]'
|
119
|
+
migration = ERB.new(
|
120
|
+
File.read('lib/generators/deimos/db_poller/templates/migration')
|
121
|
+
).result(binding)
|
122
|
+
eval(migration) # rubocop:disable Security/Eval
|
123
|
+
ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
|
124
|
+
end
|
125
|
+
|
126
|
+
# Set up the given database.
|
127
|
+
def setup_db(options)
|
128
|
+
ActiveRecord::Base.establish_connection(options)
|
129
|
+
run_db_backend_migration
|
130
|
+
run_db_poller_migration
|
113
131
|
|
114
132
|
ActiveRecord::Base.descendants.each do |klass|
|
115
133
|
klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
|
@@ -130,8 +148,11 @@ RSpec.configure do |config|
|
|
130
148
|
# true by default for RSpec 4.0
|
131
149
|
config.shared_context_metadata_behavior = :apply_to_host_groups
|
132
150
|
|
151
|
+
config.filter_run(focus: true)
|
152
|
+
config.run_all_when_everything_filtered = true
|
153
|
+
|
133
154
|
config.before(:all) do
|
134
|
-
Time.zone = '
|
155
|
+
Time.zone = 'Eastern Time (US & Canada)'
|
135
156
|
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
136
157
|
ActiveRecord::Base.establish_connection(
|
137
158
|
'adapter' => 'sqlite3',
|
@@ -141,9 +162,10 @@ RSpec.configure do |config|
|
|
141
162
|
config.include Deimos::TestHelpers
|
142
163
|
config.include ActiveSupport::Testing::TimeHelpers
|
143
164
|
config.before(:suite) do
|
144
|
-
Time.zone = 'EST'
|
145
|
-
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
146
165
|
setup_db(DbConfigs::DB_OPTIONS.last)
|
166
|
+
|
167
|
+
DatabaseCleaner.strategy = :transaction
|
168
|
+
DatabaseCleaner.clean_with(:truncation)
|
147
169
|
end
|
148
170
|
|
149
171
|
config.mock_with(:rspec) do |mocks|
|
@@ -164,6 +186,39 @@ RSpec.configure do |config|
|
|
164
186
|
deimos_config.schema.backend = :avro_validation
|
165
187
|
end
|
166
188
|
end
|
189
|
+
|
190
|
+
config.around(:each) do |example|
|
191
|
+
use_cleaner = !example.metadata[:integration]
|
192
|
+
|
193
|
+
DatabaseCleaner.start if use_cleaner
|
194
|
+
|
195
|
+
example.run
|
196
|
+
|
197
|
+
DatabaseCleaner.clean if use_cleaner
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
RSpec.shared_context('with widgets') do
|
202
|
+
before(:all) do
|
203
|
+
ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
|
204
|
+
t.string(:test_id)
|
205
|
+
t.integer(:some_int)
|
206
|
+
t.boolean(:some_bool)
|
207
|
+
t.timestamps
|
208
|
+
end
|
209
|
+
|
210
|
+
# :nodoc:
|
211
|
+
class Widget < ActiveRecord::Base
|
212
|
+
# @return [String]
|
213
|
+
def generated_id
|
214
|
+
'generated_id'
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
after(:all) do
|
220
|
+
ActiveRecord::Base.connection.drop_table(:widgets)
|
221
|
+
end
|
167
222
|
end
|
168
223
|
|
169
224
|
RSpec.shared_context('with DB') do
|
@@ -0,0 +1,320 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# @param seconds [Integer]
|
4
|
+
# @return [Time]
|
5
|
+
def time_value(secs: 0, mins: 0)
|
6
|
+
Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
|
7
|
+
end
|
8
|
+
|
9
|
+
each_db_config(Deimos::Utils::DbPoller) do
|
10
|
+
|
11
|
+
before(:each) do
|
12
|
+
Deimos::PollInfo.delete_all
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '#start!' do
|
16
|
+
|
17
|
+
before(:each) do
|
18
|
+
producer_class = Class.new(Deimos::Producer) do
|
19
|
+
schema 'MySchema'
|
20
|
+
namespace 'com.my-namespace'
|
21
|
+
topic 'my-topic'
|
22
|
+
key_config field: 'test_id'
|
23
|
+
end
|
24
|
+
stub_const('MyProducer', producer_class)
|
25
|
+
|
26
|
+
producer_class = Class.new(Deimos::Producer) do
|
27
|
+
schema 'MySchemaWithId'
|
28
|
+
namespace 'com.my-namespace'
|
29
|
+
topic 'my-topic'
|
30
|
+
key_config plain: true
|
31
|
+
end
|
32
|
+
stub_const('MyProducerWithID', producer_class)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should raise an error if no pollers configured' do
|
36
|
+
Deimos.configure {}
|
37
|
+
expect { described_class.start! }.to raise_error('No pollers configured!')
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'should start pollers as configured' do
|
41
|
+
Deimos.configure do
|
42
|
+
db_poller do
|
43
|
+
producer_class 'MyProducer'
|
44
|
+
end
|
45
|
+
db_poller do
|
46
|
+
producer_class 'MyProducerWithID'
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
allow(Deimos::Utils::DbPoller).to receive(:new)
|
51
|
+
signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
|
52
|
+
allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
|
53
|
+
described_class.start!
|
54
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).twice
|
55
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
56
|
+
with(Deimos.config.db_poller_objects[0])
|
57
|
+
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
58
|
+
with(Deimos.config.db_poller_objects[1])
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
describe 'pollers' do
|
63
|
+
include_context 'with widgets'
|
64
|
+
|
65
|
+
let(:poller) do
|
66
|
+
poller = described_class.new(config)
|
67
|
+
allow(poller).to receive(:sleep)
|
68
|
+
poller
|
69
|
+
end
|
70
|
+
|
71
|
+
let(:config) { Deimos.config.db_poller_objects.first.dup }
|
72
|
+
|
73
|
+
before(:each) do
|
74
|
+
Widget.delete_all
|
75
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
76
|
+
schema 'MySchemaWithId'
|
77
|
+
namespace 'com.my-namespace'
|
78
|
+
topic 'my-topic-with-id'
|
79
|
+
key_config none: true
|
80
|
+
record_class Widget
|
81
|
+
|
82
|
+
# :nodoc:
|
83
|
+
def self.generate_payload(attrs, widget)
|
84
|
+
super.merge(message_id: widget.generated_id)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
stub_const('MyProducer', producer_class)
|
88
|
+
|
89
|
+
Deimos.configure do
|
90
|
+
db_poller do
|
91
|
+
producer_class 'MyProducer'
|
92
|
+
run_every 1.minute
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
after(:each) do
|
98
|
+
travel_back
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should crash if initialized with an invalid producer' do
|
102
|
+
config.producer_class = 'NoProducer'
|
103
|
+
expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
|
104
|
+
end
|
105
|
+
|
106
|
+
describe '#retrieve_poll_info' do
|
107
|
+
|
108
|
+
it 'should start from beginning when configured' do
|
109
|
+
poller.retrieve_poll_info
|
110
|
+
expect(Deimos::PollInfo.count).to eq(1)
|
111
|
+
info = Deimos::PollInfo.last
|
112
|
+
expect(info.producer).to eq('MyProducer')
|
113
|
+
expect(info.last_sent).to eq(Time.new(0))
|
114
|
+
expect(info.last_sent_id).to eq(0)
|
115
|
+
end
|
116
|
+
|
117
|
+
it 'should start from now when configured' do
|
118
|
+
travel_to time_value
|
119
|
+
config.start_from_beginning = false
|
120
|
+
poller.retrieve_poll_info
|
121
|
+
expect(Deimos::PollInfo.count).to eq(1)
|
122
|
+
info = Deimos::PollInfo.last
|
123
|
+
expect(info.producer).to eq('MyProducer')
|
124
|
+
expect(info.last_sent).to eq(time_value)
|
125
|
+
expect(info.last_sent_id).to eq(0)
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|
129
|
+
|
130
|
+
specify '#start' do
|
131
|
+
i = 0
|
132
|
+
expect(poller).to receive(:process_updates).twice do
|
133
|
+
i += 1
|
134
|
+
poller.stop if i == 2
|
135
|
+
end
|
136
|
+
poller.start
|
137
|
+
end
|
138
|
+
|
139
|
+
specify '#should_run?' do
|
140
|
+
Deimos::PollInfo.create!(producer: 'MyProducer',
|
141
|
+
last_sent: time_value)
|
142
|
+
poller.retrieve_poll_info
|
143
|
+
|
144
|
+
# run_every is set to 1 minute
|
145
|
+
travel_to time_value(secs: 62)
|
146
|
+
expect(poller.should_run?).to eq(true)
|
147
|
+
|
148
|
+
travel_to time_value(secs: 30)
|
149
|
+
expect(poller.should_run?).to eq(false)
|
150
|
+
|
151
|
+
travel_to time_value(mins: -1) # this shouldn't be possible but meh
|
152
|
+
expect(poller.should_run?).to eq(false)
|
153
|
+
|
154
|
+
# take the 2 seconds of delay_time into account
|
155
|
+
travel_to time_value(secs: 60)
|
156
|
+
expect(poller.should_run?).to eq(false)
|
157
|
+
end
|
158
|
+
|
159
|
+
specify '#process_batch' do
|
160
|
+
travel_to time_value
|
161
|
+
widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
|
162
|
+
widgets.last.update_attribute(:updated_at, time_value(mins: -30))
|
163
|
+
expect(MyProducer).to receive(:send_events).with(widgets)
|
164
|
+
poller.retrieve_poll_info
|
165
|
+
poller.process_batch(widgets)
|
166
|
+
info = Deimos::PollInfo.last
|
167
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
|
168
|
+
expect(info.last_sent_id).to eq(widgets.last.id)
|
169
|
+
end
|
170
|
+
|
171
|
+
describe '#process_updates' do
|
172
|
+
before(:each) do
|
173
|
+
Deimos::PollInfo.create!(producer: 'MyProducer',
|
174
|
+
last_sent: time_value(mins: -61),
|
175
|
+
last_sent_id: 0)
|
176
|
+
poller.retrieve_poll_info
|
177
|
+
travel_to time_value
|
178
|
+
stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
|
179
|
+
end
|
180
|
+
|
181
|
+
let!(:old_widget) do
|
182
|
+
# old widget, earlier than window
|
183
|
+
Widget.create!(test_id: 'some_id', some_int: 40,
|
184
|
+
updated_at: time_value(mins: -200))
|
185
|
+
end
|
186
|
+
|
187
|
+
let!(:last_widget) do
|
188
|
+
# new widget, before delay
|
189
|
+
Widget.create!(test_id: 'some_id', some_int: 10,
|
190
|
+
updated_at: time_value(secs: -1))
|
191
|
+
end
|
192
|
+
|
193
|
+
let!(:widgets) do
|
194
|
+
(1..7).map do |i|
|
195
|
+
Widget.create!(test_id: 'some_id', some_int: i,
|
196
|
+
updated_at: time_value(mins: -61, secs: 30 + i))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
it 'should update the full table' do
|
201
|
+
info = Deimos::PollInfo.last
|
202
|
+
config.full_table = true
|
203
|
+
expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
|
204
|
+
expect(poller).to receive(:process_batch).ordered.
|
205
|
+
with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
|
206
|
+
m.call(*args)
|
207
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
|
208
|
+
expect(info.last_sent_id).to eq(widgets[1].id)
|
209
|
+
end
|
210
|
+
expect(poller).to receive(:process_batch).ordered.
|
211
|
+
with([widgets[2], widgets[3], widgets[4]]).and_call_original
|
212
|
+
expect(poller).to receive(:process_batch).ordered.
|
213
|
+
with([widgets[5], widgets[6]]).and_call_original
|
214
|
+
poller.process_updates
|
215
|
+
|
216
|
+
# this is the updated_at of widgets[6]
|
217
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
|
218
|
+
expect(info.last_sent_id).to eq(widgets[6].id)
|
219
|
+
|
220
|
+
last_widget.update_attribute(:updated_at, time_value(mins: -250))
|
221
|
+
|
222
|
+
travel 61.seconds
|
223
|
+
# should reprocess the table
|
224
|
+
expect(poller).to receive(:process_batch).ordered.
|
225
|
+
with([last_widget, old_widget, widgets[0]]).and_call_original
|
226
|
+
expect(poller).to receive(:process_batch).ordered.
|
227
|
+
with([widgets[1], widgets[2], widgets[3]]).and_call_original
|
228
|
+
expect(poller).to receive(:process_batch).ordered.
|
229
|
+
with([widgets[4], widgets[5], widgets[6]]).and_call_original
|
230
|
+
poller.process_updates
|
231
|
+
|
232
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
|
233
|
+
expect(info.last_sent_id).to eq(widgets[6].id)
|
234
|
+
end
|
235
|
+
|
236
|
+
it 'should send events across multiple batches' do
|
237
|
+
allow(MyProducer).to receive(:poll_query).and_call_original
|
238
|
+
expect(poller).to receive(:process_batch).ordered.
|
239
|
+
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
240
|
+
expect(poller).to receive(:process_batch).ordered.
|
241
|
+
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
242
|
+
expect(poller).to receive(:process_batch).ordered.
|
243
|
+
with([widgets[6]]).and_call_original
|
244
|
+
poller.process_updates
|
245
|
+
|
246
|
+
expect(MyProducer).to have_received(:poll_query).
|
247
|
+
with(time_from: time_value(mins: -61),
|
248
|
+
time_to: time_value(secs: -2),
|
249
|
+
column_name: :updated_at,
|
250
|
+
min_id: 0)
|
251
|
+
|
252
|
+
travel 61.seconds
|
253
|
+
# process the last widget which came in during the delay
|
254
|
+
expect(poller).to receive(:process_batch).with([last_widget]).
|
255
|
+
and_call_original
|
256
|
+
poller.process_updates
|
257
|
+
|
258
|
+
# widgets[6] updated_at value
|
259
|
+
expect(MyProducer).to have_received(:poll_query).
|
260
|
+
with(time_from: time_value(mins: -61, secs: 37),
|
261
|
+
time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
|
262
|
+
column_name: :updated_at,
|
263
|
+
min_id: widgets[6].id)
|
264
|
+
|
265
|
+
travel 61.seconds
|
266
|
+
# nothing else to process
|
267
|
+
expect(poller).not_to receive(:process_batch)
|
268
|
+
poller.process_updates
|
269
|
+
poller.process_updates
|
270
|
+
|
271
|
+
expect(MyProducer).to have_received(:poll_query).twice.
|
272
|
+
with(time_from: time_value(secs: -1),
|
273
|
+
time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
|
274
|
+
column_name: :updated_at,
|
275
|
+
min_id: last_widget.id)
|
276
|
+
end
|
277
|
+
|
278
|
+
it 'should recover correctly with errors and save the right ID' do
|
279
|
+
widgets.each do |w|
|
280
|
+
w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
|
281
|
+
end
|
282
|
+
allow(MyProducer).to receive(:poll_query).and_call_original
|
283
|
+
expect(poller).to receive(:process_batch).ordered.
|
284
|
+
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
285
|
+
expect(poller).to receive(:process_batch).ordered.
|
286
|
+
with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
|
287
|
+
|
288
|
+
expect { poller.process_updates }.to raise_exception('OH NOES')
|
289
|
+
|
290
|
+
expect(MyProducer).to have_received(:poll_query).
|
291
|
+
with(time_from: time_value(mins: -61),
|
292
|
+
time_to: time_value(secs: -2),
|
293
|
+
column_name: :updated_at,
|
294
|
+
min_id: 0)
|
295
|
+
|
296
|
+
info = Deimos::PollInfo.last
|
297
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
|
298
|
+
expect(info.last_sent_id).to eq(widgets[2].id)
|
299
|
+
|
300
|
+
travel 61.seconds
|
301
|
+
# process the last widget which came in during the delay
|
302
|
+
expect(poller).to receive(:process_batch).ordered.
|
303
|
+
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
304
|
+
expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
|
305
|
+
and_call_original
|
306
|
+
poller.process_updates
|
307
|
+
expect(MyProducer).to have_received(:poll_query).
|
308
|
+
with(time_from: time_value(mins: -61, secs: 30),
|
309
|
+
time_to: time_value(secs: 59),
|
310
|
+
column_name: :updated_at,
|
311
|
+
min_id: widgets[2].id)
|
312
|
+
|
313
|
+
expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
|
314
|
+
expect(info.last_sent_id).to eq(last_widget.id)
|
315
|
+
end
|
316
|
+
|
317
|
+
end
|
318
|
+
|
319
|
+
end
|
320
|
+
end
|