deimos-ruby 1.6.2 → 1.8.0.pre.beta2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +9 -0
  3. data/.rubocop.yml +15 -13
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +31 -0
  6. data/Gemfile.lock +43 -36
  7. data/README.md +141 -16
  8. data/Rakefile +1 -1
  9. data/deimos-ruby.gemspec +2 -1
  10. data/docs/ARCHITECTURE.md +144 -0
  11. data/docs/CONFIGURATION.md +27 -0
  12. data/lib/deimos.rb +7 -6
  13. data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
  14. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  15. data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
  16. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  17. data/lib/deimos/active_record_consumer.rb +33 -75
  18. data/lib/deimos/active_record_producer.rb +23 -0
  19. data/lib/deimos/batch_consumer.rb +2 -140
  20. data/lib/deimos/config/configuration.rb +28 -10
  21. data/lib/deimos/consume/batch_consumption.rb +150 -0
  22. data/lib/deimos/consume/message_consumption.rb +94 -0
  23. data/lib/deimos/consumer.rb +79 -69
  24. data/lib/deimos/kafka_message.rb +1 -1
  25. data/lib/deimos/kafka_topic_info.rb +1 -1
  26. data/lib/deimos/message.rb +6 -1
  27. data/lib/deimos/metrics/provider.rb +0 -2
  28. data/lib/deimos/poll_info.rb +9 -0
  29. data/lib/deimos/tracing/provider.rb +0 -2
  30. data/lib/deimos/utils/db_poller.rb +149 -0
  31. data/lib/deimos/utils/db_producer.rb +8 -3
  32. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  33. data/lib/deimos/utils/lag_reporter.rb +19 -26
  34. data/lib/deimos/version.rb +1 -1
  35. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  36. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  37. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  38. data/lib/tasks/deimos.rake +7 -0
  39. data/spec/active_record_batch_consumer_spec.rb +481 -0
  40. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  41. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  42. data/spec/active_record_consumer_spec.rb +3 -11
  43. data/spec/active_record_producer_spec.rb +66 -88
  44. data/spec/batch_consumer_spec.rb +24 -7
  45. data/spec/config/configuration_spec.rb +4 -0
  46. data/spec/consumer_spec.rb +8 -8
  47. data/spec/deimos_spec.rb +57 -49
  48. data/spec/handlers/my_batch_consumer.rb +6 -1
  49. data/spec/handlers/my_consumer.rb +6 -1
  50. data/spec/message_spec.rb +19 -0
  51. data/spec/producer_spec.rb +3 -3
  52. data/spec/rake_spec.rb +1 -1
  53. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  54. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  55. data/spec/spec_helper.rb +61 -6
  56. data/spec/utils/db_poller_spec.rb +320 -0
  57. data/spec/utils/deadlock_retry_spec.rb +74 -0
  58. data/spec/utils/lag_reporter_spec.rb +29 -22
  59. metadata +55 -20
  60. data/lib/deimos/base_consumer.rb +0 -104
  61. data/lib/deimos/utils/executor.rb +0 -124
  62. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  63. data/lib/deimos/utils/signal_handler.rb +0 -68
  64. data/spec/utils/executor_spec.rb +0 -53
  65. data/spec/utils/signal_handler_spec.rb +0 -16
@@ -68,11 +68,11 @@ describe Deimos do
68
68
 
69
69
  describe '#start_db_backend!' do
70
70
  it 'should start if backend is db and thread_count is > 0' do
71
- signal_handler = instance_double(Deimos::Utils::SignalHandler)
71
+ signal_handler = instance_double(Sigurd::SignalHandler)
72
72
  allow(signal_handler).to receive(:run!)
73
- expect(Deimos::Utils::Executor).to receive(:new).
73
+ expect(Sigurd::Executor).to receive(:new).
74
74
  with(anything, sleep_seconds: 5, logger: anything).and_call_original
75
- expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
75
+ expect(Sigurd::SignalHandler).to receive(:new) do |executor|
76
76
  expect(executor.runners.size).to eq(2)
77
77
  signal_handler
78
78
  end
@@ -83,7 +83,7 @@ describe Deimos do
83
83
  end
84
84
 
85
85
  it 'should not start if backend is not db' do
86
- expect(Deimos::Utils::SignalHandler).not_to receive(:new)
86
+ expect(Sigurd::SignalHandler).not_to receive(:new)
87
87
  described_class.configure do |config|
88
88
  config.producers.backend = :kafka
89
89
  end
@@ -92,7 +92,7 @@ describe Deimos do
92
92
  end
93
93
 
94
94
  it 'should not start if thread_count is nil' do
95
- expect(Deimos::Utils::SignalHandler).not_to receive(:new)
95
+ expect(Sigurd::SignalHandler).not_to receive(:new)
96
96
  described_class.configure do |config|
97
97
  config.producers.backend = :db
98
98
  end
@@ -101,61 +101,69 @@ describe Deimos do
101
101
  end
102
102
 
103
103
  it 'should not start if thread_count is 0' do
104
- expect(Deimos::Utils::SignalHandler).not_to receive(:new)
104
+ expect(Sigurd::SignalHandler).not_to receive(:new)
105
105
  described_class.configure do |config|
106
106
  config.producers.backend = :db
107
107
  end
108
108
  expect { described_class.start_db_backend!(thread_count: 0) }.
109
109
  to raise_error('Thread count is not given or set to zero, exiting')
110
110
  end
111
+ end
111
112
 
112
- describe 'delivery configuration' do
113
- before(:each) do
114
- allow(YAML).to receive(:load).and_return(phobos_configuration)
115
- end
116
-
117
- it 'should not raise an error with properly configured handlers' do
118
- path = config_path # for scope issues in the block below
119
- # Add explicit consumers
120
- phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
121
- 'delivery' => 'message' }
122
- phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
123
- 'delivery' => 'batch' }
124
-
125
- expect {
126
- described_class.configure { |c| c.phobos_config_file = path }
127
- }.not_to raise_error
128
- end
129
-
130
- it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
131
- path = config_path # for scope issues in the block below
132
- phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
133
- 'delivery' => 'message' }]
134
-
135
- expect {
136
- described_class.configure { |c| c.phobos_config_file = path }
137
- }.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
138
- end
139
-
140
- it 'should raise an error if Consumers do not have message or batch delivery' do
141
- path = config_path # for scope issues in the block below
142
- phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
143
- 'delivery' => 'inline_batch' }]
113
+ describe 'delivery configuration' do
114
+ before(:each) do
115
+ allow(YAML).to receive(:load).and_return(phobos_configuration)
116
+ end
144
117
 
145
- expect {
146
- described_class.configure { |c| c.phobos_config_file = path }
147
- }.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
148
- end
118
+ it 'should not raise an error with properly configured handlers' do
119
+ expect {
120
+ described_class.configure do
121
+ consumer do
122
+ class_name 'ConsumerTest::MyConsumer'
123
+ delivery :message
124
+ end
125
+ consumer do
126
+ class_name 'ConsumerTest::MyConsumer'
127
+ delivery :batch
128
+ end
129
+ consumer do
130
+ class_name 'ConsumerTest::MyBatchConsumer'
131
+ delivery :inline_batch
132
+ end
133
+ end
134
+ }.not_to raise_error
135
+ end
149
136
 
150
- it 'should treat nil as `batch`' do
151
- path = config_path # for scope issues in the block below
152
- phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
137
+ it 'should raise an error if inline_batch listeners do not implement consume_batch' do
138
+ expect {
139
+ described_class.configure do
140
+ consumer do
141
+ class_name 'ConsumerTest::MyConsumer'
142
+ delivery :inline_batch
143
+ end
144
+ end
145
+ }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
146
+ end
153
147
 
154
- expect {
155
- described_class.configure { |c| c.phobos_config_file = path }
156
- }.not_to raise_error
157
- end
148
+ it 'should raise an error if Consumers do not have message or batch delivery' do
149
+ expect {
150
+ described_class.configure do
151
+ consumer do
152
+ class_name 'ConsumerTest::MyBatchConsumer'
153
+ delivery :message
154
+ end
155
+ end
156
+ }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
157
+ end
158
158
 
159
+ it 'should treat nil as `batch`' do
160
+ expect {
161
+ described_class.configure do
162
+ consumer do
163
+ class_name 'ConsumerTest::MyConsumer'
164
+ end
165
+ end
166
+ }.not_to raise_error
159
167
  end
160
168
  end
161
169
  end
@@ -1,5 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ConsumerTest
4
- class MyBatchConsumer < Deimos::BatchConsumer; end
4
+ # Mock consumer
5
+ class MyBatchConsumer < Deimos::Consumer
6
+ # :no-doc:
7
+ def consume_batch
8
+ end
9
+ end
5
10
  end
@@ -1,5 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ConsumerTest
4
- class MyConsumer < Deimos::Consumer; end
4
+ # Mock consumer
5
+ class MyConsumer < Deimos::Consumer
6
+ # :no-doc:
7
+ def consume
8
+ end
9
+ end
5
10
  end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe(Deimos::Message) do
4
+ it 'should detect tombstones' do
5
+ expect(described_class.new(nil, nil, key: 'key1')).
6
+ to be_tombstone
7
+ expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
8
+ not_to be_tombstone
9
+ expect(described_class.new({ v: '' }, nil, key: 'key1')).
10
+ not_to be_tombstone
11
+ expect(described_class.new({ v: 'val1' }, nil, key: nil)).
12
+ not_to be_tombstone
13
+ end
14
+
15
+ it 'can support complex keys/values' do
16
+ expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
17
+ not_to raise_exception
18
+ end
19
+ end
@@ -148,7 +148,7 @@ module ProducerTest
148
148
  Deimos.disable_producers do
149
149
  raise 'OH NOES'
150
150
  end
151
- } .to raise_error('OH NOES')
151
+ }.to raise_error('OH NOES')
152
152
  expect(Deimos).not_to be_producers_disabled
153
153
  end
154
154
 
@@ -246,7 +246,7 @@ module ProducerTest
246
246
  MyNonEncodedProducer.publish_list(
247
247
  [{ 'test_id' => 'foo', 'some_int' => 123 }]
248
248
  )
249
- } .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
249
+ }.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
250
250
  end
251
251
 
252
252
  it 'should allow nil keys if none: true is configured' do
@@ -254,7 +254,7 @@ module ProducerTest
254
254
  MyNoKeyProducer.publish_list(
255
255
  [{ 'test_id' => 'foo', 'some_int' => 123 }]
256
256
  )
257
- } .not_to raise_error
257
+ }.not_to raise_error
258
258
  end
259
259
 
260
260
  it 'should use a partition key' do
@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
9
9
  Rake::Task.define_task(:environment)
10
10
  end
11
11
 
12
- describe 'Rakefile' do # rubocop:disable RSpec/DescribeClass
12
+ describe 'Rakefile' do
13
13
  it 'should start listeners' do
14
14
  runner = instance_double(Phobos::CLI::Runner)
15
15
  expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
@@ -0,0 +1,18 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaCompound-key",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "part_one",
9
+ "type": "string",
10
+ "doc": "test string one"
11
+ },
12
+ {
13
+ "name": "part_two",
14
+ "type": "string",
15
+ "doc": "test string two"
16
+ }
17
+ ]
18
+ }
@@ -0,0 +1,43 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "Wibble",
4
+ "type": "record",
5
+ "fields": [
6
+ {
7
+ "name": "id",
8
+ "type": "long"
9
+ },
10
+ {
11
+ "name": "wibble_id",
12
+ "type": "long"
13
+ },
14
+ {
15
+ "name": "name",
16
+ "type": "string"
17
+ },
18
+ {
19
+ "name": "floop",
20
+ "type": "string"
21
+ },
22
+ {
23
+ "name": "birthday_int",
24
+ "type": "int"
25
+ },
26
+ {
27
+ "name": "birthday_long",
28
+ "type": "long"
29
+ },
30
+ {
31
+ "name": "birthday_optional",
32
+ "type": ["null", "int"]
33
+ },
34
+ {
35
+ "name": "updated_at",
36
+ "type": "long"
37
+ },
38
+ {
39
+ "name": "created_at",
40
+ "type": "long"
41
+ }
42
+ ]
43
+ }
@@ -2,6 +2,7 @@
2
2
 
3
3
  $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
4
4
  require 'active_record'
5
+ require 'database_cleaner'
5
6
  require 'deimos'
6
7
  require 'deimos/metrics/mock'
7
8
  require 'deimos/tracing/mock'
@@ -100,9 +101,8 @@ module DbConfigs
100
101
  end
101
102
  end
102
103
 
103
- # Set up the given database.
104
- def setup_db(options)
105
- ActiveRecord::Base.establish_connection(options)
104
+ # :nodoc:
105
+ def run_db_backend_migration
106
106
  migration_class_name = 'DbBackendMigration'
107
107
  migration_version = '[5.2]'
108
108
  migration = ERB.new(
@@ -110,6 +110,24 @@ module DbConfigs
110
110
  ).result(binding)
111
111
  eval(migration) # rubocop:disable Security/Eval
112
112
  ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
113
+ end
114
+
115
+ # :nodoc:
116
+ def run_db_poller_migration
117
+ migration_class_name = 'DbPollerMigration'
118
+ migration_version = '[5.2]'
119
+ migration = ERB.new(
120
+ File.read('lib/generators/deimos/db_poller/templates/migration')
121
+ ).result(binding)
122
+ eval(migration) # rubocop:disable Security/Eval
123
+ ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
124
+ end
125
+
126
+ # Set up the given database.
127
+ def setup_db(options)
128
+ ActiveRecord::Base.establish_connection(options)
129
+ run_db_backend_migration
130
+ run_db_poller_migration
113
131
 
114
132
  ActiveRecord::Base.descendants.each do |klass|
115
133
  klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
@@ -130,8 +148,11 @@ RSpec.configure do |config|
130
148
  # true by default for RSpec 4.0
131
149
  config.shared_context_metadata_behavior = :apply_to_host_groups
132
150
 
151
+ config.filter_run(focus: true)
152
+ config.run_all_when_everything_filtered = true
153
+
133
154
  config.before(:all) do
134
- Time.zone = 'EST'
155
+ Time.zone = 'Eastern Time (US & Canada)'
135
156
  ActiveRecord::Base.logger = Logger.new('/dev/null')
136
157
  ActiveRecord::Base.establish_connection(
137
158
  'adapter' => 'sqlite3',
@@ -141,9 +162,10 @@ RSpec.configure do |config|
141
162
  config.include Deimos::TestHelpers
142
163
  config.include ActiveSupport::Testing::TimeHelpers
143
164
  config.before(:suite) do
144
- Time.zone = 'EST'
145
- ActiveRecord::Base.logger = Logger.new('/dev/null')
146
165
  setup_db(DbConfigs::DB_OPTIONS.last)
166
+
167
+ DatabaseCleaner.strategy = :transaction
168
+ DatabaseCleaner.clean_with(:truncation)
147
169
  end
148
170
 
149
171
  config.mock_with(:rspec) do |mocks|
@@ -164,6 +186,39 @@ RSpec.configure do |config|
164
186
  deimos_config.schema.backend = :avro_validation
165
187
  end
166
188
  end
189
+
190
+ config.around(:each) do |example|
191
+ use_cleaner = !example.metadata[:integration]
192
+
193
+ DatabaseCleaner.start if use_cleaner
194
+
195
+ example.run
196
+
197
+ DatabaseCleaner.clean if use_cleaner
198
+ end
199
+ end
200
+
201
+ RSpec.shared_context('with widgets') do
202
+ before(:all) do
203
+ ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
204
+ t.string(:test_id)
205
+ t.integer(:some_int)
206
+ t.boolean(:some_bool)
207
+ t.timestamps
208
+ end
209
+
210
+ # :nodoc:
211
+ class Widget < ActiveRecord::Base
212
+ # @return [String]
213
+ def generated_id
214
+ 'generated_id'
215
+ end
216
+ end
217
+ end
218
+
219
+ after(:all) do
220
+ ActiveRecord::Base.connection.drop_table(:widgets)
221
+ end
167
222
  end
168
223
 
169
224
  RSpec.shared_context('with DB') do
@@ -0,0 +1,320 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @param seconds [Integer]
4
+ # @return [Time]
5
+ def time_value(secs: 0, mins: 0)
6
+ Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
7
+ end
8
+
9
+ each_db_config(Deimos::Utils::DbPoller) do
10
+
11
+ before(:each) do
12
+ Deimos::PollInfo.delete_all
13
+ end
14
+
15
+ describe '#start!' do
16
+
17
+ before(:each) do
18
+ producer_class = Class.new(Deimos::Producer) do
19
+ schema 'MySchema'
20
+ namespace 'com.my-namespace'
21
+ topic 'my-topic'
22
+ key_config field: 'test_id'
23
+ end
24
+ stub_const('MyProducer', producer_class)
25
+
26
+ producer_class = Class.new(Deimos::Producer) do
27
+ schema 'MySchemaWithId'
28
+ namespace 'com.my-namespace'
29
+ topic 'my-topic'
30
+ key_config plain: true
31
+ end
32
+ stub_const('MyProducerWithID', producer_class)
33
+ end
34
+
35
+ it 'should raise an error if no pollers configured' do
36
+ Deimos.configure {}
37
+ expect { described_class.start! }.to raise_error('No pollers configured!')
38
+ end
39
+
40
+ it 'should start pollers as configured' do
41
+ Deimos.configure do
42
+ db_poller do
43
+ producer_class 'MyProducer'
44
+ end
45
+ db_poller do
46
+ producer_class 'MyProducerWithID'
47
+ end
48
+ end
49
+
50
+ allow(Deimos::Utils::DbPoller).to receive(:new)
51
+ signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
52
+ allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
53
+ described_class.start!
54
+ expect(Deimos::Utils::DbPoller).to have_received(:new).twice
55
+ expect(Deimos::Utils::DbPoller).to have_received(:new).
56
+ with(Deimos.config.db_poller_objects[0])
57
+ expect(Deimos::Utils::DbPoller).to have_received(:new).
58
+ with(Deimos.config.db_poller_objects[1])
59
+ end
60
+ end
61
+
62
+ describe 'pollers' do
63
+ include_context 'with widgets'
64
+
65
+ let(:poller) do
66
+ poller = described_class.new(config)
67
+ allow(poller).to receive(:sleep)
68
+ poller
69
+ end
70
+
71
+ let(:config) { Deimos.config.db_poller_objects.first.dup }
72
+
73
+ before(:each) do
74
+ Widget.delete_all
75
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
76
+ schema 'MySchemaWithId'
77
+ namespace 'com.my-namespace'
78
+ topic 'my-topic-with-id'
79
+ key_config none: true
80
+ record_class Widget
81
+
82
+ # :nodoc:
83
+ def self.generate_payload(attrs, widget)
84
+ super.merge(message_id: widget.generated_id)
85
+ end
86
+ end
87
+ stub_const('MyProducer', producer_class)
88
+
89
+ Deimos.configure do
90
+ db_poller do
91
+ producer_class 'MyProducer'
92
+ run_every 1.minute
93
+ end
94
+ end
95
+ end
96
+
97
+ after(:each) do
98
+ travel_back
99
+ end
100
+
101
+ it 'should crash if initialized with an invalid producer' do
102
+ config.producer_class = 'NoProducer'
103
+ expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
104
+ end
105
+
106
+ describe '#retrieve_poll_info' do
107
+
108
+ it 'should start from beginning when configured' do
109
+ poller.retrieve_poll_info
110
+ expect(Deimos::PollInfo.count).to eq(1)
111
+ info = Deimos::PollInfo.last
112
+ expect(info.producer).to eq('MyProducer')
113
+ expect(info.last_sent).to eq(Time.new(0))
114
+ expect(info.last_sent_id).to eq(0)
115
+ end
116
+
117
+ it 'should start from now when configured' do
118
+ travel_to time_value
119
+ config.start_from_beginning = false
120
+ poller.retrieve_poll_info
121
+ expect(Deimos::PollInfo.count).to eq(1)
122
+ info = Deimos::PollInfo.last
123
+ expect(info.producer).to eq('MyProducer')
124
+ expect(info.last_sent).to eq(time_value)
125
+ expect(info.last_sent_id).to eq(0)
126
+ end
127
+
128
+ end
129
+
130
+ specify '#start' do
131
+ i = 0
132
+ expect(poller).to receive(:process_updates).twice do
133
+ i += 1
134
+ poller.stop if i == 2
135
+ end
136
+ poller.start
137
+ end
138
+
139
+ specify '#should_run?' do
140
+ Deimos::PollInfo.create!(producer: 'MyProducer',
141
+ last_sent: time_value)
142
+ poller.retrieve_poll_info
143
+
144
+ # run_every is set to 1 minute
145
+ travel_to time_value(secs: 62)
146
+ expect(poller.should_run?).to eq(true)
147
+
148
+ travel_to time_value(secs: 30)
149
+ expect(poller.should_run?).to eq(false)
150
+
151
+ travel_to time_value(mins: -1) # this shouldn't be possible but meh
152
+ expect(poller.should_run?).to eq(false)
153
+
154
+ # take the 2 seconds of delay_time into account
155
+ travel_to time_value(secs: 60)
156
+ expect(poller.should_run?).to eq(false)
157
+ end
158
+
159
+ specify '#process_batch' do
160
+ travel_to time_value
161
+ widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
162
+ widgets.last.update_attribute(:updated_at, time_value(mins: -30))
163
+ expect(MyProducer).to receive(:send_events).with(widgets)
164
+ poller.retrieve_poll_info
165
+ poller.process_batch(widgets)
166
+ info = Deimos::PollInfo.last
167
+ expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
168
+ expect(info.last_sent_id).to eq(widgets.last.id)
169
+ end
170
+
171
+ describe '#process_updates' do
172
+ before(:each) do
173
+ Deimos::PollInfo.create!(producer: 'MyProducer',
174
+ last_sent: time_value(mins: -61),
175
+ last_sent_id: 0)
176
+ poller.retrieve_poll_info
177
+ travel_to time_value
178
+ stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
179
+ end
180
+
181
+ let!(:old_widget) do
182
+ # old widget, earlier than window
183
+ Widget.create!(test_id: 'some_id', some_int: 40,
184
+ updated_at: time_value(mins: -200))
185
+ end
186
+
187
+ let!(:last_widget) do
188
+ # new widget, before delay
189
+ Widget.create!(test_id: 'some_id', some_int: 10,
190
+ updated_at: time_value(secs: -1))
191
+ end
192
+
193
+ let!(:widgets) do
194
+ (1..7).map do |i|
195
+ Widget.create!(test_id: 'some_id', some_int: i,
196
+ updated_at: time_value(mins: -61, secs: 30 + i))
197
+ end
198
+ end
199
+
200
+ it 'should update the full table' do
201
+ info = Deimos::PollInfo.last
202
+ config.full_table = true
203
+ expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
204
+ expect(poller).to receive(:process_batch).ordered.
205
+ with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
206
+ m.call(*args)
207
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
208
+ expect(info.last_sent_id).to eq(widgets[1].id)
209
+ end
210
+ expect(poller).to receive(:process_batch).ordered.
211
+ with([widgets[2], widgets[3], widgets[4]]).and_call_original
212
+ expect(poller).to receive(:process_batch).ordered.
213
+ with([widgets[5], widgets[6]]).and_call_original
214
+ poller.process_updates
215
+
216
+ # this is the updated_at of widgets[6]
217
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
218
+ expect(info.last_sent_id).to eq(widgets[6].id)
219
+
220
+ last_widget.update_attribute(:updated_at, time_value(mins: -250))
221
+
222
+ travel 61.seconds
223
+ # should reprocess the table
224
+ expect(poller).to receive(:process_batch).ordered.
225
+ with([last_widget, old_widget, widgets[0]]).and_call_original
226
+ expect(poller).to receive(:process_batch).ordered.
227
+ with([widgets[1], widgets[2], widgets[3]]).and_call_original
228
+ expect(poller).to receive(:process_batch).ordered.
229
+ with([widgets[4], widgets[5], widgets[6]]).and_call_original
230
+ poller.process_updates
231
+
232
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
233
+ expect(info.last_sent_id).to eq(widgets[6].id)
234
+ end
235
+
236
+ it 'should send events across multiple batches' do
237
+ allow(MyProducer).to receive(:poll_query).and_call_original
238
+ expect(poller).to receive(:process_batch).ordered.
239
+ with([widgets[0], widgets[1], widgets[2]]).and_call_original
240
+ expect(poller).to receive(:process_batch).ordered.
241
+ with([widgets[3], widgets[4], widgets[5]]).and_call_original
242
+ expect(poller).to receive(:process_batch).ordered.
243
+ with([widgets[6]]).and_call_original
244
+ poller.process_updates
245
+
246
+ expect(MyProducer).to have_received(:poll_query).
247
+ with(time_from: time_value(mins: -61),
248
+ time_to: time_value(secs: -2),
249
+ column_name: :updated_at,
250
+ min_id: 0)
251
+
252
+ travel 61.seconds
253
+ # process the last widget which came in during the delay
254
+ expect(poller).to receive(:process_batch).with([last_widget]).
255
+ and_call_original
256
+ poller.process_updates
257
+
258
+ # widgets[6] updated_at value
259
+ expect(MyProducer).to have_received(:poll_query).
260
+ with(time_from: time_value(mins: -61, secs: 37),
261
+ time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
262
+ column_name: :updated_at,
263
+ min_id: widgets[6].id)
264
+
265
+ travel 61.seconds
266
+ # nothing else to process
267
+ expect(poller).not_to receive(:process_batch)
268
+ poller.process_updates
269
+ poller.process_updates
270
+
271
+ expect(MyProducer).to have_received(:poll_query).twice.
272
+ with(time_from: time_value(secs: -1),
273
+ time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
274
+ column_name: :updated_at,
275
+ min_id: last_widget.id)
276
+ end
277
+
278
+ it 'should recover correctly with errors and save the right ID' do
279
+ widgets.each do |w|
280
+ w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
281
+ end
282
+ allow(MyProducer).to receive(:poll_query).and_call_original
283
+ expect(poller).to receive(:process_batch).ordered.
284
+ with([widgets[0], widgets[1], widgets[2]]).and_call_original
285
+ expect(poller).to receive(:process_batch).ordered.
286
+ with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
287
+
288
+ expect { poller.process_updates }.to raise_exception('OH NOES')
289
+
290
+ expect(MyProducer).to have_received(:poll_query).
291
+ with(time_from: time_value(mins: -61),
292
+ time_to: time_value(secs: -2),
293
+ column_name: :updated_at,
294
+ min_id: 0)
295
+
296
+ info = Deimos::PollInfo.last
297
+ expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
298
+ expect(info.last_sent_id).to eq(widgets[2].id)
299
+
300
+ travel 61.seconds
301
+ # process the last widget which came in during the delay
302
+ expect(poller).to receive(:process_batch).ordered.
303
+ with([widgets[3], widgets[4], widgets[5]]).and_call_original
304
+ expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
305
+ and_call_original
306
+ poller.process_updates
307
+ expect(MyProducer).to have_received(:poll_query).
308
+ with(time_from: time_value(mins: -61, secs: 30),
309
+ time_to: time_value(secs: 59),
310
+ column_name: :updated_at,
311
+ min_id: widgets[2].id)
312
+
313
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
314
+ expect(info.last_sent_id).to eq(last_widget.id)
315
+ end
316
+
317
+ end
318
+
319
+ end
320
+ end