deimos-ruby 1.6.2 → 1.8.0.pre.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +9 -0
  3. data/.rubocop.yml +15 -13
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +31 -0
  6. data/Gemfile.lock +43 -36
  7. data/README.md +141 -16
  8. data/Rakefile +1 -1
  9. data/deimos-ruby.gemspec +2 -1
  10. data/docs/ARCHITECTURE.md +144 -0
  11. data/docs/CONFIGURATION.md +27 -0
  12. data/lib/deimos.rb +7 -6
  13. data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
  14. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  15. data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
  16. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  17. data/lib/deimos/active_record_consumer.rb +33 -75
  18. data/lib/deimos/active_record_producer.rb +23 -0
  19. data/lib/deimos/batch_consumer.rb +2 -140
  20. data/lib/deimos/config/configuration.rb +28 -10
  21. data/lib/deimos/consume/batch_consumption.rb +150 -0
  22. data/lib/deimos/consume/message_consumption.rb +94 -0
  23. data/lib/deimos/consumer.rb +79 -69
  24. data/lib/deimos/kafka_message.rb +1 -1
  25. data/lib/deimos/kafka_topic_info.rb +1 -1
  26. data/lib/deimos/message.rb +6 -1
  27. data/lib/deimos/metrics/provider.rb +0 -2
  28. data/lib/deimos/poll_info.rb +9 -0
  29. data/lib/deimos/tracing/provider.rb +0 -2
  30. data/lib/deimos/utils/db_poller.rb +149 -0
  31. data/lib/deimos/utils/db_producer.rb +8 -3
  32. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  33. data/lib/deimos/utils/lag_reporter.rb +19 -26
  34. data/lib/deimos/version.rb +1 -1
  35. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  36. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  37. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  38. data/lib/tasks/deimos.rake +7 -0
  39. data/spec/active_record_batch_consumer_spec.rb +481 -0
  40. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  41. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  42. data/spec/active_record_consumer_spec.rb +3 -11
  43. data/spec/active_record_producer_spec.rb +66 -88
  44. data/spec/batch_consumer_spec.rb +24 -7
  45. data/spec/config/configuration_spec.rb +4 -0
  46. data/spec/consumer_spec.rb +8 -8
  47. data/spec/deimos_spec.rb +57 -49
  48. data/spec/handlers/my_batch_consumer.rb +6 -1
  49. data/spec/handlers/my_consumer.rb +6 -1
  50. data/spec/message_spec.rb +19 -0
  51. data/spec/producer_spec.rb +3 -3
  52. data/spec/rake_spec.rb +1 -1
  53. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  54. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  55. data/spec/spec_helper.rb +61 -6
  56. data/spec/utils/db_poller_spec.rb +320 -0
  57. data/spec/utils/deadlock_retry_spec.rb +74 -0
  58. data/spec/utils/lag_reporter_spec.rb +29 -22
  59. metadata +55 -20
  60. data/lib/deimos/base_consumer.rb +0 -104
  61. data/lib/deimos/utils/executor.rb +0 -124
  62. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  63. data/lib/deimos/utils/signal_handler.rb +0 -68
  64. data/spec/utils/executor_spec.rb +0 -53
  65. data/spec/utils/signal_handler_spec.rb +0 -16
@@ -68,11 +68,11 @@ describe Deimos do
68
68
 
69
69
  describe '#start_db_backend!' do
70
70
  it 'should start if backend is db and thread_count is > 0' do
71
- signal_handler = instance_double(Deimos::Utils::SignalHandler)
71
+ signal_handler = instance_double(Sigurd::SignalHandler)
72
72
  allow(signal_handler).to receive(:run!)
73
- expect(Deimos::Utils::Executor).to receive(:new).
73
+ expect(Sigurd::Executor).to receive(:new).
74
74
  with(anything, sleep_seconds: 5, logger: anything).and_call_original
75
- expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
75
+ expect(Sigurd::SignalHandler).to receive(:new) do |executor|
76
76
  expect(executor.runners.size).to eq(2)
77
77
  signal_handler
78
78
  end
@@ -83,7 +83,7 @@ describe Deimos do
83
83
  end
84
84
 
85
85
  it 'should not start if backend is not db' do
86
- expect(Deimos::Utils::SignalHandler).not_to receive(:new)
86
+ expect(Sigurd::SignalHandler).not_to receive(:new)
87
87
  described_class.configure do |config|
88
88
  config.producers.backend = :kafka
89
89
  end
@@ -92,7 +92,7 @@ describe Deimos do
92
92
  end
93
93
 
94
94
  it 'should not start if thread_count is nil' do
95
- expect(Deimos::Utils::SignalHandler).not_to receive(:new)
95
+ expect(Sigurd::SignalHandler).not_to receive(:new)
96
96
  described_class.configure do |config|
97
97
  config.producers.backend = :db
98
98
  end
@@ -101,61 +101,69 @@ describe Deimos do
101
101
  end
102
102
 
103
103
  it 'should not start if thread_count is 0' do
104
- expect(Deimos::Utils::SignalHandler).not_to receive(:new)
104
+ expect(Sigurd::SignalHandler).not_to receive(:new)
105
105
  described_class.configure do |config|
106
106
  config.producers.backend = :db
107
107
  end
108
108
  expect { described_class.start_db_backend!(thread_count: 0) }.
109
109
  to raise_error('Thread count is not given or set to zero, exiting')
110
110
  end
111
+ end
111
112
 
112
- describe 'delivery configuration' do
113
- before(:each) do
114
- allow(YAML).to receive(:load).and_return(phobos_configuration)
115
- end
116
-
117
- it 'should not raise an error with properly configured handlers' do
118
- path = config_path # for scope issues in the block below
119
- # Add explicit consumers
120
- phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
121
- 'delivery' => 'message' }
122
- phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
123
- 'delivery' => 'batch' }
124
-
125
- expect {
126
- described_class.configure { |c| c.phobos_config_file = path }
127
- }.not_to raise_error
128
- end
129
-
130
- it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
131
- path = config_path # for scope issues in the block below
132
- phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
133
- 'delivery' => 'message' }]
134
-
135
- expect {
136
- described_class.configure { |c| c.phobos_config_file = path }
137
- }.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
138
- end
139
-
140
- it 'should raise an error if Consumers do not have message or batch delivery' do
141
- path = config_path # for scope issues in the block below
142
- phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
143
- 'delivery' => 'inline_batch' }]
113
+ describe 'delivery configuration' do
114
+ before(:each) do
115
+ allow(YAML).to receive(:load).and_return(phobos_configuration)
116
+ end
144
117
 
145
- expect {
146
- described_class.configure { |c| c.phobos_config_file = path }
147
- }.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
148
- end
118
+ it 'should not raise an error with properly configured handlers' do
119
+ expect {
120
+ described_class.configure do
121
+ consumer do
122
+ class_name 'ConsumerTest::MyConsumer'
123
+ delivery :message
124
+ end
125
+ consumer do
126
+ class_name 'ConsumerTest::MyConsumer'
127
+ delivery :batch
128
+ end
129
+ consumer do
130
+ class_name 'ConsumerTest::MyBatchConsumer'
131
+ delivery :inline_batch
132
+ end
133
+ end
134
+ }.not_to raise_error
135
+ end
149
136
 
150
- it 'should treat nil as `batch`' do
151
- path = config_path # for scope issues in the block below
152
- phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
137
+ it 'should raise an error if inline_batch listeners do not implement consume_batch' do
138
+ expect {
139
+ described_class.configure do
140
+ consumer do
141
+ class_name 'ConsumerTest::MyConsumer'
142
+ delivery :inline_batch
143
+ end
144
+ end
145
+ }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
146
+ end
153
147
 
154
- expect {
155
- described_class.configure { |c| c.phobos_config_file = path }
156
- }.not_to raise_error
157
- end
148
+ it 'should raise an error if Consumers do not have message or batch delivery' do
149
+ expect {
150
+ described_class.configure do
151
+ consumer do
152
+ class_name 'ConsumerTest::MyBatchConsumer'
153
+ delivery :message
154
+ end
155
+ end
156
+ }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
157
+ end
158
158
 
159
+ it 'should treat nil as `batch`' do
160
+ expect {
161
+ described_class.configure do
162
+ consumer do
163
+ class_name 'ConsumerTest::MyConsumer'
164
+ end
165
+ end
166
+ }.not_to raise_error
159
167
  end
160
168
  end
161
169
  end
@@ -1,5 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ConsumerTest
4
- class MyBatchConsumer < Deimos::BatchConsumer; end
4
+ # Mock consumer
5
+ class MyBatchConsumer < Deimos::Consumer
6
+ # :no-doc:
7
+ def consume_batch
8
+ end
9
+ end
5
10
  end
@@ -1,5 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ConsumerTest
4
- class MyConsumer < Deimos::Consumer; end
4
+ # Mock consumer
5
+ class MyConsumer < Deimos::Consumer
6
+ # :no-doc:
7
+ def consume
8
+ end
9
+ end
5
10
  end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe(Deimos::Message) do
4
+ it 'should detect tombstones' do
5
+ expect(described_class.new(nil, nil, key: 'key1')).
6
+ to be_tombstone
7
+ expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
8
+ not_to be_tombstone
9
+ expect(described_class.new({ v: '' }, nil, key: 'key1')).
10
+ not_to be_tombstone
11
+ expect(described_class.new({ v: 'val1' }, nil, key: nil)).
12
+ not_to be_tombstone
13
+ end
14
+
15
+ it 'can support complex keys/values' do
16
+ expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
17
+ not_to raise_exception
18
+ end
19
+ end
@@ -148,7 +148,7 @@ module ProducerTest
148
148
  Deimos.disable_producers do
149
149
  raise 'OH NOES'
150
150
  end
151
- } .to raise_error('OH NOES')
151
+ }.to raise_error('OH NOES')
152
152
  expect(Deimos).not_to be_producers_disabled
153
153
  end
154
154
 
@@ -246,7 +246,7 @@ module ProducerTest
246
246
  MyNonEncodedProducer.publish_list(
247
247
  [{ 'test_id' => 'foo', 'some_int' => 123 }]
248
248
  )
249
- } .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
249
+ }.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
250
250
  end
251
251
 
252
252
  it 'should allow nil keys if none: true is configured' do
@@ -254,7 +254,7 @@ module ProducerTest
254
254
  MyNoKeyProducer.publish_list(
255
255
  [{ 'test_id' => 'foo', 'some_int' => 123 }]
256
256
  )
257
- } .not_to raise_error
257
+ }.not_to raise_error
258
258
  end
259
259
 
260
260
  it 'should use a partition key' do
@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
9
9
  Rake::Task.define_task(:environment)
10
10
  end
11
11
 
12
- describe 'Rakefile' do # rubocop:disable RSpec/DescribeClass
12
+ describe 'Rakefile' do
13
13
  it 'should start listeners' do
14
14
  runner = instance_double(Phobos::CLI::Runner)
15
15
  expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
@@ -0,0 +1,18 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaCompound-key",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "part_one",
9
+ "type": "string",
10
+ "doc": "test string one"
11
+ },
12
+ {
13
+ "name": "part_two",
14
+ "type": "string",
15
+ "doc": "test string two"
16
+ }
17
+ ]
18
+ }
@@ -0,0 +1,43 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "Wibble",
4
+ "type": "record",
5
+ "fields": [
6
+ {
7
+ "name": "id",
8
+ "type": "long"
9
+ },
10
+ {
11
+ "name": "wibble_id",
12
+ "type": "long"
13
+ },
14
+ {
15
+ "name": "name",
16
+ "type": "string"
17
+ },
18
+ {
19
+ "name": "floop",
20
+ "type": "string"
21
+ },
22
+ {
23
+ "name": "birthday_int",
24
+ "type": "int"
25
+ },
26
+ {
27
+ "name": "birthday_long",
28
+ "type": "long"
29
+ },
30
+ {
31
+ "name": "birthday_optional",
32
+ "type": ["null", "int"]
33
+ },
34
+ {
35
+ "name": "updated_at",
36
+ "type": "long"
37
+ },
38
+ {
39
+ "name": "created_at",
40
+ "type": "long"
41
+ }
42
+ ]
43
+ }
@@ -2,6 +2,7 @@
2
2
 
3
3
  $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
4
4
  require 'active_record'
5
+ require 'database_cleaner'
5
6
  require 'deimos'
6
7
  require 'deimos/metrics/mock'
7
8
  require 'deimos/tracing/mock'
@@ -100,9 +101,8 @@ module DbConfigs
100
101
  end
101
102
  end
102
103
 
103
- # Set up the given database.
104
- def setup_db(options)
105
- ActiveRecord::Base.establish_connection(options)
104
+ # :nodoc:
105
+ def run_db_backend_migration
106
106
  migration_class_name = 'DbBackendMigration'
107
107
  migration_version = '[5.2]'
108
108
  migration = ERB.new(
@@ -110,6 +110,24 @@ module DbConfigs
110
110
  ).result(binding)
111
111
  eval(migration) # rubocop:disable Security/Eval
112
112
  ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
113
+ end
114
+
115
+ # :nodoc:
116
+ def run_db_poller_migration
117
+ migration_class_name = 'DbPollerMigration'
118
+ migration_version = '[5.2]'
119
+ migration = ERB.new(
120
+ File.read('lib/generators/deimos/db_poller/templates/migration')
121
+ ).result(binding)
122
+ eval(migration) # rubocop:disable Security/Eval
123
+ ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
124
+ end
125
+
126
+ # Set up the given database.
127
+ def setup_db(options)
128
+ ActiveRecord::Base.establish_connection(options)
129
+ run_db_backend_migration
130
+ run_db_poller_migration
113
131
 
114
132
  ActiveRecord::Base.descendants.each do |klass|
115
133
  klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
@@ -130,8 +148,11 @@ RSpec.configure do |config|
130
148
  # true by default for RSpec 4.0
131
149
  config.shared_context_metadata_behavior = :apply_to_host_groups
132
150
 
151
+ config.filter_run(focus: true)
152
+ config.run_all_when_everything_filtered = true
153
+
133
154
  config.before(:all) do
134
- Time.zone = 'EST'
155
+ Time.zone = 'Eastern Time (US & Canada)'
135
156
  ActiveRecord::Base.logger = Logger.new('/dev/null')
136
157
  ActiveRecord::Base.establish_connection(
137
158
  'adapter' => 'sqlite3',
@@ -141,9 +162,10 @@ RSpec.configure do |config|
141
162
  config.include Deimos::TestHelpers
142
163
  config.include ActiveSupport::Testing::TimeHelpers
143
164
  config.before(:suite) do
144
- Time.zone = 'EST'
145
- ActiveRecord::Base.logger = Logger.new('/dev/null')
146
165
  setup_db(DbConfigs::DB_OPTIONS.last)
166
+
167
+ DatabaseCleaner.strategy = :transaction
168
+ DatabaseCleaner.clean_with(:truncation)
147
169
  end
148
170
 
149
171
  config.mock_with(:rspec) do |mocks|
@@ -164,6 +186,39 @@ RSpec.configure do |config|
164
186
  deimos_config.schema.backend = :avro_validation
165
187
  end
166
188
  end
189
+
190
+ config.around(:each) do |example|
191
+ use_cleaner = !example.metadata[:integration]
192
+
193
+ DatabaseCleaner.start if use_cleaner
194
+
195
+ example.run
196
+
197
+ DatabaseCleaner.clean if use_cleaner
198
+ end
199
+ end
200
+
201
+ RSpec.shared_context('with widgets') do
202
+ before(:all) do
203
+ ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
204
+ t.string(:test_id)
205
+ t.integer(:some_int)
206
+ t.boolean(:some_bool)
207
+ t.timestamps
208
+ end
209
+
210
+ # :nodoc:
211
+ class Widget < ActiveRecord::Base
212
+ # @return [String]
213
+ def generated_id
214
+ 'generated_id'
215
+ end
216
+ end
217
+ end
218
+
219
+ after(:all) do
220
+ ActiveRecord::Base.connection.drop_table(:widgets)
221
+ end
167
222
  end
168
223
 
169
224
  RSpec.shared_context('with DB') do
@@ -0,0 +1,320 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @param seconds [Integer]
4
+ # @return [Time]
5
+ def time_value(secs: 0, mins: 0)
6
+ Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
7
+ end
8
+
9
+ each_db_config(Deimos::Utils::DbPoller) do
10
+
11
+ before(:each) do
12
+ Deimos::PollInfo.delete_all
13
+ end
14
+
15
+ describe '#start!' do
16
+
17
+ before(:each) do
18
+ producer_class = Class.new(Deimos::Producer) do
19
+ schema 'MySchema'
20
+ namespace 'com.my-namespace'
21
+ topic 'my-topic'
22
+ key_config field: 'test_id'
23
+ end
24
+ stub_const('MyProducer', producer_class)
25
+
26
+ producer_class = Class.new(Deimos::Producer) do
27
+ schema 'MySchemaWithId'
28
+ namespace 'com.my-namespace'
29
+ topic 'my-topic'
30
+ key_config plain: true
31
+ end
32
+ stub_const('MyProducerWithID', producer_class)
33
+ end
34
+
35
+ it 'should raise an error if no pollers configured' do
36
+ Deimos.configure {}
37
+ expect { described_class.start! }.to raise_error('No pollers configured!')
38
+ end
39
+
40
+ it 'should start pollers as configured' do
41
+ Deimos.configure do
42
+ db_poller do
43
+ producer_class 'MyProducer'
44
+ end
45
+ db_poller do
46
+ producer_class 'MyProducerWithID'
47
+ end
48
+ end
49
+
50
+ allow(Deimos::Utils::DbPoller).to receive(:new)
51
+ signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
52
+ allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
53
+ described_class.start!
54
+ expect(Deimos::Utils::DbPoller).to have_received(:new).twice
55
+ expect(Deimos::Utils::DbPoller).to have_received(:new).
56
+ with(Deimos.config.db_poller_objects[0])
57
+ expect(Deimos::Utils::DbPoller).to have_received(:new).
58
+ with(Deimos.config.db_poller_objects[1])
59
+ end
60
+ end
61
+
62
+ describe 'pollers' do
63
+ include_context 'with widgets'
64
+
65
+ let(:poller) do
66
+ poller = described_class.new(config)
67
+ allow(poller).to receive(:sleep)
68
+ poller
69
+ end
70
+
71
+ let(:config) { Deimos.config.db_poller_objects.first.dup }
72
+
73
+ before(:each) do
74
+ Widget.delete_all
75
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
76
+ schema 'MySchemaWithId'
77
+ namespace 'com.my-namespace'
78
+ topic 'my-topic-with-id'
79
+ key_config none: true
80
+ record_class Widget
81
+
82
+ # :nodoc:
83
+ def self.generate_payload(attrs, widget)
84
+ super.merge(message_id: widget.generated_id)
85
+ end
86
+ end
87
+ stub_const('MyProducer', producer_class)
88
+
89
+ Deimos.configure do
90
+ db_poller do
91
+ producer_class 'MyProducer'
92
+ run_every 1.minute
93
+ end
94
+ end
95
+ end
96
+
97
+ after(:each) do
98
+ travel_back
99
+ end
100
+
101
+ it 'should crash if initialized with an invalid producer' do
102
+ config.producer_class = 'NoProducer'
103
+ expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
104
+ end
105
+
106
+ describe '#retrieve_poll_info' do
107
+
108
+ it 'should start from beginning when configured' do
109
+ poller.retrieve_poll_info
110
+ expect(Deimos::PollInfo.count).to eq(1)
111
+ info = Deimos::PollInfo.last
112
+ expect(info.producer).to eq('MyProducer')
113
+ expect(info.last_sent).to eq(Time.new(0))
114
+ expect(info.last_sent_id).to eq(0)
115
+ end
116
+
117
+ it 'should start from now when configured' do
118
+ travel_to time_value
119
+ config.start_from_beginning = false
120
+ poller.retrieve_poll_info
121
+ expect(Deimos::PollInfo.count).to eq(1)
122
+ info = Deimos::PollInfo.last
123
+ expect(info.producer).to eq('MyProducer')
124
+ expect(info.last_sent).to eq(time_value)
125
+ expect(info.last_sent_id).to eq(0)
126
+ end
127
+
128
+ end
129
+
130
+ specify '#start' do
131
+ i = 0
132
+ expect(poller).to receive(:process_updates).twice do
133
+ i += 1
134
+ poller.stop if i == 2
135
+ end
136
+ poller.start
137
+ end
138
+
139
+ specify '#should_run?' do
140
+ Deimos::PollInfo.create!(producer: 'MyProducer',
141
+ last_sent: time_value)
142
+ poller.retrieve_poll_info
143
+
144
+ # run_every is set to 1 minute
145
+ travel_to time_value(secs: 62)
146
+ expect(poller.should_run?).to eq(true)
147
+
148
+ travel_to time_value(secs: 30)
149
+ expect(poller.should_run?).to eq(false)
150
+
151
+ travel_to time_value(mins: -1) # this shouldn't be possible but meh
152
+ expect(poller.should_run?).to eq(false)
153
+
154
+ # take the 2 seconds of delay_time into account
155
+ travel_to time_value(secs: 60)
156
+ expect(poller.should_run?).to eq(false)
157
+ end
158
+
159
+ specify '#process_batch' do
160
+ travel_to time_value
161
+ widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
162
+ widgets.last.update_attribute(:updated_at, time_value(mins: -30))
163
+ expect(MyProducer).to receive(:send_events).with(widgets)
164
+ poller.retrieve_poll_info
165
+ poller.process_batch(widgets)
166
+ info = Deimos::PollInfo.last
167
+ expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
168
+ expect(info.last_sent_id).to eq(widgets.last.id)
169
+ end
170
+
171
+ describe '#process_updates' do
172
+ before(:each) do
173
+ Deimos::PollInfo.create!(producer: 'MyProducer',
174
+ last_sent: time_value(mins: -61),
175
+ last_sent_id: 0)
176
+ poller.retrieve_poll_info
177
+ travel_to time_value
178
+ stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
179
+ end
180
+
181
+ let!(:old_widget) do
182
+ # old widget, earlier than window
183
+ Widget.create!(test_id: 'some_id', some_int: 40,
184
+ updated_at: time_value(mins: -200))
185
+ end
186
+
187
+ let!(:last_widget) do
188
+ # new widget, before delay
189
+ Widget.create!(test_id: 'some_id', some_int: 10,
190
+ updated_at: time_value(secs: -1))
191
+ end
192
+
193
+ let!(:widgets) do
194
+ (1..7).map do |i|
195
+ Widget.create!(test_id: 'some_id', some_int: i,
196
+ updated_at: time_value(mins: -61, secs: 30 + i))
197
+ end
198
+ end
199
+
200
+ it 'should update the full table' do
201
+ info = Deimos::PollInfo.last
202
+ config.full_table = true
203
+ expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
204
+ expect(poller).to receive(:process_batch).ordered.
205
+ with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
206
+ m.call(*args)
207
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
208
+ expect(info.last_sent_id).to eq(widgets[1].id)
209
+ end
210
+ expect(poller).to receive(:process_batch).ordered.
211
+ with([widgets[2], widgets[3], widgets[4]]).and_call_original
212
+ expect(poller).to receive(:process_batch).ordered.
213
+ with([widgets[5], widgets[6]]).and_call_original
214
+ poller.process_updates
215
+
216
+ # this is the updated_at of widgets[6]
217
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
218
+ expect(info.last_sent_id).to eq(widgets[6].id)
219
+
220
+ last_widget.update_attribute(:updated_at, time_value(mins: -250))
221
+
222
+ travel 61.seconds
223
+ # should reprocess the table
224
+ expect(poller).to receive(:process_batch).ordered.
225
+ with([last_widget, old_widget, widgets[0]]).and_call_original
226
+ expect(poller).to receive(:process_batch).ordered.
227
+ with([widgets[1], widgets[2], widgets[3]]).and_call_original
228
+ expect(poller).to receive(:process_batch).ordered.
229
+ with([widgets[4], widgets[5], widgets[6]]).and_call_original
230
+ poller.process_updates
231
+
232
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
233
+ expect(info.last_sent_id).to eq(widgets[6].id)
234
+ end
235
+
236
+ it 'should send events across multiple batches' do
237
+ allow(MyProducer).to receive(:poll_query).and_call_original
238
+ expect(poller).to receive(:process_batch).ordered.
239
+ with([widgets[0], widgets[1], widgets[2]]).and_call_original
240
+ expect(poller).to receive(:process_batch).ordered.
241
+ with([widgets[3], widgets[4], widgets[5]]).and_call_original
242
+ expect(poller).to receive(:process_batch).ordered.
243
+ with([widgets[6]]).and_call_original
244
+ poller.process_updates
245
+
246
+ expect(MyProducer).to have_received(:poll_query).
247
+ with(time_from: time_value(mins: -61),
248
+ time_to: time_value(secs: -2),
249
+ column_name: :updated_at,
250
+ min_id: 0)
251
+
252
+ travel 61.seconds
253
+ # process the last widget which came in during the delay
254
+ expect(poller).to receive(:process_batch).with([last_widget]).
255
+ and_call_original
256
+ poller.process_updates
257
+
258
+ # widgets[6] updated_at value
259
+ expect(MyProducer).to have_received(:poll_query).
260
+ with(time_from: time_value(mins: -61, secs: 37),
261
+ time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
262
+ column_name: :updated_at,
263
+ min_id: widgets[6].id)
264
+
265
+ travel 61.seconds
266
+ # nothing else to process
267
+ expect(poller).not_to receive(:process_batch)
268
+ poller.process_updates
269
+ poller.process_updates
270
+
271
+ expect(MyProducer).to have_received(:poll_query).twice.
272
+ with(time_from: time_value(secs: -1),
273
+ time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
274
+ column_name: :updated_at,
275
+ min_id: last_widget.id)
276
+ end
277
+
278
+ it 'should recover correctly with errors and save the right ID' do
279
+ widgets.each do |w|
280
+ w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
281
+ end
282
+ allow(MyProducer).to receive(:poll_query).and_call_original
283
+ expect(poller).to receive(:process_batch).ordered.
284
+ with([widgets[0], widgets[1], widgets[2]]).and_call_original
285
+ expect(poller).to receive(:process_batch).ordered.
286
+ with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
287
+
288
+ expect { poller.process_updates }.to raise_exception('OH NOES')
289
+
290
+ expect(MyProducer).to have_received(:poll_query).
291
+ with(time_from: time_value(mins: -61),
292
+ time_to: time_value(secs: -2),
293
+ column_name: :updated_at,
294
+ min_id: 0)
295
+
296
+ info = Deimos::PollInfo.last
297
+ expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
298
+ expect(info.last_sent_id).to eq(widgets[2].id)
299
+
300
+ travel 61.seconds
301
+ # process the last widget which came in during the delay
302
+ expect(poller).to receive(:process_batch).ordered.
303
+ with([widgets[3], widgets[4], widgets[5]]).and_call_original
304
+ expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
305
+ and_call_original
306
+ poller.process_updates
307
+ expect(MyProducer).to have_received(:poll_query).
308
+ with(time_from: time_value(mins: -61, secs: 30),
309
+ time_to: time_value(secs: 59),
310
+ column_name: :updated_at,
311
+ min_id: widgets[2].id)
312
+
313
+ expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
314
+ expect(info.last_sent_id).to eq(last_widget.id)
315
+ end
316
+
317
+ end
318
+
319
+ end
320
+ end