deimos-ruby 1.6.1 → 1.8.0.pre.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +9 -0
  3. data/.rubocop.yml +15 -13
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +30 -0
  6. data/Gemfile.lock +87 -80
  7. data/README.md +139 -15
  8. data/Rakefile +1 -1
  9. data/deimos-ruby.gemspec +3 -2
  10. data/docs/ARCHITECTURE.md +144 -0
  11. data/docs/CONFIGURATION.md +27 -0
  12. data/lib/deimos.rb +7 -6
  13. data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
  14. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  15. data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
  16. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  17. data/lib/deimos/active_record_consumer.rb +33 -75
  18. data/lib/deimos/active_record_producer.rb +23 -0
  19. data/lib/deimos/batch_consumer.rb +2 -140
  20. data/lib/deimos/config/configuration.rb +28 -10
  21. data/lib/deimos/consume/batch_consumption.rb +148 -0
  22. data/lib/deimos/consume/message_consumption.rb +93 -0
  23. data/lib/deimos/consumer.rb +79 -69
  24. data/lib/deimos/kafka_message.rb +1 -1
  25. data/lib/deimos/kafka_source.rb +29 -23
  26. data/lib/deimos/kafka_topic_info.rb +1 -1
  27. data/lib/deimos/message.rb +6 -1
  28. data/lib/deimos/metrics/provider.rb +0 -2
  29. data/lib/deimos/poll_info.rb +9 -0
  30. data/lib/deimos/tracing/provider.rb +0 -2
  31. data/lib/deimos/utils/db_poller.rb +149 -0
  32. data/lib/deimos/utils/db_producer.rb +8 -3
  33. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  34. data/lib/deimos/utils/lag_reporter.rb +19 -26
  35. data/lib/deimos/version.rb +1 -1
  36. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  37. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  38. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  39. data/lib/tasks/deimos.rake +7 -0
  40. data/spec/active_record_batch_consumer_spec.rb +481 -0
  41. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  42. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  43. data/spec/active_record_consumer_spec.rb +22 -11
  44. data/spec/active_record_producer_spec.rb +66 -88
  45. data/spec/batch_consumer_spec.rb +23 -7
  46. data/spec/config/configuration_spec.rb +4 -0
  47. data/spec/consumer_spec.rb +8 -8
  48. data/spec/deimos_spec.rb +57 -49
  49. data/spec/handlers/my_batch_consumer.rb +6 -1
  50. data/spec/handlers/my_consumer.rb +6 -1
  51. data/spec/kafka_source_spec.rb +53 -0
  52. data/spec/message_spec.rb +19 -0
  53. data/spec/producer_spec.rb +3 -3
  54. data/spec/rake_spec.rb +1 -1
  55. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  56. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  57. data/spec/spec_helper.rb +61 -6
  58. data/spec/utils/db_poller_spec.rb +320 -0
  59. data/spec/utils/deadlock_retry_spec.rb +74 -0
  60. data/spec/utils/lag_reporter_spec.rb +29 -22
  61. metadata +61 -20
  62. data/lib/deimos/base_consumer.rb +0 -104
  63. data/lib/deimos/utils/executor.rb +0 -124
  64. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  65. data/lib/deimos/utils/signal_handler.rb +0 -68
  66. data/spec/utils/executor_spec.rb +0 -53
  67. data/spec/utils/signal_handler_spec.rb +0 -16
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
4
+ describe '#slice' do
5
+ let(:batch) do
6
+ [
7
+ Deimos::Message.new({ v: 1 }, nil, key: 'C'),
8
+ Deimos::Message.new({ v: 123 }, nil, key: 'A'),
9
+ Deimos::Message.new({ v: 999 }, nil, key: 'B'),
10
+ Deimos::Message.new({ v: 456 }, nil, key: 'A'),
11
+ Deimos::Message.new({ v: 2 }, nil, key: 'C'),
12
+ Deimos::Message.new({ v: 3 }, nil, key: 'C')
13
+ ]
14
+ end
15
+
16
+ it 'should slice a batch by key' do
17
+ slices = described_class.slice(batch)
18
+
19
+ expect(slices).
20
+ to match([
21
+ match_array([
22
+ Deimos::Message.new({ v: 1 }, nil, key: 'C'),
23
+ Deimos::Message.new({ v: 123 }, nil, key: 'A'),
24
+ Deimos::Message.new({ v: 999 }, nil, key: 'B')
25
+ ]),
26
+ match_array([
27
+ Deimos::Message.new({ v: 456 }, nil, key: 'A'),
28
+ Deimos::Message.new({ v: 2 }, nil, key: 'C')
29
+ ]),
30
+ match_array([
31
+ Deimos::Message.new({ v: 3 }, nil, key: 'C')
32
+ ])
33
+ ])
34
+ end
35
+
36
+ it 'should handle empty batches' do
37
+ slices = described_class.slice([])
38
+
39
+ expect(slices).to be_empty
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'deimos/active_record_consume/schema_model_converter'
4
+ require 'deimos/schema_backends/avro_local'
5
+
6
+ # Wrapped in a module to prevent class leakage
7
+ module SchemaModelConverterTest
8
+ describe Deimos::ActiveRecordConsume::SchemaModelConverter do
9
+ # Create ActiveRecord table and model
10
+ before(:all) do
11
+ ActiveRecord::Base.connection.create_table(:wibbles, force: true) do |t|
12
+ t.integer(:wibble_id)
13
+ t.string(:name)
14
+ t.integer(:bar)
15
+ t.datetime(:birthday_int)
16
+ t.datetime(:birthday_long)
17
+ t.datetime(:birthday_optional)
18
+ t.timestamps
19
+ end
20
+
21
+ # :nodoc:
22
+ class Wibble < ActiveRecord::Base
23
+ end
24
+ Wibble.reset_column_information
25
+ end
26
+
27
+ after(:all) do
28
+ ActiveRecord::Base.connection.drop_table(:wibbles)
29
+ end
30
+
31
+ let(:schema) { Deimos::SchemaBackends::AvroLocal.new(schema: 'Wibble', namespace: 'com.my-namespace') }
32
+ let(:inst) { described_class.new(schema, Wibble) }
33
+
34
+ describe '#convert' do
35
+ it 'should extract attributes from the payload' do
36
+ payload = { 'id' => 123, 'wibble_id' => 456, 'name' => 'wibble' }
37
+
38
+ expect(inst.convert(payload)).to include('id' => 123, 'wibble_id' => 456, 'name' => 'wibble')
39
+ end
40
+
41
+ it 'should ignore payload fields that don\'t exist' do
42
+ payload = { 'foo' => 'abc' }
43
+
44
+ expect(inst.convert(payload)).not_to include('foo')
45
+ end
46
+
47
+ it 'should ignore fields in the schema but not in the model' do
48
+ payload = { 'floop' => 'def' }
49
+
50
+ expect(inst.convert(payload)).not_to include('floop')
51
+ end
52
+
53
+ it 'should ignore model fields not in the schema' do
54
+ payload = { 'bar' => 'xyz' }
55
+
56
+ expect(inst.convert(payload)).not_to include('bar')
57
+ end
58
+
59
+ it 'should handle nils' do
60
+ payload = { 'name' => nil }
61
+
62
+ expect(inst.convert(payload)['name']).to be_nil
63
+ end
64
+
65
+ describe 'timestamps' do
66
+ it 'should ignore AR timestamp fields' do
67
+ payload = { 'updated_at' => '1234567890', 'created_at' => '2345678901' }
68
+
69
+ expect(inst.convert(payload)).not_to include('updated_at', 'created_at')
70
+ end
71
+
72
+ it 'should parse timestamps' do
73
+ first = Time.zone.local(2019, 1, 1, 11, 12, 13)
74
+ second = Time.zone.local(2019, 2, 2, 12, 13, 14)
75
+
76
+ payload = { 'birthday_int' => first.to_i, 'birthday_long' => second.to_i }
77
+
78
+ expect(inst.convert(payload)).to include('birthday_int' => first, 'birthday_long' => second)
79
+ end
80
+
81
+ it 'should parse integer-string timestamps' do
82
+ date = Time.zone.local(2019, 1, 1, 11, 12, 13)
83
+
84
+ payload = { 'birthday_int' => '1546359133' }
85
+
86
+ expect(inst.convert(payload)).to include('birthday_int' => date)
87
+ end
88
+
89
+ it 'should ignore other strings for timestamps' do
90
+ payload = { 'birthday_int' => 'some-other-val' }
91
+
92
+ expect(inst.convert(payload)).to include('birthday_int' => 'some-other-val')
93
+ end
94
+ end
95
+
96
+ it 'should coerce nullable unions' do
97
+ date = Time.zone.local(2019, 1, 1, 11, 12, 13)
98
+
99
+ payload = { 'birthday_optional' => date.to_i }
100
+
101
+ expect(inst.convert(payload)).to include('birthday_optional' => date)
102
+ end
103
+ end
104
+ end
105
+ end
@@ -2,9 +2,9 @@
2
2
 
3
3
  require 'date'
4
4
 
5
- # :nodoc:
6
- module ActiveRecordProducerTest
7
- describe Deimos::ActiveRecordConsumer do
5
+ # Wrapped in a module to prevent class leakage
6
+ module ActiveRecordConsumerTest
7
+ describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
8
8
 
9
9
  before(:all) do
10
10
  ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
@@ -22,6 +22,10 @@ module ActiveRecordProducerTest
22
22
  Widget.reset_column_information
23
23
  end
24
24
 
25
+ before(:each) do
26
+ Widget.delete_all
27
+ end
28
+
25
29
  after(:all) do
26
30
  ActiveRecord::Base.connection.drop_table(:widgets)
27
31
  end
@@ -101,6 +105,21 @@ module ActiveRecordProducerTest
101
105
 
102
106
  end
103
107
 
108
+ it 'should update only updated_at' do
109
+ travel_to Time.local(2020, 5, 5, 5, 5, 5)
110
+ widget1 = Widget.create!(test_id: 'id1', some_int: 3)
111
+ expect(widget1.updated_at.in_time_zone).to eq(Time.local(2020, 5, 5, 5, 5, 5))
112
+
113
+ travel 1.day
114
+ test_consume_message(MyCustomFetchConsumer, {
115
+ test_id: 'id1',
116
+ some_int: 3
117
+ }, { call_original: true })
118
+ expect(widget1.reload.updated_at.in_time_zone).
119
+ to eq(Time.local(2020, 5, 6, 5, 5, 5))
120
+ travel_back
121
+ end
122
+
104
123
  it 'should find widgets by custom logic' do
105
124
  widget1 = Widget.create!(test_id: 'id1')
106
125
  expect(widget1.some_int).to be_nil
@@ -118,13 +137,5 @@ module ActiveRecordProducerTest
118
137
  expect(Widget.find_by_test_id('id1').some_int).to eq(3)
119
138
  expect(Widget.find_by_test_id('id2').some_int).to eq(4)
120
139
  end
121
-
122
- it 'should coerce int values to datetimes' do
123
- column = Widget.columns.find { |c| c.name == 'some_datetime_int' }
124
- expect(MyConsumer.new.send(:_coerce_field, column, 1_579_046_400)).to eq('2020-01-14 19:00:00 -0500')
125
- expect(MyConsumer.new.send(:_coerce_field, column, '1579046400')).to eq('2020-01-14 19:00:00 -0500')
126
- expect(MyConsumer.new.send(:_coerce_field, column, 'some-other-val')).to eq('some-other-val')
127
- end
128
-
129
140
  end
130
141
  end
@@ -1,107 +1,85 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # :nodoc:
4
- module ActiveRecordProducerTest
5
- describe Deimos::ActiveRecordProducer do
3
+ describe Deimos::ActiveRecordProducer do
6
4
 
7
- before(:all) do
8
- ActiveRecord::Base.connection.create_table(:widgets) do |t|
9
- t.string(:test_id)
10
- t.integer(:some_int)
11
- t.boolean(:some_bool)
12
- t.timestamps
13
- end
5
+ include_context 'with widgets'
14
6
 
15
- # :nodoc:
16
- class Widget < ActiveRecord::Base
17
- # @return [String]
18
- def generated_id
19
- 'generated_id'
20
- end
21
- end
22
- end
7
+ prepend_before(:each) do
23
8
 
24
- after(:all) do
25
- ActiveRecord::Base.connection.drop_table(:widgets)
9
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
10
+ schema 'MySchema'
11
+ namespace 'com.my-namespace'
12
+ topic 'my-topic'
13
+ key_config none: true
26
14
  end
15
+ stub_const('MyProducer', producer_class)
27
16
 
28
- prepend_before(:each) do
29
-
30
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
31
- schema 'MySchema'
32
- namespace 'com.my-namespace'
33
- topic 'my-topic'
34
- key_config none: true
35
- end
36
- stub_const('MyProducer', producer_class)
37
-
38
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
39
- schema 'MySchemaWithBooleans'
40
- namespace 'com.my-namespace'
41
- topic 'my-topic-with-boolean'
42
- key_config none: true
43
- end
44
- stub_const('MyBooleanProducer', producer_class)
45
-
46
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
47
- schema 'MySchemaWithId'
48
- namespace 'com.my-namespace'
49
- topic 'my-topic-with-id'
50
- key_config none: true
51
- record_class Widget
17
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
18
+ schema 'MySchemaWithBooleans'
19
+ namespace 'com.my-namespace'
20
+ topic 'my-topic-with-boolean'
21
+ key_config none: true
22
+ end
23
+ stub_const('MyBooleanProducer', producer_class)
52
24
 
53
- # :nodoc:
54
- def self.generate_payload(attrs, widget)
55
- super.merge(message_id: widget.generated_id)
56
- end
25
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
26
+ schema 'MySchemaWithId'
27
+ namespace 'com.my-namespace'
28
+ topic 'my-topic-with-id'
29
+ key_config none: true
30
+ record_class Widget
57
31
 
32
+ # :nodoc:
33
+ def self.generate_payload(attrs, widget)
34
+ super.merge(message_id: widget.generated_id)
58
35
  end
59
- stub_const('MyProducerWithID', producer_class)
60
36
 
61
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
62
- schema 'MySchemaWithUniqueId'
63
- namespace 'com.my-namespace'
64
- topic 'my-topic-with-unique-id'
65
- key_config field: :id
66
- record_class Widget
67
- end
68
- stub_const('MyProducerWithUniqueID', producer_class)
69
37
  end
70
-
71
- it 'should send events correctly' do
72
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
73
- expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
38
+ stub_const('MyProducerWithID', producer_class)
39
+
40
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
41
+ schema 'MySchemaWithUniqueId'
42
+ namespace 'com.my-namespace'
43
+ topic 'my-topic-with-unique-id'
44
+ key_config field: :id
45
+ record_class Widget
74
46
  end
47
+ stub_const('MyProducerWithUniqueID', producer_class)
48
+ end
75
49
 
76
- it 'should coerce values' do
77
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
78
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
79
- expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
80
- expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
81
- expect {
82
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
83
- }.to raise_error(Avro::SchemaValidator::ValidationError)
84
-
85
- MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
86
- MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
87
- expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
88
- expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
89
- end
50
+ it 'should send events correctly' do
51
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
52
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
53
+ end
90
54
 
91
- it 'should be able to call the record' do
92
- widget = Widget.create!(test_id: 'abc2', some_int: 3)
93
- MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
94
- expect('my-topic-with-id').to have_sent(
95
- test_id: 'abc2',
96
- some_int: 3,
97
- message_id: 'generated_id',
98
- timestamp: anything
99
- )
100
- end
55
+ it 'should coerce values' do
56
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
57
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
58
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
59
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
60
+ expect {
61
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
62
+ }.to raise_error(Avro::SchemaValidator::ValidationError)
63
+
64
+ MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
65
+ MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
66
+ expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
67
+ expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
68
+ end
101
69
 
102
- specify '#watched_attributes' do
103
- expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
104
- end
70
+ it 'should be able to call the record' do
71
+ widget = Widget.create!(test_id: 'abc2', some_int: 3)
72
+ MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
73
+ expect('my-topic-with-id').to have_sent(
74
+ test_id: 'abc2',
75
+ some_int: 3,
76
+ message_id: 'generated_id',
77
+ timestamp: anything
78
+ )
79
+ end
105
80
 
81
+ specify '#watched_attributes' do
82
+ expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
106
83
  end
84
+
107
85
  end
@@ -1,14 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'deimos/batch_consumer'
4
-
5
3
  # :nodoc:
6
4
  module ConsumerTest
7
- describe Deimos::BatchConsumer do
5
+ describe Deimos::Consumer, 'Batch Consumer' do
8
6
 
9
7
  prepend_before(:each) do
10
8
  # :nodoc:
11
- consumer_class = Class.new(Deimos::BatchConsumer) do
9
+ consumer_class = Class.new(described_class) do
12
10
  schema 'MySchema'
13
11
  namespace 'com.my-namespace'
14
12
  key_config field: 'test_id'
@@ -32,6 +30,24 @@ module ConsumerTest
32
30
  batch.concat([{ 'invalid' => 'key' }])
33
31
  end
34
32
 
33
+ it 'should provide backwards compatibility for BatchConsumer class' do
34
+ consumer_class = Class.new(Deimos::BatchConsumer) do
35
+ schema 'MySchema'
36
+ namespace 'com.my-namespace'
37
+ key_config field: 'test_id'
38
+
39
+ # :nodoc:
40
+ def consume_batch(_payloads, _metadata)
41
+ raise 'This should not be called unless call_original is set'
42
+ end
43
+ end
44
+ stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
45
+
46
+ test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
47
+ expect(received).to eq(batch)
48
+ end
49
+ end
50
+
35
51
  it 'should consume a batch of messages' do
36
52
  test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
37
53
  expect(received).to eq(batch)
@@ -99,7 +115,7 @@ module ConsumerTest
99
115
  end
100
116
 
101
117
  it 'should decode plain keys for all messages in the batch' do
102
- consumer_class = Class.new(Deimos::BatchConsumer) do
118
+ consumer_class = Class.new(described_class) do
103
119
  schema 'MySchema'
104
120
  namespace 'com.my-namespace'
105
121
  key_config plain: true
@@ -115,7 +131,7 @@ module ConsumerTest
115
131
  describe 'timestamps' do
116
132
  before(:each) do
117
133
  # :nodoc:
118
- consumer_class = Class.new(Deimos::BatchConsumer) do
134
+ consumer_class = Class.new(described_class) do
119
135
  schema 'MySchemaWithDateTimes'
120
136
  namespace 'com.my-namespace'
121
137
  key_config plain: true
@@ -197,7 +213,7 @@ module ConsumerTest
197
213
  describe 'logging' do
198
214
  before(:each) do
199
215
  # :nodoc:
200
- consumer_class = Class.new(Deimos::BatchConsumer) do
216
+ consumer_class = Class.new(described_class) do
201
217
  schema 'MySchemaWithUniqueId'
202
218
  namespace 'com.my-namespace'
203
219
  key_config plain: true
@@ -1,6 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # Mock consumer
3
4
  class MyConfigConsumer < Deimos::Consumer
5
+ # :no-doc:
6
+ def consume
7
+ end
4
8
  end
5
9
  describe Deimos, 'configuration' do
6
10
  it 'should configure with deprecated fields' do