deimos-ruby 1.11.0 → 1.12.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/Gemfile.lock +8 -8
- data/README.md +149 -66
- data/deimos-ruby.gemspec +1 -1
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +1 -1
- data/lib/deimos/active_record_consume/message_consumption.rb +4 -3
- data/lib/deimos/active_record_consumer.rb +2 -2
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/config/configuration.rb +29 -0
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/consume/message_consumption.rb +2 -2
- data/lib/deimos/consumer.rb +10 -0
- data/lib/deimos/producer.rb +4 -3
- data/lib/deimos/schema_backends/avro_base.rb +64 -1
- data/lib/deimos/schema_backends/base.rb +18 -2
- data/lib/deimos/schema_class/base.rb +62 -0
- data/lib/deimos/schema_class/enum.rb +24 -0
- data/lib/deimos/schema_class/record.rb +66 -0
- data/lib/deimos/shared_config.rb +5 -0
- data/lib/deimos/test_helpers.rb +43 -7
- data/lib/deimos/utils/schema_class.rb +29 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -0
- data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +15 -0
- data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt +21 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +65 -0
- data/lib/generators/deimos/schema_class_generator.rb +247 -0
- data/lib/tasks/deimos.rake +8 -0
- data/spec/active_record_batch_consumer_spec.rb +120 -110
- data/spec/active_record_consumer_spec.rb +97 -88
- data/spec/active_record_producer_spec.rb +38 -27
- data/spec/batch_consumer_spec.rb +37 -28
- data/spec/config/configuration_spec.rb +10 -3
- data/spec/consumer_spec.rb +95 -84
- data/spec/generators/active_record_generator_spec.rb +1 -0
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +206 -0
- data/spec/generators/schema_class_generator_spec.rb +186 -0
- data/spec/producer_spec.rb +110 -0
- data/spec/schema_classes/generated.rb +156 -0
- data/spec/schema_classes/my_nested_schema.rb +114 -0
- data/spec/schema_classes/my_schema.rb +53 -0
- data/spec/schema_classes/my_schema_key.rb +35 -0
- data/spec/schema_classes/my_schema_with_complex_types.rb +172 -0
- data/spec/schemas/com/my-namespace/Generated.avsc +6 -0
- data/spec/schemas/com/my-namespace/MySchemaWithComplexTypes.avsc +95 -0
- data/spec/spec_helper.rb +6 -1
- metadata +28 -4
@@ -5,7 +5,6 @@ require 'date'
|
|
5
5
|
# Wrapped in a module to prevent class leakage
|
6
6
|
module ActiveRecordConsumerTest
|
7
7
|
describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
|
8
|
-
|
9
8
|
before(:all) do
|
10
9
|
ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
|
11
10
|
t.string(:test_id)
|
@@ -61,94 +60,104 @@ module ActiveRecordConsumerTest
|
|
61
60
|
Time.zone = 'Eastern Time (US & Canada)'
|
62
61
|
end
|
63
62
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
63
|
+
describe 'consume' do
|
64
|
+
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
65
|
+
context "with Schema Class consumption #{setting}" do
|
66
|
+
before(:each) do
|
67
|
+
Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'should receive events correctly' do
|
71
|
+
travel 1.day do
|
72
|
+
expect(Widget.count).to eq(0)
|
73
|
+
test_consume_message(MyConsumer, {
|
74
|
+
test_id: 'abc',
|
75
|
+
some_int: 3,
|
76
|
+
updated_at: 1.day.ago.to_i,
|
77
|
+
some_datetime_int: Time.zone.now.to_i,
|
78
|
+
timestamp: 2.minutes.ago.to_s
|
79
|
+
}, { call_original: true, key: 5 })
|
80
|
+
|
81
|
+
expect(Widget.count).to eq(1)
|
82
|
+
widget = Widget.last
|
83
|
+
expect(widget.id).to eq(5)
|
84
|
+
expect(widget.test_id).to eq('abc')
|
85
|
+
expect(widget.some_int).to eq(3)
|
86
|
+
expect(widget.some_datetime_int).to eq(Time.zone.now)
|
87
|
+
expect(widget.some_bool).to eq(false)
|
88
|
+
expect(widget.updated_at).to eq(Time.zone.now)
|
89
|
+
|
90
|
+
# test unscoped
|
91
|
+
widget.update_attribute(:some_bool, true)
|
92
|
+
|
93
|
+
# test update
|
94
|
+
test_consume_message(MyConsumer, {
|
95
|
+
test_id: 'abcd',
|
96
|
+
some_int: 3,
|
97
|
+
some_datetime_int: Time.zone.now.to_i,
|
98
|
+
timestamp: 2.minutes.ago.to_s
|
99
|
+
}, { call_original: true, key: 5 })
|
100
|
+
expect(Widget.unscoped.count).to eq(1)
|
101
|
+
widget = Widget.unscoped.last
|
102
|
+
expect(widget.id).to eq(5)
|
103
|
+
expect(widget.test_id).to eq('abcd')
|
104
|
+
expect(widget.some_int).to eq(3)
|
105
|
+
|
106
|
+
# test delete
|
107
|
+
test_consume_message(MyConsumer, nil, call_original: true, key: 5)
|
108
|
+
expect(Widget.count).to eq(0)
|
109
|
+
|
110
|
+
end
|
111
|
+
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'should update only updated_at' do
|
115
|
+
travel_to Time.local(2020, 5, 5, 5, 5, 5)
|
116
|
+
widget1 = Widget.create!(test_id: 'id1', some_int: 3)
|
117
|
+
expect(widget1.updated_at.in_time_zone).to eq(Time.local(2020, 5, 5, 5, 5, 5))
|
118
|
+
|
119
|
+
travel 1.day
|
120
|
+
test_consume_message(MyCustomFetchConsumer, {
|
121
|
+
test_id: 'id1',
|
122
|
+
some_int: 3
|
123
|
+
}, { call_original: true })
|
124
|
+
expect(widget1.reload.updated_at.in_time_zone).
|
125
|
+
to eq(Time.local(2020, 5, 6, 5, 5, 5))
|
126
|
+
travel_back
|
127
|
+
end
|
128
|
+
|
129
|
+
it 'should find widgets by custom logic' do
|
130
|
+
widget1 = Widget.create!(test_id: 'id1')
|
131
|
+
expect(widget1.some_int).to be_nil
|
132
|
+
test_consume_message(MyCustomFetchConsumer, {
|
133
|
+
test_id: 'id1',
|
134
|
+
some_int: 3
|
135
|
+
}, { call_original: true })
|
136
|
+
expect(widget1.reload.some_int).to eq(3)
|
137
|
+
expect(Widget.count).to eq(1)
|
138
|
+
test_consume_message(MyCustomFetchConsumer, {
|
139
|
+
test_id: 'id2',
|
140
|
+
some_int: 4
|
141
|
+
}, { call_original: true })
|
142
|
+
expect(Widget.count).to eq(2)
|
143
|
+
expect(Widget.find_by_test_id('id1').some_int).to eq(3)
|
144
|
+
expect(Widget.find_by_test_id('id2').some_int).to eq(4)
|
145
|
+
end
|
146
|
+
|
147
|
+
it 'should not create record of process_message returns false' do
|
148
|
+
allow_any_instance_of(MyConsumer).to receive(:process_message?).and_return(false)
|
149
|
+
expect(Widget.count).to eq(0)
|
150
|
+
test_consume_message(MyConsumer, {
|
151
|
+
test_id: 'abc',
|
152
|
+
some_int: 3,
|
153
|
+
updated_at: 1.day.ago.to_i,
|
154
|
+
some_datetime_int: Time.zone.now.to_i,
|
155
|
+
timestamp: 2.minutes.ago.to_s
|
156
|
+
}, { call_original: true, key: 5 })
|
157
|
+
expect(Widget.count).to eq(0)
|
158
|
+
end
|
159
|
+
end
|
104
160
|
end
|
105
|
-
|
106
|
-
end
|
107
|
-
|
108
|
-
it 'should update only updated_at' do
|
109
|
-
travel_to Time.local(2020, 5, 5, 5, 5, 5)
|
110
|
-
widget1 = Widget.create!(test_id: 'id1', some_int: 3)
|
111
|
-
expect(widget1.updated_at.in_time_zone).to eq(Time.local(2020, 5, 5, 5, 5, 5))
|
112
|
-
|
113
|
-
travel 1.day
|
114
|
-
test_consume_message(MyCustomFetchConsumer, {
|
115
|
-
test_id: 'id1',
|
116
|
-
some_int: 3
|
117
|
-
}, { call_original: true })
|
118
|
-
expect(widget1.reload.updated_at.in_time_zone).
|
119
|
-
to eq(Time.local(2020, 5, 6, 5, 5, 5))
|
120
|
-
travel_back
|
121
|
-
end
|
122
|
-
|
123
|
-
it 'should find widgets by custom logic' do
|
124
|
-
widget1 = Widget.create!(test_id: 'id1')
|
125
|
-
expect(widget1.some_int).to be_nil
|
126
|
-
test_consume_message(MyCustomFetchConsumer, {
|
127
|
-
test_id: 'id1',
|
128
|
-
some_int: 3
|
129
|
-
}, { call_original: true })
|
130
|
-
expect(widget1.reload.some_int).to eq(3)
|
131
|
-
expect(Widget.count).to eq(1)
|
132
|
-
test_consume_message(MyCustomFetchConsumer, {
|
133
|
-
test_id: 'id2',
|
134
|
-
some_int: 4
|
135
|
-
}, { call_original: true })
|
136
|
-
expect(Widget.count).to eq(2)
|
137
|
-
expect(Widget.find_by_test_id('id1').some_int).to eq(3)
|
138
|
-
expect(Widget.find_by_test_id('id2').some_int).to eq(4)
|
139
|
-
end
|
140
|
-
|
141
|
-
it 'should not create record of process_message returns false' do
|
142
|
-
MyConsumer.any_instance.stub(:process_message?).and_return(false)
|
143
|
-
expect(Widget.count).to eq(0)
|
144
|
-
test_consume_message(MyConsumer, {
|
145
|
-
test_id: 'abc',
|
146
|
-
some_int: 3,
|
147
|
-
updated_at: 1.day.ago.to_i,
|
148
|
-
some_datetime_int: Time.zone.now.to_i,
|
149
|
-
timestamp: 2.minutes.ago.to_s
|
150
|
-
}, { call_original: true, key: 5 })
|
151
|
-
expect(Widget.count).to eq(0)
|
152
161
|
end
|
153
162
|
end
|
154
163
|
end
|
@@ -47,35 +47,46 @@ describe Deimos::ActiveRecordProducer do
|
|
47
47
|
stub_const('MyProducerWithUniqueID', producer_class)
|
48
48
|
end
|
49
49
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
50
|
+
describe 'produce' do
|
51
|
+
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
52
|
+
context "with Schema Class consumption #{setting}" do
|
53
|
+
before(:each) do
|
54
|
+
Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
|
55
|
+
end
|
54
56
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
57
|
+
it 'should send events correctly' do
|
58
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
|
59
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'should coerce values' do
|
63
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
|
64
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
|
65
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
|
66
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
|
67
|
+
expect {
|
68
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
|
69
|
+
}.to raise_error(Avro::SchemaValidator::ValidationError)
|
70
|
+
|
71
|
+
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
|
72
|
+
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
|
73
|
+
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
|
74
|
+
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
|
75
|
+
end
|
69
76
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
77
|
+
it 'should be able to call the record' do
|
78
|
+
widget = Widget.create!(test_id: 'abc2', some_int: 3)
|
79
|
+
MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
|
80
|
+
expect('my-topic-with-id').to have_sent(
|
81
|
+
test_id: 'abc2',
|
82
|
+
some_int: 3,
|
83
|
+
message_id: 'generated_id',
|
84
|
+
timestamp: anything
|
85
|
+
)
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|
79
90
|
end
|
80
91
|
|
81
92
|
specify '#watched_attributes' do
|
data/spec/batch_consumer_spec.rb
CHANGED
@@ -3,7 +3,6 @@
|
|
3
3
|
# :nodoc:
|
4
4
|
module ConsumerTest
|
5
5
|
describe Deimos::Consumer, 'Batch Consumer' do
|
6
|
-
|
7
6
|
prepend_before(:each) do
|
8
7
|
# :nodoc:
|
9
8
|
consumer_class = Class.new(described_class) do
|
@@ -30,40 +29,50 @@ module ConsumerTest
|
|
30
29
|
batch.concat([{ 'invalid' => 'key' }])
|
31
30
|
end
|
32
31
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
32
|
+
describe 'consume_batch' do
|
33
|
+
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
34
|
+
context "with Schema Class consumption #{setting}" do
|
35
|
+
before(:each) do
|
36
|
+
Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
|
37
|
+
end
|
38
38
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
39
|
+
it 'should provide backwards compatibility for BatchConsumer class' do
|
40
|
+
consumer_class = Class.new(Deimos::BatchConsumer) do
|
41
|
+
schema 'MySchema'
|
42
|
+
namespace 'com.my-namespace'
|
43
|
+
key_config field: 'test_id'
|
44
|
+
|
45
|
+
# :nodoc:
|
46
|
+
def consume_batch(_payloads, _metadata)
|
47
|
+
raise 'This should not be called unless call_original is set'
|
48
|
+
end
|
49
|
+
end
|
50
|
+
stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
|
51
|
+
|
52
|
+
test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
|
53
|
+
expect(received).to eq(batch)
|
54
|
+
end
|
55
|
+
end
|
45
56
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
57
|
+
it 'should consume a batch of messages' do
|
58
|
+
test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
|
59
|
+
expect(received).to eq(batch)
|
60
|
+
end
|
61
|
+
end
|
50
62
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
63
|
+
it 'should consume a message on a topic' do
|
64
|
+
test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
|
65
|
+
expect(received).to eq(batch)
|
66
|
+
end
|
67
|
+
end
|
56
68
|
|
57
|
-
|
58
|
-
|
59
|
-
|
69
|
+
it 'should fail on an invalid message in the batch' do
|
70
|
+
test_consume_batch_invalid_message(MyBatchConsumer, batch.concat(invalid_payloads))
|
71
|
+
end
|
72
|
+
end
|
60
73
|
end
|
61
74
|
end
|
62
75
|
|
63
|
-
it 'should fail on an invalid message in the batch' do
|
64
|
-
test_consume_batch_invalid_message(MyBatchConsumer, batch.concat(invalid_payloads))
|
65
|
-
end
|
66
|
-
|
67
76
|
describe 'when reraising errors is disabled' do
|
68
77
|
before(:each) do
|
69
78
|
Deimos.configure { |config| config.consumers.reraise_errors = false }
|
@@ -21,6 +21,7 @@ describe Deimos, 'configuration' do
|
|
21
21
|
kafka_logger logger
|
22
22
|
reraise_consumer_errors true
|
23
23
|
schema_registry_url 'http://schema.registry'
|
24
|
+
schema.use_schema_classes false
|
24
25
|
seed_broker 'whatever'
|
25
26
|
schema_path 'some_path'
|
26
27
|
producer_schema_namespace 'namespace'
|
@@ -37,6 +38,7 @@ describe Deimos, 'configuration' do
|
|
37
38
|
expect(described_class.config.kafka.logger).to eq(logger)
|
38
39
|
expect(described_class.config.consumers.reraise_errors).to eq(true)
|
39
40
|
expect(described_class.config.schema.registry_url).to eq('http://schema.registry')
|
41
|
+
expect(described_class.config.schema.use_schema_classes).to eq(false)
|
40
42
|
expect(described_class.config.kafka.seed_brokers).to eq('whatever')
|
41
43
|
expect(described_class.config.producers.schema_namespace).to eq('namespace')
|
42
44
|
expect(described_class.config.producers.topic_prefix).to eq('prefix')
|
@@ -86,7 +88,8 @@ describe Deimos, 'configuration' do
|
|
86
88
|
offset_commit_threshold: 0,
|
87
89
|
offset_retention_time: nil,
|
88
90
|
heartbeat_interval: 10,
|
89
|
-
handler: 'ConsumerTest::MyConsumer'
|
91
|
+
handler: 'ConsumerTest::MyConsumer',
|
92
|
+
use_schema_classes: nil
|
90
93
|
}, {
|
91
94
|
topic: 'my_batch_consume_topic',
|
92
95
|
group_id: 'my_batch_group_id',
|
@@ -102,7 +105,8 @@ describe Deimos, 'configuration' do
|
|
102
105
|
offset_commit_threshold: 0,
|
103
106
|
offset_retention_time: nil,
|
104
107
|
heartbeat_interval: 10,
|
105
|
-
handler: 'ConsumerTest::MyBatchConsumer'
|
108
|
+
handler: 'ConsumerTest::MyBatchConsumer',
|
109
|
+
use_schema_classes: nil
|
106
110
|
}
|
107
111
|
],
|
108
112
|
producer: {
|
@@ -178,6 +182,7 @@ describe Deimos, 'configuration' do
|
|
178
182
|
offset_commit_threshold 13
|
179
183
|
offset_retention_time 13
|
180
184
|
heartbeat_interval 13
|
185
|
+
use_schema_classes false
|
181
186
|
end
|
182
187
|
consumer do
|
183
188
|
disabled true
|
@@ -185,6 +190,7 @@ describe Deimos, 'configuration' do
|
|
185
190
|
schema 'blah2'
|
186
191
|
topic 'blah2'
|
187
192
|
group_id 'myconsumerid2'
|
193
|
+
use_schema_classes false
|
188
194
|
end
|
189
195
|
end
|
190
196
|
|
@@ -227,7 +233,8 @@ describe Deimos, 'configuration' do
|
|
227
233
|
offset_commit_threshold: 13,
|
228
234
|
offset_retention_time: 13,
|
229
235
|
heartbeat_interval: 13,
|
230
|
-
handler: 'MyConfigConsumer'
|
236
|
+
handler: 'MyConfigConsumer',
|
237
|
+
use_schema_classes: false
|
231
238
|
}
|
232
239
|
],
|
233
240
|
producer: {
|
data/spec/consumer_spec.rb
CHANGED
@@ -3,7 +3,6 @@
|
|
3
3
|
# :nodoc:
|
4
4
|
module ConsumerTest
|
5
5
|
describe Deimos::Consumer, 'Message Consumer' do
|
6
|
-
|
7
6
|
prepend_before(:each) do
|
8
7
|
# :nodoc:
|
9
8
|
consumer_class = Class.new(described_class) do
|
@@ -13,7 +12,7 @@ module ConsumerTest
|
|
13
12
|
|
14
13
|
# :nodoc:
|
15
14
|
def fatal_error?(_exception, payload, _metadata)
|
16
|
-
payload == 'fatal'
|
15
|
+
payload.to_s == 'fatal'
|
17
16
|
end
|
18
17
|
|
19
18
|
# :nodoc:
|
@@ -24,98 +23,110 @@ module ConsumerTest
|
|
24
23
|
stub_const('ConsumerTest::MyConsumer', consumer_class)
|
25
24
|
end
|
26
25
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
26
|
+
describe 'consume' do
|
27
|
+
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
28
|
+
context "with Schema Class consumption #{setting}" do
|
29
|
+
before(:each) do
|
30
|
+
Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
|
31
|
+
end
|
34
32
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
33
|
+
it 'should consume a message' do
|
34
|
+
test_consume_message(MyConsumer,
|
35
|
+
'test_id' => 'foo',
|
36
|
+
'some_int' => 123) do |payload, _metadata|
|
37
|
+
expect(payload['test_id']).to eq('foo')
|
38
|
+
expect(payload['some_int']).to eq(123)
|
39
|
+
end
|
40
|
+
end
|
40
41
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
k['test_id']
|
47
|
-
end
|
48
|
-
MyConsumer.new.around_consume({ 'test_id' => 'foo',
|
49
|
-
'some_int' => 123 }, test_metadata) do |_payload, metadata|
|
50
|
-
expect(metadata[:key]).to eq('foo')
|
51
|
-
end
|
52
|
-
MyConsumer.new.around_consume({ 'test_id' => 'foo',
|
53
|
-
'some_int' => 123 }, test_metadata) do |_payload, metadata|
|
54
|
-
expect(metadata[:key]).to eq('foo')
|
55
|
-
end
|
56
|
-
end
|
42
|
+
it 'should consume a nil message' do
|
43
|
+
test_consume_message(MyConsumer, nil) do |payload, _metadata|
|
44
|
+
expect(payload).to be_nil
|
45
|
+
end
|
46
|
+
end
|
57
47
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
48
|
+
it 'should consume a message idempotently' do
|
49
|
+
# testing for a crash and re-consuming the same message/metadata
|
50
|
+
key = { 'test_id' => 'foo' }
|
51
|
+
test_metadata = { key: key }
|
52
|
+
allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
|
53
|
+
k['test_id']
|
54
|
+
end
|
55
|
+
MyConsumer.new.around_consume({ 'test_id' => 'foo',
|
56
|
+
'some_int' => 123 }, test_metadata) do |_payload, metadata|
|
57
|
+
expect(metadata[:key]).to eq('foo')
|
58
|
+
end
|
59
|
+
MyConsumer.new.around_consume({ 'test_id' => 'foo',
|
60
|
+
'some_int' => 123 }, test_metadata) do |_payload, metadata|
|
61
|
+
expect(metadata[:key]).to eq('foo')
|
62
|
+
end
|
63
|
+
end
|
65
64
|
|
66
|
-
|
67
|
-
|
68
|
-
|
65
|
+
it 'should consume a message on a topic' do
|
66
|
+
test_consume_message('my_consume_topic',
|
67
|
+
'test_id' => 'foo',
|
68
|
+
'some_int' => 123) do |payload, _metadata|
|
69
|
+
expect(payload['test_id']).to eq('foo')
|
70
|
+
expect(payload['some_int']).to eq(123)
|
71
|
+
end
|
72
|
+
end
|
69
73
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
end
|
74
|
+
it 'should fail on invalid message' do
|
75
|
+
test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
|
76
|
+
end
|
74
77
|
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
end
|
80
|
-
test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
|
81
|
-
end
|
78
|
+
it 'should fail if reraise is false but fatal_error is true' do
|
79
|
+
Deimos.configure { |config| config.consumers.reraise_errors = false }
|
80
|
+
test_consume_invalid_message(MyConsumer, 'fatal')
|
81
|
+
end
|
82
82
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
it 'should fail if fatal_error is true globally' do
|
84
|
+
Deimos.configure do |config|
|
85
|
+
config.consumers.fatal_error = proc { true }
|
86
|
+
config.consumers.reraise_errors = false
|
87
|
+
end
|
88
|
+
test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
|
89
|
+
end
|
89
90
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
'some_int' => 123 },
|
97
|
-
{ skip_expectation: true }
|
98
|
-
) { raise 'OH NOES' }
|
99
|
-
}.not_to raise_error
|
100
|
-
end
|
91
|
+
it 'should fail on message with extra fields' do
|
92
|
+
test_consume_invalid_message(MyConsumer,
|
93
|
+
'test_id' => 'foo',
|
94
|
+
'some_int' => 123,
|
95
|
+
'extra_field' => 'field name')
|
96
|
+
end
|
101
97
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
98
|
+
it 'should not fail when before_consume fails without reraising errors' do
|
99
|
+
Deimos.configure { |config| config.consumers.reraise_errors = false }
|
100
|
+
expect {
|
101
|
+
test_consume_message(
|
102
|
+
MyConsumer,
|
103
|
+
{ 'test_id' => 'foo',
|
104
|
+
'some_int' => 123 },
|
105
|
+
{ skip_expectation: true }
|
106
|
+
) { raise 'OH NOES' }
|
107
|
+
}.not_to raise_error
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'should not fail when consume fails without reraising errors' do
|
111
|
+
Deimos.configure { |config| config.consumers.reraise_errors = false }
|
112
|
+
expect {
|
113
|
+
test_consume_message(
|
114
|
+
MyConsumer,
|
115
|
+
{ 'invalid' => 'key' },
|
116
|
+
{ skip_expectation: true }
|
117
|
+
)
|
118
|
+
}.not_to raise_error
|
119
|
+
end
|
112
120
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
121
|
+
it 'should call original' do
|
122
|
+
expect {
|
123
|
+
test_consume_message(MyConsumer,
|
124
|
+
{ 'test_id' => 'foo', 'some_int' => 123 },
|
125
|
+
{ call_original: true })
|
126
|
+
}.to raise_error('This should not be called unless call_original is set')
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
119
130
|
end
|
120
131
|
|
121
132
|
describe 'decode_key' do
|