deimos-ruby 1.24.3 → 2.0.0.pre.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -17
- data/.tool-versions +1 -0
- data/CHANGELOG.md +1 -1
- data/README.md +287 -498
- data/deimos-ruby.gemspec +4 -4
- data/docs/CONFIGURATION.md +133 -227
- data/docs/UPGRADING.md +237 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
- data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
- data/lib/deimos/active_record_consumer.rb +36 -26
- data/lib/deimos/active_record_producer.rb +28 -9
- data/lib/deimos/backends/base.rb +4 -35
- data/lib/deimos/backends/kafka.rb +6 -22
- data/lib/deimos/backends/kafka_async.rb +6 -22
- data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
- data/lib/deimos/config/configuration.rb +116 -385
- data/lib/deimos/consume/batch_consumption.rb +24 -124
- data/lib/deimos/consume/message_consumption.rb +36 -63
- data/lib/deimos/consumer.rb +16 -75
- data/lib/deimos/ext/consumer_route.rb +35 -0
- data/lib/deimos/ext/producer_middleware.rb +94 -0
- data/lib/deimos/ext/producer_route.rb +22 -0
- data/lib/deimos/ext/redraw.rb +29 -0
- data/lib/deimos/ext/routing_defaults.rb +72 -0
- data/lib/deimos/ext/schema_route.rb +70 -0
- data/lib/deimos/kafka_message.rb +2 -2
- data/lib/deimos/kafka_source.rb +2 -7
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +71 -0
- data/lib/deimos/message.rb +2 -11
- data/lib/deimos/metrics/datadog.rb +40 -1
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +39 -116
- data/lib/deimos/railtie.rb +6 -0
- data/lib/deimos/schema_backends/avro_base.rb +21 -21
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
- data/lib/deimos/schema_backends/avro_validation.rb +2 -2
- data/lib/deimos/schema_backends/base.rb +19 -12
- data/lib/deimos/schema_backends/mock.rb +6 -1
- data/lib/deimos/schema_backends/plain.rb +47 -0
- data/lib/deimos/schema_class/base.rb +2 -2
- data/lib/deimos/schema_class/enum.rb +1 -1
- data/lib/deimos/schema_class/record.rb +2 -2
- data/lib/deimos/test_helpers.rb +95 -320
- data/lib/deimos/tracing/provider.rb +6 -6
- data/lib/deimos/transcoder.rb +88 -0
- data/lib/deimos/utils/db_poller/base.rb +16 -14
- data/lib/deimos/utils/db_poller/state_based.rb +3 -3
- data/lib/deimos/utils/db_poller/time_based.rb +4 -4
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/utils/deadlock_retry.rb +1 -1
- data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
- data/lib/deimos/utils/schema_class.rb +0 -7
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +79 -26
- data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
- data/lib/generators/deimos/schema_class_generator.rb +0 -1
- data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
- data/lib/generators/deimos/v2_generator.rb +193 -0
- data/lib/tasks/deimos.rake +5 -7
- data/spec/active_record_batch_consumer_association_spec.rb +22 -13
- data/spec/active_record_batch_consumer_spec.rb +84 -65
- data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
- data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
- data/spec/active_record_consumer_spec.rb +29 -13
- data/spec/active_record_producer_spec.rb +36 -26
- data/spec/backends/base_spec.rb +0 -23
- data/spec/backends/kafka_async_spec.rb +1 -3
- data/spec/backends/kafka_spec.rb +1 -3
- data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
- data/spec/batch_consumer_spec.rb +66 -116
- data/spec/consumer_spec.rb +53 -147
- data/spec/deimos_spec.rb +10 -126
- data/spec/kafka_source_spec.rb +19 -52
- data/spec/karafka/karafka.rb +69 -0
- data/spec/karafka_config/karafka_spec.rb +97 -0
- data/spec/logging_spec.rb +25 -0
- data/spec/message_spec.rb +9 -9
- data/spec/producer_spec.rb +112 -254
- data/spec/rake_spec.rb +1 -3
- data/spec/schema_backends/avro_validation_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
- data/spec/snapshots/consumers-no-nest.snap +49 -0
- data/spec/snapshots/consumers.snap +49 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
- data/spec/snapshots/consumers_and_producers.snap +49 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
- data/spec/snapshots/consumers_circular.snap +49 -0
- data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
- data/spec/snapshots/consumers_complex_types.snap +49 -0
- data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
- data/spec/snapshots/consumers_nested.snap +49 -0
- data/spec/snapshots/namespace_folders.snap +49 -0
- data/spec/snapshots/namespace_map.snap +49 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
- data/spec/snapshots/producers_with_key.snap +49 -0
- data/spec/spec_helper.rb +61 -29
- data/spec/utils/db_poller_spec.rb +49 -39
- data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
- metadata +58 -67
- data/lib/deimos/batch_consumer.rb +0 -7
- data/lib/deimos/config/phobos_config.rb +0 -164
- data/lib/deimos/instrumentation.rb +0 -95
- data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
- data/lib/deimos/utils/inline_consumer.rb +0 -158
- data/lib/deimos/utils/lag_reporter.rb +0 -186
- data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
- data/spec/config/configuration_spec.rb +0 -329
- data/spec/kafka_listener_spec.rb +0 -55
- data/spec/phobos.bad_db.yml +0 -73
- data/spec/phobos.yml +0 -77
- data/spec/utils/inline_consumer_spec.rb +0 -31
- data/spec/utils/lag_reporter_spec.rb +0 -76
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/schema_controller_mixin_spec.rb +0 -84
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
data/spec/producer_spec.rb
CHANGED
@@ -5,27 +5,13 @@ module ProducerTest
|
|
5
5
|
describe Deimos::Producer do
|
6
6
|
|
7
7
|
prepend_before(:each) do
|
8
|
-
producer_class = Class.new(Deimos::Producer)
|
9
|
-
schema 'MySchema'
|
10
|
-
namespace 'com.my-namespace'
|
11
|
-
topic 'my-topic'
|
12
|
-
key_config field: 'test_id'
|
13
|
-
end
|
8
|
+
producer_class = Class.new(Deimos::Producer)
|
14
9
|
stub_const('MyProducer', producer_class)
|
15
10
|
|
16
|
-
producer_class = Class.new(Deimos::Producer)
|
17
|
-
schema 'MySchemaWithId'
|
18
|
-
namespace 'com.my-namespace'
|
19
|
-
topic 'my-topic'
|
20
|
-
key_config plain: true
|
21
|
-
end
|
11
|
+
producer_class = Class.new(Deimos::Producer)
|
22
12
|
stub_const('MyProducerWithID', producer_class)
|
23
13
|
|
24
14
|
producer_class = Class.new(Deimos::Producer) do
|
25
|
-
schema 'MySchema'
|
26
|
-
namespace 'com.my-namespace'
|
27
|
-
topic 'my-topic'
|
28
|
-
key_config plain: true
|
29
15
|
# :nodoc:
|
30
16
|
def self.partition_key(payload)
|
31
17
|
payload[:payload_key] ? payload[:payload_key] + '1' : nil
|
@@ -33,71 +19,90 @@ module ProducerTest
|
|
33
19
|
end
|
34
20
|
stub_const('MyNonEncodedProducer', producer_class)
|
35
21
|
|
36
|
-
producer_class = Class.new(Deimos::Producer)
|
37
|
-
schema 'MySchema'
|
38
|
-
namespace 'com.my-namespace'
|
39
|
-
topic 'my-topic2'
|
40
|
-
key_config none: true
|
41
|
-
end
|
22
|
+
producer_class = Class.new(Deimos::Producer)
|
42
23
|
stub_const('MyNoKeyProducer', producer_class)
|
43
24
|
|
44
|
-
producer_class = Class.new(Deimos::Producer)
|
45
|
-
schema 'MyNestedSchema'
|
46
|
-
namespace 'com.my-namespace'
|
47
|
-
topic 'my-topic'
|
48
|
-
key_config field: 'test_id'
|
49
|
-
end
|
25
|
+
producer_class = Class.new(Deimos::Producer)
|
50
26
|
stub_const('MyNestedSchemaProducer', producer_class)
|
51
27
|
|
52
|
-
producer_class = Class.new(Deimos::Producer)
|
53
|
-
schema 'MySchema'
|
54
|
-
namespace 'com.my-namespace'
|
55
|
-
topic 'my-topic2'
|
56
|
-
key_config schema: 'MySchema_key'
|
57
|
-
end
|
28
|
+
producer_class = Class.new(Deimos::Producer)
|
58
29
|
stub_const('MySchemaProducer', producer_class)
|
59
30
|
|
60
|
-
producer_class = Class.new(Deimos::Producer)
|
61
|
-
schema 'MySchema'
|
62
|
-
namespace 'com.my-namespace'
|
63
|
-
topic 'my-topic'
|
64
|
-
end
|
31
|
+
producer_class = Class.new(Deimos::Producer)
|
65
32
|
stub_const('MyErrorProducer', producer_class)
|
66
33
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
34
|
+
Karafka::App.routes.redraw do
|
35
|
+
topic 'my-topic' do
|
36
|
+
producer_class MyProducer
|
37
|
+
schema 'MySchema'
|
38
|
+
namespace 'com.my-namespace'
|
39
|
+
key_config field: 'test_id'
|
40
|
+
end
|
41
|
+
topic 'a-new-topic' do
|
42
|
+
producer_class MyProducer
|
43
|
+
schema 'MySchema'
|
44
|
+
namespace 'com.my-namespace'
|
45
|
+
key_config field: 'test_id'
|
46
|
+
end
|
47
|
+
topic 'my-topic-with-id' do
|
48
|
+
producer_class MyProducerWithID
|
49
|
+
schema 'MySchemaWithId'
|
50
|
+
namespace 'com.my-namespace'
|
51
|
+
key_config plain: true
|
52
|
+
end
|
53
|
+
topic 'my-topic-non-encoded' do
|
54
|
+
producer_class MyNonEncodedProducer
|
55
|
+
schema 'MySchema'
|
56
|
+
namespace 'com.my-namespace'
|
57
|
+
key_config plain: true
|
58
|
+
end
|
59
|
+
topic 'my-topic-no-key' do
|
60
|
+
producer_class MyNoKeyProducer
|
61
|
+
schema 'MySchema'
|
62
|
+
namespace 'com.my-namespace'
|
63
|
+
key_config none: true
|
64
|
+
end
|
65
|
+
topic 'my-topic-nested-schema' do
|
66
|
+
producer_class MyNestedSchemaProducer
|
67
|
+
schema 'MyNestedSchema'
|
68
|
+
namespace 'com.my-namespace'
|
69
|
+
key_config field: 'test_id'
|
70
|
+
end
|
71
|
+
topic 'my-topic-schema' do
|
72
|
+
producer_class MySchemaProducer
|
73
|
+
schema 'MySchema'
|
74
|
+
namespace 'com.my-namespace'
|
75
|
+
key_config schema: 'MySchema_key'
|
76
|
+
end
|
77
|
+
topic 'my-topic-error' do
|
78
|
+
schema 'MySchema'
|
79
|
+
namespace 'com.my-namespace'
|
80
|
+
producer_class MyErrorProducer
|
81
|
+
end
|
72
82
|
end
|
73
|
-
stub_const('MyNoTopicProducer', producer_class)
|
74
83
|
|
75
84
|
end
|
76
85
|
|
77
|
-
it 'should fail on invalid message
|
78
|
-
|
79
|
-
expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
|
80
|
-
end
|
81
|
-
expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
|
86
|
+
it 'should fail on invalid message' do
|
87
|
+
expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
|
82
88
|
expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
|
83
89
|
to raise_error('OH NOES')
|
84
|
-
Deimos.unsubscribe(subscriber)
|
85
90
|
end
|
86
91
|
|
87
92
|
it 'should produce a message' do
|
88
|
-
expect(
|
89
|
-
Deimos::Backends::
|
93
|
+
expect(MyProducer).to receive(:produce_batch).once.with(
|
94
|
+
Deimos::Backends::Kafka,
|
90
95
|
[
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
96
|
+
hash_including({
|
97
|
+
payload: { 'test_id' => 'foo', 'some_int' => 123 },
|
98
|
+
topic: 'my-topic',
|
99
|
+
partition_key: nil
|
100
|
+
}),
|
101
|
+
hash_including({
|
102
|
+
payload: { 'test_id' => 'bar', 'some_int' => 124 },
|
103
|
+
topic: 'my-topic',
|
104
|
+
partition_key: nil
|
105
|
+
})
|
101
106
|
]
|
102
107
|
).and_call_original
|
103
108
|
|
@@ -105,27 +110,27 @@ module ProducerTest
|
|
105
110
|
[{ 'test_id' => 'foo', 'some_int' => 123 },
|
106
111
|
{ 'test_id' => 'bar', 'some_int' => 124 }]
|
107
112
|
)
|
108
|
-
expect('my-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
|
113
|
+
expect('my-topic').to have_sent({'test_id' => 'foo', 'some_int' => 123}, 'foo', 'foo')
|
109
114
|
expect('your-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
|
110
115
|
expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
|
111
116
|
end
|
112
117
|
|
113
118
|
it 'should allow setting the topic and headers from publish_list' do
|
114
|
-
expect(
|
115
|
-
Deimos::Backends::
|
119
|
+
expect(MyProducer).to receive(:produce_batch).once.with(
|
120
|
+
Deimos::Backends::Kafka,
|
116
121
|
[
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
122
|
+
hash_including({
|
123
|
+
payload: { 'test_id' => 'foo', 'some_int' => 123 },
|
124
|
+
topic: 'a-new-topic',
|
125
|
+
headers: { 'foo' => 'bar' },
|
126
|
+
partition_key: nil
|
127
|
+
}),
|
128
|
+
hash_including({
|
129
|
+
payload: { 'test_id' => 'bar', 'some_int' => 124 },
|
130
|
+
topic: 'a-new-topic',
|
131
|
+
headers: { 'foo' => 'bar' },
|
132
|
+
partition_key: nil
|
133
|
+
})
|
129
134
|
]
|
130
135
|
).and_call_original
|
131
136
|
|
@@ -145,27 +150,14 @@ module ProducerTest
|
|
145
150
|
'some_int' => 123,
|
146
151
|
'message_id' => a_kind_of(String),
|
147
152
|
'timestamp' => a_kind_of(String) }
|
148
|
-
expect(described_class).to receive(:produce_batch).once do |_, messages|
|
149
|
-
expect(messages.size).to eq(1)
|
150
|
-
expect(messages[0].to_h).
|
151
|
-
to match(
|
152
|
-
payload: payload,
|
153
|
-
topic: 'my-topic',
|
154
|
-
partition_key: 'key',
|
155
|
-
metadata: {
|
156
|
-
producer_name: 'MyProducerWithID',
|
157
|
-
decoded_payload: payload
|
158
|
-
},
|
159
|
-
key: 'key'
|
160
|
-
)
|
161
|
-
end
|
162
153
|
MyProducerWithID.publish_list(
|
163
154
|
[{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'key' }]
|
164
155
|
)
|
156
|
+
expect(MyProducerWithID.topic).to have_sent(payload, 'key', 'key')
|
157
|
+
|
165
158
|
end
|
166
159
|
|
167
160
|
it 'should not publish if publish disabled' do
|
168
|
-
expect(described_class).not_to receive(:produce_batch)
|
169
161
|
Deimos.configure { |c| c.producers.disabled = true }
|
170
162
|
MyProducer.publish_list(
|
171
163
|
[{ 'test_id' => 'foo', 'some_int' => 123 },
|
@@ -201,102 +193,22 @@ module ProducerTest
|
|
201
193
|
it 'should produce to a prefixed topic' do
|
202
194
|
Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
|
203
195
|
payload = { 'test_id' => 'foo', 'some_int' => 123 }
|
204
|
-
expect(described_class).to receive(:produce_batch).once do |_, messages|
|
205
|
-
expect(messages.size).to eq(1)
|
206
|
-
expect(messages[0].to_h).
|
207
|
-
to eq(
|
208
|
-
payload: payload,
|
209
|
-
topic: 'prefix.my-topic',
|
210
|
-
partition_key: 'foo',
|
211
|
-
metadata: {
|
212
|
-
producer_name: 'MyProducer',
|
213
|
-
decoded_payload: payload
|
214
|
-
},
|
215
|
-
key: 'foo'
|
216
|
-
)
|
217
|
-
end
|
218
196
|
|
219
197
|
MyProducer.publish_list([payload])
|
220
|
-
|
221
|
-
expect(
|
222
|
-
|
223
|
-
expect(messages[0].to_h).
|
224
|
-
to eq(
|
225
|
-
payload: payload,
|
226
|
-
topic: 'my-topic',
|
227
|
-
partition_key: 'foo',
|
228
|
-
metadata: {
|
229
|
-
producer_name: 'MyProducer',
|
230
|
-
decoded_payload: payload
|
231
|
-
},
|
232
|
-
key: 'foo'
|
233
|
-
)
|
234
|
-
end
|
198
|
+
expect('prefix.my-topic').to have_sent(payload, 'foo', 'foo')
|
199
|
+
expect(karafka.produced_messages.size).to eq(1)
|
200
|
+
karafka.produced_messages.clear
|
235
201
|
|
236
|
-
MyProducer.publish_list(
|
237
|
-
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
238
|
-
)
|
239
|
-
end
|
240
|
-
|
241
|
-
it 'should encode the key' do
|
242
202
|
Deimos.configure { |c| c.producers.topic_prefix = nil }
|
243
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
244
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
|
245
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
246
|
-
'test_id' => 'foo',
|
247
|
-
'some_int' => 123
|
248
|
-
}, { topic: 'my-topic-value' })
|
249
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
250
|
-
'test_id' => 'bar',
|
251
|
-
'some_int' => 124
|
252
|
-
}, { topic: 'my-topic-value' })
|
253
203
|
|
254
204
|
MyProducer.publish_list(
|
255
|
-
[{ 'test_id' => 'foo', 'some_int' => 123 }
|
256
|
-
{ 'test_id' => 'bar', 'some_int' => 124 }]
|
257
|
-
)
|
258
|
-
end
|
259
|
-
|
260
|
-
it 'should encode the key with topic prefix' do
|
261
|
-
Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
|
262
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
|
263
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
|
264
|
-
expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'foo',
|
265
|
-
'some_int' => 123 },
|
266
|
-
{ topic: 'prefix.my-topic-value' })
|
267
|
-
expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'bar',
|
268
|
-
'some_int' => 124 },
|
269
|
-
{ topic: 'prefix.my-topic-value' })
|
270
|
-
|
271
|
-
MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
|
272
|
-
{ 'test_id' => 'bar', 'some_int' => 124 }])
|
273
|
-
end
|
274
|
-
|
275
|
-
it 'should not encode with plaintext key' do
|
276
|
-
expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
|
277
|
-
|
278
|
-
MyNonEncodedProducer.publish_list(
|
279
|
-
[{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'foo_key' },
|
280
|
-
{ 'test_id' => 'bar', 'some_int' => 124, :payload_key => 'bar_key' }]
|
281
|
-
)
|
282
|
-
end
|
283
|
-
|
284
|
-
it 'should encode with a schema' do
|
285
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
|
286
|
-
{ topic: 'my-topic2-key' })
|
287
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
|
288
|
-
{ topic: 'my-topic2-key' })
|
289
|
-
|
290
|
-
MySchemaProducer.publish_list(
|
291
|
-
[{ 'test_id' => 'foo', 'some_int' => 123,
|
292
|
-
:payload_key => { 'test_id' => 'foo_key' } },
|
293
|
-
{ 'test_id' => 'bar', 'some_int' => 124,
|
294
|
-
:payload_key => { 'test_id' => 'bar_key' } }]
|
205
|
+
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
295
206
|
)
|
207
|
+
expect('my-topic').to have_sent(payload, 'foo', 'foo')
|
208
|
+
expect(karafka.produced_messages.size).to eq(1)
|
296
209
|
end
|
297
210
|
|
298
211
|
it 'should properly encode and coerce values with a nested record' do
|
299
|
-
expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
300
212
|
MyNestedSchemaProducer.publish({
|
301
213
|
'test_id' => 'foo',
|
302
214
|
'test_float' => BigDecimal('123.456'),
|
@@ -323,39 +235,6 @@ module ProducerTest
|
|
323
235
|
)
|
324
236
|
end
|
325
237
|
|
326
|
-
it 'should raise error if blank topic is passed in explicitly' do
|
327
|
-
expect {
|
328
|
-
MyProducer.publish_list(
|
329
|
-
[{ 'test_id' => 'foo',
|
330
|
-
'some_int' => 123 },
|
331
|
-
{ 'test_id' => 'bar',
|
332
|
-
'some_int' => 124 }],
|
333
|
-
topic: ''
|
334
|
-
)
|
335
|
-
}.to raise_error(RuntimeError,
|
336
|
-
'Topic not specified. Please specify the topic.')
|
337
|
-
end
|
338
|
-
|
339
|
-
it 'should raise error if the producer has not been initialized with a topic' do
|
340
|
-
expect {
|
341
|
-
MyNoTopicProducer.publish_list(
|
342
|
-
[{ 'test_id' => 'foo',
|
343
|
-
'some_int' => 123 },
|
344
|
-
{ 'test_id' => 'bar',
|
345
|
-
'some_int' => 124 }]
|
346
|
-
)
|
347
|
-
}.to raise_error(RuntimeError,
|
348
|
-
'Topic not specified. Please specify the topic.')
|
349
|
-
end
|
350
|
-
|
351
|
-
it 'should error with nothing set' do
|
352
|
-
expect {
|
353
|
-
MyErrorProducer.publish_list(
|
354
|
-
[{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => '123' }]
|
355
|
-
)
|
356
|
-
}.to raise_error('No key config given - if you are not encoding keys, please use `key_config plain: true`')
|
357
|
-
end
|
358
|
-
|
359
238
|
it 'should error if no key given and none is not the config' do
|
360
239
|
expect {
|
361
240
|
MyNonEncodedProducer.publish_list(
|
@@ -395,30 +274,25 @@ module ProducerTest
|
|
395
274
|
|
396
275
|
context 'with Schema Class payloads' do
|
397
276
|
it 'should fail on invalid message with error handler' do
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
|
402
|
-
expect {
|
403
|
-
MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid'))
|
404
|
-
}.to raise_error('OH NOES')
|
405
|
-
Deimos.unsubscribe(subscriber)
|
277
|
+
expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
|
278
|
+
expect { MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid')) }.
|
279
|
+
to raise_error('OH NOES')
|
406
280
|
end
|
407
281
|
|
408
282
|
it 'should produce a message' do
|
409
|
-
expect(
|
410
|
-
Deimos::Backends::
|
283
|
+
expect(MyProducer).to receive(:produce_batch).once.with(
|
284
|
+
Deimos::Backends::Kafka,
|
411
285
|
[
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
286
|
+
hash_including({
|
287
|
+
payload: { 'test_id' => 'foo', 'some_int' => 123, 'payload_key' => nil },
|
288
|
+
topic: 'my-topic',
|
289
|
+
partition_key: nil,
|
290
|
+
}),
|
291
|
+
hash_including({
|
292
|
+
payload: { 'test_id' => 'bar', 'some_int' => 124, 'payload_key' => nil },
|
293
|
+
topic: 'my-topic',
|
294
|
+
partition_key: nil
|
295
|
+
})
|
422
296
|
]
|
423
297
|
).and_call_original
|
424
298
|
|
@@ -432,7 +306,7 @@ module ProducerTest
|
|
432
306
|
end
|
433
307
|
|
434
308
|
it 'should not publish if publish disabled' do
|
435
|
-
expect(
|
309
|
+
expect(MyProducer).not_to receive(:produce_batch)
|
436
310
|
Deimos.configure { |c| c.producers.disabled = true }
|
437
311
|
MyProducer.publish_list(
|
438
312
|
[Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
|
@@ -443,16 +317,6 @@ module ProducerTest
|
|
443
317
|
|
444
318
|
it 'should encode the key' do
|
445
319
|
Deimos.configure { |c| c.producers.topic_prefix = nil }
|
446
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
447
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
|
448
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
449
|
-
'test_id' => 'foo',
|
450
|
-
'some_int' => 123
|
451
|
-
}, { topic: 'my-topic-value' })
|
452
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
453
|
-
'test_id' => 'bar',
|
454
|
-
'some_int' => 124
|
455
|
-
}, { topic: 'my-topic-value' })
|
456
320
|
|
457
321
|
MyProducer.publish_list(
|
458
322
|
[Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
|
@@ -461,11 +325,6 @@ module ProducerTest
|
|
461
325
|
end
|
462
326
|
|
463
327
|
it 'should encode with a schema' do
|
464
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
|
465
|
-
{ topic: 'my-topic2-key' })
|
466
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
|
467
|
-
{ topic: 'my-topic2-key' })
|
468
|
-
|
469
328
|
MySchemaProducer.publish_list(
|
470
329
|
[Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123, payload_key: { 'test_id' => 'foo_key' }),
|
471
330
|
Schemas::MyNamespace::MySchema.new(test_id: 'bar', some_int: 124, payload_key: { 'test_id' => 'bar_key' })]
|
@@ -473,7 +332,6 @@ module ProducerTest
|
|
473
332
|
end
|
474
333
|
|
475
334
|
it 'should properly encode and coerce values with a nested record' do
|
476
|
-
expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
477
335
|
MyNestedSchemaProducer.publish(
|
478
336
|
Schemas::MyNamespace::MyNestedSchema.new(
|
479
337
|
test_id: 'foo',
|
@@ -524,7 +382,7 @@ module ProducerTest
|
|
524
382
|
MyProducerWithID.publish({
|
525
383
|
'test_id' => 'foo', 'some_int' => 123, :payload_key => 123
|
526
384
|
})
|
527
|
-
expect('my-topic').
|
385
|
+
expect('my-topic-with-id').
|
528
386
|
to have_sent('test_id' => 'foo', 'some_int' => 123,
|
529
387
|
'message_id' => anything, 'timestamp' => anything)
|
530
388
|
expect(Deimos).not_to be_producers_disabled
|
@@ -544,7 +402,7 @@ module ProducerTest
|
|
544
402
|
:payload_key => '123'
|
545
403
|
})
|
546
404
|
expect('my-topic').not_to have_sent(anything)
|
547
|
-
expect('my-
|
405
|
+
expect('my-topic-schema').to have_sent('test_id' => 'foo', 'some_int' => 123)
|
548
406
|
expect(Deimos).not_to be_producers_disabled
|
549
407
|
expect(Deimos).to be_producers_disabled(MyProducer)
|
550
408
|
expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
|
@@ -582,11 +440,11 @@ module ProducerTest
|
|
582
440
|
end
|
583
441
|
|
584
442
|
it 'should return db if db is set' do
|
585
|
-
Deimos.configure { producers.backend = :
|
443
|
+
Deimos.configure { producers.backend = :outbox }
|
586
444
|
expect(described_class.determine_backend_class(true, false)).
|
587
|
-
to eq(Deimos::Backends::
|
445
|
+
to eq(Deimos::Backends::Outbox)
|
588
446
|
expect(described_class.determine_backend_class(false, false)).
|
589
|
-
to eq(Deimos::Backends::
|
447
|
+
to eq(Deimos::Backends::Outbox)
|
590
448
|
end
|
591
449
|
|
592
450
|
it 'should return kafka if force_send is true' do
|
data/spec/rake_spec.rb
CHANGED
@@ -11,9 +11,7 @@ end
|
|
11
11
|
|
12
12
|
describe 'Rakefile' do
|
13
13
|
it 'should start listeners' do
|
14
|
-
|
15
|
-
expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
|
16
|
-
expect(runner).to receive(:run!)
|
14
|
+
expect(Karafka::Server).to receive(:run)
|
17
15
|
Rake::Task['deimos:start'].invoke
|
18
16
|
end
|
19
17
|
end
|
@@ -16,7 +16,7 @@ RSpec.describe Deimos::SchemaBackends::AvroValidation do
|
|
16
16
|
|
17
17
|
it 'should encode and decode correctly' do
|
18
18
|
results = backend.encode(payload)
|
19
|
-
expect(results).to eq(payload)
|
19
|
+
expect(results).to eq(payload.to_json)
|
20
20
|
results = backend.decode(results)
|
21
21
|
expect(results).to eq(payload)
|
22
22
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
{
|
2
|
+
"namespace": "com.my-namespace",
|
3
|
+
"name": "MySchemaWithTitle",
|
4
|
+
"type": "record",
|
5
|
+
"doc": "Test schema",
|
6
|
+
"fields": [
|
7
|
+
{
|
8
|
+
"name": "test_id",
|
9
|
+
"type": "string",
|
10
|
+
"doc": "test string"
|
11
|
+
},
|
12
|
+
{
|
13
|
+
"name": "some_int",
|
14
|
+
"type": "int",
|
15
|
+
"doc": "test int"
|
16
|
+
},
|
17
|
+
{
|
18
|
+
"name": "title",
|
19
|
+
"type": "string"
|
20
|
+
}
|
21
|
+
]
|
22
|
+
}
|
@@ -950,6 +950,55 @@ module Schemas
|
|
950
950
|
end
|
951
951
|
|
952
952
|
|
953
|
+
spec/app/lib/schema_classes/my_schema_with_title.rb:
|
954
|
+
# frozen_string_literal: true
|
955
|
+
|
956
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
957
|
+
module Schemas
|
958
|
+
### Primary Schema Class ###
|
959
|
+
# Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
|
960
|
+
class MySchemaWithTitle < Deimos::SchemaClass::Record
|
961
|
+
|
962
|
+
### Attribute Accessors ###
|
963
|
+
# @return [String]
|
964
|
+
attr_accessor :test_id
|
965
|
+
# @return [Integer]
|
966
|
+
attr_accessor :some_int
|
967
|
+
# @return [String]
|
968
|
+
attr_accessor :title
|
969
|
+
|
970
|
+
# @override
|
971
|
+
def initialize(test_id: nil,
|
972
|
+
some_int: nil,
|
973
|
+
title: nil)
|
974
|
+
super
|
975
|
+
self.test_id = test_id
|
976
|
+
self.some_int = some_int
|
977
|
+
self.title = title
|
978
|
+
end
|
979
|
+
|
980
|
+
# @override
|
981
|
+
def schema
|
982
|
+
'MySchemaWithTitle'
|
983
|
+
end
|
984
|
+
|
985
|
+
# @override
|
986
|
+
def namespace
|
987
|
+
'com.my-namespace'
|
988
|
+
end
|
989
|
+
|
990
|
+
# @override
|
991
|
+
def as_json(_opts={})
|
992
|
+
{
|
993
|
+
'test_id' => @test_id,
|
994
|
+
'some_int' => @some_int,
|
995
|
+
'title' => @title
|
996
|
+
}
|
997
|
+
end
|
998
|
+
end
|
999
|
+
end
|
1000
|
+
|
1001
|
+
|
953
1002
|
spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
|
954
1003
|
# frozen_string_literal: true
|
955
1004
|
|
@@ -1010,6 +1010,55 @@ module Schemas
|
|
1010
1010
|
end
|
1011
1011
|
|
1012
1012
|
|
1013
|
+
spec/app/lib/schema_classes/my_schema_with_title.rb:
|
1014
|
+
# frozen_string_literal: true
|
1015
|
+
|
1016
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
1017
|
+
module Schemas
|
1018
|
+
### Primary Schema Class ###
|
1019
|
+
# Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
|
1020
|
+
class MySchemaWithTitle < Deimos::SchemaClass::Record
|
1021
|
+
|
1022
|
+
### Attribute Accessors ###
|
1023
|
+
# @return [String]
|
1024
|
+
attr_accessor :test_id
|
1025
|
+
# @return [Integer]
|
1026
|
+
attr_accessor :some_int
|
1027
|
+
# @return [String]
|
1028
|
+
attr_accessor :title
|
1029
|
+
|
1030
|
+
# @override
|
1031
|
+
def initialize(test_id: nil,
|
1032
|
+
some_int: nil,
|
1033
|
+
title: nil)
|
1034
|
+
super
|
1035
|
+
self.test_id = test_id
|
1036
|
+
self.some_int = some_int
|
1037
|
+
self.title = title
|
1038
|
+
end
|
1039
|
+
|
1040
|
+
# @override
|
1041
|
+
def schema
|
1042
|
+
'MySchemaWithTitle'
|
1043
|
+
end
|
1044
|
+
|
1045
|
+
# @override
|
1046
|
+
def namespace
|
1047
|
+
'com.my-namespace'
|
1048
|
+
end
|
1049
|
+
|
1050
|
+
# @override
|
1051
|
+
def as_json(_opts={})
|
1052
|
+
{
|
1053
|
+
'test_id' => @test_id,
|
1054
|
+
'some_int' => @some_int,
|
1055
|
+
'title' => @title
|
1056
|
+
}
|
1057
|
+
end
|
1058
|
+
end
|
1059
|
+
end
|
1060
|
+
|
1061
|
+
|
1013
1062
|
spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
|
1014
1063
|
# frozen_string_literal: true
|
1015
1064
|
|