deimos-ruby 1.24.2 → 2.0.0.pre.alpha1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -17
- data/.tool-versions +1 -0
- data/CHANGELOG.md +5 -0
- data/README.md +287 -498
- data/deimos-ruby.gemspec +4 -4
- data/docs/CONFIGURATION.md +133 -226
- data/docs/UPGRADING.md +237 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
- data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
- data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
- data/lib/deimos/active_record_consumer.rb +36 -21
- data/lib/deimos/active_record_producer.rb +28 -9
- data/lib/deimos/backends/base.rb +4 -35
- data/lib/deimos/backends/kafka.rb +6 -22
- data/lib/deimos/backends/kafka_async.rb +6 -22
- data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
- data/lib/deimos/config/configuration.rb +116 -379
- data/lib/deimos/consume/batch_consumption.rb +24 -124
- data/lib/deimos/consume/message_consumption.rb +36 -63
- data/lib/deimos/consumer.rb +16 -75
- data/lib/deimos/ext/consumer_route.rb +35 -0
- data/lib/deimos/ext/producer_middleware.rb +94 -0
- data/lib/deimos/ext/producer_route.rb +22 -0
- data/lib/deimos/ext/redraw.rb +29 -0
- data/lib/deimos/ext/routing_defaults.rb +72 -0
- data/lib/deimos/ext/schema_route.rb +70 -0
- data/lib/deimos/kafka_message.rb +2 -2
- data/lib/deimos/kafka_source.rb +2 -7
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +71 -0
- data/lib/deimos/message.rb +2 -11
- data/lib/deimos/metrics/datadog.rb +40 -1
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +39 -116
- data/lib/deimos/railtie.rb +6 -0
- data/lib/deimos/schema_backends/avro_base.rb +21 -21
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
- data/lib/deimos/schema_backends/avro_validation.rb +2 -2
- data/lib/deimos/schema_backends/base.rb +19 -12
- data/lib/deimos/schema_backends/mock.rb +6 -1
- data/lib/deimos/schema_backends/plain.rb +47 -0
- data/lib/deimos/schema_class/base.rb +2 -2
- data/lib/deimos/schema_class/enum.rb +1 -1
- data/lib/deimos/schema_class/record.rb +2 -2
- data/lib/deimos/test_helpers.rb +95 -320
- data/lib/deimos/tracing/provider.rb +6 -6
- data/lib/deimos/transcoder.rb +88 -0
- data/lib/deimos/utils/db_poller/base.rb +16 -14
- data/lib/deimos/utils/db_poller/state_based.rb +3 -3
- data/lib/deimos/utils/db_poller/time_based.rb +4 -4
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/utils/deadlock_retry.rb +1 -1
- data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
- data/lib/deimos/utils/schema_class.rb +0 -7
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +79 -26
- data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
- data/lib/generators/deimos/schema_class_generator.rb +0 -1
- data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
- data/lib/generators/deimos/v2_generator.rb +193 -0
- data/lib/tasks/deimos.rake +5 -7
- data/spec/active_record_batch_consumer_association_spec.rb +22 -13
- data/spec/active_record_batch_consumer_spec.rb +84 -65
- data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
- data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
- data/spec/active_record_consume/mass_updater_spec.rb +137 -0
- data/spec/active_record_consumer_spec.rb +29 -13
- data/spec/active_record_producer_spec.rb +36 -26
- data/spec/backends/base_spec.rb +0 -23
- data/spec/backends/kafka_async_spec.rb +1 -3
- data/spec/backends/kafka_spec.rb +1 -3
- data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
- data/spec/batch_consumer_spec.rb +66 -116
- data/spec/consumer_spec.rb +53 -147
- data/spec/deimos_spec.rb +10 -126
- data/spec/kafka_source_spec.rb +19 -52
- data/spec/karafka/karafka.rb +69 -0
- data/spec/karafka_config/karafka_spec.rb +97 -0
- data/spec/logging_spec.rb +25 -0
- data/spec/message_spec.rb +9 -9
- data/spec/producer_spec.rb +112 -254
- data/spec/rake_spec.rb +1 -3
- data/spec/schema_backends/avro_validation_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
- data/spec/snapshots/consumers-no-nest.snap +49 -0
- data/spec/snapshots/consumers.snap +49 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
- data/spec/snapshots/consumers_and_producers.snap +49 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
- data/spec/snapshots/consumers_circular.snap +49 -0
- data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
- data/spec/snapshots/consumers_complex_types.snap +49 -0
- data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
- data/spec/snapshots/consumers_nested.snap +49 -0
- data/spec/snapshots/namespace_folders.snap +49 -0
- data/spec/snapshots/namespace_map.snap +49 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
- data/spec/snapshots/producers_with_key.snap +49 -0
- data/spec/spec_helper.rb +61 -29
- data/spec/utils/db_poller_spec.rb +49 -39
- data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
- metadata +58 -67
- data/lib/deimos/batch_consumer.rb +0 -7
- data/lib/deimos/config/phobos_config.rb +0 -163
- data/lib/deimos/instrumentation.rb +0 -95
- data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
- data/lib/deimos/utils/inline_consumer.rb +0 -158
- data/lib/deimos/utils/lag_reporter.rb +0 -186
- data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
- data/spec/config/configuration_spec.rb +0 -321
- data/spec/kafka_listener_spec.rb +0 -55
- data/spec/phobos.bad_db.yml +0 -73
- data/spec/phobos.yml +0 -77
- data/spec/utils/inline_consumer_spec.rb +0 -31
- data/spec/utils/lag_reporter_spec.rb +0 -76
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/schema_controller_mixin_spec.rb +0 -84
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
data/spec/producer_spec.rb
CHANGED
@@ -5,27 +5,13 @@ module ProducerTest
|
|
5
5
|
describe Deimos::Producer do
|
6
6
|
|
7
7
|
prepend_before(:each) do
|
8
|
-
producer_class = Class.new(Deimos::Producer)
|
9
|
-
schema 'MySchema'
|
10
|
-
namespace 'com.my-namespace'
|
11
|
-
topic 'my-topic'
|
12
|
-
key_config field: 'test_id'
|
13
|
-
end
|
8
|
+
producer_class = Class.new(Deimos::Producer)
|
14
9
|
stub_const('MyProducer', producer_class)
|
15
10
|
|
16
|
-
producer_class = Class.new(Deimos::Producer)
|
17
|
-
schema 'MySchemaWithId'
|
18
|
-
namespace 'com.my-namespace'
|
19
|
-
topic 'my-topic'
|
20
|
-
key_config plain: true
|
21
|
-
end
|
11
|
+
producer_class = Class.new(Deimos::Producer)
|
22
12
|
stub_const('MyProducerWithID', producer_class)
|
23
13
|
|
24
14
|
producer_class = Class.new(Deimos::Producer) do
|
25
|
-
schema 'MySchema'
|
26
|
-
namespace 'com.my-namespace'
|
27
|
-
topic 'my-topic'
|
28
|
-
key_config plain: true
|
29
15
|
# :nodoc:
|
30
16
|
def self.partition_key(payload)
|
31
17
|
payload[:payload_key] ? payload[:payload_key] + '1' : nil
|
@@ -33,71 +19,90 @@ module ProducerTest
|
|
33
19
|
end
|
34
20
|
stub_const('MyNonEncodedProducer', producer_class)
|
35
21
|
|
36
|
-
producer_class = Class.new(Deimos::Producer)
|
37
|
-
schema 'MySchema'
|
38
|
-
namespace 'com.my-namespace'
|
39
|
-
topic 'my-topic2'
|
40
|
-
key_config none: true
|
41
|
-
end
|
22
|
+
producer_class = Class.new(Deimos::Producer)
|
42
23
|
stub_const('MyNoKeyProducer', producer_class)
|
43
24
|
|
44
|
-
producer_class = Class.new(Deimos::Producer)
|
45
|
-
schema 'MyNestedSchema'
|
46
|
-
namespace 'com.my-namespace'
|
47
|
-
topic 'my-topic'
|
48
|
-
key_config field: 'test_id'
|
49
|
-
end
|
25
|
+
producer_class = Class.new(Deimos::Producer)
|
50
26
|
stub_const('MyNestedSchemaProducer', producer_class)
|
51
27
|
|
52
|
-
producer_class = Class.new(Deimos::Producer)
|
53
|
-
schema 'MySchema'
|
54
|
-
namespace 'com.my-namespace'
|
55
|
-
topic 'my-topic2'
|
56
|
-
key_config schema: 'MySchema_key'
|
57
|
-
end
|
28
|
+
producer_class = Class.new(Deimos::Producer)
|
58
29
|
stub_const('MySchemaProducer', producer_class)
|
59
30
|
|
60
|
-
producer_class = Class.new(Deimos::Producer)
|
61
|
-
schema 'MySchema'
|
62
|
-
namespace 'com.my-namespace'
|
63
|
-
topic 'my-topic'
|
64
|
-
end
|
31
|
+
producer_class = Class.new(Deimos::Producer)
|
65
32
|
stub_const('MyErrorProducer', producer_class)
|
66
33
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
34
|
+
Karafka::App.routes.redraw do
|
35
|
+
topic 'my-topic' do
|
36
|
+
producer_class MyProducer
|
37
|
+
schema 'MySchema'
|
38
|
+
namespace 'com.my-namespace'
|
39
|
+
key_config field: 'test_id'
|
40
|
+
end
|
41
|
+
topic 'a-new-topic' do
|
42
|
+
producer_class MyProducer
|
43
|
+
schema 'MySchema'
|
44
|
+
namespace 'com.my-namespace'
|
45
|
+
key_config field: 'test_id'
|
46
|
+
end
|
47
|
+
topic 'my-topic-with-id' do
|
48
|
+
producer_class MyProducerWithID
|
49
|
+
schema 'MySchemaWithId'
|
50
|
+
namespace 'com.my-namespace'
|
51
|
+
key_config plain: true
|
52
|
+
end
|
53
|
+
topic 'my-topic-non-encoded' do
|
54
|
+
producer_class MyNonEncodedProducer
|
55
|
+
schema 'MySchema'
|
56
|
+
namespace 'com.my-namespace'
|
57
|
+
key_config plain: true
|
58
|
+
end
|
59
|
+
topic 'my-topic-no-key' do
|
60
|
+
producer_class MyNoKeyProducer
|
61
|
+
schema 'MySchema'
|
62
|
+
namespace 'com.my-namespace'
|
63
|
+
key_config none: true
|
64
|
+
end
|
65
|
+
topic 'my-topic-nested-schema' do
|
66
|
+
producer_class MyNestedSchemaProducer
|
67
|
+
schema 'MyNestedSchema'
|
68
|
+
namespace 'com.my-namespace'
|
69
|
+
key_config field: 'test_id'
|
70
|
+
end
|
71
|
+
topic 'my-topic-schema' do
|
72
|
+
producer_class MySchemaProducer
|
73
|
+
schema 'MySchema'
|
74
|
+
namespace 'com.my-namespace'
|
75
|
+
key_config schema: 'MySchema_key'
|
76
|
+
end
|
77
|
+
topic 'my-topic-error' do
|
78
|
+
schema 'MySchema'
|
79
|
+
namespace 'com.my-namespace'
|
80
|
+
producer_class MyErrorProducer
|
81
|
+
end
|
72
82
|
end
|
73
|
-
stub_const('MyNoTopicProducer', producer_class)
|
74
83
|
|
75
84
|
end
|
76
85
|
|
77
|
-
it 'should fail on invalid message
|
78
|
-
|
79
|
-
expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
|
80
|
-
end
|
81
|
-
expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
|
86
|
+
it 'should fail on invalid message' do
|
87
|
+
expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
|
82
88
|
expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
|
83
89
|
to raise_error('OH NOES')
|
84
|
-
Deimos.unsubscribe(subscriber)
|
85
90
|
end
|
86
91
|
|
87
92
|
it 'should produce a message' do
|
88
|
-
expect(
|
89
|
-
Deimos::Backends::
|
93
|
+
expect(MyProducer).to receive(:produce_batch).once.with(
|
94
|
+
Deimos::Backends::Kafka,
|
90
95
|
[
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
96
|
+
hash_including({
|
97
|
+
payload: { 'test_id' => 'foo', 'some_int' => 123 },
|
98
|
+
topic: 'my-topic',
|
99
|
+
partition_key: nil
|
100
|
+
}),
|
101
|
+
hash_including({
|
102
|
+
payload: { 'test_id' => 'bar', 'some_int' => 124 },
|
103
|
+
topic: 'my-topic',
|
104
|
+
partition_key: nil
|
105
|
+
})
|
101
106
|
]
|
102
107
|
).and_call_original
|
103
108
|
|
@@ -105,27 +110,27 @@ module ProducerTest
|
|
105
110
|
[{ 'test_id' => 'foo', 'some_int' => 123 },
|
106
111
|
{ 'test_id' => 'bar', 'some_int' => 124 }]
|
107
112
|
)
|
108
|
-
expect('my-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
|
113
|
+
expect('my-topic').to have_sent({'test_id' => 'foo', 'some_int' => 123}, 'foo', 'foo')
|
109
114
|
expect('your-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
|
110
115
|
expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
|
111
116
|
end
|
112
117
|
|
113
118
|
it 'should allow setting the topic and headers from publish_list' do
|
114
|
-
expect(
|
115
|
-
Deimos::Backends::
|
119
|
+
expect(MyProducer).to receive(:produce_batch).once.with(
|
120
|
+
Deimos::Backends::Kafka,
|
116
121
|
[
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
122
|
+
hash_including({
|
123
|
+
payload: { 'test_id' => 'foo', 'some_int' => 123 },
|
124
|
+
topic: 'a-new-topic',
|
125
|
+
headers: { 'foo' => 'bar' },
|
126
|
+
partition_key: nil
|
127
|
+
}),
|
128
|
+
hash_including({
|
129
|
+
payload: { 'test_id' => 'bar', 'some_int' => 124 },
|
130
|
+
topic: 'a-new-topic',
|
131
|
+
headers: { 'foo' => 'bar' },
|
132
|
+
partition_key: nil
|
133
|
+
})
|
129
134
|
]
|
130
135
|
).and_call_original
|
131
136
|
|
@@ -145,27 +150,14 @@ module ProducerTest
|
|
145
150
|
'some_int' => 123,
|
146
151
|
'message_id' => a_kind_of(String),
|
147
152
|
'timestamp' => a_kind_of(String) }
|
148
|
-
expect(described_class).to receive(:produce_batch).once do |_, messages|
|
149
|
-
expect(messages.size).to eq(1)
|
150
|
-
expect(messages[0].to_h).
|
151
|
-
to match(
|
152
|
-
payload: payload,
|
153
|
-
topic: 'my-topic',
|
154
|
-
partition_key: 'key',
|
155
|
-
metadata: {
|
156
|
-
producer_name: 'MyProducerWithID',
|
157
|
-
decoded_payload: payload
|
158
|
-
},
|
159
|
-
key: 'key'
|
160
|
-
)
|
161
|
-
end
|
162
153
|
MyProducerWithID.publish_list(
|
163
154
|
[{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'key' }]
|
164
155
|
)
|
156
|
+
expect(MyProducerWithID.topic).to have_sent(payload, 'key', 'key')
|
157
|
+
|
165
158
|
end
|
166
159
|
|
167
160
|
it 'should not publish if publish disabled' do
|
168
|
-
expect(described_class).not_to receive(:produce_batch)
|
169
161
|
Deimos.configure { |c| c.producers.disabled = true }
|
170
162
|
MyProducer.publish_list(
|
171
163
|
[{ 'test_id' => 'foo', 'some_int' => 123 },
|
@@ -201,102 +193,22 @@ module ProducerTest
|
|
201
193
|
it 'should produce to a prefixed topic' do
|
202
194
|
Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
|
203
195
|
payload = { 'test_id' => 'foo', 'some_int' => 123 }
|
204
|
-
expect(described_class).to receive(:produce_batch).once do |_, messages|
|
205
|
-
expect(messages.size).to eq(1)
|
206
|
-
expect(messages[0].to_h).
|
207
|
-
to eq(
|
208
|
-
payload: payload,
|
209
|
-
topic: 'prefix.my-topic',
|
210
|
-
partition_key: 'foo',
|
211
|
-
metadata: {
|
212
|
-
producer_name: 'MyProducer',
|
213
|
-
decoded_payload: payload
|
214
|
-
},
|
215
|
-
key: 'foo'
|
216
|
-
)
|
217
|
-
end
|
218
196
|
|
219
197
|
MyProducer.publish_list([payload])
|
220
|
-
|
221
|
-
expect(
|
222
|
-
|
223
|
-
expect(messages[0].to_h).
|
224
|
-
to eq(
|
225
|
-
payload: payload,
|
226
|
-
topic: 'my-topic',
|
227
|
-
partition_key: 'foo',
|
228
|
-
metadata: {
|
229
|
-
producer_name: 'MyProducer',
|
230
|
-
decoded_payload: payload
|
231
|
-
},
|
232
|
-
key: 'foo'
|
233
|
-
)
|
234
|
-
end
|
198
|
+
expect('prefix.my-topic').to have_sent(payload, 'foo', 'foo')
|
199
|
+
expect(karafka.produced_messages.size).to eq(1)
|
200
|
+
karafka.produced_messages.clear
|
235
201
|
|
236
|
-
MyProducer.publish_list(
|
237
|
-
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
238
|
-
)
|
239
|
-
end
|
240
|
-
|
241
|
-
it 'should encode the key' do
|
242
202
|
Deimos.configure { |c| c.producers.topic_prefix = nil }
|
243
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
244
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
|
245
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
246
|
-
'test_id' => 'foo',
|
247
|
-
'some_int' => 123
|
248
|
-
}, { topic: 'my-topic-value' })
|
249
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
250
|
-
'test_id' => 'bar',
|
251
|
-
'some_int' => 124
|
252
|
-
}, { topic: 'my-topic-value' })
|
253
203
|
|
254
204
|
MyProducer.publish_list(
|
255
|
-
[{ 'test_id' => 'foo', 'some_int' => 123 }
|
256
|
-
{ 'test_id' => 'bar', 'some_int' => 124 }]
|
257
|
-
)
|
258
|
-
end
|
259
|
-
|
260
|
-
it 'should encode the key with topic prefix' do
|
261
|
-
Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
|
262
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
|
263
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
|
264
|
-
expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'foo',
|
265
|
-
'some_int' => 123 },
|
266
|
-
{ topic: 'prefix.my-topic-value' })
|
267
|
-
expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'bar',
|
268
|
-
'some_int' => 124 },
|
269
|
-
{ topic: 'prefix.my-topic-value' })
|
270
|
-
|
271
|
-
MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
|
272
|
-
{ 'test_id' => 'bar', 'some_int' => 124 }])
|
273
|
-
end
|
274
|
-
|
275
|
-
it 'should not encode with plaintext key' do
|
276
|
-
expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
|
277
|
-
|
278
|
-
MyNonEncodedProducer.publish_list(
|
279
|
-
[{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'foo_key' },
|
280
|
-
{ 'test_id' => 'bar', 'some_int' => 124, :payload_key => 'bar_key' }]
|
281
|
-
)
|
282
|
-
end
|
283
|
-
|
284
|
-
it 'should encode with a schema' do
|
285
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
|
286
|
-
{ topic: 'my-topic2-key' })
|
287
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
|
288
|
-
{ topic: 'my-topic2-key' })
|
289
|
-
|
290
|
-
MySchemaProducer.publish_list(
|
291
|
-
[{ 'test_id' => 'foo', 'some_int' => 123,
|
292
|
-
:payload_key => { 'test_id' => 'foo_key' } },
|
293
|
-
{ 'test_id' => 'bar', 'some_int' => 124,
|
294
|
-
:payload_key => { 'test_id' => 'bar_key' } }]
|
205
|
+
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
295
206
|
)
|
207
|
+
expect('my-topic').to have_sent(payload, 'foo', 'foo')
|
208
|
+
expect(karafka.produced_messages.size).to eq(1)
|
296
209
|
end
|
297
210
|
|
298
211
|
it 'should properly encode and coerce values with a nested record' do
|
299
|
-
expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
300
212
|
MyNestedSchemaProducer.publish({
|
301
213
|
'test_id' => 'foo',
|
302
214
|
'test_float' => BigDecimal('123.456'),
|
@@ -323,39 +235,6 @@ module ProducerTest
|
|
323
235
|
)
|
324
236
|
end
|
325
237
|
|
326
|
-
it 'should raise error if blank topic is passed in explicitly' do
|
327
|
-
expect {
|
328
|
-
MyProducer.publish_list(
|
329
|
-
[{ 'test_id' => 'foo',
|
330
|
-
'some_int' => 123 },
|
331
|
-
{ 'test_id' => 'bar',
|
332
|
-
'some_int' => 124 }],
|
333
|
-
topic: ''
|
334
|
-
)
|
335
|
-
}.to raise_error(RuntimeError,
|
336
|
-
'Topic not specified. Please specify the topic.')
|
337
|
-
end
|
338
|
-
|
339
|
-
it 'should raise error if the producer has not been initialized with a topic' do
|
340
|
-
expect {
|
341
|
-
MyNoTopicProducer.publish_list(
|
342
|
-
[{ 'test_id' => 'foo',
|
343
|
-
'some_int' => 123 },
|
344
|
-
{ 'test_id' => 'bar',
|
345
|
-
'some_int' => 124 }]
|
346
|
-
)
|
347
|
-
}.to raise_error(RuntimeError,
|
348
|
-
'Topic not specified. Please specify the topic.')
|
349
|
-
end
|
350
|
-
|
351
|
-
it 'should error with nothing set' do
|
352
|
-
expect {
|
353
|
-
MyErrorProducer.publish_list(
|
354
|
-
[{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => '123' }]
|
355
|
-
)
|
356
|
-
}.to raise_error('No key config given - if you are not encoding keys, please use `key_config plain: true`')
|
357
|
-
end
|
358
|
-
|
359
238
|
it 'should error if no key given and none is not the config' do
|
360
239
|
expect {
|
361
240
|
MyNonEncodedProducer.publish_list(
|
@@ -395,30 +274,25 @@ module ProducerTest
|
|
395
274
|
|
396
275
|
context 'with Schema Class payloads' do
|
397
276
|
it 'should fail on invalid message with error handler' do
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
|
402
|
-
expect {
|
403
|
-
MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid'))
|
404
|
-
}.to raise_error('OH NOES')
|
405
|
-
Deimos.unsubscribe(subscriber)
|
277
|
+
expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
|
278
|
+
expect { MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid')) }.
|
279
|
+
to raise_error('OH NOES')
|
406
280
|
end
|
407
281
|
|
408
282
|
it 'should produce a message' do
|
409
|
-
expect(
|
410
|
-
Deimos::Backends::
|
283
|
+
expect(MyProducer).to receive(:produce_batch).once.with(
|
284
|
+
Deimos::Backends::Kafka,
|
411
285
|
[
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
286
|
+
hash_including({
|
287
|
+
payload: { 'test_id' => 'foo', 'some_int' => 123, 'payload_key' => nil },
|
288
|
+
topic: 'my-topic',
|
289
|
+
partition_key: nil,
|
290
|
+
}),
|
291
|
+
hash_including({
|
292
|
+
payload: { 'test_id' => 'bar', 'some_int' => 124, 'payload_key' => nil },
|
293
|
+
topic: 'my-topic',
|
294
|
+
partition_key: nil
|
295
|
+
})
|
422
296
|
]
|
423
297
|
).and_call_original
|
424
298
|
|
@@ -432,7 +306,7 @@ module ProducerTest
|
|
432
306
|
end
|
433
307
|
|
434
308
|
it 'should not publish if publish disabled' do
|
435
|
-
expect(
|
309
|
+
expect(MyProducer).not_to receive(:produce_batch)
|
436
310
|
Deimos.configure { |c| c.producers.disabled = true }
|
437
311
|
MyProducer.publish_list(
|
438
312
|
[Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
|
@@ -443,16 +317,6 @@ module ProducerTest
|
|
443
317
|
|
444
318
|
it 'should encode the key' do
|
445
319
|
Deimos.configure { |c| c.producers.topic_prefix = nil }
|
446
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
447
|
-
expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
|
448
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
449
|
-
'test_id' => 'foo',
|
450
|
-
'some_int' => 123
|
451
|
-
}, { topic: 'my-topic-value' })
|
452
|
-
expect(MyProducer.encoder).to receive(:encode).with({
|
453
|
-
'test_id' => 'bar',
|
454
|
-
'some_int' => 124
|
455
|
-
}, { topic: 'my-topic-value' })
|
456
320
|
|
457
321
|
MyProducer.publish_list(
|
458
322
|
[Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
|
@@ -461,11 +325,6 @@ module ProducerTest
|
|
461
325
|
end
|
462
326
|
|
463
327
|
it 'should encode with a schema' do
|
464
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
|
465
|
-
{ topic: 'my-topic2-key' })
|
466
|
-
expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
|
467
|
-
{ topic: 'my-topic2-key' })
|
468
|
-
|
469
328
|
MySchemaProducer.publish_list(
|
470
329
|
[Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123, payload_key: { 'test_id' => 'foo_key' }),
|
471
330
|
Schemas::MyNamespace::MySchema.new(test_id: 'bar', some_int: 124, payload_key: { 'test_id' => 'bar_key' })]
|
@@ -473,7 +332,6 @@ module ProducerTest
|
|
473
332
|
end
|
474
333
|
|
475
334
|
it 'should properly encode and coerce values with a nested record' do
|
476
|
-
expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
|
477
335
|
MyNestedSchemaProducer.publish(
|
478
336
|
Schemas::MyNamespace::MyNestedSchema.new(
|
479
337
|
test_id: 'foo',
|
@@ -524,7 +382,7 @@ module ProducerTest
|
|
524
382
|
MyProducerWithID.publish({
|
525
383
|
'test_id' => 'foo', 'some_int' => 123, :payload_key => 123
|
526
384
|
})
|
527
|
-
expect('my-topic').
|
385
|
+
expect('my-topic-with-id').
|
528
386
|
to have_sent('test_id' => 'foo', 'some_int' => 123,
|
529
387
|
'message_id' => anything, 'timestamp' => anything)
|
530
388
|
expect(Deimos).not_to be_producers_disabled
|
@@ -544,7 +402,7 @@ module ProducerTest
|
|
544
402
|
:payload_key => '123'
|
545
403
|
})
|
546
404
|
expect('my-topic').not_to have_sent(anything)
|
547
|
-
expect('my-
|
405
|
+
expect('my-topic-schema').to have_sent('test_id' => 'foo', 'some_int' => 123)
|
548
406
|
expect(Deimos).not_to be_producers_disabled
|
549
407
|
expect(Deimos).to be_producers_disabled(MyProducer)
|
550
408
|
expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
|
@@ -582,11 +440,11 @@ module ProducerTest
|
|
582
440
|
end
|
583
441
|
|
584
442
|
it 'should return db if db is set' do
|
585
|
-
Deimos.configure { producers.backend = :
|
443
|
+
Deimos.configure { producers.backend = :outbox }
|
586
444
|
expect(described_class.determine_backend_class(true, false)).
|
587
|
-
to eq(Deimos::Backends::
|
445
|
+
to eq(Deimos::Backends::Outbox)
|
588
446
|
expect(described_class.determine_backend_class(false, false)).
|
589
|
-
to eq(Deimos::Backends::
|
447
|
+
to eq(Deimos::Backends::Outbox)
|
590
448
|
end
|
591
449
|
|
592
450
|
it 'should return kafka if force_send is true' do
|
data/spec/rake_spec.rb
CHANGED
@@ -11,9 +11,7 @@ end
|
|
11
11
|
|
12
12
|
describe 'Rakefile' do
|
13
13
|
it 'should start listeners' do
|
14
|
-
|
15
|
-
expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
|
16
|
-
expect(runner).to receive(:run!)
|
14
|
+
expect(Karafka::Server).to receive(:run)
|
17
15
|
Rake::Task['deimos:start'].invoke
|
18
16
|
end
|
19
17
|
end
|
@@ -16,7 +16,7 @@ RSpec.describe Deimos::SchemaBackends::AvroValidation do
|
|
16
16
|
|
17
17
|
it 'should encode and decode correctly' do
|
18
18
|
results = backend.encode(payload)
|
19
|
-
expect(results).to eq(payload)
|
19
|
+
expect(results).to eq(payload.to_json)
|
20
20
|
results = backend.decode(results)
|
21
21
|
expect(results).to eq(payload)
|
22
22
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
{
|
2
|
+
"namespace": "com.my-namespace",
|
3
|
+
"name": "MySchemaWithTitle",
|
4
|
+
"type": "record",
|
5
|
+
"doc": "Test schema",
|
6
|
+
"fields": [
|
7
|
+
{
|
8
|
+
"name": "test_id",
|
9
|
+
"type": "string",
|
10
|
+
"doc": "test string"
|
11
|
+
},
|
12
|
+
{
|
13
|
+
"name": "some_int",
|
14
|
+
"type": "int",
|
15
|
+
"doc": "test int"
|
16
|
+
},
|
17
|
+
{
|
18
|
+
"name": "title",
|
19
|
+
"type": "string"
|
20
|
+
}
|
21
|
+
]
|
22
|
+
}
|
@@ -950,6 +950,55 @@ module Schemas
|
|
950
950
|
end
|
951
951
|
|
952
952
|
|
953
|
+
spec/app/lib/schema_classes/my_schema_with_title.rb:
|
954
|
+
# frozen_string_literal: true
|
955
|
+
|
956
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
957
|
+
module Schemas
|
958
|
+
### Primary Schema Class ###
|
959
|
+
# Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
|
960
|
+
class MySchemaWithTitle < Deimos::SchemaClass::Record
|
961
|
+
|
962
|
+
### Attribute Accessors ###
|
963
|
+
# @return [String]
|
964
|
+
attr_accessor :test_id
|
965
|
+
# @return [Integer]
|
966
|
+
attr_accessor :some_int
|
967
|
+
# @return [String]
|
968
|
+
attr_accessor :title
|
969
|
+
|
970
|
+
# @override
|
971
|
+
def initialize(test_id: nil,
|
972
|
+
some_int: nil,
|
973
|
+
title: nil)
|
974
|
+
super
|
975
|
+
self.test_id = test_id
|
976
|
+
self.some_int = some_int
|
977
|
+
self.title = title
|
978
|
+
end
|
979
|
+
|
980
|
+
# @override
|
981
|
+
def schema
|
982
|
+
'MySchemaWithTitle'
|
983
|
+
end
|
984
|
+
|
985
|
+
# @override
|
986
|
+
def namespace
|
987
|
+
'com.my-namespace'
|
988
|
+
end
|
989
|
+
|
990
|
+
# @override
|
991
|
+
def as_json(_opts={})
|
992
|
+
{
|
993
|
+
'test_id' => @test_id,
|
994
|
+
'some_int' => @some_int,
|
995
|
+
'title' => @title
|
996
|
+
}
|
997
|
+
end
|
998
|
+
end
|
999
|
+
end
|
1000
|
+
|
1001
|
+
|
953
1002
|
spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
|
954
1003
|
# frozen_string_literal: true
|
955
1004
|
|
@@ -1010,6 +1010,55 @@ module Schemas
|
|
1010
1010
|
end
|
1011
1011
|
|
1012
1012
|
|
1013
|
+
spec/app/lib/schema_classes/my_schema_with_title.rb:
|
1014
|
+
# frozen_string_literal: true
|
1015
|
+
|
1016
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
1017
|
+
module Schemas
|
1018
|
+
### Primary Schema Class ###
|
1019
|
+
# Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
|
1020
|
+
class MySchemaWithTitle < Deimos::SchemaClass::Record
|
1021
|
+
|
1022
|
+
### Attribute Accessors ###
|
1023
|
+
# @return [String]
|
1024
|
+
attr_accessor :test_id
|
1025
|
+
# @return [Integer]
|
1026
|
+
attr_accessor :some_int
|
1027
|
+
# @return [String]
|
1028
|
+
attr_accessor :title
|
1029
|
+
|
1030
|
+
# @override
|
1031
|
+
def initialize(test_id: nil,
|
1032
|
+
some_int: nil,
|
1033
|
+
title: nil)
|
1034
|
+
super
|
1035
|
+
self.test_id = test_id
|
1036
|
+
self.some_int = some_int
|
1037
|
+
self.title = title
|
1038
|
+
end
|
1039
|
+
|
1040
|
+
# @override
|
1041
|
+
def schema
|
1042
|
+
'MySchemaWithTitle'
|
1043
|
+
end
|
1044
|
+
|
1045
|
+
# @override
|
1046
|
+
def namespace
|
1047
|
+
'com.my-namespace'
|
1048
|
+
end
|
1049
|
+
|
1050
|
+
# @override
|
1051
|
+
def as_json(_opts={})
|
1052
|
+
{
|
1053
|
+
'test_id' => @test_id,
|
1054
|
+
'some_int' => @some_int,
|
1055
|
+
'title' => @title
|
1056
|
+
}
|
1057
|
+
end
|
1058
|
+
end
|
1059
|
+
end
|
1060
|
+
|
1061
|
+
|
1013
1062
|
spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
|
1014
1063
|
# frozen_string_literal: true
|
1015
1064
|
|