deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +1 -1
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -227
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
  10. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  11. data/lib/deimos/active_record_consumer.rb +36 -26
  12. data/lib/deimos/active_record_producer.rb +28 -9
  13. data/lib/deimos/backends/base.rb +4 -35
  14. data/lib/deimos/backends/kafka.rb +6 -22
  15. data/lib/deimos/backends/kafka_async.rb +6 -22
  16. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  17. data/lib/deimos/config/configuration.rb +116 -385
  18. data/lib/deimos/consume/batch_consumption.rb +24 -124
  19. data/lib/deimos/consume/message_consumption.rb +36 -63
  20. data/lib/deimos/consumer.rb +16 -75
  21. data/lib/deimos/ext/consumer_route.rb +35 -0
  22. data/lib/deimos/ext/producer_middleware.rb +94 -0
  23. data/lib/deimos/ext/producer_route.rb +22 -0
  24. data/lib/deimos/ext/redraw.rb +29 -0
  25. data/lib/deimos/ext/routing_defaults.rb +72 -0
  26. data/lib/deimos/ext/schema_route.rb +70 -0
  27. data/lib/deimos/kafka_message.rb +2 -2
  28. data/lib/deimos/kafka_source.rb +2 -7
  29. data/lib/deimos/kafka_topic_info.rb +1 -1
  30. data/lib/deimos/logging.rb +71 -0
  31. data/lib/deimos/message.rb +2 -11
  32. data/lib/deimos/metrics/datadog.rb +40 -1
  33. data/lib/deimos/metrics/provider.rb +4 -4
  34. data/lib/deimos/producer.rb +39 -116
  35. data/lib/deimos/railtie.rb +6 -0
  36. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  37. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  38. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  39. data/lib/deimos/schema_backends/base.rb +19 -12
  40. data/lib/deimos/schema_backends/mock.rb +6 -1
  41. data/lib/deimos/schema_backends/plain.rb +47 -0
  42. data/lib/deimos/schema_class/base.rb +2 -2
  43. data/lib/deimos/schema_class/enum.rb +1 -1
  44. data/lib/deimos/schema_class/record.rb +2 -2
  45. data/lib/deimos/test_helpers.rb +95 -320
  46. data/lib/deimos/tracing/provider.rb +6 -6
  47. data/lib/deimos/transcoder.rb +88 -0
  48. data/lib/deimos/utils/db_poller/base.rb +16 -14
  49. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  50. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  51. data/lib/deimos/utils/db_poller.rb +1 -1
  52. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  53. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  54. data/lib/deimos/utils/schema_class.rb +0 -7
  55. data/lib/deimos/version.rb +1 -1
  56. data/lib/deimos.rb +79 -26
  57. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  58. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  59. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  60. data/lib/generators/deimos/v2_generator.rb +193 -0
  61. data/lib/tasks/deimos.rake +5 -7
  62. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  63. data/spec/active_record_batch_consumer_spec.rb +84 -65
  64. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  65. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  66. data/spec/active_record_consumer_spec.rb +29 -13
  67. data/spec/active_record_producer_spec.rb +36 -26
  68. data/spec/backends/base_spec.rb +0 -23
  69. data/spec/backends/kafka_async_spec.rb +1 -3
  70. data/spec/backends/kafka_spec.rb +1 -3
  71. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  72. data/spec/batch_consumer_spec.rb +66 -116
  73. data/spec/consumer_spec.rb +53 -147
  74. data/spec/deimos_spec.rb +10 -126
  75. data/spec/kafka_source_spec.rb +19 -52
  76. data/spec/karafka/karafka.rb +69 -0
  77. data/spec/karafka_config/karafka_spec.rb +97 -0
  78. data/spec/logging_spec.rb +25 -0
  79. data/spec/message_spec.rb +9 -9
  80. data/spec/producer_spec.rb +112 -254
  81. data/spec/rake_spec.rb +1 -3
  82. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  83. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  84. data/spec/snapshots/consumers-no-nest.snap +49 -0
  85. data/spec/snapshots/consumers.snap +49 -0
  86. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers_and_producers.snap +49 -0
  88. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_circular.snap +49 -0
  90. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_complex_types.snap +49 -0
  92. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_nested.snap +49 -0
  94. data/spec/snapshots/namespace_folders.snap +49 -0
  95. data/spec/snapshots/namespace_map.snap +49 -0
  96. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  97. data/spec/snapshots/producers_with_key.snap +49 -0
  98. data/spec/spec_helper.rb +61 -29
  99. data/spec/utils/db_poller_spec.rb +49 -39
  100. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  101. metadata +58 -67
  102. data/lib/deimos/batch_consumer.rb +0 -7
  103. data/lib/deimos/config/phobos_config.rb +0 -164
  104. data/lib/deimos/instrumentation.rb +0 -95
  105. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  106. data/lib/deimos/utils/inline_consumer.rb +0 -158
  107. data/lib/deimos/utils/lag_reporter.rb +0 -186
  108. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  109. data/spec/config/configuration_spec.rb +0 -329
  110. data/spec/kafka_listener_spec.rb +0 -55
  111. data/spec/phobos.bad_db.yml +0 -73
  112. data/spec/phobos.yml +0 -77
  113. data/spec/utils/inline_consumer_spec.rb +0 -31
  114. data/spec/utils/lag_reporter_spec.rb +0 -76
  115. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  116. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  117. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  118. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -5,27 +5,13 @@ module ProducerTest
5
5
  describe Deimos::Producer do
6
6
 
7
7
  prepend_before(:each) do
8
- producer_class = Class.new(Deimos::Producer) do
9
- schema 'MySchema'
10
- namespace 'com.my-namespace'
11
- topic 'my-topic'
12
- key_config field: 'test_id'
13
- end
8
+ producer_class = Class.new(Deimos::Producer)
14
9
  stub_const('MyProducer', producer_class)
15
10
 
16
- producer_class = Class.new(Deimos::Producer) do
17
- schema 'MySchemaWithId'
18
- namespace 'com.my-namespace'
19
- topic 'my-topic'
20
- key_config plain: true
21
- end
11
+ producer_class = Class.new(Deimos::Producer)
22
12
  stub_const('MyProducerWithID', producer_class)
23
13
 
24
14
  producer_class = Class.new(Deimos::Producer) do
25
- schema 'MySchema'
26
- namespace 'com.my-namespace'
27
- topic 'my-topic'
28
- key_config plain: true
29
15
  # :nodoc:
30
16
  def self.partition_key(payload)
31
17
  payload[:payload_key] ? payload[:payload_key] + '1' : nil
@@ -33,71 +19,90 @@ module ProducerTest
33
19
  end
34
20
  stub_const('MyNonEncodedProducer', producer_class)
35
21
 
36
- producer_class = Class.new(Deimos::Producer) do
37
- schema 'MySchema'
38
- namespace 'com.my-namespace'
39
- topic 'my-topic2'
40
- key_config none: true
41
- end
22
+ producer_class = Class.new(Deimos::Producer)
42
23
  stub_const('MyNoKeyProducer', producer_class)
43
24
 
44
- producer_class = Class.new(Deimos::Producer) do
45
- schema 'MyNestedSchema'
46
- namespace 'com.my-namespace'
47
- topic 'my-topic'
48
- key_config field: 'test_id'
49
- end
25
+ producer_class = Class.new(Deimos::Producer)
50
26
  stub_const('MyNestedSchemaProducer', producer_class)
51
27
 
52
- producer_class = Class.new(Deimos::Producer) do
53
- schema 'MySchema'
54
- namespace 'com.my-namespace'
55
- topic 'my-topic2'
56
- key_config schema: 'MySchema_key'
57
- end
28
+ producer_class = Class.new(Deimos::Producer)
58
29
  stub_const('MySchemaProducer', producer_class)
59
30
 
60
- producer_class = Class.new(Deimos::Producer) do
61
- schema 'MySchema'
62
- namespace 'com.my-namespace'
63
- topic 'my-topic'
64
- end
31
+ producer_class = Class.new(Deimos::Producer)
65
32
  stub_const('MyErrorProducer', producer_class)
66
33
 
67
- producer_class = Class.new(Deimos::Producer) do
68
- schema 'MySchema'
69
- namespace 'com.my-namespace'
70
- topic nil
71
- key_config none: true
34
+ Karafka::App.routes.redraw do
35
+ topic 'my-topic' do
36
+ producer_class MyProducer
37
+ schema 'MySchema'
38
+ namespace 'com.my-namespace'
39
+ key_config field: 'test_id'
40
+ end
41
+ topic 'a-new-topic' do
42
+ producer_class MyProducer
43
+ schema 'MySchema'
44
+ namespace 'com.my-namespace'
45
+ key_config field: 'test_id'
46
+ end
47
+ topic 'my-topic-with-id' do
48
+ producer_class MyProducerWithID
49
+ schema 'MySchemaWithId'
50
+ namespace 'com.my-namespace'
51
+ key_config plain: true
52
+ end
53
+ topic 'my-topic-non-encoded' do
54
+ producer_class MyNonEncodedProducer
55
+ schema 'MySchema'
56
+ namespace 'com.my-namespace'
57
+ key_config plain: true
58
+ end
59
+ topic 'my-topic-no-key' do
60
+ producer_class MyNoKeyProducer
61
+ schema 'MySchema'
62
+ namespace 'com.my-namespace'
63
+ key_config none: true
64
+ end
65
+ topic 'my-topic-nested-schema' do
66
+ producer_class MyNestedSchemaProducer
67
+ schema 'MyNestedSchema'
68
+ namespace 'com.my-namespace'
69
+ key_config field: 'test_id'
70
+ end
71
+ topic 'my-topic-schema' do
72
+ producer_class MySchemaProducer
73
+ schema 'MySchema'
74
+ namespace 'com.my-namespace'
75
+ key_config schema: 'MySchema_key'
76
+ end
77
+ topic 'my-topic-error' do
78
+ schema 'MySchema'
79
+ namespace 'com.my-namespace'
80
+ producer_class MyErrorProducer
81
+ end
72
82
  end
73
- stub_const('MyNoTopicProducer', producer_class)
74
83
 
75
84
  end
76
85
 
77
- it 'should fail on invalid message with error handler' do
78
- subscriber = Deimos.subscribe('produce') do |event|
79
- expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
80
- end
81
- expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
86
+ it 'should fail on invalid message' do
87
+ expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
82
88
  expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
83
89
  to raise_error('OH NOES')
84
- Deimos.unsubscribe(subscriber)
85
90
  end
86
91
 
87
92
  it 'should produce a message' do
88
- expect(described_class).to receive(:produce_batch).once.with(
89
- Deimos::Backends::Test,
93
+ expect(MyProducer).to receive(:produce_batch).once.with(
94
+ Deimos::Backends::Kafka,
90
95
  [
91
- Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
92
- MyProducer,
93
- topic: 'my-topic',
94
- partition_key: 'foo',
95
- key: 'foo'),
96
- Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
97
- MyProducer,
98
- topic: 'my-topic',
99
- partition_key: 'bar',
100
- key: 'bar')
96
+ hash_including({
97
+ payload: { 'test_id' => 'foo', 'some_int' => 123 },
98
+ topic: 'my-topic',
99
+ partition_key: nil
100
+ }),
101
+ hash_including({
102
+ payload: { 'test_id' => 'bar', 'some_int' => 124 },
103
+ topic: 'my-topic',
104
+ partition_key: nil
105
+ })
101
106
  ]
102
107
  ).and_call_original
103
108
 
@@ -105,27 +110,27 @@ module ProducerTest
105
110
  [{ 'test_id' => 'foo', 'some_int' => 123 },
106
111
  { 'test_id' => 'bar', 'some_int' => 124 }]
107
112
  )
108
- expect('my-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
113
+ expect('my-topic').to have_sent({'test_id' => 'foo', 'some_int' => 123}, 'foo', 'foo')
109
114
  expect('your-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
110
115
  expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
111
116
  end
112
117
 
113
118
  it 'should allow setting the topic and headers from publish_list' do
114
- expect(described_class).to receive(:produce_batch).once.with(
115
- Deimos::Backends::Test,
119
+ expect(MyProducer).to receive(:produce_batch).once.with(
120
+ Deimos::Backends::Kafka,
116
121
  [
117
- Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
118
- MyProducer,
119
- topic: 'a-new-topic',
120
- headers: { 'foo' => 'bar' },
121
- partition_key: 'foo',
122
- key: 'foo'),
123
- Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
124
- MyProducer,
125
- topic: 'a-new-topic',
126
- headers: { 'foo' => 'bar' },
127
- partition_key: 'bar',
128
- key: 'bar')
122
+ hash_including({
123
+ payload: { 'test_id' => 'foo', 'some_int' => 123 },
124
+ topic: 'a-new-topic',
125
+ headers: { 'foo' => 'bar' },
126
+ partition_key: nil
127
+ }),
128
+ hash_including({
129
+ payload: { 'test_id' => 'bar', 'some_int' => 124 },
130
+ topic: 'a-new-topic',
131
+ headers: { 'foo' => 'bar' },
132
+ partition_key: nil
133
+ })
129
134
  ]
130
135
  ).and_call_original
131
136
 
@@ -145,27 +150,14 @@ module ProducerTest
145
150
  'some_int' => 123,
146
151
  'message_id' => a_kind_of(String),
147
152
  'timestamp' => a_kind_of(String) }
148
- expect(described_class).to receive(:produce_batch).once do |_, messages|
149
- expect(messages.size).to eq(1)
150
- expect(messages[0].to_h).
151
- to match(
152
- payload: payload,
153
- topic: 'my-topic',
154
- partition_key: 'key',
155
- metadata: {
156
- producer_name: 'MyProducerWithID',
157
- decoded_payload: payload
158
- },
159
- key: 'key'
160
- )
161
- end
162
153
  MyProducerWithID.publish_list(
163
154
  [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'key' }]
164
155
  )
156
+ expect(MyProducerWithID.topic).to have_sent(payload, 'key', 'key')
157
+
165
158
  end
166
159
 
167
160
  it 'should not publish if publish disabled' do
168
- expect(described_class).not_to receive(:produce_batch)
169
161
  Deimos.configure { |c| c.producers.disabled = true }
170
162
  MyProducer.publish_list(
171
163
  [{ 'test_id' => 'foo', 'some_int' => 123 },
@@ -201,102 +193,22 @@ module ProducerTest
201
193
  it 'should produce to a prefixed topic' do
202
194
  Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
203
195
  payload = { 'test_id' => 'foo', 'some_int' => 123 }
204
- expect(described_class).to receive(:produce_batch).once do |_, messages|
205
- expect(messages.size).to eq(1)
206
- expect(messages[0].to_h).
207
- to eq(
208
- payload: payload,
209
- topic: 'prefix.my-topic',
210
- partition_key: 'foo',
211
- metadata: {
212
- producer_name: 'MyProducer',
213
- decoded_payload: payload
214
- },
215
- key: 'foo'
216
- )
217
- end
218
196
 
219
197
  MyProducer.publish_list([payload])
220
- Deimos.configure { |c| c.producers.topic_prefix = nil }
221
- expect(described_class).to receive(:produce_batch).once do |_, messages|
222
- expect(messages.size).to eq(1)
223
- expect(messages[0].to_h).
224
- to eq(
225
- payload: payload,
226
- topic: 'my-topic',
227
- partition_key: 'foo',
228
- metadata: {
229
- producer_name: 'MyProducer',
230
- decoded_payload: payload
231
- },
232
- key: 'foo'
233
- )
234
- end
198
+ expect('prefix.my-topic').to have_sent(payload, 'foo', 'foo')
199
+ expect(karafka.produced_messages.size).to eq(1)
200
+ karafka.produced_messages.clear
235
201
 
236
- MyProducer.publish_list(
237
- [{ 'test_id' => 'foo', 'some_int' => 123 }]
238
- )
239
- end
240
-
241
- it 'should encode the key' do
242
202
  Deimos.configure { |c| c.producers.topic_prefix = nil }
243
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
244
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
245
- expect(MyProducer.encoder).to receive(:encode).with({
246
- 'test_id' => 'foo',
247
- 'some_int' => 123
248
- }, { topic: 'my-topic-value' })
249
- expect(MyProducer.encoder).to receive(:encode).with({
250
- 'test_id' => 'bar',
251
- 'some_int' => 124
252
- }, { topic: 'my-topic-value' })
253
203
 
254
204
  MyProducer.publish_list(
255
- [{ 'test_id' => 'foo', 'some_int' => 123 },
256
- { 'test_id' => 'bar', 'some_int' => 124 }]
257
- )
258
- end
259
-
260
- it 'should encode the key with topic prefix' do
261
- Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
262
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
263
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
264
- expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'foo',
265
- 'some_int' => 123 },
266
- { topic: 'prefix.my-topic-value' })
267
- expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'bar',
268
- 'some_int' => 124 },
269
- { topic: 'prefix.my-topic-value' })
270
-
271
- MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
272
- { 'test_id' => 'bar', 'some_int' => 124 }])
273
- end
274
-
275
- it 'should not encode with plaintext key' do
276
- expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
277
-
278
- MyNonEncodedProducer.publish_list(
279
- [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'foo_key' },
280
- { 'test_id' => 'bar', 'some_int' => 124, :payload_key => 'bar_key' }]
281
- )
282
- end
283
-
284
- it 'should encode with a schema' do
285
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
286
- { topic: 'my-topic2-key' })
287
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
288
- { topic: 'my-topic2-key' })
289
-
290
- MySchemaProducer.publish_list(
291
- [{ 'test_id' => 'foo', 'some_int' => 123,
292
- :payload_key => { 'test_id' => 'foo_key' } },
293
- { 'test_id' => 'bar', 'some_int' => 124,
294
- :payload_key => { 'test_id' => 'bar_key' } }]
205
+ [{ 'test_id' => 'foo', 'some_int' => 123 }]
295
206
  )
207
+ expect('my-topic').to have_sent(payload, 'foo', 'foo')
208
+ expect(karafka.produced_messages.size).to eq(1)
296
209
  end
297
210
 
298
211
  it 'should properly encode and coerce values with a nested record' do
299
- expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
300
212
  MyNestedSchemaProducer.publish({
301
213
  'test_id' => 'foo',
302
214
  'test_float' => BigDecimal('123.456'),
@@ -323,39 +235,6 @@ module ProducerTest
323
235
  )
324
236
  end
325
237
 
326
- it 'should raise error if blank topic is passed in explicitly' do
327
- expect {
328
- MyProducer.publish_list(
329
- [{ 'test_id' => 'foo',
330
- 'some_int' => 123 },
331
- { 'test_id' => 'bar',
332
- 'some_int' => 124 }],
333
- topic: ''
334
- )
335
- }.to raise_error(RuntimeError,
336
- 'Topic not specified. Please specify the topic.')
337
- end
338
-
339
- it 'should raise error if the producer has not been initialized with a topic' do
340
- expect {
341
- MyNoTopicProducer.publish_list(
342
- [{ 'test_id' => 'foo',
343
- 'some_int' => 123 },
344
- { 'test_id' => 'bar',
345
- 'some_int' => 124 }]
346
- )
347
- }.to raise_error(RuntimeError,
348
- 'Topic not specified. Please specify the topic.')
349
- end
350
-
351
- it 'should error with nothing set' do
352
- expect {
353
- MyErrorProducer.publish_list(
354
- [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => '123' }]
355
- )
356
- }.to raise_error('No key config given - if you are not encoding keys, please use `key_config plain: true`')
357
- end
358
-
359
238
  it 'should error if no key given and none is not the config' do
360
239
  expect {
361
240
  MyNonEncodedProducer.publish_list(
@@ -395,30 +274,25 @@ module ProducerTest
395
274
 
396
275
  context 'with Schema Class payloads' do
397
276
  it 'should fail on invalid message with error handler' do
398
- subscriber = Deimos.subscribe('produce') do |event|
399
- expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
400
- end
401
- expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
402
- expect {
403
- MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid'))
404
- }.to raise_error('OH NOES')
405
- Deimos.unsubscribe(subscriber)
277
+ expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
278
+ expect { MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid')) }.
279
+ to raise_error('OH NOES')
406
280
  end
407
281
 
408
282
  it 'should produce a message' do
409
- expect(described_class).to receive(:produce_batch).once.with(
410
- Deimos::Backends::Test,
283
+ expect(MyProducer).to receive(:produce_batch).once.with(
284
+ Deimos::Backends::Kafka,
411
285
  [
412
- Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
413
- MyProducer,
414
- topic: 'my-topic',
415
- partition_key: 'foo',
416
- key: 'foo'),
417
- Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
418
- MyProducer,
419
- topic: 'my-topic',
420
- partition_key: 'bar',
421
- key: 'bar')
286
+ hash_including({
287
+ payload: { 'test_id' => 'foo', 'some_int' => 123, 'payload_key' => nil },
288
+ topic: 'my-topic',
289
+ partition_key: nil,
290
+ }),
291
+ hash_including({
292
+ payload: { 'test_id' => 'bar', 'some_int' => 124, 'payload_key' => nil },
293
+ topic: 'my-topic',
294
+ partition_key: nil
295
+ })
422
296
  ]
423
297
  ).and_call_original
424
298
 
@@ -432,7 +306,7 @@ module ProducerTest
432
306
  end
433
307
 
434
308
  it 'should not publish if publish disabled' do
435
- expect(described_class).not_to receive(:produce_batch)
309
+ expect(MyProducer).not_to receive(:produce_batch)
436
310
  Deimos.configure { |c| c.producers.disabled = true }
437
311
  MyProducer.publish_list(
438
312
  [Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
@@ -443,16 +317,6 @@ module ProducerTest
443
317
 
444
318
  it 'should encode the key' do
445
319
  Deimos.configure { |c| c.producers.topic_prefix = nil }
446
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
447
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
448
- expect(MyProducer.encoder).to receive(:encode).with({
449
- 'test_id' => 'foo',
450
- 'some_int' => 123
451
- }, { topic: 'my-topic-value' })
452
- expect(MyProducer.encoder).to receive(:encode).with({
453
- 'test_id' => 'bar',
454
- 'some_int' => 124
455
- }, { topic: 'my-topic-value' })
456
320
 
457
321
  MyProducer.publish_list(
458
322
  [Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
@@ -461,11 +325,6 @@ module ProducerTest
461
325
  end
462
326
 
463
327
  it 'should encode with a schema' do
464
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
465
- { topic: 'my-topic2-key' })
466
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
467
- { topic: 'my-topic2-key' })
468
-
469
328
  MySchemaProducer.publish_list(
470
329
  [Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123, payload_key: { 'test_id' => 'foo_key' }),
471
330
  Schemas::MyNamespace::MySchema.new(test_id: 'bar', some_int: 124, payload_key: { 'test_id' => 'bar_key' })]
@@ -473,7 +332,6 @@ module ProducerTest
473
332
  end
474
333
 
475
334
  it 'should properly encode and coerce values with a nested record' do
476
- expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
477
335
  MyNestedSchemaProducer.publish(
478
336
  Schemas::MyNamespace::MyNestedSchema.new(
479
337
  test_id: 'foo',
@@ -524,7 +382,7 @@ module ProducerTest
524
382
  MyProducerWithID.publish({
525
383
  'test_id' => 'foo', 'some_int' => 123, :payload_key => 123
526
384
  })
527
- expect('my-topic').
385
+ expect('my-topic-with-id').
528
386
  to have_sent('test_id' => 'foo', 'some_int' => 123,
529
387
  'message_id' => anything, 'timestamp' => anything)
530
388
  expect(Deimos).not_to be_producers_disabled
@@ -544,7 +402,7 @@ module ProducerTest
544
402
  :payload_key => '123'
545
403
  })
546
404
  expect('my-topic').not_to have_sent(anything)
547
- expect('my-topic2').to have_sent('test_id' => 'foo', 'some_int' => 123)
405
+ expect('my-topic-schema').to have_sent('test_id' => 'foo', 'some_int' => 123)
548
406
  expect(Deimos).not_to be_producers_disabled
549
407
  expect(Deimos).to be_producers_disabled(MyProducer)
550
408
  expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
@@ -582,11 +440,11 @@ module ProducerTest
582
440
  end
583
441
 
584
442
  it 'should return db if db is set' do
585
- Deimos.configure { producers.backend = :db }
443
+ Deimos.configure { producers.backend = :outbox }
586
444
  expect(described_class.determine_backend_class(true, false)).
587
- to eq(Deimos::Backends::Db)
445
+ to eq(Deimos::Backends::Outbox)
588
446
  expect(described_class.determine_backend_class(false, false)).
589
- to eq(Deimos::Backends::Db)
447
+ to eq(Deimos::Backends::Outbox)
590
448
  end
591
449
 
592
450
  it 'should return kafka if force_send is true' do
data/spec/rake_spec.rb CHANGED
@@ -11,9 +11,7 @@ end
11
11
 
12
12
  describe 'Rakefile' do
13
13
  it 'should start listeners' do
14
- runner = instance_double(Phobos::CLI::Runner)
15
- expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
16
- expect(runner).to receive(:run!)
14
+ expect(Karafka::Server).to receive(:run)
17
15
  Rake::Task['deimos:start'].invoke
18
16
  end
19
17
  end
@@ -16,7 +16,7 @@ RSpec.describe Deimos::SchemaBackends::AvroValidation do
16
16
 
17
17
  it 'should encode and decode correctly' do
18
18
  results = backend.encode(payload)
19
- expect(results).to eq(payload)
19
+ expect(results).to eq(payload.to_json)
20
20
  results = backend.decode(results)
21
21
  expect(results).to eq(payload)
22
22
  end
@@ -0,0 +1,22 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaWithTitle",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "test_id",
9
+ "type": "string",
10
+ "doc": "test string"
11
+ },
12
+ {
13
+ "name": "some_int",
14
+ "type": "int",
15
+ "doc": "test int"
16
+ },
17
+ {
18
+ "name": "title",
19
+ "type": "string"
20
+ }
21
+ ]
22
+ }
@@ -950,6 +950,55 @@ module Schemas
950
950
  end
951
951
 
952
952
 
953
+ spec/app/lib/schema_classes/my_schema_with_title.rb:
954
+ # frozen_string_literal: true
955
+
956
+ # This file is autogenerated by Deimos, Do NOT modify
957
+ module Schemas
958
+ ### Primary Schema Class ###
959
+ # Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
960
+ class MySchemaWithTitle < Deimos::SchemaClass::Record
961
+
962
+ ### Attribute Accessors ###
963
+ # @return [String]
964
+ attr_accessor :test_id
965
+ # @return [Integer]
966
+ attr_accessor :some_int
967
+ # @return [String]
968
+ attr_accessor :title
969
+
970
+ # @override
971
+ def initialize(test_id: nil,
972
+ some_int: nil,
973
+ title: nil)
974
+ super
975
+ self.test_id = test_id
976
+ self.some_int = some_int
977
+ self.title = title
978
+ end
979
+
980
+ # @override
981
+ def schema
982
+ 'MySchemaWithTitle'
983
+ end
984
+
985
+ # @override
986
+ def namespace
987
+ 'com.my-namespace'
988
+ end
989
+
990
+ # @override
991
+ def as_json(_opts={})
992
+ {
993
+ 'test_id' => @test_id,
994
+ 'some_int' => @some_int,
995
+ 'title' => @title
996
+ }
997
+ end
998
+ end
999
+ end
1000
+
1001
+
953
1002
  spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
954
1003
  # frozen_string_literal: true
955
1004
 
@@ -1010,6 +1010,55 @@ module Schemas
1010
1010
  end
1011
1011
 
1012
1012
 
1013
+ spec/app/lib/schema_classes/my_schema_with_title.rb:
1014
+ # frozen_string_literal: true
1015
+
1016
+ # This file is autogenerated by Deimos, Do NOT modify
1017
+ module Schemas
1018
+ ### Primary Schema Class ###
1019
+ # Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
1020
+ class MySchemaWithTitle < Deimos::SchemaClass::Record
1021
+
1022
+ ### Attribute Accessors ###
1023
+ # @return [String]
1024
+ attr_accessor :test_id
1025
+ # @return [Integer]
1026
+ attr_accessor :some_int
1027
+ # @return [String]
1028
+ attr_accessor :title
1029
+
1030
+ # @override
1031
+ def initialize(test_id: nil,
1032
+ some_int: nil,
1033
+ title: nil)
1034
+ super
1035
+ self.test_id = test_id
1036
+ self.some_int = some_int
1037
+ self.title = title
1038
+ end
1039
+
1040
+ # @override
1041
+ def schema
1042
+ 'MySchemaWithTitle'
1043
+ end
1044
+
1045
+ # @override
1046
+ def namespace
1047
+ 'com.my-namespace'
1048
+ end
1049
+
1050
+ # @override
1051
+ def as_json(_opts={})
1052
+ {
1053
+ 'test_id' => @test_id,
1054
+ 'some_int' => @some_int,
1055
+ 'title' => @title
1056
+ }
1057
+ end
1058
+ end
1059
+ end
1060
+
1061
+
1013
1062
  spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
1014
1063
  # frozen_string_literal: true
1015
1064