deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +5 -0
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -226
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
  10. data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
  11. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  12. data/lib/deimos/active_record_consumer.rb +36 -21
  13. data/lib/deimos/active_record_producer.rb +28 -9
  14. data/lib/deimos/backends/base.rb +4 -35
  15. data/lib/deimos/backends/kafka.rb +6 -22
  16. data/lib/deimos/backends/kafka_async.rb +6 -22
  17. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  18. data/lib/deimos/config/configuration.rb +116 -379
  19. data/lib/deimos/consume/batch_consumption.rb +24 -124
  20. data/lib/deimos/consume/message_consumption.rb +36 -63
  21. data/lib/deimos/consumer.rb +16 -75
  22. data/lib/deimos/ext/consumer_route.rb +35 -0
  23. data/lib/deimos/ext/producer_middleware.rb +94 -0
  24. data/lib/deimos/ext/producer_route.rb +22 -0
  25. data/lib/deimos/ext/redraw.rb +29 -0
  26. data/lib/deimos/ext/routing_defaults.rb +72 -0
  27. data/lib/deimos/ext/schema_route.rb +70 -0
  28. data/lib/deimos/kafka_message.rb +2 -2
  29. data/lib/deimos/kafka_source.rb +2 -7
  30. data/lib/deimos/kafka_topic_info.rb +1 -1
  31. data/lib/deimos/logging.rb +71 -0
  32. data/lib/deimos/message.rb +2 -11
  33. data/lib/deimos/metrics/datadog.rb +40 -1
  34. data/lib/deimos/metrics/provider.rb +4 -4
  35. data/lib/deimos/producer.rb +39 -116
  36. data/lib/deimos/railtie.rb +6 -0
  37. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  38. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  39. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  40. data/lib/deimos/schema_backends/base.rb +19 -12
  41. data/lib/deimos/schema_backends/mock.rb +6 -1
  42. data/lib/deimos/schema_backends/plain.rb +47 -0
  43. data/lib/deimos/schema_class/base.rb +2 -2
  44. data/lib/deimos/schema_class/enum.rb +1 -1
  45. data/lib/deimos/schema_class/record.rb +2 -2
  46. data/lib/deimos/test_helpers.rb +95 -320
  47. data/lib/deimos/tracing/provider.rb +6 -6
  48. data/lib/deimos/transcoder.rb +88 -0
  49. data/lib/deimos/utils/db_poller/base.rb +16 -14
  50. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  51. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  52. data/lib/deimos/utils/db_poller.rb +1 -1
  53. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  54. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  55. data/lib/deimos/utils/schema_class.rb +0 -7
  56. data/lib/deimos/version.rb +1 -1
  57. data/lib/deimos.rb +79 -26
  58. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  59. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  60. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  61. data/lib/generators/deimos/v2_generator.rb +193 -0
  62. data/lib/tasks/deimos.rake +5 -7
  63. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  64. data/spec/active_record_batch_consumer_spec.rb +84 -65
  65. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  66. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  67. data/spec/active_record_consume/mass_updater_spec.rb +137 -0
  68. data/spec/active_record_consumer_spec.rb +29 -13
  69. data/spec/active_record_producer_spec.rb +36 -26
  70. data/spec/backends/base_spec.rb +0 -23
  71. data/spec/backends/kafka_async_spec.rb +1 -3
  72. data/spec/backends/kafka_spec.rb +1 -3
  73. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  74. data/spec/batch_consumer_spec.rb +66 -116
  75. data/spec/consumer_spec.rb +53 -147
  76. data/spec/deimos_spec.rb +10 -126
  77. data/spec/kafka_source_spec.rb +19 -52
  78. data/spec/karafka/karafka.rb +69 -0
  79. data/spec/karafka_config/karafka_spec.rb +97 -0
  80. data/spec/logging_spec.rb +25 -0
  81. data/spec/message_spec.rb +9 -9
  82. data/spec/producer_spec.rb +112 -254
  83. data/spec/rake_spec.rb +1 -3
  84. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  85. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  86. data/spec/snapshots/consumers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers.snap +49 -0
  88. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_and_producers.snap +49 -0
  90. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_circular.snap +49 -0
  92. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_complex_types.snap +49 -0
  94. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  95. data/spec/snapshots/consumers_nested.snap +49 -0
  96. data/spec/snapshots/namespace_folders.snap +49 -0
  97. data/spec/snapshots/namespace_map.snap +49 -0
  98. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  99. data/spec/snapshots/producers_with_key.snap +49 -0
  100. data/spec/spec_helper.rb +61 -29
  101. data/spec/utils/db_poller_spec.rb +49 -39
  102. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  103. metadata +58 -67
  104. data/lib/deimos/batch_consumer.rb +0 -7
  105. data/lib/deimos/config/phobos_config.rb +0 -163
  106. data/lib/deimos/instrumentation.rb +0 -95
  107. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  108. data/lib/deimos/utils/inline_consumer.rb +0 -158
  109. data/lib/deimos/utils/lag_reporter.rb +0 -186
  110. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  111. data/spec/config/configuration_spec.rb +0 -321
  112. data/spec/kafka_listener_spec.rb +0 -55
  113. data/spec/phobos.bad_db.yml +0 -73
  114. data/spec/phobos.yml +0 -77
  115. data/spec/utils/inline_consumer_spec.rb +0 -31
  116. data/spec/utils/lag_reporter_spec.rb +0 -76
  117. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  118. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  119. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  120. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -5,27 +5,13 @@ module ProducerTest
5
5
  describe Deimos::Producer do
6
6
 
7
7
  prepend_before(:each) do
8
- producer_class = Class.new(Deimos::Producer) do
9
- schema 'MySchema'
10
- namespace 'com.my-namespace'
11
- topic 'my-topic'
12
- key_config field: 'test_id'
13
- end
8
+ producer_class = Class.new(Deimos::Producer)
14
9
  stub_const('MyProducer', producer_class)
15
10
 
16
- producer_class = Class.new(Deimos::Producer) do
17
- schema 'MySchemaWithId'
18
- namespace 'com.my-namespace'
19
- topic 'my-topic'
20
- key_config plain: true
21
- end
11
+ producer_class = Class.new(Deimos::Producer)
22
12
  stub_const('MyProducerWithID', producer_class)
23
13
 
24
14
  producer_class = Class.new(Deimos::Producer) do
25
- schema 'MySchema'
26
- namespace 'com.my-namespace'
27
- topic 'my-topic'
28
- key_config plain: true
29
15
  # :nodoc:
30
16
  def self.partition_key(payload)
31
17
  payload[:payload_key] ? payload[:payload_key] + '1' : nil
@@ -33,71 +19,90 @@ module ProducerTest
33
19
  end
34
20
  stub_const('MyNonEncodedProducer', producer_class)
35
21
 
36
- producer_class = Class.new(Deimos::Producer) do
37
- schema 'MySchema'
38
- namespace 'com.my-namespace'
39
- topic 'my-topic2'
40
- key_config none: true
41
- end
22
+ producer_class = Class.new(Deimos::Producer)
42
23
  stub_const('MyNoKeyProducer', producer_class)
43
24
 
44
- producer_class = Class.new(Deimos::Producer) do
45
- schema 'MyNestedSchema'
46
- namespace 'com.my-namespace'
47
- topic 'my-topic'
48
- key_config field: 'test_id'
49
- end
25
+ producer_class = Class.new(Deimos::Producer)
50
26
  stub_const('MyNestedSchemaProducer', producer_class)
51
27
 
52
- producer_class = Class.new(Deimos::Producer) do
53
- schema 'MySchema'
54
- namespace 'com.my-namespace'
55
- topic 'my-topic2'
56
- key_config schema: 'MySchema_key'
57
- end
28
+ producer_class = Class.new(Deimos::Producer)
58
29
  stub_const('MySchemaProducer', producer_class)
59
30
 
60
- producer_class = Class.new(Deimos::Producer) do
61
- schema 'MySchema'
62
- namespace 'com.my-namespace'
63
- topic 'my-topic'
64
- end
31
+ producer_class = Class.new(Deimos::Producer)
65
32
  stub_const('MyErrorProducer', producer_class)
66
33
 
67
- producer_class = Class.new(Deimos::Producer) do
68
- schema 'MySchema'
69
- namespace 'com.my-namespace'
70
- topic nil
71
- key_config none: true
34
+ Karafka::App.routes.redraw do
35
+ topic 'my-topic' do
36
+ producer_class MyProducer
37
+ schema 'MySchema'
38
+ namespace 'com.my-namespace'
39
+ key_config field: 'test_id'
40
+ end
41
+ topic 'a-new-topic' do
42
+ producer_class MyProducer
43
+ schema 'MySchema'
44
+ namespace 'com.my-namespace'
45
+ key_config field: 'test_id'
46
+ end
47
+ topic 'my-topic-with-id' do
48
+ producer_class MyProducerWithID
49
+ schema 'MySchemaWithId'
50
+ namespace 'com.my-namespace'
51
+ key_config plain: true
52
+ end
53
+ topic 'my-topic-non-encoded' do
54
+ producer_class MyNonEncodedProducer
55
+ schema 'MySchema'
56
+ namespace 'com.my-namespace'
57
+ key_config plain: true
58
+ end
59
+ topic 'my-topic-no-key' do
60
+ producer_class MyNoKeyProducer
61
+ schema 'MySchema'
62
+ namespace 'com.my-namespace'
63
+ key_config none: true
64
+ end
65
+ topic 'my-topic-nested-schema' do
66
+ producer_class MyNestedSchemaProducer
67
+ schema 'MyNestedSchema'
68
+ namespace 'com.my-namespace'
69
+ key_config field: 'test_id'
70
+ end
71
+ topic 'my-topic-schema' do
72
+ producer_class MySchemaProducer
73
+ schema 'MySchema'
74
+ namespace 'com.my-namespace'
75
+ key_config schema: 'MySchema_key'
76
+ end
77
+ topic 'my-topic-error' do
78
+ schema 'MySchema'
79
+ namespace 'com.my-namespace'
80
+ producer_class MyErrorProducer
81
+ end
72
82
  end
73
- stub_const('MyNoTopicProducer', producer_class)
74
83
 
75
84
  end
76
85
 
77
- it 'should fail on invalid message with error handler' do
78
- subscriber = Deimos.subscribe('produce') do |event|
79
- expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
80
- end
81
- expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
86
+ it 'should fail on invalid message' do
87
+ expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
82
88
  expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
83
89
  to raise_error('OH NOES')
84
- Deimos.unsubscribe(subscriber)
85
90
  end
86
91
 
87
92
  it 'should produce a message' do
88
- expect(described_class).to receive(:produce_batch).once.with(
89
- Deimos::Backends::Test,
93
+ expect(MyProducer).to receive(:produce_batch).once.with(
94
+ Deimos::Backends::Kafka,
90
95
  [
91
- Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
92
- MyProducer,
93
- topic: 'my-topic',
94
- partition_key: 'foo',
95
- key: 'foo'),
96
- Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
97
- MyProducer,
98
- topic: 'my-topic',
99
- partition_key: 'bar',
100
- key: 'bar')
96
+ hash_including({
97
+ payload: { 'test_id' => 'foo', 'some_int' => 123 },
98
+ topic: 'my-topic',
99
+ partition_key: nil
100
+ }),
101
+ hash_including({
102
+ payload: { 'test_id' => 'bar', 'some_int' => 124 },
103
+ topic: 'my-topic',
104
+ partition_key: nil
105
+ })
101
106
  ]
102
107
  ).and_call_original
103
108
 
@@ -105,27 +110,27 @@ module ProducerTest
105
110
  [{ 'test_id' => 'foo', 'some_int' => 123 },
106
111
  { 'test_id' => 'bar', 'some_int' => 124 }]
107
112
  )
108
- expect('my-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
113
+ expect('my-topic').to have_sent({'test_id' => 'foo', 'some_int' => 123}, 'foo', 'foo')
109
114
  expect('your-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
110
115
  expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
111
116
  end
112
117
 
113
118
  it 'should allow setting the topic and headers from publish_list' do
114
- expect(described_class).to receive(:produce_batch).once.with(
115
- Deimos::Backends::Test,
119
+ expect(MyProducer).to receive(:produce_batch).once.with(
120
+ Deimos::Backends::Kafka,
116
121
  [
117
- Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
118
- MyProducer,
119
- topic: 'a-new-topic',
120
- headers: { 'foo' => 'bar' },
121
- partition_key: 'foo',
122
- key: 'foo'),
123
- Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
124
- MyProducer,
125
- topic: 'a-new-topic',
126
- headers: { 'foo' => 'bar' },
127
- partition_key: 'bar',
128
- key: 'bar')
122
+ hash_including({
123
+ payload: { 'test_id' => 'foo', 'some_int' => 123 },
124
+ topic: 'a-new-topic',
125
+ headers: { 'foo' => 'bar' },
126
+ partition_key: nil
127
+ }),
128
+ hash_including({
129
+ payload: { 'test_id' => 'bar', 'some_int' => 124 },
130
+ topic: 'a-new-topic',
131
+ headers: { 'foo' => 'bar' },
132
+ partition_key: nil
133
+ })
129
134
  ]
130
135
  ).and_call_original
131
136
 
@@ -145,27 +150,14 @@ module ProducerTest
145
150
  'some_int' => 123,
146
151
  'message_id' => a_kind_of(String),
147
152
  'timestamp' => a_kind_of(String) }
148
- expect(described_class).to receive(:produce_batch).once do |_, messages|
149
- expect(messages.size).to eq(1)
150
- expect(messages[0].to_h).
151
- to match(
152
- payload: payload,
153
- topic: 'my-topic',
154
- partition_key: 'key',
155
- metadata: {
156
- producer_name: 'MyProducerWithID',
157
- decoded_payload: payload
158
- },
159
- key: 'key'
160
- )
161
- end
162
153
  MyProducerWithID.publish_list(
163
154
  [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'key' }]
164
155
  )
156
+ expect(MyProducerWithID.topic).to have_sent(payload, 'key', 'key')
157
+
165
158
  end
166
159
 
167
160
  it 'should not publish if publish disabled' do
168
- expect(described_class).not_to receive(:produce_batch)
169
161
  Deimos.configure { |c| c.producers.disabled = true }
170
162
  MyProducer.publish_list(
171
163
  [{ 'test_id' => 'foo', 'some_int' => 123 },
@@ -201,102 +193,22 @@ module ProducerTest
201
193
  it 'should produce to a prefixed topic' do
202
194
  Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
203
195
  payload = { 'test_id' => 'foo', 'some_int' => 123 }
204
- expect(described_class).to receive(:produce_batch).once do |_, messages|
205
- expect(messages.size).to eq(1)
206
- expect(messages[0].to_h).
207
- to eq(
208
- payload: payload,
209
- topic: 'prefix.my-topic',
210
- partition_key: 'foo',
211
- metadata: {
212
- producer_name: 'MyProducer',
213
- decoded_payload: payload
214
- },
215
- key: 'foo'
216
- )
217
- end
218
196
 
219
197
  MyProducer.publish_list([payload])
220
- Deimos.configure { |c| c.producers.topic_prefix = nil }
221
- expect(described_class).to receive(:produce_batch).once do |_, messages|
222
- expect(messages.size).to eq(1)
223
- expect(messages[0].to_h).
224
- to eq(
225
- payload: payload,
226
- topic: 'my-topic',
227
- partition_key: 'foo',
228
- metadata: {
229
- producer_name: 'MyProducer',
230
- decoded_payload: payload
231
- },
232
- key: 'foo'
233
- )
234
- end
198
+ expect('prefix.my-topic').to have_sent(payload, 'foo', 'foo')
199
+ expect(karafka.produced_messages.size).to eq(1)
200
+ karafka.produced_messages.clear
235
201
 
236
- MyProducer.publish_list(
237
- [{ 'test_id' => 'foo', 'some_int' => 123 }]
238
- )
239
- end
240
-
241
- it 'should encode the key' do
242
202
  Deimos.configure { |c| c.producers.topic_prefix = nil }
243
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
244
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
245
- expect(MyProducer.encoder).to receive(:encode).with({
246
- 'test_id' => 'foo',
247
- 'some_int' => 123
248
- }, { topic: 'my-topic-value' })
249
- expect(MyProducer.encoder).to receive(:encode).with({
250
- 'test_id' => 'bar',
251
- 'some_int' => 124
252
- }, { topic: 'my-topic-value' })
253
203
 
254
204
  MyProducer.publish_list(
255
- [{ 'test_id' => 'foo', 'some_int' => 123 },
256
- { 'test_id' => 'bar', 'some_int' => 124 }]
257
- )
258
- end
259
-
260
- it 'should encode the key with topic prefix' do
261
- Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
262
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
263
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
264
- expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'foo',
265
- 'some_int' => 123 },
266
- { topic: 'prefix.my-topic-value' })
267
- expect(MyProducer.encoder).to receive(:encode).with({ 'test_id' => 'bar',
268
- 'some_int' => 124 },
269
- { topic: 'prefix.my-topic-value' })
270
-
271
- MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
272
- { 'test_id' => 'bar', 'some_int' => 124 }])
273
- end
274
-
275
- it 'should not encode with plaintext key' do
276
- expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
277
-
278
- MyNonEncodedProducer.publish_list(
279
- [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'foo_key' },
280
- { 'test_id' => 'bar', 'some_int' => 124, :payload_key => 'bar_key' }]
281
- )
282
- end
283
-
284
- it 'should encode with a schema' do
285
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
286
- { topic: 'my-topic2-key' })
287
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
288
- { topic: 'my-topic2-key' })
289
-
290
- MySchemaProducer.publish_list(
291
- [{ 'test_id' => 'foo', 'some_int' => 123,
292
- :payload_key => { 'test_id' => 'foo_key' } },
293
- { 'test_id' => 'bar', 'some_int' => 124,
294
- :payload_key => { 'test_id' => 'bar_key' } }]
205
+ [{ 'test_id' => 'foo', 'some_int' => 123 }]
295
206
  )
207
+ expect('my-topic').to have_sent(payload, 'foo', 'foo')
208
+ expect(karafka.produced_messages.size).to eq(1)
296
209
  end
297
210
 
298
211
  it 'should properly encode and coerce values with a nested record' do
299
- expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
300
212
  MyNestedSchemaProducer.publish({
301
213
  'test_id' => 'foo',
302
214
  'test_float' => BigDecimal('123.456'),
@@ -323,39 +235,6 @@ module ProducerTest
323
235
  )
324
236
  end
325
237
 
326
- it 'should raise error if blank topic is passed in explicitly' do
327
- expect {
328
- MyProducer.publish_list(
329
- [{ 'test_id' => 'foo',
330
- 'some_int' => 123 },
331
- { 'test_id' => 'bar',
332
- 'some_int' => 124 }],
333
- topic: ''
334
- )
335
- }.to raise_error(RuntimeError,
336
- 'Topic not specified. Please specify the topic.')
337
- end
338
-
339
- it 'should raise error if the producer has not been initialized with a topic' do
340
- expect {
341
- MyNoTopicProducer.publish_list(
342
- [{ 'test_id' => 'foo',
343
- 'some_int' => 123 },
344
- { 'test_id' => 'bar',
345
- 'some_int' => 124 }]
346
- )
347
- }.to raise_error(RuntimeError,
348
- 'Topic not specified. Please specify the topic.')
349
- end
350
-
351
- it 'should error with nothing set' do
352
- expect {
353
- MyErrorProducer.publish_list(
354
- [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => '123' }]
355
- )
356
- }.to raise_error('No key config given - if you are not encoding keys, please use `key_config plain: true`')
357
- end
358
-
359
238
  it 'should error if no key given and none is not the config' do
360
239
  expect {
361
240
  MyNonEncodedProducer.publish_list(
@@ -395,30 +274,25 @@ module ProducerTest
395
274
 
396
275
  context 'with Schema Class payloads' do
397
276
  it 'should fail on invalid message with error handler' do
398
- subscriber = Deimos.subscribe('produce') do |event|
399
- expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
400
- end
401
- expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
402
- expect {
403
- MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid'))
404
- }.to raise_error('OH NOES')
405
- Deimos.unsubscribe(subscriber)
277
+ expect(Deimos::ProducerMiddleware).to receive(:call).and_raise('OH NOES')
278
+ expect { MyProducer.publish(Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 'invalid')) }.
279
+ to raise_error('OH NOES')
406
280
  end
407
281
 
408
282
  it 'should produce a message' do
409
- expect(described_class).to receive(:produce_batch).once.with(
410
- Deimos::Backends::Test,
283
+ expect(MyProducer).to receive(:produce_batch).once.with(
284
+ Deimos::Backends::Kafka,
411
285
  [
412
- Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
413
- MyProducer,
414
- topic: 'my-topic',
415
- partition_key: 'foo',
416
- key: 'foo'),
417
- Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
418
- MyProducer,
419
- topic: 'my-topic',
420
- partition_key: 'bar',
421
- key: 'bar')
286
+ hash_including({
287
+ payload: { 'test_id' => 'foo', 'some_int' => 123, 'payload_key' => nil },
288
+ topic: 'my-topic',
289
+ partition_key: nil,
290
+ }),
291
+ hash_including({
292
+ payload: { 'test_id' => 'bar', 'some_int' => 124, 'payload_key' => nil },
293
+ topic: 'my-topic',
294
+ partition_key: nil
295
+ })
422
296
  ]
423
297
  ).and_call_original
424
298
 
@@ -432,7 +306,7 @@ module ProducerTest
432
306
  end
433
307
 
434
308
  it 'should not publish if publish disabled' do
435
- expect(described_class).not_to receive(:produce_batch)
309
+ expect(MyProducer).not_to receive(:produce_batch)
436
310
  Deimos.configure { |c| c.producers.disabled = true }
437
311
  MyProducer.publish_list(
438
312
  [Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
@@ -443,16 +317,6 @@ module ProducerTest
443
317
 
444
318
  it 'should encode the key' do
445
319
  Deimos.configure { |c| c.producers.topic_prefix = nil }
446
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
447
- expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
448
- expect(MyProducer.encoder).to receive(:encode).with({
449
- 'test_id' => 'foo',
450
- 'some_int' => 123
451
- }, { topic: 'my-topic-value' })
452
- expect(MyProducer.encoder).to receive(:encode).with({
453
- 'test_id' => 'bar',
454
- 'some_int' => 124
455
- }, { topic: 'my-topic-value' })
456
320
 
457
321
  MyProducer.publish_list(
458
322
  [Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123),
@@ -461,11 +325,6 @@ module ProducerTest
461
325
  end
462
326
 
463
327
  it 'should encode with a schema' do
464
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
465
- { topic: 'my-topic2-key' })
466
- expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
467
- { topic: 'my-topic2-key' })
468
-
469
328
  MySchemaProducer.publish_list(
470
329
  [Schemas::MyNamespace::MySchema.new(test_id: 'foo', some_int: 123, payload_key: { 'test_id' => 'foo_key' }),
471
330
  Schemas::MyNamespace::MySchema.new(test_id: 'bar', some_int: 124, payload_key: { 'test_id' => 'bar_key' })]
@@ -473,7 +332,6 @@ module ProducerTest
473
332
  end
474
333
 
475
334
  it 'should properly encode and coerce values with a nested record' do
476
- expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
477
335
  MyNestedSchemaProducer.publish(
478
336
  Schemas::MyNamespace::MyNestedSchema.new(
479
337
  test_id: 'foo',
@@ -524,7 +382,7 @@ module ProducerTest
524
382
  MyProducerWithID.publish({
525
383
  'test_id' => 'foo', 'some_int' => 123, :payload_key => 123
526
384
  })
527
- expect('my-topic').
385
+ expect('my-topic-with-id').
528
386
  to have_sent('test_id' => 'foo', 'some_int' => 123,
529
387
  'message_id' => anything, 'timestamp' => anything)
530
388
  expect(Deimos).not_to be_producers_disabled
@@ -544,7 +402,7 @@ module ProducerTest
544
402
  :payload_key => '123'
545
403
  })
546
404
  expect('my-topic').not_to have_sent(anything)
547
- expect('my-topic2').to have_sent('test_id' => 'foo', 'some_int' => 123)
405
+ expect('my-topic-schema').to have_sent('test_id' => 'foo', 'some_int' => 123)
548
406
  expect(Deimos).not_to be_producers_disabled
549
407
  expect(Deimos).to be_producers_disabled(MyProducer)
550
408
  expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
@@ -582,11 +440,11 @@ module ProducerTest
582
440
  end
583
441
 
584
442
  it 'should return db if db is set' do
585
- Deimos.configure { producers.backend = :db }
443
+ Deimos.configure { producers.backend = :outbox }
586
444
  expect(described_class.determine_backend_class(true, false)).
587
- to eq(Deimos::Backends::Db)
445
+ to eq(Deimos::Backends::Outbox)
588
446
  expect(described_class.determine_backend_class(false, false)).
589
- to eq(Deimos::Backends::Db)
447
+ to eq(Deimos::Backends::Outbox)
590
448
  end
591
449
 
592
450
  it 'should return kafka if force_send is true' do
data/spec/rake_spec.rb CHANGED
@@ -11,9 +11,7 @@ end
11
11
 
12
12
  describe 'Rakefile' do
13
13
  it 'should start listeners' do
14
- runner = instance_double(Phobos::CLI::Runner)
15
- expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
16
- expect(runner).to receive(:run!)
14
+ expect(Karafka::Server).to receive(:run)
17
15
  Rake::Task['deimos:start'].invoke
18
16
  end
19
17
  end
@@ -16,7 +16,7 @@ RSpec.describe Deimos::SchemaBackends::AvroValidation do
16
16
 
17
17
  it 'should encode and decode correctly' do
18
18
  results = backend.encode(payload)
19
- expect(results).to eq(payload)
19
+ expect(results).to eq(payload.to_json)
20
20
  results = backend.decode(results)
21
21
  expect(results).to eq(payload)
22
22
  end
@@ -0,0 +1,22 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaWithTitle",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "test_id",
9
+ "type": "string",
10
+ "doc": "test string"
11
+ },
12
+ {
13
+ "name": "some_int",
14
+ "type": "int",
15
+ "doc": "test int"
16
+ },
17
+ {
18
+ "name": "title",
19
+ "type": "string"
20
+ }
21
+ ]
22
+ }
@@ -950,6 +950,55 @@ module Schemas
950
950
  end
951
951
 
952
952
 
953
+ spec/app/lib/schema_classes/my_schema_with_title.rb:
954
+ # frozen_string_literal: true
955
+
956
+ # This file is autogenerated by Deimos, Do NOT modify
957
+ module Schemas
958
+ ### Primary Schema Class ###
959
+ # Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
960
+ class MySchemaWithTitle < Deimos::SchemaClass::Record
961
+
962
+ ### Attribute Accessors ###
963
+ # @return [String]
964
+ attr_accessor :test_id
965
+ # @return [Integer]
966
+ attr_accessor :some_int
967
+ # @return [String]
968
+ attr_accessor :title
969
+
970
+ # @override
971
+ def initialize(test_id: nil,
972
+ some_int: nil,
973
+ title: nil)
974
+ super
975
+ self.test_id = test_id
976
+ self.some_int = some_int
977
+ self.title = title
978
+ end
979
+
980
+ # @override
981
+ def schema
982
+ 'MySchemaWithTitle'
983
+ end
984
+
985
+ # @override
986
+ def namespace
987
+ 'com.my-namespace'
988
+ end
989
+
990
+ # @override
991
+ def as_json(_opts={})
992
+ {
993
+ 'test_id' => @test_id,
994
+ 'some_int' => @some_int,
995
+ 'title' => @title
996
+ }
997
+ end
998
+ end
999
+ end
1000
+
1001
+
953
1002
  spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
954
1003
  # frozen_string_literal: true
955
1004
 
@@ -1010,6 +1010,55 @@ module Schemas
1010
1010
  end
1011
1011
 
1012
1012
 
1013
+ spec/app/lib/schema_classes/my_schema_with_title.rb:
1014
+ # frozen_string_literal: true
1015
+
1016
+ # This file is autogenerated by Deimos, Do NOT modify
1017
+ module Schemas
1018
+ ### Primary Schema Class ###
1019
+ # Autogenerated Schema for Record at com.my-namespace.MySchemaWithTitle
1020
+ class MySchemaWithTitle < Deimos::SchemaClass::Record
1021
+
1022
+ ### Attribute Accessors ###
1023
+ # @return [String]
1024
+ attr_accessor :test_id
1025
+ # @return [Integer]
1026
+ attr_accessor :some_int
1027
+ # @return [String]
1028
+ attr_accessor :title
1029
+
1030
+ # @override
1031
+ def initialize(test_id: nil,
1032
+ some_int: nil,
1033
+ title: nil)
1034
+ super
1035
+ self.test_id = test_id
1036
+ self.some_int = some_int
1037
+ self.title = title
1038
+ end
1039
+
1040
+ # @override
1041
+ def schema
1042
+ 'MySchemaWithTitle'
1043
+ end
1044
+
1045
+ # @override
1046
+ def namespace
1047
+ 'com.my-namespace'
1048
+ end
1049
+
1050
+ # @override
1051
+ def as_json(_opts={})
1052
+ {
1053
+ 'test_id' => @test_id,
1054
+ 'some_int' => @some_int,
1055
+ 'title' => @title
1056
+ }
1057
+ end
1058
+ end
1059
+ end
1060
+
1061
+
1013
1062
  spec/app/lib/schema_classes/my_schema_with_unique_id.rb:
1014
1063
  # frozen_string_literal: true
1015
1064