deimos-ruby 2.2.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +33 -30
- data/CHANGELOG.md +6 -0
- data/Gemfile +0 -6
- data/deimos-ruby.gemspec +15 -11
- data/karafka.rb +7 -4
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -7
- data/lib/deimos/active_record_consume/batch_record.rb +2 -2
- data/lib/deimos/active_record_consume/message_consumption.rb +6 -5
- data/lib/deimos/active_record_consume/schema_model_converter.rb +2 -2
- data/lib/deimos/active_record_consumer.rb +1 -0
- data/lib/deimos/active_record_producer.rb +4 -2
- data/lib/deimos/backends/base.rb +1 -3
- data/lib/deimos/backends/outbox.rb +1 -1
- data/lib/deimos/config/configuration.rb +88 -75
- data/lib/deimos/consume/batch_consumption.rb +5 -5
- data/lib/deimos/consume/message_consumption.rb +3 -3
- data/lib/deimos/ext/consumer_route.rb +3 -3
- data/lib/deimos/ext/producer_metrics_listener.rb +2 -2
- data/lib/deimos/ext/producer_middleware.rb +19 -15
- data/lib/deimos/ext/producer_route.rb +3 -1
- data/lib/deimos/ext/routing_defaults.rb +9 -7
- data/lib/deimos/ext/schema_route.rb +22 -15
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_source.rb +36 -31
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +20 -19
- data/lib/deimos/message.rb +1 -1
- data/lib/deimos/metrics/minimal_datadog_listener.rb +19 -6
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +3 -1
- data/lib/deimos/railtie.rb +1 -1
- data/lib/deimos/schema_backends/avro_base.rb +1 -1
- data/lib/deimos/schema_backends/avro_schema_coercer.rb +46 -27
- data/lib/deimos/schema_backends/avro_schema_registry.rb +8 -8
- data/lib/deimos/schema_backends/base.rb +9 -9
- data/lib/deimos/schema_backends/plain.rb +1 -1
- data/lib/deimos/schema_backends/proto_base.rb +7 -5
- data/lib/deimos/schema_backends/proto_local.rb +0 -2
- data/lib/deimos/schema_backends/proto_schema_registry.rb +0 -2
- data/lib/deimos/schema_class/base.rb +1 -1
- data/lib/deimos/schema_class/record.rb +3 -3
- data/lib/deimos/test_helpers.rb +31 -26
- data/lib/deimos/tracing/provider.rb +5 -5
- data/lib/deimos/transcoder.rb +6 -2
- data/lib/deimos/utils/db_poller/base.rb +3 -3
- data/lib/deimos/utils/deadlock_retry.rb +2 -2
- data/lib/deimos/utils/outbox_producer.rb +14 -14
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +4 -4
- data/lib/generators/deimos/active_record_generator.rb +2 -1
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/outbox_backend_generator.rb +1 -0
- data/lib/generators/deimos/schema_class_generator.rb +3 -2
- data/lib/generators/deimos/v2_generator.rb +184 -155
- data/spec/active_record_batch_consumer_association_spec.rb +6 -2
- data/spec/active_record_batch_consumer_spec.rb +83 -106
- data/spec/active_record_consume/batch_consumption_spec.rb +27 -28
- data/spec/active_record_consume/batch_slicer_spec.rb +4 -12
- data/spec/active_record_consume/mass_updater_spec.rb +42 -46
- data/spec/active_record_consume/schema_model_converter_spec.rb +1 -1
- data/spec/active_record_consumer_spec.rb +7 -5
- data/spec/active_record_producer_spec.rb +83 -73
- data/spec/backends/outbox_spec.rb +1 -1
- data/spec/batch_consumer_spec.rb +20 -20
- data/spec/consumer_spec.rb +23 -12
- data/spec/gen/sample/v1/sample_pb.rb +3 -3
- data/spec/generators/active_record_generator_spec.rb +4 -4
- data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb +2 -1
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +9 -2
- data/spec/generators/schema_class_generator_spec.rb +5 -5
- data/spec/kafka_source_spec.rb +13 -6
- data/spec/kafka_topic_info_spec.rb +7 -7
- data/spec/karafka/karafka.rb +6 -5
- data/spec/karafka_config/karafka_spec.rb +22 -19
- data/spec/logging_spec.rb +2 -0
- data/spec/producer_spec.rb +25 -20
- data/spec/schema_backends/avro_base_shared.rb +8 -8
- data/spec/schema_backends/avro_local_spec.rb +5 -6
- data/spec/schema_backends/avro_schema_registry_spec.rb +5 -6
- data/spec/schema_backends/proto_schema_registry_spec.rb +9 -12
- data/spec/schemas/my_namespace/generated.rb +1 -2
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +5 -8
- data/spec/schemas/my_namespace/my_schema_with_union_type.rb +22 -23
- data/spec/spec_helper.rb +13 -17
- data/spec/utils/db_poller_spec.rb +5 -5
- data/spec/utils/deadlock_retry_spec.rb +1 -4
- data/spec/utils/outbox_producer_spec.rb +36 -24
- metadata +68 -161
- data/.ruby-version +0 -1
|
@@ -16,7 +16,7 @@ module ActiveRecordConsumerTest
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
# :nodoc:
|
|
19
|
-
class Widget < ActiveRecord::Base
|
|
19
|
+
class Widget < ActiveRecord::Base # rubocop:disable Lint/ConstantDefinitionInBlock
|
|
20
20
|
default_scope -> { where(some_bool: false) }
|
|
21
21
|
end
|
|
22
22
|
Widget.reset_column_information
|
|
@@ -75,6 +75,7 @@ module ActiveRecordConsumerTest
|
|
|
75
75
|
some_int: nil)
|
|
76
76
|
self.test_id = test_id
|
|
77
77
|
self.some_int = some_int
|
|
78
|
+
super
|
|
78
79
|
end
|
|
79
80
|
|
|
80
81
|
def as_json(_opts={})
|
|
@@ -107,6 +108,7 @@ module ActiveRecordConsumerTest
|
|
|
107
108
|
updated_at: nil,
|
|
108
109
|
some_datetime_int: nil,
|
|
109
110
|
timestamp: nil)
|
|
111
|
+
super
|
|
110
112
|
self.test_id = test_id
|
|
111
113
|
self.some_int = some_int
|
|
112
114
|
self.updated_at = updated_at
|
|
@@ -128,19 +130,19 @@ module ActiveRecordConsumerTest
|
|
|
128
130
|
stub_const('Schemas::MySchemaWithDateTimes', schema_datetime_class)
|
|
129
131
|
|
|
130
132
|
Karafka::App.routes.redraw do
|
|
131
|
-
topic
|
|
133
|
+
topic 'my-topic' do
|
|
132
134
|
consumer MyConsumer
|
|
133
135
|
schema 'MySchemaWithDateTimes'
|
|
134
136
|
namespace 'com.my-namespace'
|
|
135
137
|
key_config plain: true
|
|
136
138
|
end
|
|
137
|
-
topic
|
|
139
|
+
topic 'my-topic2' do
|
|
138
140
|
consumer MyConsumerWithKey
|
|
139
141
|
schema 'MySchemaWithDateTimes'
|
|
140
142
|
namespace 'com.my-namespace'
|
|
141
143
|
key_config schema: 'MySchemaId_key'
|
|
142
144
|
end
|
|
143
|
-
topic
|
|
145
|
+
topic 'my-topic3' do
|
|
144
146
|
consumer MyCustomFetchConsumer
|
|
145
147
|
schema 'MySchema'
|
|
146
148
|
namespace 'com.my-namespace'
|
|
@@ -180,7 +182,7 @@ module ActiveRecordConsumerTest
|
|
|
180
182
|
expect(widget.test_id).to eq('abc')
|
|
181
183
|
expect(widget.some_int).to eq(3)
|
|
182
184
|
expect(widget.some_datetime_int).to eq(Time.zone.now)
|
|
183
|
-
expect(widget.some_bool).to
|
|
185
|
+
expect(widget.some_bool).to be(false)
|
|
184
186
|
expect(widget.updated_at).to eq(Time.zone.now)
|
|
185
187
|
|
|
186
188
|
# test unscoped
|
|
@@ -6,10 +6,10 @@ describe Deimos::ActiveRecordProducer do
|
|
|
6
6
|
include_context 'with widget_with_union_types'
|
|
7
7
|
|
|
8
8
|
prepend_before(:each) do
|
|
9
|
-
producer_class = Class.new(
|
|
9
|
+
producer_class = Class.new(described_class)
|
|
10
10
|
stub_const('MyProducer', producer_class)
|
|
11
11
|
|
|
12
|
-
producer_class = Class.new(
|
|
12
|
+
producer_class = Class.new(described_class)
|
|
13
13
|
stub_const('MyBooleanProducer', producer_class)
|
|
14
14
|
|
|
15
15
|
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
|
@@ -104,96 +104,106 @@ describe Deimos::ActiveRecordProducer do
|
|
|
104
104
|
expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
|
|
105
105
|
end
|
|
106
106
|
|
|
107
|
-
it 'should coerce
|
|
107
|
+
it 'should coerce strings for a UnionSchema' do
|
|
108
108
|
MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
109
|
+
test_id: 'abc',
|
|
110
|
+
test_long: 399_999,
|
|
111
|
+
test_union_type: %w(hello world)
|
|
112
|
+
))
|
|
113
113
|
|
|
114
114
|
expect('my-topic-with-union-type').to have_sent(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
115
|
+
test_id: 'abc',
|
|
116
|
+
test_long: 399_999,
|
|
117
|
+
test_union_type: %w(hello world)
|
|
118
|
+
)
|
|
119
|
+
end
|
|
119
120
|
|
|
121
|
+
it 'should coerce maps for a UnionSchema' do
|
|
120
122
|
MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
|
|
121
|
-
|
|
122
|
-
|
|
123
|
+
test_id: 'abc',
|
|
124
|
+
test_long: 399_999,
|
|
125
|
+
test_union_type: {
|
|
126
|
+
record1_map: { a: 9999, b: 234 },
|
|
127
|
+
record1_id: 567
|
|
128
|
+
}
|
|
129
|
+
))
|
|
130
|
+
|
|
131
|
+
expect('my-topic-with-union-type').to have_sent(
|
|
132
|
+
test_id: 'abc',
|
|
133
|
+
test_long: 399_999,
|
|
123
134
|
test_union_type: {
|
|
124
|
-
record1_map:{ a:9999, b:234 },
|
|
135
|
+
record1_map: { a: 9999, b: 234 },
|
|
125
136
|
record1_id: 567
|
|
126
137
|
}
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
expect('my-topic-with-union-type').to have_sent(
|
|
130
|
-
test_id: "abc",
|
|
131
|
-
test_long: 399999,
|
|
132
|
-
test_union_type:{
|
|
133
|
-
record1_map:{ a:9999, b:234 },
|
|
134
|
-
record1_id: 567
|
|
135
|
-
}
|
|
136
|
-
)
|
|
138
|
+
)
|
|
139
|
+
end
|
|
137
140
|
|
|
141
|
+
it 'should coerce numbers for a UnionSchema' do
|
|
138
142
|
MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
+
test_id: 'abc',
|
|
144
|
+
test_long: 399_999,
|
|
145
|
+
test_union_type: 1_010_101
|
|
146
|
+
))
|
|
143
147
|
|
|
144
148
|
expect('my-topic-with-union-type').to have_sent(
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
+
test_id: 'abc',
|
|
150
|
+
test_long: 399_999,
|
|
151
|
+
test_union_type: 1_010_101
|
|
152
|
+
)
|
|
153
|
+
end
|
|
149
154
|
|
|
155
|
+
it 'should coerce records with strings for a UnionSchema' do
|
|
150
156
|
MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
+
test_id: 'abc',
|
|
158
|
+
test_long: 399_999,
|
|
159
|
+
test_union_type: {
|
|
160
|
+
record2_id: 'hello world'
|
|
161
|
+
}
|
|
162
|
+
))
|
|
157
163
|
|
|
158
164
|
expect('my-topic-with-union-type').to have_sent(
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
test_union_type: {
|
|
162
|
-
record2_id: "hello world"
|
|
163
|
-
}
|
|
164
|
-
)
|
|
165
|
-
|
|
166
|
-
MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
|
|
167
|
-
test_id: "abc",
|
|
168
|
-
test_long: 399999,
|
|
165
|
+
test_id: 'abc',
|
|
166
|
+
test_long: 399_999,
|
|
169
167
|
test_union_type: {
|
|
170
|
-
|
|
168
|
+
record2_id: 'hello world'
|
|
171
169
|
}
|
|
172
|
-
)
|
|
173
|
-
|
|
174
|
-
expect('my-topic-with-union-type').to have_sent(
|
|
175
|
-
test_id: "abc",
|
|
176
|
-
test_long: 399999,
|
|
177
|
-
test_union_type: {
|
|
178
|
-
record3_id:10.1010
|
|
179
|
-
}
|
|
180
|
-
)
|
|
170
|
+
)
|
|
171
|
+
end
|
|
181
172
|
|
|
173
|
+
it 'should coerce records with floats for a UnionSchema' do
|
|
182
174
|
MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
|
|
183
|
-
|
|
184
|
-
|
|
175
|
+
test_id: 'abc',
|
|
176
|
+
test_long: 399_999,
|
|
177
|
+
test_union_type: {
|
|
178
|
+
record3_id: 10.1010
|
|
179
|
+
}
|
|
180
|
+
))
|
|
181
|
+
|
|
182
|
+
expect('my-topic-with-union-type').to have_sent(
|
|
183
|
+
test_id: 'abc',
|
|
184
|
+
test_long: 399_999,
|
|
185
185
|
test_union_type: {
|
|
186
|
-
|
|
186
|
+
record3_id: 10.1010
|
|
187
187
|
}
|
|
188
|
-
)
|
|
188
|
+
)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
it 'should coerce records with ints for a UnionSchema' do
|
|
192
|
+
MyProducerWithUnionType.send_event(WidgetWithUnionType.new(
|
|
193
|
+
test_id: 'abc',
|
|
194
|
+
test_long: 399_999,
|
|
195
|
+
test_union_type: {
|
|
196
|
+
record4_id: 101_010
|
|
197
|
+
}
|
|
198
|
+
))
|
|
189
199
|
|
|
190
200
|
expect('my-topic-with-union-type').to have_sent(
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
201
|
+
test_id: 'abc',
|
|
202
|
+
test_long: 399_999,
|
|
203
|
+
test_union_type: {
|
|
204
|
+
record4_id: 101_010
|
|
205
|
+
}
|
|
206
|
+
)
|
|
197
207
|
end
|
|
198
208
|
|
|
199
209
|
it 'should coerce values' do
|
|
@@ -215,11 +225,11 @@ describe Deimos::ActiveRecordProducer do
|
|
|
215
225
|
widget = Widget.create!(test_id: 'abc2', some_int: 3)
|
|
216
226
|
MyProducerWithID.send_event({id: widget.id, test_id: 'abc2', some_int: 3})
|
|
217
227
|
expect('my-topic-with-id').to have_sent(
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
228
|
+
test_id: 'abc2',
|
|
229
|
+
some_int: 3,
|
|
230
|
+
message_id: 'generated_id',
|
|
231
|
+
timestamp: anything
|
|
232
|
+
)
|
|
223
233
|
end
|
|
224
234
|
|
|
225
235
|
it 'should post process the batch of records in #send_events' do
|
|
@@ -33,7 +33,7 @@ each_db_config(Deimos::Backends::Outbox) do
|
|
|
33
33
|
described_class.publish(producer_class: MyProducer,
|
|
34
34
|
messages: [build_message(nil, 'my-topic', 'foo1')])
|
|
35
35
|
expect(Deimos::KafkaMessage.count).to eq(1)
|
|
36
|
-
expect(Deimos::KafkaMessage.last.message).to
|
|
36
|
+
expect(Deimos::KafkaMessage.last.message).to be_nil
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
it 'should add to non-keyed messages' do
|
data/spec/batch_consumer_spec.rb
CHANGED
|
@@ -4,6 +4,15 @@
|
|
|
4
4
|
module ConsumerTest
|
|
5
5
|
describe Deimos::Consumer, 'Batch Consumer' do
|
|
6
6
|
let(:schema) { 'MySchema' }
|
|
7
|
+
let(:batch) do
|
|
8
|
+
[
|
|
9
|
+
{ 'test_id' => 'foo', 'some_int' => 123 },
|
|
10
|
+
{ 'test_id' => 'bar', 'some_int' => 456 }
|
|
11
|
+
]
|
|
12
|
+
end
|
|
13
|
+
let(:invalid_payloads) do
|
|
14
|
+
batch.push({ 'invalid' => 'key' })
|
|
15
|
+
end
|
|
7
16
|
let(:use_schema_classes) { false }
|
|
8
17
|
let(:should_reraise_errors) { false }
|
|
9
18
|
let(:key_config) { { field: 'test_id' } }
|
|
@@ -14,6 +23,7 @@ module ConsumerTest
|
|
|
14
23
|
end
|
|
15
24
|
end
|
|
16
25
|
end
|
|
26
|
+
|
|
17
27
|
before(:each) do
|
|
18
28
|
# :nodoc:
|
|
19
29
|
stub_const('MyBatchConsumer', consumer_class)
|
|
@@ -35,19 +45,8 @@ module ConsumerTest
|
|
|
35
45
|
end
|
|
36
46
|
end
|
|
37
47
|
|
|
38
|
-
let(:batch) do
|
|
39
|
-
[
|
|
40
|
-
{ 'test_id' => 'foo', 'some_int' => 123 },
|
|
41
|
-
{ 'test_id' => 'bar', 'some_int' => 456 }
|
|
42
|
-
]
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
let(:invalid_payloads) do
|
|
46
|
-
batch.concat([{ 'invalid' => 'key' }])
|
|
47
|
-
end
|
|
48
|
-
|
|
49
48
|
describe 'consume_batch' do
|
|
50
|
-
SCHEMA_CLASS_SETTINGS.
|
|
49
|
+
SCHEMA_CLASS_SETTINGS.each_key do |setting|
|
|
51
50
|
context "with Schema Class consumption #{setting}" do
|
|
52
51
|
|
|
53
52
|
let(:schema_class_batch) do
|
|
@@ -114,9 +113,10 @@ module ConsumerTest
|
|
|
114
113
|
|
|
115
114
|
context 'with plain keys' do
|
|
116
115
|
let(:key_config) { { plain: true } }
|
|
116
|
+
|
|
117
117
|
it 'should decode plain keys for all messages in the batch' do
|
|
118
118
|
test_consume_batch('my-topic', batch, keys: [1, 2]) do |received|
|
|
119
|
-
expect(received.map(&:key)).to eq(
|
|
119
|
+
expect(received.map(&:key)).to eq(%w(1 2))
|
|
120
120
|
end
|
|
121
121
|
end
|
|
122
122
|
end
|
|
@@ -124,11 +124,6 @@ module ConsumerTest
|
|
|
124
124
|
|
|
125
125
|
describe 'timestamps' do
|
|
126
126
|
let(:schema) { 'MySchemaWithDateTimes' }
|
|
127
|
-
let(:key_config) { { none: true } }
|
|
128
|
-
before(:each) do
|
|
129
|
-
allow(Deimos.config.metrics).to receive(:histogram)
|
|
130
|
-
end
|
|
131
|
-
|
|
132
127
|
let(:batch_with_time) do
|
|
133
128
|
[
|
|
134
129
|
{
|
|
@@ -145,7 +140,6 @@ module ConsumerTest
|
|
|
145
140
|
}
|
|
146
141
|
]
|
|
147
142
|
end
|
|
148
|
-
|
|
149
143
|
let(:invalid_times) do
|
|
150
144
|
[
|
|
151
145
|
{
|
|
@@ -168,6 +162,11 @@ module ConsumerTest
|
|
|
168
162
|
}
|
|
169
163
|
]
|
|
170
164
|
end
|
|
165
|
+
let(:key_config) { { none: true } }
|
|
166
|
+
|
|
167
|
+
before(:each) do
|
|
168
|
+
allow(Deimos.config.metrics).to receive(:histogram)
|
|
169
|
+
end
|
|
171
170
|
|
|
172
171
|
it 'should consume a batch' do
|
|
173
172
|
# expect(Deimos.config.metrics).
|
|
@@ -197,6 +196,7 @@ module ConsumerTest
|
|
|
197
196
|
describe 'logging' do
|
|
198
197
|
let(:schema) { 'MySchemaWithUniqueId' }
|
|
199
198
|
let(:key_config) { { plain: true } }
|
|
199
|
+
|
|
200
200
|
before(:each) do
|
|
201
201
|
allow(Deimos.config.metrics).to receive(:histogram)
|
|
202
202
|
set_karafka_config(:payload_log, :keys)
|
|
@@ -214,7 +214,7 @@ module ConsumerTest
|
|
|
214
214
|
|
|
215
215
|
expect(Deimos::Logging).
|
|
216
216
|
to receive(:log_info).
|
|
217
|
-
with(hash_including(payload_keys:
|
|
217
|
+
with(hash_including(payload_keys: %w(1 2)))
|
|
218
218
|
|
|
219
219
|
test_consume_batch('my-topic', batch_with_message_id, keys: [1, 2])
|
|
220
220
|
end
|
data/spec/consumer_spec.rb
CHANGED
|
@@ -8,6 +8,7 @@ module ConsumerTest
|
|
|
8
8
|
describe Deimos::Consumer, 'Message Consumer' do
|
|
9
9
|
let(:use_schema_classes) { false }
|
|
10
10
|
let(:reraise_errors) { false }
|
|
11
|
+
|
|
11
12
|
prepend_before(:each) do
|
|
12
13
|
# :nodoc:
|
|
13
14
|
consumer_class = Class.new(described_class) do
|
|
@@ -96,12 +97,14 @@ module ConsumerTest
|
|
|
96
97
|
end
|
|
97
98
|
end
|
|
98
99
|
|
|
99
|
-
|
|
100
|
+
expect {
|
|
101
|
+
test_consume_message(MyConsumer, { 'test_id' => 'foo',
|
|
100
102
|
'some_int' => 123 }, key: 'a key')
|
|
103
|
+
}.not_to raise_error
|
|
101
104
|
end
|
|
102
105
|
|
|
103
106
|
it 'should fail if reraise is false but fatal_error is true' do
|
|
104
|
-
expect { test_consume_message(MyConsumer, {test_id: 'fatal'}) }.
|
|
107
|
+
expect { test_consume_message(MyConsumer, { test_id: 'fatal' }) }.
|
|
105
108
|
to raise_error(Avro::SchemaValidator::ValidationError)
|
|
106
109
|
end
|
|
107
110
|
|
|
@@ -114,10 +117,12 @@ module ConsumerTest
|
|
|
114
117
|
it 'should fail on message with extra fields' do
|
|
115
118
|
allow_any_instance_of(Deimos::SchemaBackends::AvroValidation).
|
|
116
119
|
to receive(:coerce) { |_, m| m.with_indifferent_access }
|
|
117
|
-
expect {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
120
|
+
expect {
|
|
121
|
+
test_consume_message(MyConsumer,
|
|
122
|
+
{ 'test_id' => 'foo',
|
|
123
|
+
'some_int' => 123,
|
|
124
|
+
'extra_field' => 'field name' })
|
|
125
|
+
}.
|
|
121
126
|
to raise_error(Avro::SchemaValidator::ValidationError)
|
|
122
127
|
end
|
|
123
128
|
|
|
@@ -127,17 +132,22 @@ module ConsumerTest
|
|
|
127
132
|
test_consume_message(
|
|
128
133
|
MyConsumer,
|
|
129
134
|
{ 'test_id' => 'foo',
|
|
130
|
-
'some_int' => 123 }
|
|
135
|
+
'some_int' => 123 }
|
|
136
|
+
) { raise 'OH NOES' }
|
|
131
137
|
}.not_to raise_error
|
|
132
138
|
end
|
|
133
139
|
|
|
134
140
|
it 'should not fail when consume fails without reraising errors' do
|
|
135
141
|
set_karafka_config(:reraise_errors, false)
|
|
136
|
-
allow(Deimos::ProducerMiddleware).to receive(:call)
|
|
142
|
+
allow(Deimos::ProducerMiddleware).to receive(:call) do |m|
|
|
143
|
+
m[:payload] = m[:payload].to_json
|
|
144
|
+
m
|
|
145
|
+
end
|
|
137
146
|
expect {
|
|
138
147
|
test_consume_message(
|
|
139
148
|
MyConsumer,
|
|
140
|
-
{ 'invalid' => 'key' }
|
|
149
|
+
{ 'invalid' => 'key' }
|
|
150
|
+
)
|
|
141
151
|
}.not_to raise_error
|
|
142
152
|
end
|
|
143
153
|
end
|
|
@@ -171,6 +181,7 @@ module ConsumerTest
|
|
|
171
181
|
end
|
|
172
182
|
end
|
|
173
183
|
end
|
|
184
|
+
|
|
174
185
|
after(:each) do
|
|
175
186
|
Karafka::App.routes.clear
|
|
176
187
|
end
|
|
@@ -227,10 +238,10 @@ module ConsumerTest
|
|
|
227
238
|
test_consume_message('my_consume_topic',
|
|
228
239
|
{ 'test_id' => 'foo',
|
|
229
240
|
'test_float' => 4.0,
|
|
230
|
-
'test_array' =>
|
|
241
|
+
'test_array' => %w(1 2),
|
|
231
242
|
'additional_field' => 'bar',
|
|
232
243
|
'some_nested_record' => {
|
|
233
|
-
|
|
244
|
+
'some_int' => 1,
|
|
234
245
|
'some_float' => 10.0,
|
|
235
246
|
'some_string' => 'hi mom',
|
|
236
247
|
'additional_field' => 'baz'
|
|
@@ -240,7 +251,7 @@ module ConsumerTest
|
|
|
240
251
|
expect(payload['some_nested_record']['some_int']).to eq(1)
|
|
241
252
|
expect(payload.to_h).not_to have_key('additional_field')
|
|
242
253
|
expect(payload.to_h['some_nested_record']).not_to have_key('additional_field')
|
|
243
|
-
|
|
254
|
+
end
|
|
244
255
|
|
|
245
256
|
end
|
|
246
257
|
end
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
4
|
# source: sample/v1/sample.proto
|
|
4
5
|
|
|
@@ -6,7 +7,6 @@ require 'google/protobuf'
|
|
|
6
7
|
|
|
7
8
|
require 'google/protobuf/timestamp_pb'
|
|
8
9
|
|
|
9
|
-
|
|
10
10
|
descriptor_data = "\n\x16sample/v1/sample.proto\x12\tsample.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"7\n\rNestedMessage\x12\x12\n\nnested_str\x18\x01 \x01(\t\x12\x12\n\nnested_num\x18\x02 \x01(\x05\"\xdb\x02\n\rSampleMessage\x12\x0b\n\x03str\x18\x01 \x01(\t\x12\x0b\n\x03num\x18\x02 \x01(\x05\x12\x0f\n\x07str_arr\x18\x03 \x03(\t\x12\x0c\n\x04\x66lag\x18\x04 \x01(\x08\x12-\n\ttimestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x06nested\x18\x06 \x01(\x0b\x32\x18.sample.v1.NestedMessageH\x00\x12\x13\n\tunion_str\x18\x07 \x01(\tH\x00\x12\x32\n\x10non_union_nested\x18\x08 \x01(\x0b\x32\x18.sample.v1.NestedMessage\x12\x35\n\x07str_map\x18\t \x03(\x0b\x32$.sample.v1.SampleMessage.StrMapEntry\x1a-\n\x0bStrMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x07\n\x05unionb\x06proto3"
|
|
11
11
|
|
|
12
12
|
pool = Google::Protobuf::DescriptorPool.generated_pool
|
|
@@ -14,7 +14,7 @@ pool.add_serialized_file(descriptor_data)
|
|
|
14
14
|
|
|
15
15
|
module Sample
|
|
16
16
|
module V1
|
|
17
|
-
NestedMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(
|
|
18
|
-
SampleMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(
|
|
17
|
+
NestedMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup('sample.v1.NestedMessage').msgclass
|
|
18
|
+
SampleMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup('sample.v1.SampleMessage').msgclass
|
|
19
19
|
end
|
|
20
20
|
end
|
|
@@ -5,8 +5,8 @@ require 'generators/deimos/active_record_generator'
|
|
|
5
5
|
RSpec.describe Deimos::Generators::ActiveRecordGenerator do
|
|
6
6
|
|
|
7
7
|
after(:each) do
|
|
8
|
-
FileUtils.rm_rf('db')
|
|
9
|
-
FileUtils.rm_rf('app')
|
|
8
|
+
FileUtils.rm_rf('db')
|
|
9
|
+
FileUtils.rm_rf('app')
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
it 'should generate a migration' do
|
|
@@ -39,12 +39,12 @@ RSpec.describe Deimos::Generators::ActiveRecordGenerator do
|
|
|
39
39
|
# TODO add indexes as necessary
|
|
40
40
|
end
|
|
41
41
|
end
|
|
42
|
-
|
|
42
|
+
|
|
43
43
|
def down
|
|
44
44
|
return unless table_exists?(:generated_table)
|
|
45
45
|
drop_table :generated_table
|
|
46
46
|
end
|
|
47
|
-
|
|
47
|
+
|
|
48
48
|
end
|
|
49
49
|
MIGRATION
|
|
50
50
|
expect(File.read(files[0])).to eq(results)
|
|
@@ -90,7 +90,8 @@ RSpec.describe Schemas::MyNamespace::MySchemaWithCircularReference do
|
|
|
90
90
|
end
|
|
91
91
|
|
|
92
92
|
it 'should return a JSON string of the payload' do
|
|
93
|
-
s = '{"properties":{"a_boolean":{"property":true},"an_integer":{"property":1},"a_float":{"property":4.5},"
|
|
93
|
+
s = '{"properties":{"a_boolean":{"property":true},"an_integer":{"property":1},"a_float":{"property":4.5},"a_str' \
|
|
94
|
+
'ing":{"property":"string"},"an_array":{"property":[1,2,3]},"an_hash":{"property":{"a_key":"a_value"}}}}'
|
|
94
95
|
expect(klass.to_json).to eq(s)
|
|
95
96
|
end
|
|
96
97
|
end
|
|
@@ -109,7 +109,14 @@ RSpec.describe Schemas::MyNamespace::MySchemaWithComplexType do
|
|
|
109
109
|
end
|
|
110
110
|
|
|
111
111
|
it 'should return a JSON string of the payload' do
|
|
112
|
-
s = '{"test_id":"test
|
|
112
|
+
s = '{"test_id":"test ' \
|
|
113
|
+
'id","union_string":"","test_float":1.2,"test_string_array":["abc","def"],"test_int_array":[123,456],"test_' \
|
|
114
|
+
'optional_i' \
|
|
115
|
+
'nt":123,"some_integer_map":{"int_1":1,"int_2":2},"some_record":{"a_record_field":"field ' \
|
|
116
|
+
'1"},"some_optional_record":{"a_record_field":"field 2"},"some_record_array":[{"a_record_field":"field ' \
|
|
117
|
+
'3"},{"a_record_field":"field 4"}],"some_record_map":{"record_1":{"a_record_field":"field ' \
|
|
118
|
+
'5"},"record_2":{"a_record_field":"field ' \
|
|
119
|
+
'6"}},"some_enum_array":["sym1","sym2"],"some_optional_enum":null,"some_enum_with_default":"sym6"}'
|
|
113
120
|
expect(klass.to_json).to eq(s)
|
|
114
121
|
end
|
|
115
122
|
end
|
|
@@ -129,7 +136,7 @@ RSpec.describe Schemas::MyNamespace::MySchemaWithComplexType do
|
|
|
129
136
|
end
|
|
130
137
|
|
|
131
138
|
it 'should set some_record to nil' do
|
|
132
|
-
klass = described_class.new(**payload_hash
|
|
139
|
+
klass = described_class.new(**payload_hash, some_record: nil)
|
|
133
140
|
expect(klass.some_record).to be_nil
|
|
134
141
|
end
|
|
135
142
|
end
|
|
@@ -4,15 +4,15 @@ require 'generators/deimos/schema_class_generator'
|
|
|
4
4
|
require 'fileutils'
|
|
5
5
|
|
|
6
6
|
class MultiFileSerializer
|
|
7
|
-
def process_string(
|
|
7
|
+
def process_string(str)
|
|
8
8
|
# Ruby 3.4 changes how hashes are printed
|
|
9
9
|
if Gem::Version.new(RUBY_VERSION) > Gem::Version.new('3.4.0')
|
|
10
|
-
|
|
10
|
+
str.gsub(/{"(.*)" => /, '{"\1"=>')
|
|
11
11
|
else
|
|
12
|
-
|
|
12
|
+
str
|
|
13
13
|
end
|
|
14
|
-
|
|
15
14
|
end
|
|
15
|
+
|
|
16
16
|
def dump(value)
|
|
17
17
|
value.keys.sort.map { |k| "#{k}:\n#{process_string(value[k])}\n" }.join("\n")
|
|
18
18
|
end
|
|
@@ -32,7 +32,7 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
|
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
after(:each) do
|
|
35
|
-
FileUtils.rm_rf('spec/app')
|
|
35
|
+
FileUtils.rm_rf('spec/app')
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
context 'with a Consumers Schema' do
|
data/spec/kafka_source_spec.rb
CHANGED
|
@@ -5,6 +5,7 @@ require 'activerecord-import'
|
|
|
5
5
|
# Wrap in a module so our classes don't leak out afterwards
|
|
6
6
|
module KafkaSourceSpec
|
|
7
7
|
RSpec.describe Deimos::KafkaSource do
|
|
8
|
+
# rubocop:disable Lint/ConstantDefinitionInBlock
|
|
8
9
|
before(:all) do
|
|
9
10
|
ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
|
|
10
11
|
t.integer(:widget_id)
|
|
@@ -40,8 +41,8 @@ module KafkaSourceSpec
|
|
|
40
41
|
end
|
|
41
42
|
end
|
|
42
43
|
Widget.reset_column_information
|
|
43
|
-
|
|
44
44
|
end
|
|
45
|
+
# rubocop:enable Lint/ConstantDefinitionInBlock
|
|
45
46
|
|
|
46
47
|
after(:all) do
|
|
47
48
|
ActiveRecord::Base.connection.drop_table(:widgets)
|
|
@@ -111,22 +112,24 @@ module KafkaSourceSpec
|
|
|
111
112
|
expect('my-topic-the-second').to have_sent(nil, widget.id)
|
|
112
113
|
end
|
|
113
114
|
|
|
114
|
-
|
|
115
|
+
describe 'multi-producer model using two different key fields' do
|
|
115
116
|
before(:each) do
|
|
116
|
-
|
|
117
|
+
klass = Class.new(ActiveRecord::Base) do
|
|
117
118
|
include Deimos::KafkaSource
|
|
119
|
+
|
|
118
120
|
self.table_name = 'widgets'
|
|
119
121
|
|
|
120
122
|
def self.kafka_producers
|
|
121
123
|
[WidgetProducer, WidgetStringKeyProducer]
|
|
122
124
|
end
|
|
123
125
|
end
|
|
126
|
+
stub_const('MultiKeyWidget', klass)
|
|
124
127
|
|
|
125
|
-
|
|
128
|
+
klass = Class.new(Deimos::ActiveRecordProducer) do
|
|
126
129
|
class << self
|
|
127
130
|
|
|
128
131
|
def generate_payload(attributes, record)
|
|
129
|
-
payload = super
|
|
132
|
+
payload = super
|
|
130
133
|
payload.merge('id' => record.model_id)
|
|
131
134
|
end
|
|
132
135
|
|
|
@@ -135,6 +138,7 @@ module KafkaSourceSpec
|
|
|
135
138
|
end
|
|
136
139
|
end
|
|
137
140
|
end
|
|
141
|
+
stub_const('WidgetStringKeyProducer', klass)
|
|
138
142
|
|
|
139
143
|
Karafka::App.routes.redraw do
|
|
140
144
|
topic 'my-topic' do
|
|
@@ -365,8 +369,9 @@ module KafkaSourceSpec
|
|
|
365
369
|
before(:each) do
|
|
366
370
|
# Dummy class we can include the mixin in. Has a backing table created
|
|
367
371
|
# earlier and has the import hook disabled
|
|
368
|
-
|
|
372
|
+
klass = Class.new(ActiveRecord::Base) do
|
|
369
373
|
include Deimos::KafkaSource
|
|
374
|
+
|
|
370
375
|
self.table_name = 'widgets'
|
|
371
376
|
|
|
372
377
|
# :nodoc:
|
|
@@ -385,6 +390,7 @@ module KafkaSourceSpec
|
|
|
385
390
|
end
|
|
386
391
|
end
|
|
387
392
|
|
|
393
|
+
stub_const('WidgetNoImportHook', klass)
|
|
388
394
|
WidgetNoImportHook.reset_column_information
|
|
389
395
|
end
|
|
390
396
|
|
|
@@ -451,6 +457,7 @@ module KafkaSourceSpec
|
|
|
451
457
|
# earlier and has the import hook disabled
|
|
452
458
|
buggy_class = Class.new(ActiveRecord::Base) do
|
|
453
459
|
include Deimos::KafkaSource
|
|
460
|
+
|
|
454
461
|
self.table_name = 'widgets'
|
|
455
462
|
|
|
456
463
|
# :nodoc:
|