deimos-ruby 1.22.5 → 1.23.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +12 -0
- data/README.md +6 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +43 -20
- data/lib/deimos/active_record_consume/batch_record.rb +5 -4
- data/lib/deimos/active_record_consume/batch_record_list.rb +11 -2
- data/lib/deimos/active_record_consume/mass_updater.rb +13 -4
- data/lib/deimos/active_record_consumer.rb +10 -0
- data/lib/deimos/config/configuration.rb +25 -2
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/test_helpers.rb +1 -0
- data/lib/deimos/tracing/datadog.rb +12 -6
- data/lib/deimos/tracing/mock.rb +31 -2
- data/lib/deimos/tracing/provider.rb +6 -0
- data/lib/deimos/utils/schema_class.rb +10 -2
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +14 -1
- data/spec/active_record_batch_consumer_association_spec.rb +32 -5
- data/spec/active_record_batch_consumer_spec.rb +376 -59
- data/spec/active_record_consume/mass_updater_spec.rb +46 -3
- data/spec/active_record_consumer_spec.rb +74 -1
- data/spec/active_record_producer_spec.rb +4 -1
- data/spec/batch_consumer_spec.rb +4 -1
- data/spec/config/configuration_spec.rb +42 -3
- data/spec/consumer_spec.rb +42 -1
- data/spec/schemas/my_namespace/my_updated_schema.rb +18 -0
- metadata +5 -3
@@ -91,7 +91,8 @@ describe Deimos, 'configuration' do
|
|
91
91
|
heartbeat_interval: 10,
|
92
92
|
handler: 'ConsumerTest::MyConsumer',
|
93
93
|
use_schema_classes: nil,
|
94
|
-
max_db_batch_size: nil
|
94
|
+
max_db_batch_size: nil,
|
95
|
+
bulk_import_id_generator: nil
|
95
96
|
}, {
|
96
97
|
topic: 'my_batch_consume_topic',
|
97
98
|
group_id: 'my_batch_group_id',
|
@@ -109,7 +110,8 @@ describe Deimos, 'configuration' do
|
|
109
110
|
heartbeat_interval: 10,
|
110
111
|
handler: 'ConsumerTest::MyBatchConsumer',
|
111
112
|
use_schema_classes: nil,
|
112
|
-
max_db_batch_size: nil
|
113
|
+
max_db_batch_size: nil,
|
114
|
+
bulk_import_id_generator: nil
|
113
115
|
}
|
114
116
|
],
|
115
117
|
producer: {
|
@@ -261,7 +263,8 @@ describe Deimos, 'configuration' do
|
|
261
263
|
heartbeat_interval: 13,
|
262
264
|
handler: 'MyConfigConsumer',
|
263
265
|
use_schema_classes: false,
|
264
|
-
max_db_batch_size: nil
|
266
|
+
max_db_batch_size: nil,
|
267
|
+
bulk_import_id_generator: nil
|
265
268
|
}
|
266
269
|
],
|
267
270
|
producer: {
|
@@ -279,4 +282,40 @@ describe Deimos, 'configuration' do
|
|
279
282
|
}
|
280
283
|
)
|
281
284
|
end
|
285
|
+
|
286
|
+
it 'should override global configurations' do
|
287
|
+
described_class.configure do
|
288
|
+
consumers.bulk_import_id_generator(-> { 'global' })
|
289
|
+
consumers.replace_associations true
|
290
|
+
|
291
|
+
consumer do
|
292
|
+
class_name 'MyConfigConsumer'
|
293
|
+
schema 'blah'
|
294
|
+
topic 'blah'
|
295
|
+
group_id 'myconsumerid'
|
296
|
+
bulk_import_id_generator(-> { 'consumer' })
|
297
|
+
replace_associations false
|
298
|
+
end
|
299
|
+
|
300
|
+
consumer do
|
301
|
+
class_name 'MyConfigConsumer2'
|
302
|
+
schema 'blah'
|
303
|
+
topic 'blah'
|
304
|
+
group_id 'myconsumerid'
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
consumers = described_class.config.consumers
|
309
|
+
expect(consumers.replace_associations).to eq(true)
|
310
|
+
expect(consumers.bulk_import_id_generator.call).to eq('global')
|
311
|
+
|
312
|
+
custom = MyConfigConsumer.config
|
313
|
+
expect(custom[:replace_associations]).to eq(false)
|
314
|
+
expect(custom[:bulk_import_id_generator].call).to eq('consumer')
|
315
|
+
|
316
|
+
default = MyConfigConsumer2.config
|
317
|
+
expect(default[:replace_associations]).to eq(true)
|
318
|
+
expect(default[:bulk_import_id_generator].call).to eq('global')
|
319
|
+
|
320
|
+
end
|
282
321
|
end
|
data/spec/consumer_spec.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# :nodoc:
|
4
|
+
# rubocop:disable Metrics/ModuleLength
|
4
5
|
module ConsumerTest
|
5
6
|
describe Deimos::Consumer, 'Message Consumer' do
|
6
7
|
prepend_before(:each) do
|
@@ -26,8 +27,12 @@ module ConsumerTest
|
|
26
27
|
describe 'consume' do
|
27
28
|
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
28
29
|
context "with Schema Class consumption #{setting}" do
|
30
|
+
|
29
31
|
before(:each) do
|
30
|
-
Deimos.configure
|
32
|
+
Deimos.configure do |config|
|
33
|
+
config.schema.use_schema_classes = use_schema_classes
|
34
|
+
config.schema.generate_namespace_folders = true
|
35
|
+
end
|
31
36
|
end
|
32
37
|
|
33
38
|
it 'should consume a message' do
|
@@ -127,6 +132,41 @@ module ConsumerTest
|
|
127
132
|
end
|
128
133
|
end
|
129
134
|
end
|
135
|
+
|
136
|
+
context 'with overriden schema classes' do
|
137
|
+
|
138
|
+
before(:each) do
|
139
|
+
Deimos.configure do |config|
|
140
|
+
config.schema.use_schema_classes = true
|
141
|
+
config.schema.generate_namespace_folders = true
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
prepend_before(:each) do
|
146
|
+
consumer_class = Class.new(described_class) do
|
147
|
+
schema 'MyUpdatedSchema'
|
148
|
+
namespace 'com.my-namespace'
|
149
|
+
key_config field: 'test_id'
|
150
|
+
|
151
|
+
# :nodoc:
|
152
|
+
def consume(_payload, _metadata)
|
153
|
+
raise 'This should not be called unless call_original is set'
|
154
|
+
end
|
155
|
+
end
|
156
|
+
stub_const('ConsumerTest::MyConsumer', consumer_class)
|
157
|
+
end
|
158
|
+
|
159
|
+
it 'should consume messages' do
|
160
|
+
test_consume_message('my_consume_topic',
|
161
|
+
{ 'test_id' => 'foo',
|
162
|
+
'some_int' => 1 }) do |payload, _metadata|
|
163
|
+
expect(payload['test_id']).to eq('foo')
|
164
|
+
expect(payload['some_int']).to eq(1)
|
165
|
+
expect(payload['super_int']).to eq(9000)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
130
170
|
end
|
131
171
|
|
132
172
|
describe 'decode_key' do
|
@@ -218,3 +258,4 @@ module ConsumerTest
|
|
218
258
|
end
|
219
259
|
end
|
220
260
|
end
|
261
|
+
# rubocop:enable Metrics/ModuleLength
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
4
|
+
module Schemas; module MyNamespace
|
5
|
+
### Primary Schema Class ###
|
6
|
+
# Autogenerated Schema for Record at com.my-namespace.MySchema
|
7
|
+
class MyUpdatedSchema < Schemas::MyNamespace::MySchema
|
8
|
+
|
9
|
+
attr_accessor :super_int
|
10
|
+
|
11
|
+
def initialize(test_id: nil,
|
12
|
+
some_int: nil)
|
13
|
+
super
|
14
|
+
self.super_int = some_int.nil? ? 10 : some_int * 9000
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.23.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -579,6 +579,7 @@ files:
|
|
579
579
|
- spec/schemas/my_namespace/my_schema_with_date_time.rb
|
580
580
|
- spec/schemas/my_namespace/my_schema_with_id.rb
|
581
581
|
- spec/schemas/my_namespace/my_schema_with_unique_id.rb
|
582
|
+
- spec/schemas/my_namespace/my_updated_schema.rb
|
582
583
|
- spec/schemas/my_namespace/wibble.rb
|
583
584
|
- spec/schemas/my_namespace/widget.rb
|
584
585
|
- spec/schemas/my_namespace/widget_the_second.rb
|
@@ -632,7 +633,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
632
633
|
- !ruby/object:Gem::Version
|
633
634
|
version: '0'
|
634
635
|
requirements: []
|
635
|
-
rubygems_version: 3.
|
636
|
+
rubygems_version: 3.1.2
|
636
637
|
signing_key:
|
637
638
|
specification_version: 4
|
638
639
|
summary: Kafka libraries for Ruby.
|
@@ -706,6 +707,7 @@ test_files:
|
|
706
707
|
- spec/schemas/my_namespace/my_schema_with_date_time.rb
|
707
708
|
- spec/schemas/my_namespace/my_schema_with_id.rb
|
708
709
|
- spec/schemas/my_namespace/my_schema_with_unique_id.rb
|
710
|
+
- spec/schemas/my_namespace/my_updated_schema.rb
|
709
711
|
- spec/schemas/my_namespace/wibble.rb
|
710
712
|
- spec/schemas/my_namespace/widget.rb
|
711
713
|
- spec/schemas/my_namespace/widget_the_second.rb
|