deimos-ruby 2.2.0.pre.beta5 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +33 -30
- data/CHANGELOG.md +8 -0
- data/Gemfile +0 -6
- data/deimos-ruby.gemspec +15 -10
- data/karafka.rb +7 -4
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -7
- data/lib/deimos/active_record_consume/batch_record.rb +2 -2
- data/lib/deimos/active_record_consume/message_consumption.rb +6 -5
- data/lib/deimos/active_record_consume/schema_model_converter.rb +2 -2
- data/lib/deimos/active_record_consumer.rb +1 -0
- data/lib/deimos/active_record_producer.rb +4 -2
- data/lib/deimos/backends/base.rb +1 -3
- data/lib/deimos/backends/outbox.rb +1 -1
- data/lib/deimos/config/configuration.rb +88 -75
- data/lib/deimos/consume/batch_consumption.rb +7 -5
- data/lib/deimos/consume/message_consumption.rb +5 -3
- data/lib/deimos/ext/consumer_route.rb +3 -3
- data/lib/deimos/ext/producer_metrics_listener.rb +2 -2
- data/lib/deimos/ext/producer_middleware.rb +19 -15
- data/lib/deimos/ext/producer_route.rb +3 -1
- data/lib/deimos/ext/routing_defaults.rb +9 -7
- data/lib/deimos/ext/schema_route.rb +22 -15
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_source.rb +36 -31
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +20 -19
- data/lib/deimos/message.rb +1 -1
- data/lib/deimos/metrics/minimal_datadog_listener.rb +19 -6
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +3 -1
- data/lib/deimos/railtie.rb +1 -1
- data/lib/deimos/schema_backends/avro_base.rb +1 -1
- data/lib/deimos/schema_backends/avro_schema_coercer.rb +46 -27
- data/lib/deimos/schema_backends/avro_schema_registry.rb +8 -8
- data/lib/deimos/schema_backends/base.rb +9 -9
- data/lib/deimos/schema_backends/mock.rb +4 -0
- data/lib/deimos/schema_backends/plain.rb +1 -1
- data/lib/deimos/schema_backends/proto_base.rb +7 -5
- data/lib/deimos/schema_backends/proto_local.rb +0 -2
- data/lib/deimos/schema_backends/proto_schema_registry.rb +0 -2
- data/lib/deimos/schema_class/base.rb +1 -1
- data/lib/deimos/schema_class/record.rb +3 -3
- data/lib/deimos/test_helpers.rb +31 -26
- data/lib/deimos/tracing/provider.rb +5 -5
- data/lib/deimos/transcoder.rb +6 -2
- data/lib/deimos/utils/db_poller/base.rb +3 -3
- data/lib/deimos/utils/deadlock_retry.rb +2 -2
- data/lib/deimos/utils/outbox_producer.rb +14 -14
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +4 -4
- data/lib/generators/deimos/active_record_generator.rb +2 -1
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/outbox_backend_generator.rb +1 -0
- data/lib/generators/deimos/schema_class_generator.rb +3 -2
- data/lib/generators/deimos/v2_generator.rb +184 -155
- data/spec/active_record_batch_consumer_association_spec.rb +6 -2
- data/spec/active_record_batch_consumer_spec.rb +83 -106
- data/spec/active_record_consume/batch_consumption_spec.rb +27 -28
- data/spec/active_record_consume/batch_slicer_spec.rb +4 -12
- data/spec/active_record_consume/mass_updater_spec.rb +42 -46
- data/spec/active_record_consume/schema_model_converter_spec.rb +1 -1
- data/spec/active_record_consumer_spec.rb +7 -5
- data/spec/active_record_producer_spec.rb +83 -73
- data/spec/backends/outbox_spec.rb +1 -1
- data/spec/batch_consumer_spec.rb +20 -20
- data/spec/consumer_spec.rb +23 -12
- data/spec/gen/sample/v1/sample_pb.rb +3 -3
- data/spec/generators/active_record_generator_spec.rb +5 -5
- data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb +2 -1
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +9 -2
- data/spec/generators/schema_class_generator_spec.rb +5 -5
- data/spec/kafka_source_spec.rb +13 -6
- data/spec/kafka_topic_info_spec.rb +7 -7
- data/spec/karafka/karafka.rb +6 -5
- data/spec/karafka_config/karafka_spec.rb +22 -19
- data/spec/logging_spec.rb +2 -0
- data/spec/producer_spec.rb +25 -20
- data/spec/schema_backends/avro_base_shared.rb +8 -8
- data/spec/schema_backends/avro_local_spec.rb +5 -6
- data/spec/schema_backends/avro_schema_registry_spec.rb +5 -6
- data/spec/schema_backends/proto_schema_registry_spec.rb +9 -12
- data/spec/schemas/my_namespace/generated.rb +1 -2
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +5 -8
- data/spec/schemas/my_namespace/my_schema_with_union_type.rb +22 -23
- data/spec/spec_helper.rb +13 -17
- data/spec/utils/db_poller_spec.rb +5 -5
- data/spec/utils/deadlock_retry_spec.rb +1 -4
- data/spec/utils/outbox_producer_spec.rb +36 -24
- metadata +79 -158
- data/.ruby-version +0 -1
|
@@ -9,13 +9,13 @@ module Schemas; module MyNamespace
|
|
|
9
9
|
### Secondary Schema Classes ###
|
|
10
10
|
# Autogenerated Schema for Record at com.flipp.content.Record1
|
|
11
11
|
class Record1 < Deimos::SchemaClass::Record
|
|
12
|
-
|
|
12
|
+
|
|
13
13
|
### Attribute Accessors ###
|
|
14
14
|
# @return [Hash<String, Integer>]
|
|
15
15
|
attr_accessor :record1_map
|
|
16
16
|
# @return [Integer]
|
|
17
17
|
attr_accessor :record1_id
|
|
18
|
-
|
|
18
|
+
|
|
19
19
|
# @override
|
|
20
20
|
def initialize(_from_message: false, record1_map: {},
|
|
21
21
|
record1_id: 0)
|
|
@@ -24,17 +24,17 @@ module Schemas; module MyNamespace
|
|
|
24
24
|
self.record1_map = record1_map
|
|
25
25
|
self.record1_id = record1_id
|
|
26
26
|
end
|
|
27
|
-
|
|
27
|
+
|
|
28
28
|
# @override
|
|
29
29
|
def schema
|
|
30
30
|
'Record1'
|
|
31
31
|
end
|
|
32
|
-
|
|
32
|
+
|
|
33
33
|
# @override
|
|
34
34
|
def namespace
|
|
35
35
|
'com.flipp.content'
|
|
36
36
|
end
|
|
37
|
-
|
|
37
|
+
|
|
38
38
|
# @override
|
|
39
39
|
def as_json(_opts={})
|
|
40
40
|
{
|
|
@@ -46,28 +46,28 @@ module Schemas; module MyNamespace
|
|
|
46
46
|
|
|
47
47
|
# Autogenerated Schema for Record at com.flipp.content.Record2
|
|
48
48
|
class Record2 < Deimos::SchemaClass::Record
|
|
49
|
-
|
|
49
|
+
|
|
50
50
|
### Attribute Accessors ###
|
|
51
51
|
# @return [String]
|
|
52
52
|
attr_accessor :record2_id
|
|
53
|
-
|
|
53
|
+
|
|
54
54
|
# @override
|
|
55
|
-
def initialize(_from_message: false, record2_id:
|
|
55
|
+
def initialize(_from_message: false, record2_id: '')
|
|
56
56
|
@_from_message = _from_message
|
|
57
57
|
super
|
|
58
58
|
self.record2_id = record2_id
|
|
59
59
|
end
|
|
60
|
-
|
|
60
|
+
|
|
61
61
|
# @override
|
|
62
62
|
def schema
|
|
63
63
|
'Record2'
|
|
64
64
|
end
|
|
65
|
-
|
|
65
|
+
|
|
66
66
|
# @override
|
|
67
67
|
def namespace
|
|
68
68
|
'com.flipp.content'
|
|
69
69
|
end
|
|
70
|
-
|
|
70
|
+
|
|
71
71
|
# @override
|
|
72
72
|
def as_json(_opts={})
|
|
73
73
|
{
|
|
@@ -78,28 +78,28 @@ module Schemas; module MyNamespace
|
|
|
78
78
|
|
|
79
79
|
# Autogenerated Schema for Record at com.flipp.content.Record3
|
|
80
80
|
class Record3 < Deimos::SchemaClass::Record
|
|
81
|
-
|
|
81
|
+
|
|
82
82
|
### Attribute Accessors ###
|
|
83
83
|
# @return [Float]
|
|
84
84
|
attr_accessor :record3_id
|
|
85
|
-
|
|
85
|
+
|
|
86
86
|
# @override
|
|
87
87
|
def initialize(_from_message: false, record3_id: 0.0)
|
|
88
88
|
@_from_message = _from_message
|
|
89
89
|
super
|
|
90
90
|
self.record3_id = record3_id
|
|
91
91
|
end
|
|
92
|
-
|
|
92
|
+
|
|
93
93
|
# @override
|
|
94
94
|
def schema
|
|
95
95
|
'Record3'
|
|
96
96
|
end
|
|
97
|
-
|
|
97
|
+
|
|
98
98
|
# @override
|
|
99
99
|
def namespace
|
|
100
100
|
'com.flipp.content'
|
|
101
101
|
end
|
|
102
|
-
|
|
102
|
+
|
|
103
103
|
# @override
|
|
104
104
|
def as_json(_opts={})
|
|
105
105
|
{
|
|
@@ -110,28 +110,28 @@ module Schemas; module MyNamespace
|
|
|
110
110
|
|
|
111
111
|
# Autogenerated Schema for Record at com.flipp.content.Record4
|
|
112
112
|
class Record4 < Deimos::SchemaClass::Record
|
|
113
|
-
|
|
113
|
+
|
|
114
114
|
### Attribute Accessors ###
|
|
115
115
|
# @return [Integer]
|
|
116
116
|
attr_accessor :record4_id
|
|
117
|
-
|
|
117
|
+
|
|
118
118
|
# @override
|
|
119
119
|
def initialize(_from_message: false, record4_id: 0)
|
|
120
120
|
@_from_message = _from_message
|
|
121
121
|
super
|
|
122
122
|
self.record4_id = record4_id
|
|
123
123
|
end
|
|
124
|
-
|
|
124
|
+
|
|
125
125
|
# @override
|
|
126
126
|
def schema
|
|
127
127
|
'Record4'
|
|
128
128
|
end
|
|
129
|
-
|
|
129
|
+
|
|
130
130
|
# @override
|
|
131
131
|
def namespace
|
|
132
132
|
'com.flipp.content'
|
|
133
133
|
end
|
|
134
|
-
|
|
134
|
+
|
|
135
135
|
# @override
|
|
136
136
|
def as_json(_opts={})
|
|
137
137
|
{
|
|
@@ -140,7 +140,6 @@ module Schemas; module MyNamespace
|
|
|
140
140
|
end
|
|
141
141
|
end
|
|
142
142
|
|
|
143
|
-
|
|
144
143
|
### Attribute Readers ###
|
|
145
144
|
# @return [nil, Record1, Record2, Record3, Record4, Integer, Array<String>]
|
|
146
145
|
attr_reader :test_union_type
|
|
@@ -173,7 +172,7 @@ module Schemas; module MyNamespace
|
|
|
173
172
|
end
|
|
174
173
|
|
|
175
174
|
# @override
|
|
176
|
-
def initialize(_from_message: false, test_id:
|
|
175
|
+
def initialize(_from_message: false, test_id: '',
|
|
177
176
|
test_long: nil,
|
|
178
177
|
test_union_type: nil)
|
|
179
178
|
@_from_message = _from_message
|
data/spec/spec_helper.rb
CHANGED
|
@@ -16,8 +16,6 @@ require 'handlers/my_consumer'
|
|
|
16
16
|
require 'rspec/rails'
|
|
17
17
|
require 'rspec/snapshot'
|
|
18
18
|
require 'karafka/testing/rspec/helpers'
|
|
19
|
-
require "trilogy_adapter/connection"
|
|
20
|
-
ActiveRecord::Base.public_send :extend, TrilogyAdapter::Connection
|
|
21
19
|
Dir['./spec/schemas/**/*.rb'].sort.each { |f| require f }
|
|
22
20
|
|
|
23
21
|
# Constants used for consumer specs
|
|
@@ -31,18 +29,17 @@ end
|
|
|
31
29
|
DeimosApp.initialize!
|
|
32
30
|
|
|
33
31
|
module Helpers
|
|
34
|
-
|
|
35
32
|
def set_karafka_config(method, val)
|
|
36
33
|
Deimos.karafka_configs.each { |c| c.send(method.to_sym, val) }
|
|
37
34
|
end
|
|
38
35
|
|
|
39
|
-
def register_consumer(klass, schema, namespace='com.my-namespace', key_config:{none: true}, configs: {})
|
|
36
|
+
def register_consumer(klass, schema, namespace='com.my-namespace', key_config: { none: true }, configs: {})
|
|
40
37
|
Karafka::App.routes.redraw do
|
|
41
|
-
topic
|
|
42
|
-
consumer
|
|
43
|
-
schema
|
|
44
|
-
namespace
|
|
45
|
-
key_config
|
|
38
|
+
topic('my-topic') do
|
|
39
|
+
consumer(klass)
|
|
40
|
+
schema(schema)
|
|
41
|
+
namespace(namespace)
|
|
42
|
+
key_config(key_config)
|
|
46
43
|
configs.each do |k, v|
|
|
47
44
|
public_send(k, v)
|
|
48
45
|
end
|
|
@@ -99,7 +96,7 @@ module DbConfigs
|
|
|
99
96
|
# @param topic [String]
|
|
100
97
|
# @param key [String]
|
|
101
98
|
def build_message(payload, topic, key)
|
|
102
|
-
{ payload: payload, topic: topic, key: key}
|
|
99
|
+
{ payload: payload, topic: topic, key: key }
|
|
103
100
|
end
|
|
104
101
|
|
|
105
102
|
DB_OPTIONS = [
|
|
@@ -201,7 +198,7 @@ RSpec.configure do |config|
|
|
|
201
198
|
|
|
202
199
|
config.before(:all) do
|
|
203
200
|
Time.zone = 'Eastern Time (US & Canada)'
|
|
204
|
-
ActiveRecord::Base.logger = Logger.new(
|
|
201
|
+
ActiveRecord::Base.logger = Logger.new(File::NULL)
|
|
205
202
|
ActiveRecord::Base.establish_connection(
|
|
206
203
|
'adapter' => 'sqlite3',
|
|
207
204
|
'database' => 'test.sqlite3'
|
|
@@ -229,7 +226,7 @@ RSpec.configure do |config|
|
|
|
229
226
|
deimos_config.schema.nest_child_schemas = true
|
|
230
227
|
deimos_config.schema.path = File.join(File.expand_path(__dir__), 'schemas')
|
|
231
228
|
deimos_config.schema.registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
|
|
232
|
-
deimos_config.logger = Logger.new(
|
|
229
|
+
deimos_config.logger = Logger.new(File::NULL)
|
|
233
230
|
deimos_config.logger.level = Logger::INFO
|
|
234
231
|
deimos_config.schema.backend = :avro_validation
|
|
235
232
|
deimos_config.schema.generated_class_path = 'spec/schemas'
|
|
@@ -263,7 +260,7 @@ RSpec.shared_context('with widgets') do
|
|
|
263
260
|
end
|
|
264
261
|
|
|
265
262
|
# :nodoc:
|
|
266
|
-
class Widget < ActiveRecord::Base
|
|
263
|
+
class Widget < ActiveRecord::Base # rubocop:disable Lint/ConstantDefinitionInBlock
|
|
267
264
|
# @return [String]
|
|
268
265
|
def generated_id
|
|
269
266
|
'generated_id'
|
|
@@ -287,7 +284,7 @@ RSpec.shared_context('with widget_with_union_types') do
|
|
|
287
284
|
end
|
|
288
285
|
|
|
289
286
|
# :nodoc:
|
|
290
|
-
class WidgetWithUnionType < ActiveRecord::Base
|
|
287
|
+
class WidgetWithUnionType < ActiveRecord::Base # rubocop:disable Lint/ConstantDefinitionInBlock
|
|
291
288
|
# @return [String]
|
|
292
289
|
def generated_id
|
|
293
290
|
'generated_id'
|
|
@@ -300,7 +297,6 @@ RSpec.shared_context('with widget_with_union_types') do
|
|
|
300
297
|
end
|
|
301
298
|
end
|
|
302
299
|
|
|
303
|
-
|
|
304
300
|
RSpec.shared_context('with DB') do
|
|
305
301
|
before(:all) do
|
|
306
302
|
setup_db(self.class.metadata[:db_config] || DbConfigs::DB_OPTIONS.last)
|
|
@@ -346,7 +342,7 @@ end
|
|
|
346
342
|
|
|
347
343
|
RSpec::Matchers.define :match_message do |msg|
|
|
348
344
|
match do |actual|
|
|
349
|
-
|
|
345
|
+
|
|
350
346
|
return false unless actual.is_a?(Hash)
|
|
351
347
|
|
|
352
348
|
actual[:payloads]&.each do |p|
|
|
@@ -354,6 +350,6 @@ RSpec::Matchers.define :match_message do |msg|
|
|
|
354
350
|
p[:payload].delete('message_id')
|
|
355
351
|
end
|
|
356
352
|
expect(actual).to match(a_hash_including(msg))
|
|
357
|
-
|
|
353
|
+
|
|
358
354
|
end
|
|
359
355
|
end
|
|
@@ -44,7 +44,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
|
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
it 'should raise an error if no pollers configured' do
|
|
47
|
-
Deimos.configure {}
|
|
47
|
+
Deimos.configure {} # rubocop:disable Lint/EmptyBlock
|
|
48
48
|
expect { Deimos::Utils::DbPoller.start! }.to raise_error('No pollers configured!')
|
|
49
49
|
end
|
|
50
50
|
|
|
@@ -159,17 +159,17 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
|
|
|
159
159
|
|
|
160
160
|
# run_every is set to 1 minute
|
|
161
161
|
travel_to time_value(secs: 62)
|
|
162
|
-
expect(poller.should_run?).to
|
|
162
|
+
expect(poller.should_run?).to be(true)
|
|
163
163
|
|
|
164
164
|
travel_to time_value(secs: 30)
|
|
165
|
-
expect(poller.should_run?).to
|
|
165
|
+
expect(poller.should_run?).to be(false)
|
|
166
166
|
|
|
167
167
|
travel_to time_value(mins: -1) # this shouldn't be possible but meh
|
|
168
|
-
expect(poller.should_run?).to
|
|
168
|
+
expect(poller.should_run?).to be(false)
|
|
169
169
|
|
|
170
170
|
# take the 2 seconds of delay_time into account
|
|
171
171
|
travel_to time_value(secs: 60)
|
|
172
|
-
expect(poller.should_run?).to
|
|
172
|
+
expect(poller.should_run?).to be(false)
|
|
173
173
|
end
|
|
174
174
|
|
|
175
175
|
describe '#process_batch' do
|
|
@@ -52,10 +52,7 @@ RSpec.describe Deimos::Utils::DeadlockRetry do
|
|
|
52
52
|
Widget.create(test_id: 'second')
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
-
expect(Widget.all).to
|
|
56
|
-
have_attributes(test_id: 'first'),
|
|
57
|
-
have_attributes(test_id: 'second')
|
|
58
|
-
])
|
|
55
|
+
expect(Widget.all).to contain_exactly(have_attributes(test_id: 'first'), have_attributes(test_id: 'second'))
|
|
59
56
|
end
|
|
60
57
|
|
|
61
58
|
it 'should not retry non-deadlock exceptions' do
|
|
@@ -7,7 +7,7 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
7
7
|
producer
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
-
let(:logger) { instance_double(Logger, error: nil, info: nil, debug: nil
|
|
10
|
+
let(:logger) { instance_double(Logger, error: nil, info: nil, debug: nil) }
|
|
11
11
|
|
|
12
12
|
before(:each) do
|
|
13
13
|
stub_const('Deimos::Utils::OutboxProducer::BATCH_SIZE', 2)
|
|
@@ -15,11 +15,15 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
specify '#process_next_messages' do
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
allow(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
|
|
19
|
+
allow(producer).to receive(:process_topic)
|
|
20
|
+
allow(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics)
|
|
21
|
+
allow(producer).to receive(:sleep)
|
|
22
22
|
producer.process_next_messages
|
|
23
|
+
expect(producer).to have_received(:retrieve_topics)
|
|
24
|
+
expect(producer).to have_received(:process_topic).twice
|
|
25
|
+
expect(Deimos::KafkaTopicInfo).to have_received(:ping_empty_topics).with(%w(topic1 topic2))
|
|
26
|
+
expect(producer).to have_received(:sleep).with(0.5)
|
|
23
27
|
end
|
|
24
28
|
|
|
25
29
|
specify '#retrieve_topics' do
|
|
@@ -45,7 +49,7 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
45
49
|
producer.current_topic = 'topic1'
|
|
46
50
|
messages = producer.retrieve_messages
|
|
47
51
|
expect(messages.size).to eq(3)
|
|
48
|
-
expect(messages).to all(
|
|
52
|
+
expect(messages).to all(be_a(Deimos::KafkaMessage))
|
|
49
53
|
end
|
|
50
54
|
|
|
51
55
|
describe '#produce_messages' do
|
|
@@ -125,13 +129,16 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
125
129
|
end
|
|
126
130
|
|
|
127
131
|
it 'should do nothing if lock fails' do
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
expect(producer).not_to receive(:retrieve_messages)
|
|
132
|
+
allow(Deimos::KafkaTopicInfo).to receive(:lock).and_return(false)
|
|
133
|
+
allow(producer).to receive(:retrieve_messages)
|
|
131
134
|
producer.process_topic('my-topic')
|
|
135
|
+
expect(Deimos::KafkaTopicInfo).to have_received(:lock).with('my-topic', 'abc')
|
|
136
|
+
expect(producer).not_to have_received(:retrieve_messages)
|
|
132
137
|
end
|
|
133
138
|
|
|
134
|
-
|
|
139
|
+
# using expect(x).to have_received(:y).ordered doesn't work
|
|
140
|
+
# rubocop:disable RSpec/StubbedMock
|
|
141
|
+
it 'should complete successfully' do # rubocop:disable RSpec/ExampleLength
|
|
135
142
|
messages = (1..4).map do |i|
|
|
136
143
|
Deimos::KafkaMessage.new(
|
|
137
144
|
topic: 'my-topic',
|
|
@@ -196,13 +203,16 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
196
203
|
with('my-topic', 'abc').once
|
|
197
204
|
producer.process_topic('my-topic')
|
|
198
205
|
end
|
|
206
|
+
# rubocop:enable RSpec/StubbedMock
|
|
199
207
|
|
|
200
208
|
it 'should register an error if it gets an error' do
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
expect(producer).not_to receive(:produce_messages)
|
|
209
|
+
allow(producer).to receive(:retrieve_messages).and_raise('OH NOES')
|
|
210
|
+
allow(Deimos::KafkaTopicInfo).to receive(:register_error)
|
|
211
|
+
allow(producer).to receive(:produce_messages)
|
|
205
212
|
producer.process_topic('my-topic')
|
|
213
|
+
expect(producer).to have_received(:retrieve_messages)
|
|
214
|
+
expect(Deimos::KafkaTopicInfo).to have_received(:register_error).with('my-topic', 'abc')
|
|
215
|
+
expect(producer).not_to have_received(:produce_messages)
|
|
206
216
|
end
|
|
207
217
|
|
|
208
218
|
it 'should move on if it gets a partial batch' do
|
|
@@ -225,10 +235,9 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
225
235
|
)
|
|
226
236
|
end
|
|
227
237
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
expect(producer).to receive(:retrieve_messages).and_return(messages)
|
|
238
|
+
allow(Deimos::KafkaTopicInfo).to receive(:lock).and_return(true)
|
|
239
|
+
allow(producer).to receive(:produce_messages).and_raise('OH NOES')
|
|
240
|
+
allow(producer).to receive(:retrieve_messages).and_return(messages)
|
|
232
241
|
expect(Deimos::KafkaTopicInfo).to receive(:register_error)
|
|
233
242
|
|
|
234
243
|
expect(Deimos::KafkaMessage.count).to eq(4)
|
|
@@ -237,6 +246,9 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
237
246
|
expect(event.payload[:messages]).to eq(messages)
|
|
238
247
|
end
|
|
239
248
|
producer.process_topic('my-topic')
|
|
249
|
+
expect(Deimos::KafkaTopicInfo).to have_received(:lock).with('my-topic', 'abc')
|
|
250
|
+
expect(producer).to have_received(:produce_messages)
|
|
251
|
+
expect(producer).to have_received(:retrieve_messages)
|
|
240
252
|
# don't delete for regular errors
|
|
241
253
|
expect(Deimos::KafkaMessage.count).to eq(4)
|
|
242
254
|
Karafka.monitor.notifications_bus.clear('deimos.outbox.produce')
|
|
@@ -269,14 +281,14 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
269
281
|
m.call(*args)
|
|
270
282
|
end
|
|
271
283
|
|
|
272
|
-
|
|
273
|
-
with('my-topic', 'abc').and_return(true)
|
|
284
|
+
allow(Deimos::KafkaTopicInfo).to receive(:lock).and_return(true)
|
|
274
285
|
expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
|
|
275
286
|
expect(producer).to receive(:retrieve_messages).ordered.and_return([])
|
|
276
287
|
expect(Karafka.producer).to receive(:produce_many_sync).once.with(messages.map(&:karafka_message))
|
|
277
288
|
|
|
278
289
|
expect(Deimos::KafkaMessage.count).to eq(8)
|
|
279
290
|
producer.process_topic('my-topic')
|
|
291
|
+
expect(Deimos::KafkaTopicInfo).to have_received(:lock).with('my-topic', 'abc')
|
|
280
292
|
expect(Deimos::KafkaMessage.count).to eq(4)
|
|
281
293
|
end
|
|
282
294
|
|
|
@@ -340,9 +352,9 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
340
352
|
end
|
|
341
353
|
|
|
342
354
|
example 'Full integration test' do
|
|
343
|
-
Deimos::KafkaMessage.create!(topic:
|
|
344
|
-
message:
|
|
345
|
-
partition_key:
|
|
355
|
+
Deimos::KafkaMessage.create!(topic: 'topic1',
|
|
356
|
+
message: 'mess1',
|
|
357
|
+
partition_key: 'key1')
|
|
346
358
|
producer.process_next_messages
|
|
347
359
|
expect(Deimos::KafkaTopicInfo.count).to eq(1)
|
|
348
360
|
expect(Deimos::KafkaTopicInfo.first.topic).to eq('topic1')
|
|
@@ -350,7 +362,7 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
350
362
|
expect('topic1').to have_sent('mess1')
|
|
351
363
|
end
|
|
352
364
|
|
|
353
|
-
example 'Integration test - batching' do
|
|
365
|
+
example 'Integration test - batching' do # rubocop:disable RSpec/ExampleLength
|
|
354
366
|
(1..4).each do |i|
|
|
355
367
|
(1..2).each do |j|
|
|
356
368
|
Deimos::KafkaMessage.create!(topic: "topic#{j}",
|