deimos-ruby 2.4.0.pre.beta9 → 2.4.0.pre.beta10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/deimos/config/configuration.rb +3 -0
- data/lib/deimos/schema_backends/base.rb +5 -0
- data/lib/deimos/schema_backends/proto_schema_registry.rb +1 -1
- data/lib/deimos/test_helpers.rb +1 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/tasks/deimos.rake +1 -1
- data/spec/deimos_spec.rb +70 -0
- data/spec/gen/sample/v1/sample_pb.rb +5 -5
- data/spec/message_spec.rb +129 -0
- data/spec/protos/sample/v1/sample.proto +1 -0
- data/spec/rake_spec.rb +173 -0
- data/spec/schema_backends/proto_schema_registry_spec.rb +6 -0
- data/spec/test_helpers_spec.rb +75 -0
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2bfb935df5a67962364fbb6774c4dd03e2e56cbdb4d6dad7d15cb034b9522dee
|
|
4
|
+
data.tar.gz: c55979be4de60e902b02f80c48bce656d550d89fdb712938462d1f8b1af1f38b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9fc0b3ed398049f73002d77c8b103f69b1b569616a9bfa960921edc3947702fc3affdc7d39676db43e8e935df2e08b2bb97726ea93060503d3f53f00a36a79bc
|
|
7
|
+
data.tar.gz: cae1f740a93966da3076095c1adbb1e9fd125ee4531947d15913d98795d460c21d3a98a6840137ef40a1d8d13374833065a59ee6e04196cb58e1c9b3f4acc1b8
|
|
@@ -190,6 +190,9 @@ module Deimos
|
|
|
190
190
|
# would replace a prefixed with the given key with the module name SchemaClasses.
|
|
191
191
|
# @return [Hash]
|
|
192
192
|
setting :schema_namespace_map, {}
|
|
193
|
+
|
|
194
|
+
# The base directory for generated protobuf key schemas.
|
|
195
|
+
setting :proto_schema_key_path, 'protos'
|
|
193
196
|
end
|
|
194
197
|
|
|
195
198
|
# The configured metrics provider.
|
|
@@ -41,6 +41,11 @@ module Deimos
|
|
|
41
41
|
@namespace = namespace
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
+
# @return [String]
|
|
45
|
+
def inspect
|
|
46
|
+
"Type #{self.class.name.demodulize} Schema: #{@namespace}.#{@schema} Key schema: #{@key_schema}"
|
|
47
|
+
end
|
|
48
|
+
|
|
44
49
|
# @return [Boolean]
|
|
45
50
|
def supports_key_schemas?
|
|
46
51
|
false
|
|
@@ -74,7 +74,7 @@ module Deimos
|
|
|
74
74
|
writer.indent
|
|
75
75
|
SchemaRegistry::Output::ProtoText.write_field(info, field)
|
|
76
76
|
writer.dedent
|
|
77
|
-
writer.write_line(
|
|
77
|
+
writer.write_line('}')
|
|
78
78
|
path = "#{file}/#{package.gsub('.', '/')}/#{proto.to_proto.name.underscore}_key.proto"
|
|
79
79
|
File.write(path, writer.string)
|
|
80
80
|
end
|
data/lib/deimos/test_helpers.rb
CHANGED
data/lib/deimos/version.rb
CHANGED
data/lib/tasks/deimos.rake
CHANGED
|
@@ -51,7 +51,7 @@ namespace :deimos do
|
|
|
51
51
|
|
|
52
52
|
puts "Writing key proto for #{config.name}"
|
|
53
53
|
backend = Deimos.schema_backend_for(config.name)
|
|
54
|
-
backend.write_key_proto(
|
|
54
|
+
backend.write_key_proto(Deimos.config.schema.proto_schema_key_path, config.deserializers[:key].key_field)
|
|
55
55
|
end
|
|
56
56
|
end
|
|
57
57
|
end
|
data/spec/deimos_spec.rb
CHANGED
|
@@ -82,4 +82,74 @@ describe Deimos do
|
|
|
82
82
|
expect(producer3.config.kafka[:'bootstrap.servers']).to eq('broker2:9092')
|
|
83
83
|
end
|
|
84
84
|
|
|
85
|
+
describe '#schema_backend_for' do
|
|
86
|
+
it 'should return a schema backend for a given topic' do
|
|
87
|
+
Karafka::App.routes.redraw do
|
|
88
|
+
topic 'backend-test-topic' do
|
|
89
|
+
active false
|
|
90
|
+
schema 'MySchema'
|
|
91
|
+
namespace 'com.my-namespace'
|
|
92
|
+
key_config none: true
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
backend = described_class.schema_backend_for('backend-test-topic')
|
|
97
|
+
expect(backend.inspect).to eq('Type AvroValidation Schema: com.my-namespace.MySchema Key schema: ')
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
it 'should use topic-specific schema_backend if configured' do
|
|
101
|
+
Karafka::App.routes.redraw do
|
|
102
|
+
topic 'backend-test-topic-local' do
|
|
103
|
+
active false
|
|
104
|
+
schema 'MySchema'
|
|
105
|
+
namespace 'com.my-namespace'
|
|
106
|
+
key_config none: true
|
|
107
|
+
schema_backend :avro_local
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
backend = described_class.schema_backend_for('backend-test-topic-local')
|
|
112
|
+
expect(backend).to be_a(Deimos::SchemaBackends::AvroLocal)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
it 'should fall back to global schema backend config' do
|
|
116
|
+
described_class.config.schema.backend = :avro_validation
|
|
117
|
+
Karafka::App.routes.redraw do
|
|
118
|
+
topic 'backend-test-topic-global' do
|
|
119
|
+
active false
|
|
120
|
+
schema 'MySchema'
|
|
121
|
+
namespace 'com.my-namespace'
|
|
122
|
+
key_config none: true
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
backend = described_class.schema_backend_for('backend-test-topic-global')
|
|
127
|
+
expect(backend).to be_a(Deimos::SchemaBackends::AvroValidation)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
describe '#schema_backend_class with mock_backends' do
|
|
132
|
+
after(:each) do
|
|
133
|
+
described_class.mock_backends = false
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
it 'should return the mock backend when mock_backends is true' do
|
|
137
|
+
described_class.mock_backends = true
|
|
138
|
+
klass = described_class.schema_backend_class(backend: :avro_schema_registry)
|
|
139
|
+
expect(klass).to eq(Deimos::SchemaBackends::AvroValidation)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
it 'should return the real backend when mock_backends is false' do
|
|
143
|
+
described_class.mock_backends = false
|
|
144
|
+
klass = described_class.schema_backend_class(backend: :avro_schema_registry)
|
|
145
|
+
expect(klass).to eq(Deimos::SchemaBackends::AvroSchemaRegistry)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
it 'should use proto_local for proto_schema_registry when mocked' do
|
|
149
|
+
described_class.mock_backends = true
|
|
150
|
+
klass = described_class.schema_backend_class(backend: :proto_schema_registry)
|
|
151
|
+
expect(klass).to eq(Deimos::SchemaBackends::ProtoLocal)
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
85
155
|
end
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
-
|
|
3
2
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
4
3
|
# source: sample/v1/sample.proto
|
|
5
4
|
|
|
@@ -7,14 +6,15 @@ require 'google/protobuf'
|
|
|
7
6
|
|
|
8
7
|
require 'google/protobuf/timestamp_pb'
|
|
9
8
|
|
|
10
|
-
descriptor_data = "\n\x16sample/v1/sample.proto\x12\tsample.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"7\n\rNestedMessage\x12\x12\n\nnested_str\x18\x01 \x01(\t\x12\x12\n\nnested_num\x18\x02 \x01(\x05\"\xdb\x02\n\rSampleMessage\x12\x0b\n\x03str\x18\x01 \x01(\t\x12\x0b\n\x03num\x18\x02 \x01(\x05\x12\x0f\n\x07str_arr\x18\x03 \x03(\t\x12\x0c\n\x04\x66lag\x18\x04 \x01(\x08\x12-\n\ttimestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x06nested\x18\x06 \x01(\x0b\x32\x18.sample.v1.NestedMessageH\x00\x12\x13\n\tunion_str\x18\x07 \x01(\tH\x00\x12\x32\n\x10non_union_nested\x18\x08 \x01(\x0b\x32\x18.sample.v1.NestedMessage\x12\x35\n\x07str_map\x18\t \x03(\x0b\x32$.sample.v1.SampleMessage.StrMapEntry\x1a-\n\x0bStrMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x07\n\x05unionb\x06proto3"
|
|
11
9
|
|
|
12
|
-
|
|
10
|
+
descriptor_data = "\n\x16sample/v1/sample.proto\x12\tsample.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"7\n\rNestedMessage\x12\x12\n\nnested_str\x18\x01 \x01(\t\x12\x12\n\nnested_num\x18\x02 \x01(\x05\"\xef\x02\n\rSampleMessage\x12\x0b\n\x03str\x18\x01 \x01(\t\x12\x0b\n\x03num\x18\x02 \x01(\x05\x12\x0f\n\x07str_arr\x18\x03 \x03(\t\x12\x0c\n\x04\x66lag\x18\x04 \x01(\x08\x12-\n\ttimestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x06nested\x18\x06 \x01(\x0b\x32\x18.sample.v1.NestedMessageH\x00\x12\x13\n\tunion_str\x18\x07 \x01(\tH\x00\x12\x32\n\x10non_union_nested\x18\x08 \x01(\x0b\x32\x18.sample.v1.NestedMessage\x12\x35\n\x07str_map\x18\t \x03(\x0b\x32$.sample.v1.SampleMessage.StrMapEntry\x12\x12\n\nmessage_id\x18\n \x01(\t\x1a-\n\x0bStrMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x07\n\x05unionb\x06proto3"
|
|
11
|
+
|
|
12
|
+
pool = ::Google::Protobuf::DescriptorPool.generated_pool
|
|
13
13
|
pool.add_serialized_file(descriptor_data)
|
|
14
14
|
|
|
15
15
|
module Sample
|
|
16
16
|
module V1
|
|
17
|
-
NestedMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(
|
|
18
|
-
SampleMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(
|
|
17
|
+
NestedMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("sample.v1.NestedMessage").msgclass
|
|
18
|
+
SampleMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("sample.v1.SampleMessage").msgclass
|
|
19
19
|
end
|
|
20
20
|
end
|
data/spec/message_spec.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'gen/sample/v1/sample_pb'
|
|
4
|
+
|
|
3
5
|
RSpec.describe(Deimos::Message) do
|
|
4
6
|
it 'should detect tombstones' do
|
|
5
7
|
expect(described_class.new(nil, key: 'key1')).
|
|
@@ -36,4 +38,131 @@ RSpec.describe(Deimos::Message) do
|
|
|
36
38
|
to include(headers: { a: 1 })
|
|
37
39
|
end
|
|
38
40
|
end
|
|
41
|
+
|
|
42
|
+
describe '#add_fields' do
|
|
43
|
+
context 'with Hash payloads' do
|
|
44
|
+
it 'should add message_id if not present' do
|
|
45
|
+
message = described_class.new({ some_field: 'value' }, key: 'key1')
|
|
46
|
+
message.add_fields(%w(message_id timestamp))
|
|
47
|
+
expect(message.payload['message_id']).to be_present
|
|
48
|
+
expect(message.payload['message_id']).to match(/\A[0-9a-f-]{36}\z/)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
it 'should not overwrite existing message_id' do
|
|
52
|
+
message = described_class.new({ some_field: 'value', message_id: 'existing-id' }, key: 'key1')
|
|
53
|
+
message.add_fields(%w(message_id timestamp))
|
|
54
|
+
expect(message.payload['message_id']).to eq('existing-id')
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
it 'should add timestamp if not present' do
|
|
58
|
+
freeze_time = Time.zone.local(2025, 1, 1, 12, 0, 0)
|
|
59
|
+
travel_to(freeze_time) do
|
|
60
|
+
message = described_class.new({ some_field: 'value' }, key: 'key1')
|
|
61
|
+
message.add_fields(%w(message_id timestamp))
|
|
62
|
+
expect(message.payload['timestamp']).to be_present
|
|
63
|
+
expect(message.payload['timestamp']).to include('2025')
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
it 'should not overwrite existing timestamp' do
|
|
68
|
+
message = described_class.new({ some_field: 'value', timestamp: '2020-01-01' }, key: 'key1')
|
|
69
|
+
message.add_fields(%w(message_id timestamp))
|
|
70
|
+
expect(message.payload['timestamp']).to eq('2020-01-01')
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it 'should not add fields that are not in the list' do
|
|
74
|
+
message = described_class.new({ some_field: 'value' }, key: 'key1')
|
|
75
|
+
message.add_fields(%w(other_field))
|
|
76
|
+
expect(message.payload['message_id']).to be_nil
|
|
77
|
+
expect(message.payload['timestamp']).to be_nil
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
it 'should do nothing for empty payload' do
|
|
81
|
+
message = described_class.new({}, key: 'key1')
|
|
82
|
+
message.add_fields(%w(message_id timestamp))
|
|
83
|
+
expect(message.payload['message_id']).to be_nil
|
|
84
|
+
expect(message.payload['timestamp']).to be_nil
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
it 'should do nothing for nil payload' do
|
|
88
|
+
message = described_class.new(nil, key: 'key1')
|
|
89
|
+
message.add_fields(%w(message_id timestamp))
|
|
90
|
+
expect(message.payload).to be_nil
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
context 'with Protobuf payloads' do
|
|
95
|
+
it 'should add message_id to protobuf objects if not present' do
|
|
96
|
+
proto_payload = Sample::V1::SampleMessage.new(str: 'test', num: 123)
|
|
97
|
+
expect(proto_payload.message_id).to eq('')
|
|
98
|
+
|
|
99
|
+
message = described_class.new(proto_payload, key: 'key1')
|
|
100
|
+
message.add_fields(%w(message_id))
|
|
101
|
+
|
|
102
|
+
expect(proto_payload.message_id).to match(/\A[0-9a-f-]{36}\z/)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
it 'should not overwrite existing message_id on protobuf objects' do
|
|
106
|
+
proto_payload = Sample::V1::SampleMessage.new(str: 'test', num: 123, message_id: 'existing-id')
|
|
107
|
+
|
|
108
|
+
message = described_class.new(proto_payload, key: 'key1')
|
|
109
|
+
message.add_fields(%w(message_id))
|
|
110
|
+
|
|
111
|
+
expect(proto_payload.message_id).to eq('existing-id')
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
it 'should add timestamp to protobuf objects if not present' do
|
|
115
|
+
proto_payload = Sample::V1::SampleMessage.new(str: 'test', num: 123)
|
|
116
|
+
expect(proto_payload.timestamp).to be_nil
|
|
117
|
+
|
|
118
|
+
message = described_class.new(proto_payload, key: 'key1')
|
|
119
|
+
message.add_fields(%w(timestamp))
|
|
120
|
+
|
|
121
|
+
expect(proto_payload.timestamp).to be_a(Google::Protobuf::Timestamp)
|
|
122
|
+
expect(proto_payload.timestamp.seconds).to be > 0
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
it 'should not overwrite existing timestamp on protobuf objects' do
|
|
126
|
+
existing_time = Google::Protobuf::Timestamp.new(seconds: 1_577_836_800) # 2020-01-01
|
|
127
|
+
proto_payload = Sample::V1::SampleMessage.new(str: 'test', num: 123, timestamp: existing_time)
|
|
128
|
+
|
|
129
|
+
message = described_class.new(proto_payload, key: 'key1')
|
|
130
|
+
message.add_fields(%w(timestamp))
|
|
131
|
+
|
|
132
|
+
expect(proto_payload.timestamp.seconds).to eq(1_577_836_800)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
it 'should add both message_id and timestamp to protobuf objects' do
|
|
136
|
+
proto_payload = Sample::V1::SampleMessage.new(str: 'test', num: 123)
|
|
137
|
+
|
|
138
|
+
message = described_class.new(proto_payload, key: 'key1')
|
|
139
|
+
message.add_fields(%w(message_id timestamp))
|
|
140
|
+
|
|
141
|
+
expect(proto_payload.message_id).to match(/\A[0-9a-f-]{36}\z/)
|
|
142
|
+
expect(proto_payload.timestamp).to be_a(Google::Protobuf::Timestamp)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
it 'should not modify protobuf when field not in list' do
|
|
146
|
+
proto_payload = Sample::V1::SampleMessage.new(str: 'test', num: 123)
|
|
147
|
+
|
|
148
|
+
message = described_class.new(proto_payload, key: 'key1')
|
|
149
|
+
message.add_fields(%w(other_field))
|
|
150
|
+
|
|
151
|
+
expect(proto_payload.message_id).to eq('')
|
|
152
|
+
expect(proto_payload.timestamp).to be_nil
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
it 'should handle protobuf with nested messages' do
|
|
156
|
+
nested = Sample::V1::NestedMessage.new(nested_str: 'nested', nested_num: 456)
|
|
157
|
+
proto_payload = Sample::V1::SampleMessage.new(str: 'test', num: 123, non_union_nested: nested)
|
|
158
|
+
|
|
159
|
+
message = described_class.new(proto_payload, key: 'key1')
|
|
160
|
+
message.add_fields(%w(message_id timestamp))
|
|
161
|
+
|
|
162
|
+
expect(proto_payload.message_id).to match(/\A[0-9a-f-]{36}\z/)
|
|
163
|
+
expect(proto_payload.timestamp).to be_a(Google::Protobuf::Timestamp)
|
|
164
|
+
expect(proto_payload.non_union_nested.nested_str).to eq('nested')
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
39
168
|
end
|
data/spec/rake_spec.rb
CHANGED
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
require 'rake'
|
|
4
4
|
require 'rails'
|
|
5
|
+
require 'deimos/schema_backends/proto_schema_registry'
|
|
6
|
+
require 'deimos/schema_backends/avro_validation'
|
|
7
|
+
require_relative 'gen/sample/v1/sample_pb'
|
|
8
|
+
require 'fileutils'
|
|
9
|
+
require 'tmpdir'
|
|
10
|
+
|
|
5
11
|
Rails.logger = Logger.new(STDOUT)
|
|
6
12
|
load("#{__dir__}/../lib/tasks/deimos.rake")
|
|
7
13
|
|
|
@@ -14,4 +20,171 @@ describe 'Rakefile' do
|
|
|
14
20
|
expect(Karafka::Server).to receive(:run)
|
|
15
21
|
Rake::Task['deimos:start'].invoke
|
|
16
22
|
end
|
|
23
|
+
|
|
24
|
+
describe 'generate_key_protos' do
|
|
25
|
+
let(:temp_dir) { Dir.mktmpdir('deimos_rake_test') }
|
|
26
|
+
|
|
27
|
+
before(:each) do
|
|
28
|
+
Rake::Task['deimos:generate_key_protos'].reenable
|
|
29
|
+
# Create the package directory structure
|
|
30
|
+
FileUtils.mkdir_p(File.join(temp_dir, 'sample/v1'))
|
|
31
|
+
# Configure Deimos to use the temp directory
|
|
32
|
+
Deimos.config.schema.proto_schema_key_path = temp_dir
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
after(:each) do
|
|
36
|
+
FileUtils.rm_rf(temp_dir)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
it 'should skip configs without producer_class' do
|
|
40
|
+
Karafka::App.routes.redraw do
|
|
41
|
+
topic 'no-producer-topic' do
|
|
42
|
+
active false
|
|
43
|
+
schema 'sample.v1.SampleMessage'
|
|
44
|
+
namespace 'sample.v1'
|
|
45
|
+
key_config field: 'str'
|
|
46
|
+
schema_backend :proto_schema_registry
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
Rake::Task['deimos:generate_key_protos'].invoke
|
|
51
|
+
|
|
52
|
+
# No files should be written since there's no producer_class
|
|
53
|
+
expect(Dir.glob("#{temp_dir}/**/*.proto")).to be_empty
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
it 'should skip non-protobuf backends' do
|
|
57
|
+
producer_class = Class.new(Deimos::Producer)
|
|
58
|
+
stub_const('AvroProducer', producer_class)
|
|
59
|
+
|
|
60
|
+
Karafka::App.routes.redraw do
|
|
61
|
+
topic 'avro-topic' do
|
|
62
|
+
active false
|
|
63
|
+
schema 'MySchema'
|
|
64
|
+
namespace 'com.my-namespace'
|
|
65
|
+
key_config field: 'test_id'
|
|
66
|
+
producer_class AvroProducer
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
Rake::Task['deimos:generate_key_protos'].invoke
|
|
71
|
+
|
|
72
|
+
# No files should be written for non-protobuf backends
|
|
73
|
+
expect(Dir.glob("#{temp_dir}/**/*.proto")).to be_empty
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
it 'should skip configs without key_field' do
|
|
77
|
+
producer_class = Class.new(Deimos::Producer)
|
|
78
|
+
stub_const('NoKeyProducer', producer_class)
|
|
79
|
+
|
|
80
|
+
Karafka::App.routes.redraw do
|
|
81
|
+
topic 'no-key-topic' do
|
|
82
|
+
active false
|
|
83
|
+
schema 'sample.v1.SampleMessage'
|
|
84
|
+
namespace 'sample.v1'
|
|
85
|
+
key_config none: true
|
|
86
|
+
producer_class NoKeyProducer
|
|
87
|
+
schema_backend :proto_schema_registry
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
Rake::Task['deimos:generate_key_protos'].invoke
|
|
92
|
+
|
|
93
|
+
# No files should be written since there's no key_field
|
|
94
|
+
expect(Dir.glob("#{temp_dir}/**/*.proto")).to be_empty
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
it 'should write key proto for protobuf topics with string key_field' do
|
|
98
|
+
producer_class = Class.new(Deimos::Producer)
|
|
99
|
+
stub_const('ProtoProducer', producer_class)
|
|
100
|
+
|
|
101
|
+
Karafka::App.routes.redraw do
|
|
102
|
+
topic 'proto-topic' do
|
|
103
|
+
active false
|
|
104
|
+
schema 'sample.v1.SampleMessage'
|
|
105
|
+
namespace 'sample.v1'
|
|
106
|
+
key_config field: 'str'
|
|
107
|
+
producer_class ProtoProducer
|
|
108
|
+
schema_backend :proto_schema_registry
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
Rake::Task['deimos:generate_key_protos'].invoke
|
|
113
|
+
|
|
114
|
+
# Verify the file was created with correct content
|
|
115
|
+
proto_file = File.join(temp_dir, 'sample/v1/sample_message_key.proto')
|
|
116
|
+
expect(File).to exist(proto_file)
|
|
117
|
+
|
|
118
|
+
content = File.read(proto_file)
|
|
119
|
+
expect(content).to include('syntax = "proto3";')
|
|
120
|
+
expect(content).to include('package sample.v1;')
|
|
121
|
+
expect(content).to include('message SampleMessageKey {')
|
|
122
|
+
expect(content).to include('string str = 1;')
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
it 'should write key proto for protobuf topics with int32 key_field' do
|
|
126
|
+
producer_class = Class.new(Deimos::Producer)
|
|
127
|
+
stub_const('ProtoProducer', producer_class)
|
|
128
|
+
|
|
129
|
+
Karafka::App.routes.redraw do
|
|
130
|
+
topic 'proto-topic-int' do
|
|
131
|
+
active false
|
|
132
|
+
schema 'sample.v1.SampleMessage'
|
|
133
|
+
namespace 'sample.v1'
|
|
134
|
+
key_config field: 'num'
|
|
135
|
+
producer_class ProtoProducer
|
|
136
|
+
schema_backend :proto_schema_registry
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
Rake::Task['deimos:generate_key_protos'].invoke
|
|
141
|
+
|
|
142
|
+
proto_file = File.join(temp_dir, 'sample/v1/sample_message_key.proto')
|
|
143
|
+
expect(File).to exist(proto_file)
|
|
144
|
+
|
|
145
|
+
content = File.read(proto_file)
|
|
146
|
+
expect(content).to include('syntax = "proto3";')
|
|
147
|
+
expect(content).to include('package sample.v1;')
|
|
148
|
+
expect(content).to include('message SampleMessageKey {')
|
|
149
|
+
expect(content).to include('int32 num = 2;')
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
it 'should process multiple topics' do
|
|
153
|
+
producer_class1 = Class.new(Deimos::Producer)
|
|
154
|
+
producer_class2 = Class.new(Deimos::Producer)
|
|
155
|
+
stub_const('ProtoProducer1', producer_class1)
|
|
156
|
+
stub_const('ProtoProducer2', producer_class2)
|
|
157
|
+
|
|
158
|
+
Karafka::App.routes.redraw do
|
|
159
|
+
topic 'proto-topic-1' do
|
|
160
|
+
active false
|
|
161
|
+
schema 'sample.v1.SampleMessage'
|
|
162
|
+
namespace 'sample.v1'
|
|
163
|
+
key_config field: 'str'
|
|
164
|
+
producer_class ProtoProducer1
|
|
165
|
+
schema_backend :proto_schema_registry
|
|
166
|
+
end
|
|
167
|
+
topic 'proto-topic-2' do
|
|
168
|
+
active false
|
|
169
|
+
schema 'sample.v1.SampleMessage'
|
|
170
|
+
namespace 'sample.v1'
|
|
171
|
+
key_config field: 'num'
|
|
172
|
+
producer_class ProtoProducer2
|
|
173
|
+
schema_backend :proto_schema_registry
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
Rake::Task['deimos:generate_key_protos'].invoke
|
|
178
|
+
|
|
179
|
+
# Both topics use the same schema, so only one file is created
|
|
180
|
+
# but it should be called twice (once per topic)
|
|
181
|
+
proto_file = File.join(temp_dir, 'sample/v1/sample_message_key.proto')
|
|
182
|
+
expect(File).to exist(proto_file)
|
|
183
|
+
|
|
184
|
+
# The second topic overwrites the first, so we should see the 'num' field
|
|
185
|
+
content = File.read(proto_file)
|
|
186
|
+
expect(content).to include('message SampleMessageKey {')
|
|
187
|
+
expect(content).to include('int32 num = 2;')
|
|
188
|
+
end
|
|
189
|
+
end
|
|
17
190
|
end
|
|
@@ -247,4 +247,10 @@ RSpec.describe Deimos::SchemaBackends::ProtoSchemaRegistry do
|
|
|
247
247
|
expect(fields.map(&:name)).to include('str', 'num', 'str_arr', 'flag', 'timestamp')
|
|
248
248
|
end
|
|
249
249
|
end
|
|
250
|
+
|
|
251
|
+
describe '.mock_backend' do
|
|
252
|
+
it 'should return :proto_local' do
|
|
253
|
+
expect(described_class.mock_backend).to eq(:proto_local)
|
|
254
|
+
end
|
|
255
|
+
end
|
|
250
256
|
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'deimos/test_helpers'
|
|
4
|
+
|
|
5
|
+
RSpec.describe Deimos::TestHelpers do
|
|
6
|
+
|
|
7
|
+
describe '.with_mock_backends' do
|
|
8
|
+
before(:each) do
|
|
9
|
+
Karafka::App.routes.redraw do
|
|
10
|
+
topic 'mock-backend-test' do
|
|
11
|
+
active false
|
|
12
|
+
schema 'MySchema'
|
|
13
|
+
namespace 'com.my-namespace'
|
|
14
|
+
key_config field: 'test_id'
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
after(:each) do
|
|
20
|
+
Deimos.mock_backends = false
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
it 'should set mock_backends to true during the block' do
|
|
24
|
+
expect(Deimos.mock_backends).to be_falsey
|
|
25
|
+
|
|
26
|
+
described_class.with_mock_backends do
|
|
27
|
+
expect(Deimos.mock_backends).to be(true)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
expect(Deimos.mock_backends).to be(false)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it 'should reset mock_backends after the block completes' do
|
|
34
|
+
described_class.with_mock_backends do
|
|
35
|
+
# noop
|
|
36
|
+
end
|
|
37
|
+
expect(Deimos.mock_backends).to be(false)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
it 'should reset mock_backends even if an error occurs in the block' do
|
|
41
|
+
expect {
|
|
42
|
+
described_class.with_mock_backends do
|
|
43
|
+
raise 'Test error'
|
|
44
|
+
end
|
|
45
|
+
}.to raise_error('Test error')
|
|
46
|
+
expect(Deimos.mock_backends).to be(false)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
it 'should reset backends on deserializers' do
|
|
50
|
+
config = Deimos.karafka_config_for(topic: 'mock-backend-test')
|
|
51
|
+
payload_transcoder = config.deserializers[:payload]
|
|
52
|
+
key_transcoder = config.deserializers[:key]
|
|
53
|
+
|
|
54
|
+
expect(payload_transcoder).to receive(:reset_backend)
|
|
55
|
+
expect(key_transcoder).to receive(:reset_backend) if key_transcoder.respond_to?(:reset_backend)
|
|
56
|
+
|
|
57
|
+
described_class.with_mock_backends do
|
|
58
|
+
# noop
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
it 'should use mock backends for schema_backend_class' do
|
|
63
|
+
described_class.with_mock_backends do
|
|
64
|
+
# avro_schema_registry should return avro_validation when mocked
|
|
65
|
+
klass = Deimos.schema_backend_class(backend: :avro_schema_registry)
|
|
66
|
+
expect(klass).to eq(Deimos::SchemaBackends::AvroValidation)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Outside the block, should return the real class
|
|
70
|
+
klass = Deimos.schema_backend_class(backend: :avro_schema_registry)
|
|
71
|
+
expect(klass).to eq(Deimos::SchemaBackends::AvroSchemaRegistry)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: deimos-ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.4.0.pre.
|
|
4
|
+
version: 2.4.0.pre.beta10
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Orner
|
|
@@ -669,6 +669,7 @@ files:
|
|
|
669
669
|
- spec/snapshots/producers_with_key-no-nest.snap
|
|
670
670
|
- spec/snapshots/producers_with_key.snap
|
|
671
671
|
- spec/spec_helper.rb
|
|
672
|
+
- spec/test_helpers_spec.rb
|
|
672
673
|
- spec/utils/db_poller_spec.rb
|
|
673
674
|
- spec/utils/deadlock_retry_spec.rb
|
|
674
675
|
- spec/utils/outbox_producer_spec.rb
|