avro_turf 0.8.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.circleci/config.yml +36 -0
- data/.github/workflows/ruby.yml +20 -0
- data/.github/workflows/stale.yml +19 -0
- data/CHANGELOG.md +30 -1
- data/Gemfile +0 -3
- data/README.md +62 -0
- data/avro_turf.gemspec +7 -6
- data/lib/avro_turf.rb +14 -3
- data/lib/avro_turf/cached_confluent_schema_registry.rb +18 -6
- data/lib/avro_turf/confluent_schema_registry.rb +23 -4
- data/lib/avro_turf/disk_cache.rb +83 -0
- data/lib/avro_turf/in_memory_cache.rb +38 -0
- data/lib/avro_turf/messaging.rb +109 -16
- data/lib/avro_turf/mutable_schema_store.rb +18 -0
- data/lib/avro_turf/schema_store.rb +58 -22
- data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb +15 -3
- data/lib/avro_turf/version.rb +1 -1
- data/spec/cached_confluent_schema_registry_spec.rb +24 -2
- data/spec/confluent_schema_registry_spec.rb +13 -1
- data/spec/disk_cached_confluent_schema_registry_spec.rb +159 -0
- data/spec/messaging_spec.rb +205 -17
- data/spec/schema_store_spec.rb +134 -0
- data/spec/spec_helper.rb +8 -0
- data/spec/support/confluent_schema_registry_context.rb +8 -5
- data/spec/test/fake_confluent_schema_registry_server_spec.rb +40 -0
- metadata +39 -16
- data/circle.yml +0 -4
data/spec/messaging_spec.rb
CHANGED
@@ -4,29 +4,25 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
|
|
4
4
|
|
5
5
|
describe AvroTurf::Messaging do
|
6
6
|
let(:registry_url) { "http://registry.example.com" }
|
7
|
+
let(:client_cert) { "test client cert" }
|
8
|
+
let(:client_key) { "test client key" }
|
9
|
+
let(:client_key_pass) { "test client key password" }
|
7
10
|
let(:logger) { Logger.new(StringIO.new) }
|
8
11
|
|
9
12
|
let(:avro) {
|
10
13
|
AvroTurf::Messaging.new(
|
11
14
|
registry_url: registry_url,
|
12
15
|
schemas_path: "spec/schemas",
|
13
|
-
logger: logger
|
16
|
+
logger: logger,
|
17
|
+
client_cert: client_cert,
|
18
|
+
client_key: client_key,
|
19
|
+
client_key_pass: client_key_pass
|
14
20
|
)
|
15
21
|
}
|
16
22
|
|
17
23
|
let(:message) { { "full_name" => "John Doe" } }
|
18
|
-
|
19
|
-
|
20
|
-
FileUtils.mkdir_p("spec/schemas")
|
21
|
-
end
|
22
|
-
|
23
|
-
before do
|
24
|
-
stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
|
25
|
-
FakeConfluentSchemaRegistryServer.clear
|
26
|
-
end
|
27
|
-
|
28
|
-
before do
|
29
|
-
define_schema "person.avsc", <<-AVSC
|
24
|
+
let(:schema_json) do
|
25
|
+
<<-AVSC
|
30
26
|
{
|
31
27
|
"name": "person",
|
32
28
|
"type": "record",
|
@@ -39,8 +35,22 @@ describe AvroTurf::Messaging do
|
|
39
35
|
}
|
40
36
|
AVSC
|
41
37
|
end
|
38
|
+
let(:schema) { Avro::Schema.parse(schema_json) }
|
42
39
|
|
43
|
-
|
40
|
+
before do
|
41
|
+
FileUtils.mkdir_p("spec/schemas")
|
42
|
+
end
|
43
|
+
|
44
|
+
before do
|
45
|
+
stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
|
46
|
+
FakeConfluentSchemaRegistryServer.clear
|
47
|
+
end
|
48
|
+
|
49
|
+
before do
|
50
|
+
define_schema "person.avsc", schema_json
|
51
|
+
end
|
52
|
+
|
53
|
+
shared_examples_for "encoding and decoding with the schema from schema store" do
|
44
54
|
it "encodes and decodes messages" do
|
45
55
|
data = avro.encode(message, schema_name: "person")
|
46
56
|
expect(avro.decode(data)).to eq message
|
@@ -60,7 +70,66 @@ describe AvroTurf::Messaging do
|
|
60
70
|
end
|
61
71
|
end
|
62
72
|
|
63
|
-
|
73
|
+
shared_examples_for 'encoding and decoding with the schema from registry' do
|
74
|
+
before do
|
75
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
76
|
+
registry.register('person', schema)
|
77
|
+
registry.register('people', schema)
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'encodes and decodes messages' do
|
81
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
82
|
+
expect(avro.decode(data)).to eq message
|
83
|
+
end
|
84
|
+
|
85
|
+
it "allows specifying a reader's schema by subject and version" do
|
86
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
87
|
+
expect(avro.decode(data, schema_name: 'person')).to eq message
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
|
91
|
+
expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
|
92
|
+
end
|
93
|
+
|
94
|
+
it 'caches parsed schemas for decoding' do
|
95
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
96
|
+
avro.decode(data)
|
97
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
98
|
+
expect(avro.decode(data)).to eq message
|
99
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
shared_examples_for 'encoding and decoding with the schema_id from registry' do
|
104
|
+
before do
|
105
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
106
|
+
registry.register('person', schema)
|
107
|
+
registry.register('people', schema)
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'encodes and decodes messages' do
|
111
|
+
data = avro.encode(message, schema_id: 1)
|
112
|
+
expect(avro.decode(data)).to eq message
|
113
|
+
end
|
114
|
+
|
115
|
+
it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
|
116
|
+
expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
|
117
|
+
end
|
118
|
+
|
119
|
+
it 'caches parsed schemas for decoding' do
|
120
|
+
data = avro.encode(message, schema_id: 1)
|
121
|
+
avro.decode(data)
|
122
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
123
|
+
expect(avro.decode(data)).to eq message
|
124
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
129
|
+
|
130
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
131
|
+
|
132
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
64
133
|
|
65
134
|
context "with a provided registry" do
|
66
135
|
let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
@@ -73,7 +142,11 @@ describe AvroTurf::Messaging do
|
|
73
142
|
)
|
74
143
|
end
|
75
144
|
|
76
|
-
it_behaves_like "encoding and decoding"
|
145
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
146
|
+
|
147
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
148
|
+
|
149
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
77
150
|
|
78
151
|
it "uses the provided registry" do
|
79
152
|
allow(registry).to receive(:register).and_call_original
|
@@ -101,7 +174,7 @@ describe AvroTurf::Messaging do
|
|
101
174
|
)
|
102
175
|
end
|
103
176
|
|
104
|
-
it_behaves_like "encoding and decoding"
|
177
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
105
178
|
|
106
179
|
it "uses the provided schema store" do
|
107
180
|
allow(schema_store).to receive(:find).and_call_original
|
@@ -109,4 +182,119 @@ describe AvroTurf::Messaging do
|
|
109
182
|
expect(schema_store).to have_received(:find).with("person", nil)
|
110
183
|
end
|
111
184
|
end
|
185
|
+
|
186
|
+
describe 'decoding with #decode_message' do
|
187
|
+
shared_examples_for "encoding and decoding with the schema from schema store" do
|
188
|
+
it "encodes and decodes messages" do
|
189
|
+
data = avro.encode(message, schema_name: "person")
|
190
|
+
result = avro.decode_message(data)
|
191
|
+
expect(result.message).to eq message
|
192
|
+
expect(result.schema_id).to eq 0
|
193
|
+
expect(result.writer_schema).to eq schema
|
194
|
+
expect(result.reader_schema).to eq nil
|
195
|
+
end
|
196
|
+
|
197
|
+
it "allows specifying a reader's schema" do
|
198
|
+
data = avro.encode(message, schema_name: "person")
|
199
|
+
result = avro.decode_message(data, schema_name: "person")
|
200
|
+
expect(result.message).to eq message
|
201
|
+
expect(result.writer_schema).to eq schema
|
202
|
+
expect(result.reader_schema).to eq schema
|
203
|
+
end
|
204
|
+
|
205
|
+
it "caches parsed schemas for decoding" do
|
206
|
+
data = avro.encode(message, schema_name: "person")
|
207
|
+
avro.decode_message(data)
|
208
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
209
|
+
expect(avro.decode_message(data).message).to eq message
|
210
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
shared_examples_for 'encoding and decoding with the schema from registry' do
|
215
|
+
before do
|
216
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
217
|
+
registry.register('person', schema)
|
218
|
+
registry.register('people', schema)
|
219
|
+
end
|
220
|
+
|
221
|
+
it 'encodes and decodes messages' do
|
222
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
223
|
+
result = avro.decode_message(data)
|
224
|
+
expect(result.message).to eq message
|
225
|
+
expect(result.schema_id).to eq 0
|
226
|
+
end
|
227
|
+
|
228
|
+
it "allows specifying a reader's schema by subject and version" do
|
229
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
230
|
+
expect(avro.decode_message(data, schema_name: 'person').message).to eq message
|
231
|
+
end
|
232
|
+
|
233
|
+
it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
|
234
|
+
expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
|
235
|
+
end
|
236
|
+
|
237
|
+
it 'caches parsed schemas for decoding' do
|
238
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
239
|
+
avro.decode_message(data)
|
240
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
241
|
+
expect(avro.decode_message(data).message).to eq message
|
242
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
247
|
+
|
248
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
249
|
+
|
250
|
+
context "with a provided registry" do
|
251
|
+
let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
252
|
+
|
253
|
+
let(:avro) do
|
254
|
+
AvroTurf::Messaging.new(
|
255
|
+
registry: registry,
|
256
|
+
schemas_path: "spec/schemas",
|
257
|
+
logger: logger
|
258
|
+
)
|
259
|
+
end
|
260
|
+
|
261
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
262
|
+
|
263
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
264
|
+
|
265
|
+
it "uses the provided registry" do
|
266
|
+
allow(registry).to receive(:register).and_call_original
|
267
|
+
message = { "full_name" => "John Doe" }
|
268
|
+
avro.encode(message, schema_name: "person")
|
269
|
+
expect(registry).to have_received(:register).with("person", anything)
|
270
|
+
end
|
271
|
+
|
272
|
+
it "allows specifying a schema registry subject" do
|
273
|
+
allow(registry).to receive(:register).and_call_original
|
274
|
+
message = { "full_name" => "John Doe" }
|
275
|
+
avro.encode(message, schema_name: "person", subject: "people")
|
276
|
+
expect(registry).to have_received(:register).with("people", anything)
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
context "with a provided schema store" do
|
281
|
+
let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
|
282
|
+
|
283
|
+
let(:avro) do
|
284
|
+
AvroTurf::Messaging.new(
|
285
|
+
registry_url: registry_url,
|
286
|
+
schema_store: schema_store,
|
287
|
+
logger: logger
|
288
|
+
)
|
289
|
+
end
|
290
|
+
|
291
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
292
|
+
|
293
|
+
it "uses the provided schema store" do
|
294
|
+
allow(schema_store).to receive(:find).and_call_original
|
295
|
+
avro.encode(message, schema_name: "person")
|
296
|
+
expect(schema_store).to have_received(:find).with("person", nil)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
112
300
|
end
|
data/spec/schema_store_spec.rb
CHANGED
@@ -197,6 +197,140 @@ describe AvroTurf::SchemaStore do
|
|
197
197
|
schema = store.find("person")
|
198
198
|
expect(schema.fullname).to eq "person"
|
199
199
|
end
|
200
|
+
|
201
|
+
# This test would fail under avro_turf <= v0.11.0
|
202
|
+
it "does NOT cache *nested* schemas in memory" do
|
203
|
+
FileUtils.mkdir_p("spec/schemas/test")
|
204
|
+
|
205
|
+
define_schema "test/person.avsc", <<-AVSC
|
206
|
+
{
|
207
|
+
"name": "person",
|
208
|
+
"namespace": "test",
|
209
|
+
"type": "record",
|
210
|
+
"fields": [
|
211
|
+
{
|
212
|
+
"name": "address",
|
213
|
+
"type": {
|
214
|
+
"name": "address",
|
215
|
+
"type": "record",
|
216
|
+
"fields": [
|
217
|
+
{ "name": "addr1", "type": "string" },
|
218
|
+
{ "name": "addr2", "type": "string" },
|
219
|
+
{ "name": "city", "type": "string" },
|
220
|
+
{ "name": "zip", "type": "string" }
|
221
|
+
]
|
222
|
+
}
|
223
|
+
}
|
224
|
+
]
|
225
|
+
}
|
226
|
+
AVSC
|
227
|
+
|
228
|
+
schema = store.find('person', 'test')
|
229
|
+
expect(schema.fullname).to eq "test.person"
|
230
|
+
|
231
|
+
expect { store.find('address', 'test') }.
|
232
|
+
to raise_error(AvroTurf::SchemaNotFoundError)
|
233
|
+
end
|
234
|
+
|
235
|
+
# This test would fail under avro_turf <= v0.11.0
|
236
|
+
it "allows two different avsc files to define nested sub-schemas with the same fullname" do
|
237
|
+
FileUtils.mkdir_p("spec/schemas/test")
|
238
|
+
|
239
|
+
define_schema "test/person.avsc", <<-AVSC
|
240
|
+
{
|
241
|
+
"name": "person",
|
242
|
+
"namespace": "test",
|
243
|
+
"type": "record",
|
244
|
+
"fields": [
|
245
|
+
{
|
246
|
+
"name": "location",
|
247
|
+
"type": {
|
248
|
+
"name": "location",
|
249
|
+
"type": "record",
|
250
|
+
"fields": [
|
251
|
+
{ "name": "city", "type": "string" },
|
252
|
+
{ "name": "zipcode", "type": "string" }
|
253
|
+
]
|
254
|
+
}
|
255
|
+
}
|
256
|
+
]
|
257
|
+
}
|
258
|
+
AVSC
|
259
|
+
|
260
|
+
define_schema "test/company.avsc", <<-AVSC
|
261
|
+
{
|
262
|
+
"name": "company",
|
263
|
+
"namespace": "test",
|
264
|
+
"type": "record",
|
265
|
+
"fields": [
|
266
|
+
{
|
267
|
+
"name": "headquarters",
|
268
|
+
"type": {
|
269
|
+
"name": "location",
|
270
|
+
"type": "record",
|
271
|
+
"fields": [
|
272
|
+
{ "name": "city", "type": "string" },
|
273
|
+
{ "name": "postcode", "type": "string" }
|
274
|
+
]
|
275
|
+
}
|
276
|
+
}
|
277
|
+
]
|
278
|
+
}
|
279
|
+
AVSC
|
280
|
+
|
281
|
+
company = nil
|
282
|
+
person = store.find('person', 'test')
|
283
|
+
|
284
|
+
# This should *NOT* raise the error:
|
285
|
+
# #<Avro::SchemaParseError: The name "test.location" is already in use.>
|
286
|
+
expect { company = store.find('company', 'test') }.not_to raise_error
|
287
|
+
|
288
|
+
person_location_field = person.fields_hash['location']
|
289
|
+
expect(person_location_field.type.name).to eq('location')
|
290
|
+
expect(person_location_field.type.fields_hash).to include('zipcode')
|
291
|
+
expect(person_location_field.type.fields_hash).not_to include('postcode')
|
292
|
+
|
293
|
+
company_headquarters_field = company.fields_hash['headquarters']
|
294
|
+
expect(company_headquarters_field.type.name).to eq('location')
|
295
|
+
expect(company_headquarters_field.type.fields_hash).to include('postcode')
|
296
|
+
expect(company_headquarters_field.type.fields_hash).not_to include('zipcode')
|
297
|
+
end
|
298
|
+
|
299
|
+
it "is thread safe" do
|
300
|
+
define_schema "address.avsc", <<-AVSC
|
301
|
+
{
|
302
|
+
"type": "record",
|
303
|
+
"name": "address",
|
304
|
+
"fields": []
|
305
|
+
}
|
306
|
+
AVSC
|
307
|
+
|
308
|
+
# Set a Thread breakpoint right in the core place of race condition
|
309
|
+
expect(Avro::Name)
|
310
|
+
.to receive(:add_name)
|
311
|
+
.and_wrap_original { |m, *args|
|
312
|
+
Thread.stop
|
313
|
+
m.call(*args)
|
314
|
+
}
|
315
|
+
|
316
|
+
# Run two concurring threads which both will trigger the same schema loading
|
317
|
+
threads = 2.times.map { Thread.new { store.find("address") } }
|
318
|
+
# Wait for the moment when both threads will reach the breakpoint
|
319
|
+
sleep 0.001 until threads.all?(&:stop?)
|
320
|
+
|
321
|
+
expect {
|
322
|
+
# Resume the threads evaluation, one after one
|
323
|
+
threads.each do |thread|
|
324
|
+
next unless thread.status == 'sleep'
|
325
|
+
|
326
|
+
thread.run
|
327
|
+
sleep 0.001 until thread.stop?
|
328
|
+
end
|
329
|
+
|
330
|
+
# Ensure that threads are finished
|
331
|
+
threads.each(&:join)
|
332
|
+
}.to_not raise_error
|
333
|
+
end
|
200
334
|
end
|
201
335
|
|
202
336
|
describe "#load_schemas!" do
|
data/spec/spec_helper.rb
CHANGED
@@ -12,6 +12,14 @@ module Helpers
|
|
12
12
|
f.write(content)
|
13
13
|
end
|
14
14
|
end
|
15
|
+
|
16
|
+
def store_cache(path, hash)
|
17
|
+
File.write(File.join("spec/cache", path), JSON.generate(hash))
|
18
|
+
end
|
19
|
+
|
20
|
+
def load_cache(path)
|
21
|
+
JSON.parse(File.read(File.join("spec/cache", path)))
|
22
|
+
end
|
15
23
|
end
|
16
24
|
|
17
25
|
RSpec.configure do |config|
|
@@ -88,16 +88,18 @@ shared_examples_for "a confluent schema registry client" do
|
|
88
88
|
end
|
89
89
|
|
90
90
|
describe "#subject_version" do
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
91
|
+
let!(:schema_id1) do
|
92
|
+
registry.register(subject_name, { type: :record, name: "r0", fields: [] }.to_json)
|
93
|
+
end
|
94
|
+
let!(:schema_id2) do
|
95
|
+
registry.register(subject_name, { type: :record, name: "r1", fields: [] }.to_json)
|
96
96
|
end
|
97
|
+
|
97
98
|
let(:expected) do
|
98
99
|
{
|
99
100
|
name: subject_name,
|
100
101
|
version: 1,
|
102
|
+
id: schema_id1,
|
101
103
|
schema: { type: :record, name: "r0", fields: [] }.to_json
|
102
104
|
}.to_json
|
103
105
|
end
|
@@ -112,6 +114,7 @@ shared_examples_for "a confluent schema registry client" do
|
|
112
114
|
{
|
113
115
|
name: subject_name,
|
114
116
|
version: 2,
|
117
|
+
id: schema_id2,
|
115
118
|
schema: { type: :record, name: "r1", fields: [] }.to_json
|
116
119
|
}.to_json
|
117
120
|
end
|