avro_turf 0.8.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,29 +4,25 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
4
4
 
5
5
  describe AvroTurf::Messaging do
6
6
  let(:registry_url) { "http://registry.example.com" }
7
+ let(:client_cert) { "test client cert" }
8
+ let(:client_key) { "test client key" }
9
+ let(:client_key_pass) { "test client key password" }
7
10
  let(:logger) { Logger.new(StringIO.new) }
8
11
 
9
12
  let(:avro) {
10
13
  AvroTurf::Messaging.new(
11
14
  registry_url: registry_url,
12
15
  schemas_path: "spec/schemas",
13
- logger: logger
16
+ logger: logger,
17
+ client_cert: client_cert,
18
+ client_key: client_key,
19
+ client_key_pass: client_key_pass
14
20
  )
15
21
  }
16
22
 
17
23
  let(:message) { { "full_name" => "John Doe" } }
18
-
19
- before do
20
- FileUtils.mkdir_p("spec/schemas")
21
- end
22
-
23
- before do
24
- stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
25
- FakeConfluentSchemaRegistryServer.clear
26
- end
27
-
28
- before do
29
- define_schema "person.avsc", <<-AVSC
24
+ let(:schema_json) do
25
+ <<-AVSC
30
26
  {
31
27
  "name": "person",
32
28
  "type": "record",
@@ -39,8 +35,22 @@ describe AvroTurf::Messaging do
39
35
  }
40
36
  AVSC
41
37
  end
38
+ let(:schema) { Avro::Schema.parse(schema_json) }
42
39
 
43
- shared_examples_for "encoding and decoding" do
40
+ before do
41
+ FileUtils.mkdir_p("spec/schemas")
42
+ end
43
+
44
+ before do
45
+ stub_request(:any, /^#{registry_url}/).to_rack(FakeConfluentSchemaRegistryServer)
46
+ FakeConfluentSchemaRegistryServer.clear
47
+ end
48
+
49
+ before do
50
+ define_schema "person.avsc", schema_json
51
+ end
52
+
53
+ shared_examples_for "encoding and decoding with the schema from schema store" do
44
54
  it "encodes and decodes messages" do
45
55
  data = avro.encode(message, schema_name: "person")
46
56
  expect(avro.decode(data)).to eq message
@@ -60,7 +70,66 @@ describe AvroTurf::Messaging do
60
70
  end
61
71
  end
62
72
 
63
- it_behaves_like "encoding and decoding"
73
+ shared_examples_for 'encoding and decoding with the schema from registry' do
74
+ before do
75
+ registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
76
+ registry.register('person', schema)
77
+ registry.register('people', schema)
78
+ end
79
+
80
+ it 'encodes and decodes messages' do
81
+ data = avro.encode(message, subject: 'person', version: 1)
82
+ expect(avro.decode(data)).to eq message
83
+ end
84
+
85
+ it "allows specifying a reader's schema by subject and version" do
86
+ data = avro.encode(message, subject: 'person', version: 1)
87
+ expect(avro.decode(data, schema_name: 'person')).to eq message
88
+ end
89
+
90
+ it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
91
+ expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
92
+ end
93
+
94
+ it 'caches parsed schemas for decoding' do
95
+ data = avro.encode(message, subject: 'person', version: 1)
96
+ avro.decode(data)
97
+ allow(Avro::Schema).to receive(:parse).and_call_original
98
+ expect(avro.decode(data)).to eq message
99
+ expect(Avro::Schema).not_to have_received(:parse)
100
+ end
101
+ end
102
+
103
+ shared_examples_for 'encoding and decoding with the schema_id from registry' do
104
+ before do
105
+ registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
106
+ registry.register('person', schema)
107
+ registry.register('people', schema)
108
+ end
109
+
110
+ it 'encodes and decodes messages' do
111
+ data = avro.encode(message, schema_id: 1)
112
+ expect(avro.decode(data)).to eq message
113
+ end
114
+
115
+ it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
116
+ expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
117
+ end
118
+
119
+ it 'caches parsed schemas for decoding' do
120
+ data = avro.encode(message, schema_id: 1)
121
+ avro.decode(data)
122
+ allow(Avro::Schema).to receive(:parse).and_call_original
123
+ expect(avro.decode(data)).to eq message
124
+ expect(Avro::Schema).not_to have_received(:parse)
125
+ end
126
+ end
127
+
128
+ it_behaves_like "encoding and decoding with the schema from schema store"
129
+
130
+ it_behaves_like 'encoding and decoding with the schema from registry'
131
+
132
+ it_behaves_like 'encoding and decoding with the schema_id from registry'
64
133
 
65
134
  context "with a provided registry" do
66
135
  let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
@@ -73,7 +142,11 @@ describe AvroTurf::Messaging do
73
142
  )
74
143
  end
75
144
 
76
- it_behaves_like "encoding and decoding"
145
+ it_behaves_like "encoding and decoding with the schema from schema store"
146
+
147
+ it_behaves_like 'encoding and decoding with the schema from registry'
148
+
149
+ it_behaves_like 'encoding and decoding with the schema_id from registry'
77
150
 
78
151
  it "uses the provided registry" do
79
152
  allow(registry).to receive(:register).and_call_original
@@ -101,7 +174,7 @@ describe AvroTurf::Messaging do
101
174
  )
102
175
  end
103
176
 
104
- it_behaves_like "encoding and decoding"
177
+ it_behaves_like "encoding and decoding with the schema from schema store"
105
178
 
106
179
  it "uses the provided schema store" do
107
180
  allow(schema_store).to receive(:find).and_call_original
@@ -109,4 +182,119 @@ describe AvroTurf::Messaging do
109
182
  expect(schema_store).to have_received(:find).with("person", nil)
110
183
  end
111
184
  end
185
+
186
+ describe 'decoding with #decode_message' do
187
+ shared_examples_for "encoding and decoding with the schema from schema store" do
188
+ it "encodes and decodes messages" do
189
+ data = avro.encode(message, schema_name: "person")
190
+ result = avro.decode_message(data)
191
+ expect(result.message).to eq message
192
+ expect(result.schema_id).to eq 0
193
+ expect(result.writer_schema).to eq schema
194
+ expect(result.reader_schema).to eq nil
195
+ end
196
+
197
+ it "allows specifying a reader's schema" do
198
+ data = avro.encode(message, schema_name: "person")
199
+ result = avro.decode_message(data, schema_name: "person")
200
+ expect(result.message).to eq message
201
+ expect(result.writer_schema).to eq schema
202
+ expect(result.reader_schema).to eq schema
203
+ end
204
+
205
+ it "caches parsed schemas for decoding" do
206
+ data = avro.encode(message, schema_name: "person")
207
+ avro.decode_message(data)
208
+ allow(Avro::Schema).to receive(:parse).and_call_original
209
+ expect(avro.decode_message(data).message).to eq message
210
+ expect(Avro::Schema).not_to have_received(:parse)
211
+ end
212
+ end
213
+
214
+ shared_examples_for 'encoding and decoding with the schema from registry' do
215
+ before do
216
+ registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
217
+ registry.register('person', schema)
218
+ registry.register('people', schema)
219
+ end
220
+
221
+ it 'encodes and decodes messages' do
222
+ data = avro.encode(message, subject: 'person', version: 1)
223
+ result = avro.decode_message(data)
224
+ expect(result.message).to eq message
225
+ expect(result.schema_id).to eq 0
226
+ end
227
+
228
+ it "allows specifying a reader's schema by subject and version" do
229
+ data = avro.encode(message, subject: 'person', version: 1)
230
+ expect(avro.decode_message(data, schema_name: 'person').message).to eq message
231
+ end
232
+
233
+ it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
234
+ expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
235
+ end
236
+
237
+ it 'caches parsed schemas for decoding' do
238
+ data = avro.encode(message, subject: 'person', version: 1)
239
+ avro.decode_message(data)
240
+ allow(Avro::Schema).to receive(:parse).and_call_original
241
+ expect(avro.decode_message(data).message).to eq message
242
+ expect(Avro::Schema).not_to have_received(:parse)
243
+ end
244
+ end
245
+
246
+ it_behaves_like "encoding and decoding with the schema from schema store"
247
+
248
+ it_behaves_like 'encoding and decoding with the schema from registry'
249
+
250
+ context "with a provided registry" do
251
+ let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
252
+
253
+ let(:avro) do
254
+ AvroTurf::Messaging.new(
255
+ registry: registry,
256
+ schemas_path: "spec/schemas",
257
+ logger: logger
258
+ )
259
+ end
260
+
261
+ it_behaves_like "encoding and decoding with the schema from schema store"
262
+
263
+ it_behaves_like 'encoding and decoding with the schema from registry'
264
+
265
+ it "uses the provided registry" do
266
+ allow(registry).to receive(:register).and_call_original
267
+ message = { "full_name" => "John Doe" }
268
+ avro.encode(message, schema_name: "person")
269
+ expect(registry).to have_received(:register).with("person", anything)
270
+ end
271
+
272
+ it "allows specifying a schema registry subject" do
273
+ allow(registry).to receive(:register).and_call_original
274
+ message = { "full_name" => "John Doe" }
275
+ avro.encode(message, schema_name: "person", subject: "people")
276
+ expect(registry).to have_received(:register).with("people", anything)
277
+ end
278
+ end
279
+
280
+ context "with a provided schema store" do
281
+ let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
282
+
283
+ let(:avro) do
284
+ AvroTurf::Messaging.new(
285
+ registry_url: registry_url,
286
+ schema_store: schema_store,
287
+ logger: logger
288
+ )
289
+ end
290
+
291
+ it_behaves_like "encoding and decoding with the schema from schema store"
292
+
293
+ it "uses the provided schema store" do
294
+ allow(schema_store).to receive(:find).and_call_original
295
+ avro.encode(message, schema_name: "person")
296
+ expect(schema_store).to have_received(:find).with("person", nil)
297
+ end
298
+ end
299
+ end
112
300
  end
@@ -197,6 +197,140 @@ describe AvroTurf::SchemaStore do
197
197
  schema = store.find("person")
198
198
  expect(schema.fullname).to eq "person"
199
199
  end
200
+
201
+ # This test would fail under avro_turf <= v0.11.0
202
+ it "does NOT cache *nested* schemas in memory" do
203
+ FileUtils.mkdir_p("spec/schemas/test")
204
+
205
+ define_schema "test/person.avsc", <<-AVSC
206
+ {
207
+ "name": "person",
208
+ "namespace": "test",
209
+ "type": "record",
210
+ "fields": [
211
+ {
212
+ "name": "address",
213
+ "type": {
214
+ "name": "address",
215
+ "type": "record",
216
+ "fields": [
217
+ { "name": "addr1", "type": "string" },
218
+ { "name": "addr2", "type": "string" },
219
+ { "name": "city", "type": "string" },
220
+ { "name": "zip", "type": "string" }
221
+ ]
222
+ }
223
+ }
224
+ ]
225
+ }
226
+ AVSC
227
+
228
+ schema = store.find('person', 'test')
229
+ expect(schema.fullname).to eq "test.person"
230
+
231
+ expect { store.find('address', 'test') }.
232
+ to raise_error(AvroTurf::SchemaNotFoundError)
233
+ end
234
+
235
+ # This test would fail under avro_turf <= v0.11.0
236
+ it "allows two different avsc files to define nested sub-schemas with the same fullname" do
237
+ FileUtils.mkdir_p("spec/schemas/test")
238
+
239
+ define_schema "test/person.avsc", <<-AVSC
240
+ {
241
+ "name": "person",
242
+ "namespace": "test",
243
+ "type": "record",
244
+ "fields": [
245
+ {
246
+ "name": "location",
247
+ "type": {
248
+ "name": "location",
249
+ "type": "record",
250
+ "fields": [
251
+ { "name": "city", "type": "string" },
252
+ { "name": "zipcode", "type": "string" }
253
+ ]
254
+ }
255
+ }
256
+ ]
257
+ }
258
+ AVSC
259
+
260
+ define_schema "test/company.avsc", <<-AVSC
261
+ {
262
+ "name": "company",
263
+ "namespace": "test",
264
+ "type": "record",
265
+ "fields": [
266
+ {
267
+ "name": "headquarters",
268
+ "type": {
269
+ "name": "location",
270
+ "type": "record",
271
+ "fields": [
272
+ { "name": "city", "type": "string" },
273
+ { "name": "postcode", "type": "string" }
274
+ ]
275
+ }
276
+ }
277
+ ]
278
+ }
279
+ AVSC
280
+
281
+ company = nil
282
+ person = store.find('person', 'test')
283
+
284
+ # This should *NOT* raise the error:
285
+ # #<Avro::SchemaParseError: The name "test.location" is already in use.>
286
+ expect { company = store.find('company', 'test') }.not_to raise_error
287
+
288
+ person_location_field = person.fields_hash['location']
289
+ expect(person_location_field.type.name).to eq('location')
290
+ expect(person_location_field.type.fields_hash).to include('zipcode')
291
+ expect(person_location_field.type.fields_hash).not_to include('postcode')
292
+
293
+ company_headquarters_field = company.fields_hash['headquarters']
294
+ expect(company_headquarters_field.type.name).to eq('location')
295
+ expect(company_headquarters_field.type.fields_hash).to include('postcode')
296
+ expect(company_headquarters_field.type.fields_hash).not_to include('zipcode')
297
+ end
298
+
299
+ it "is thread safe" do
300
+ define_schema "address.avsc", <<-AVSC
301
+ {
302
+ "type": "record",
303
+ "name": "address",
304
+ "fields": []
305
+ }
306
+ AVSC
307
+
308
+ # Set a Thread breakpoint right in the core place of race condition
309
+ expect(Avro::Name)
310
+ .to receive(:add_name)
311
+ .and_wrap_original { |m, *args|
312
+ Thread.stop
313
+ m.call(*args)
314
+ }
315
+
316
+ # Run two concurring threads which both will trigger the same schema loading
317
+ threads = 2.times.map { Thread.new { store.find("address") } }
318
+ # Wait for the moment when both threads will reach the breakpoint
319
+ sleep 0.001 until threads.all?(&:stop?)
320
+
321
+ expect {
322
+ # Resume the threads evaluation, one after one
323
+ threads.each do |thread|
324
+ next unless thread.status == 'sleep'
325
+
326
+ thread.run
327
+ sleep 0.001 until thread.stop?
328
+ end
329
+
330
+ # Ensure that threads are finished
331
+ threads.each(&:join)
332
+ }.to_not raise_error
333
+ end
200
334
  end
201
335
 
202
336
  describe "#load_schemas!" do
@@ -12,6 +12,14 @@ module Helpers
12
12
  f.write(content)
13
13
  end
14
14
  end
15
+
16
+ def store_cache(path, hash)
17
+ File.write(File.join("spec/cache", path), JSON.generate(hash))
18
+ end
19
+
20
+ def load_cache(path)
21
+ JSON.parse(File.read(File.join("spec/cache", path)))
22
+ end
15
23
  end
16
24
 
17
25
  RSpec.configure do |config|
@@ -88,16 +88,18 @@ shared_examples_for "a confluent schema registry client" do
88
88
  end
89
89
 
90
90
  describe "#subject_version" do
91
- before do
92
- 2.times do |n|
93
- registry.register(subject_name,
94
- { type: :record, name: "r#{n}", fields: [] }.to_json)
95
- end
91
+ let!(:schema_id1) do
92
+ registry.register(subject_name, { type: :record, name: "r0", fields: [] }.to_json)
93
+ end
94
+ let!(:schema_id2) do
95
+ registry.register(subject_name, { type: :record, name: "r1", fields: [] }.to_json)
96
96
  end
97
+
97
98
  let(:expected) do
98
99
  {
99
100
  name: subject_name,
100
101
  version: 1,
102
+ id: schema_id1,
101
103
  schema: { type: :record, name: "r0", fields: [] }.to_json
102
104
  }.to_json
103
105
  end
@@ -112,6 +114,7 @@ shared_examples_for "a confluent schema registry client" do
112
114
  {
113
115
  name: subject_name,
114
116
  version: 2,
117
+ id: schema_id2,
115
118
  schema: { type: :record, name: "r1", fields: [] }.to_json
116
119
  }.to_json
117
120
  end