avro_turf 1.19.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +20 -11
- data/CHANGELOG.md +6 -0
- data/Gemfile +5 -2
- data/Rakefile +2 -1
- data/avro_turf.gemspec +16 -16
- data/lib/avro_turf/cached_confluent_schema_registry.rb +9 -8
- data/lib/avro_turf/cached_schema_registry.rb +3 -1
- data/lib/avro_turf/confluent_schema_registry.rb +23 -17
- data/lib/avro_turf/core_ext/date.rb +2 -0
- data/lib/avro_turf/core_ext/enumerable.rb +2 -0
- data/lib/avro_turf/core_ext/false_class.rb +2 -0
- data/lib/avro_turf/core_ext/hash.rb +4 -2
- data/lib/avro_turf/core_ext/nil_class.rb +2 -0
- data/lib/avro_turf/core_ext/numeric.rb +2 -0
- data/lib/avro_turf/core_ext/string.rb +2 -0
- data/lib/avro_turf/core_ext/symbol.rb +2 -0
- data/lib/avro_turf/core_ext/time.rb +2 -0
- data/lib/avro_turf/core_ext/true_class.rb +2 -0
- data/lib/avro_turf/core_ext.rb +12 -10
- data/lib/avro_turf/disk_cache.rb +13 -12
- data/lib/avro_turf/in_memory_cache.rb +2 -0
- data/lib/avro_turf/messaging.rb +22 -14
- data/lib/avro_turf/mutable_schema_store.rb +25 -4
- data/lib/avro_turf/schema_registry.rb +3 -1
- data/lib/avro_turf/schema_store.rb +3 -2
- data/lib/avro_turf/schema_to_avro_patch.rb +14 -12
- data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb +24 -23
- data/lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb +12 -10
- data/lib/avro_turf/test/fake_schema_registry_server.rb +3 -1
- data/lib/avro_turf/version.rb +3 -1
- data/lib/avro_turf.rb +15 -13
- data/perf/encoding_size.rb +4 -2
- data/perf/encoding_speed.rb +4 -2
- data/spec/avro_turf_spec.rb +24 -23
- data/spec/cached_confluent_schema_registry_spec.rb +9 -7
- data/spec/confluent_schema_registry_spec.rb +31 -10
- data/spec/core_ext/date_spec.rb +2 -0
- data/spec/core_ext/enumerable_spec.rb +2 -0
- data/spec/core_ext/false_class_spec.rb +2 -0
- data/spec/core_ext/hash_spec.rb +3 -1
- data/spec/core_ext/nil_class_spec.rb +2 -0
- data/spec/core_ext/numeric_spec.rb +2 -0
- data/spec/core_ext/string_spec.rb +2 -0
- data/spec/core_ext/symbol_spec.rb +2 -0
- data/spec/core_ext/time_spec.rb +2 -0
- data/spec/core_ext/true_class_spec.rb +2 -0
- data/spec/disk_cached_confluent_schema_registry_spec.rb +23 -21
- data/spec/messaging_spec.rb +124 -99
- data/spec/mutable_schema_store_spec.rb +134 -0
- data/spec/schema_store_spec.rb +23 -21
- data/spec/schema_to_avro_patch_spec.rb +8 -7
- data/spec/spec_helper.rb +9 -9
- data/spec/support/authorized_fake_confluent_schema_registry_server.rb +4 -2
- data/spec/support/authorized_fake_prefixed_confluent_schema_registry_server.rb +4 -2
- data/spec/support/confluent_schema_registry_context.rb +32 -30
- data/spec/test/fake_confluent_schema_registry_server_spec.rb +97 -94
- metadata +5 -26
data/spec/schema_store_spec.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "avro_turf/schema_store"
|
2
4
|
|
3
5
|
describe AvroTurf::SchemaStore do
|
4
6
|
let(:store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
|
@@ -26,7 +28,7 @@ describe AvroTurf::SchemaStore do
|
|
26
28
|
expect(schema.fullname).to eq "message"
|
27
29
|
end
|
28
30
|
|
29
|
-
it
|
31
|
+
it "searches for nested types with the correct namespace" do
|
30
32
|
define_schema "foo/bar.avsc", <<-AVSC
|
31
33
|
{
|
32
34
|
"type": "record",
|
@@ -50,7 +52,7 @@ describe AvroTurf::SchemaStore do
|
|
50
52
|
}
|
51
53
|
AVSC
|
52
54
|
|
53
|
-
schema = store.find(
|
55
|
+
schema = store.find("foo.bar")
|
54
56
|
expect(schema.fullname).to eq "foo.bar"
|
55
57
|
expect(schema.fields.first.type.fullname).to eq "foo.another_schema"
|
56
58
|
end
|
@@ -281,11 +283,11 @@ describe AvroTurf::SchemaStore do
|
|
281
283
|
}
|
282
284
|
AVSC
|
283
285
|
|
284
|
-
schema = store.find(
|
286
|
+
schema = store.find("person", "test")
|
285
287
|
expect(schema.fullname).to eq "test.person"
|
286
288
|
|
287
|
-
expect { store.find(
|
288
|
-
to raise_error(AvroTurf::SchemaNotFoundError)
|
289
|
+
expect { store.find("address", "test") }
|
290
|
+
.to raise_error(AvroTurf::SchemaNotFoundError)
|
289
291
|
end
|
290
292
|
|
291
293
|
# This test would fail under avro_turf <= v0.11.0
|
@@ -335,21 +337,21 @@ describe AvroTurf::SchemaStore do
|
|
335
337
|
AVSC
|
336
338
|
|
337
339
|
company = nil
|
338
|
-
person = store.find(
|
340
|
+
person = store.find("person", "test")
|
339
341
|
|
340
342
|
# This should *NOT* raise the error:
|
341
343
|
# #<Avro::SchemaParseError: The name "test.location" is already in use.>
|
342
|
-
expect { company = store.find(
|
344
|
+
expect { company = store.find("company", "test") }.not_to raise_error
|
343
345
|
|
344
|
-
person_location_field = person.fields_hash[
|
345
|
-
expect(person_location_field.type.name).to eq(
|
346
|
-
expect(person_location_field.type.fields_hash).to include(
|
347
|
-
expect(person_location_field.type.fields_hash).not_to include(
|
346
|
+
person_location_field = person.fields_hash["location"]
|
347
|
+
expect(person_location_field.type.name).to eq("location")
|
348
|
+
expect(person_location_field.type.fields_hash).to include("zipcode")
|
349
|
+
expect(person_location_field.type.fields_hash).not_to include("postcode")
|
348
350
|
|
349
|
-
company_headquarters_field = company.fields_hash[
|
350
|
-
expect(company_headquarters_field.type.name).to eq(
|
351
|
-
expect(company_headquarters_field.type.fields_hash).to include(
|
352
|
-
expect(company_headquarters_field.type.fields_hash).not_to include(
|
351
|
+
company_headquarters_field = company.fields_hash["headquarters"]
|
352
|
+
expect(company_headquarters_field.type.name).to eq("location")
|
353
|
+
expect(company_headquarters_field.type.fields_hash).to include("postcode")
|
354
|
+
expect(company_headquarters_field.type.fields_hash).not_to include("zipcode")
|
353
355
|
end
|
354
356
|
|
355
357
|
it "is thread safe" do
|
@@ -364,10 +366,10 @@ describe AvroTurf::SchemaStore do
|
|
364
366
|
# Set a Thread breakpoint right in the core place of race condition
|
365
367
|
expect(Avro::Name)
|
366
368
|
.to receive(:add_name)
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
369
|
+
.and_wrap_original { |m, *args|
|
370
|
+
Thread.stop
|
371
|
+
m.call(*args)
|
372
|
+
}
|
371
373
|
|
372
374
|
# Run two concurring threads which both will trigger the same schema loading
|
373
375
|
threads = 2.times.map { Thread.new { store.find("address") } }
|
@@ -377,7 +379,7 @@ describe AvroTurf::SchemaStore do
|
|
377
379
|
expect {
|
378
380
|
# Resume the threads evaluation, one after one
|
379
381
|
threads.each do |thread|
|
380
|
-
next unless thread.status ==
|
382
|
+
next unless thread.status == "sleep"
|
381
383
|
|
382
384
|
thread.run
|
383
385
|
sleep 0.001 until thread.stop?
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "webmock/rspec"
|
2
4
|
|
3
5
|
# This spec verifies the monkey-patch that we have to apply until the avro
|
4
6
|
# gem releases a fix for bug AVRO-1848:
|
@@ -13,17 +15,16 @@ describe Avro::Schema do
|
|
13
15
|
]
|
14
16
|
}
|
15
17
|
SCHEMA
|
16
|
-
|
18
|
+
|
17
19
|
expect(schema.to_avro).to eq({
|
18
|
-
|
19
|
-
|
20
|
-
{
|
20
|
+
"type" => "record", "name" => "Record", "namespace" => "my.name.space",
|
21
|
+
"fields" => [
|
22
|
+
{"name" => "is_usable", "type" => "boolean", "default" => false}
|
21
23
|
]
|
22
24
|
})
|
23
25
|
end
|
24
26
|
end
|
25
27
|
|
26
|
-
|
27
28
|
describe Avro::IO::DatumReader do
|
28
29
|
let(:writer_schema) do
|
29
30
|
Avro::Schema.parse <<-AVSC
|
@@ -53,7 +54,7 @@ describe Avro::IO::DatumReader do
|
|
53
54
|
stream = StringIO.new
|
54
55
|
writer = Avro::IO::DatumWriter.new(writer_schema)
|
55
56
|
encoder = Avro::IO::BinaryEncoder.new(stream)
|
56
|
-
writer.write({
|
57
|
+
writer.write({"one" => "first"}, encoder)
|
57
58
|
encoded = stream.string
|
58
59
|
|
59
60
|
stream = StringIO.new(encoded)
|
data/spec/spec_helper.rb
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "logger"
|
5
|
+
require "json_spec"
|
6
|
+
require "pp" # Require pp before fakefs to fix TypeError: superclass mismatch for class File
|
7
|
+
require "fakefs/spec_helpers"
|
8
|
+
require "avro_turf"
|
7
9
|
|
8
10
|
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
9
11
|
|
@@ -12,9 +14,7 @@ module Helpers
|
|
12
14
|
file = File.join("spec/schemas", path)
|
13
15
|
dir = File.dirname(file)
|
14
16
|
FileUtils.mkdir_p(dir)
|
15
|
-
File.
|
16
|
-
f.write(content)
|
17
|
-
end
|
17
|
+
File.write(file, content)
|
18
18
|
end
|
19
19
|
|
20
20
|
def store_cache(path, hash)
|
@@ -1,5 +1,7 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "avro_turf/test/fake_confluent_schema_registry_server"
|
2
4
|
|
3
5
|
class AuthorizedFakeConfluentSchemaRegistryServer < FakeConfluentSchemaRegistryServer
|
4
|
-
set :host_authorization, permitted_hosts: [
|
6
|
+
set :host_authorization, permitted_hosts: ["example.org", "registry.example.com"]
|
5
7
|
end
|
@@ -1,5 +1,7 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "avro_turf/test/fake_prefixed_confluent_schema_registry_server"
|
2
4
|
|
3
5
|
class AuthorizedFakePrefixedConfluentSchemaRegistryServer < FakePrefixedConfluentSchemaRegistryServer
|
4
|
-
set :host_authorization, permitted_hosts: [
|
6
|
+
set :host_authorization, permitted_hosts: ["example.org", "registry.example.com"]
|
5
7
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# This shared example expects a registry variable to be defined
|
2
4
|
# with an instance of the registry class being tested.
|
3
5
|
shared_examples_for "a confluent schema registry client" do |schema_context: nil|
|
@@ -10,16 +12,16 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
10
12
|
type: "record",
|
11
13
|
name: "person",
|
12
14
|
fields: [
|
13
|
-
{
|
15
|
+
{name: "name", type: "string"}
|
14
16
|
]
|
15
17
|
}.to_json
|
16
18
|
end
|
17
19
|
let(:headers) do
|
18
20
|
{
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
21
|
+
"Accept" => "*/*",
|
22
|
+
"Content-Type" => AvroTurf::ConfluentSchemaRegistry::CONTENT_TYPE,
|
23
|
+
"Host" => URI.parse(registry_url).host.to_s,
|
24
|
+
"User-Agent" => "excon/#{Excon::VERSION}"
|
23
25
|
}
|
24
26
|
end
|
25
27
|
|
@@ -81,21 +83,21 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
81
83
|
|
82
84
|
describe "#schema_subject_versions" do
|
83
85
|
it "returns subject and version for a schema id" do
|
84
|
-
schema_id1 = registry.register(subject_name, {
|
85
|
-
registry.register(subject_name, {
|
86
|
+
schema_id1 = registry.register(subject_name, {type: :record, name: "r1", fields: []}.to_json)
|
87
|
+
registry.register(subject_name, {type: :record, name: "r2", fields: []}.to_json)
|
86
88
|
other_subject_name = "other#{subject_name}"
|
87
|
-
schema_id2 = registry.register(other_subject_name, {
|
89
|
+
schema_id2 = registry.register(other_subject_name, {type: :record, name: "r2", fields: []}.to_json)
|
88
90
|
expect(registry.schema_subject_versions(schema_id1)).to eq([
|
89
|
-
|
90
|
-
|
91
|
+
"subject" => expected_subject_name,
|
92
|
+
"version" => 1
|
91
93
|
])
|
92
94
|
expect(registry.schema_subject_versions(schema_id2)).to include({
|
93
|
-
|
94
|
-
|
95
|
-
},{
|
96
|
-
|
97
|
-
|
98
|
-
}
|
95
|
+
"subject" => expected_subject_name,
|
96
|
+
"version" => 2
|
97
|
+
}, {
|
98
|
+
"subject" => with_schema_context_if_applicable(schema_context, other_subject_name),
|
99
|
+
"version" => 1
|
100
|
+
})
|
99
101
|
end
|
100
102
|
|
101
103
|
context "when the schema does not exist" do
|
@@ -111,14 +113,14 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
111
113
|
it "lists all the versions for the subject" do
|
112
114
|
2.times do |n|
|
113
115
|
registry.register(subject_name,
|
114
|
-
|
116
|
+
{type: :record, name: "r#{n}", fields: []}.to_json)
|
115
117
|
end
|
116
118
|
expect(registry.subject_versions(subject_name))
|
117
119
|
.to be_json_eql((1..2).to_a.to_json)
|
118
120
|
end
|
119
121
|
|
120
122
|
context "when the subject does not exist" do
|
121
|
-
let(:subject_name) {
|
123
|
+
let(:subject_name) { "missing" }
|
122
124
|
|
123
125
|
it "raises an error" do
|
124
126
|
expect do
|
@@ -130,10 +132,10 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
130
132
|
|
131
133
|
describe "#subject_version" do
|
132
134
|
let!(:schema_id1) do
|
133
|
-
registry.register(subject_name, {
|
135
|
+
registry.register(subject_name, {type: :record, name: "r0", fields: []}.to_json)
|
134
136
|
end
|
135
137
|
let!(:schema_id2) do
|
136
|
-
registry.register(subject_name, {
|
138
|
+
registry.register(subject_name, {type: :record, name: "r1", fields: []}.to_json)
|
137
139
|
end
|
138
140
|
|
139
141
|
let(:expected) do
|
@@ -141,7 +143,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
141
143
|
subject: expected_subject_name,
|
142
144
|
version: 1,
|
143
145
|
id: schema_id1,
|
144
|
-
schema: {
|
146
|
+
schema: {type: :record, name: "r0", fields: []}.to_json
|
145
147
|
}.to_json
|
146
148
|
end
|
147
149
|
|
@@ -156,7 +158,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
156
158
|
subject: expected_subject_name,
|
157
159
|
version: 2,
|
158
160
|
id: schema_id2,
|
159
|
-
schema: {
|
161
|
+
schema: {type: :record, name: "r1", fields: []}.to_json
|
160
162
|
}.to_json
|
161
163
|
end
|
162
164
|
|
@@ -169,7 +171,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
169
171
|
context "when the subject does not exist" do
|
170
172
|
it "raises an error" do
|
171
173
|
expect do
|
172
|
-
registry.subject_version(
|
174
|
+
registry.subject_version("missing")
|
173
175
|
end.to raise_error(Excon::Errors::NotFound)
|
174
176
|
end
|
175
177
|
end
|
@@ -224,7 +226,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
224
226
|
|
225
227
|
describe "#global_config" do
|
226
228
|
let(:expected) do
|
227
|
-
{
|
229
|
+
{compatibility: "BACKWARD"}.to_json
|
228
230
|
end
|
229
231
|
|
230
232
|
it "returns the global configuration" do
|
@@ -234,7 +236,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
234
236
|
|
235
237
|
describe "#update_global_config" do
|
236
238
|
let(:config) do
|
237
|
-
{
|
239
|
+
{compatibility: "FORWARD"}
|
238
240
|
end
|
239
241
|
let(:expected) { config.to_json }
|
240
242
|
|
@@ -246,7 +248,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
246
248
|
|
247
249
|
describe "#subject_config" do
|
248
250
|
let(:expected) do
|
249
|
-
{
|
251
|
+
{compatibility: "BACKWARD"}.to_json
|
250
252
|
end
|
251
253
|
|
252
254
|
context "when the subject config is not set" do
|
@@ -257,7 +259,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
257
259
|
|
258
260
|
context "when the subject config is set" do
|
259
261
|
let(:config) do
|
260
|
-
{
|
262
|
+
{compatibility: "FULL"}
|
261
263
|
end
|
262
264
|
let(:expected) { config.to_json }
|
263
265
|
|
@@ -273,7 +275,7 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
273
275
|
|
274
276
|
describe "#update_subject_config" do
|
275
277
|
let(:config) do
|
276
|
-
{
|
278
|
+
{compatibility: "NONE"}
|
277
279
|
end
|
278
280
|
let(:expected) { config.to_json }
|
279
281
|
|
@@ -287,8 +289,8 @@ shared_examples_for "a confluent schema registry client" do |schema_context: nil
|
|
287
289
|
# active_support/core_ext.
|
288
290
|
def break_to_json(avro_schema)
|
289
291
|
def avro_schema.to_json(*args)
|
290
|
-
instance_variables.each_with_object(
|
291
|
-
result[ivar.to_s.sub(
|
292
|
+
instance_variables.each_with_object({}) do |ivar, result|
|
293
|
+
result[ivar.to_s.sub("@", "")] = instance_variable_get(ivar)
|
292
294
|
end.to_json(*args)
|
293
295
|
end
|
294
296
|
end
|