schema_registry_client 0.0.11.pre.beta1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +1 -1
- data/.gitignore +1 -0
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +1 -1
- data/lib/schema_registry_client/avro_schema_store.rb +0 -7
- data/lib/schema_registry_client/cached_confluent_schema_registry.rb +26 -11
- data/lib/schema_registry_client/confluent_schema_registry.rb +0 -4
- data/lib/schema_registry_client/schema/avro.rb +12 -6
- data/lib/schema_registry_client/version.rb +1 -1
- data/lib/schema_registry_client.rb +6 -3
- data/schema_registry_client.gemspec +1 -1
- data/spec/decoding_spec.rb +123 -5
- data/spec/encoding_spec.rb +174 -9
- data/spec/schemas/cross_file/v1/Inner.avsc +11 -0
- data/spec/schemas/cross_file/v1/Outer.avsc +11 -0
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ade8a75ca2eb7bacb07566bd5eaa2899bfd69c2ed34dd7c1bc7e186b531b055e
|
|
4
|
+
data.tar.gz: 17174e9182ce1f875006bbe302f8ad7cc9809af3d415e2a9500ccfda114ec95d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4ccea73cc7df695421518ad82de2507950d7d6e300146d5210c982addea11396ad4b4ee154f58ed38aa81891a13df20b5e60f50e84a74d74e7f6cd86c39aa1cb
|
|
7
|
+
data.tar.gz: 0daa9b8085d05991b1be936e8ce0aa209ab0973a05103a8dc8368e394988ec52ef7066110d912542b891342e2842b92b4e7e24d29b8d597a3a39d315d1b8d0bc
|
data/.github/workflows/test.yml
CHANGED
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## UNRELEASED
|
|
9
9
|
|
|
10
|
+
# 0.1.0 - 2026-06-24
|
|
11
|
+
|
|
12
|
+
* Avro: register the fully-resolved (inlined) schema so schemas that reference named types defined in other `.avsc` files are valid standalone.
|
|
13
|
+
|
|
14
|
+
# 0.0.11 - 2026-02-20
|
|
15
|
+
|
|
16
|
+
* Fixes for caching working correctly and ensuring we don't recalculate / re-parse Avro schemas
|
|
17
|
+
|
|
10
18
|
# 0.0.10 - 2026-02-11
|
|
11
19
|
|
|
12
20
|
* Fix: Do not send `schemaType` or `references` if schema type is `AVRO`, for backwards compatibility with older schema registries.
|
data/Gemfile.lock
CHANGED
|
@@ -7,16 +7,11 @@ module SchemaRegistry
|
|
|
7
7
|
def initialize(path: nil)
|
|
8
8
|
@path = path or raise "Please specify a schema path"
|
|
9
9
|
@schemas = {}
|
|
10
|
-
@schema_text = {}
|
|
11
10
|
@mutex = Mutex.new
|
|
12
11
|
end
|
|
13
12
|
|
|
14
13
|
attr_accessor :schemas
|
|
15
14
|
|
|
16
|
-
def find_text(name)
|
|
17
|
-
@schema_text[name]
|
|
18
|
-
end
|
|
19
|
-
|
|
20
15
|
# Resolves and returns a schema.
|
|
21
16
|
#
|
|
22
17
|
# schema_name - The String name of the schema to resolve.
|
|
@@ -64,7 +59,6 @@ module SchemaRegistry
|
|
|
64
59
|
# we will discard it.
|
|
65
60
|
schema = Avro::Schema.real_parse(schema_hash, @schemas.dup)
|
|
66
61
|
@schemas[full_name] = schema
|
|
67
|
-
@schema_text[full_name] = JSON.pretty_generate(schema_hash)
|
|
68
62
|
|
|
69
63
|
schema
|
|
70
64
|
end
|
|
@@ -96,7 +90,6 @@ module SchemaRegistry
|
|
|
96
90
|
# Essentially, the only schemas that should be resolvable in @schemas
|
|
97
91
|
# are those that have their own .avsc files on disk.
|
|
98
92
|
@schemas[fullname] = schema
|
|
99
|
-
@schema_text[fullname] = schema_text
|
|
100
93
|
|
|
101
94
|
schema
|
|
102
95
|
rescue ::Avro::UnknownSchemaError => e
|
|
@@ -8,6 +8,7 @@ module SchemaRegistry
|
|
|
8
8
|
@schemas_by_id = {}
|
|
9
9
|
@ids_by_schema = {}
|
|
10
10
|
@versions_by_subject_and_id = {}
|
|
11
|
+
@mutex = Mutex.new
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
# Delegate the following methods to the upstream
|
|
@@ -20,25 +21,37 @@ module SchemaRegistry
|
|
|
20
21
|
# @param id [Integer] the schema ID to fetch
|
|
21
22
|
# @return [String] the schema string stored in the registry for the given id
|
|
22
23
|
def fetch(id)
|
|
23
|
-
@
|
|
24
|
+
@mutex.synchronize do
|
|
25
|
+
@schemas_by_id[id] ||= @upstream.fetch(id)
|
|
26
|
+
end
|
|
24
27
|
end
|
|
25
28
|
|
|
26
29
|
# @param id [Integer] the schema ID to fetch
|
|
27
30
|
# @param subject [String] the subject to fetch the version for
|
|
28
31
|
# @return [Integer, nil] the version of the schema for the given subject and id, or nil if not found
|
|
29
32
|
def fetch_version(id, subject)
|
|
30
|
-
|
|
31
|
-
|
|
33
|
+
@mutex.synchronize do
|
|
34
|
+
key = [subject, id]
|
|
35
|
+
return @versions_by_subject_and_id[key] if @versions_by_subject_and_id[key]
|
|
32
36
|
|
|
33
|
-
|
|
34
|
-
|
|
37
|
+
results = @upstream.schema_subject_versions(id)
|
|
38
|
+
@versions_by_subject_and_id[key] = results&.find { |r| r["subject"] == subject }&.dig("version")
|
|
39
|
+
end
|
|
35
40
|
end
|
|
36
41
|
|
|
37
42
|
# @param subject [String] the subject to check
|
|
38
43
|
# @param schema [String] the schema text to check
|
|
39
44
|
# @return [Boolean] true if we know the schema has been registered for that subject.
|
|
40
45
|
def registered?(subject, schema)
|
|
41
|
-
@
|
|
46
|
+
@mutex.synchronize do
|
|
47
|
+
@ids_by_schema.key?([subject, schema])
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def fetch_id(subject, schema)
|
|
52
|
+
@mutex.synchronize do
|
|
53
|
+
@ids_by_schema[[subject, schema]]
|
|
54
|
+
end
|
|
42
55
|
end
|
|
43
56
|
|
|
44
57
|
# @param subject [String] the subject to register the schema under
|
|
@@ -46,12 +59,14 @@ module SchemaRegistry
|
|
|
46
59
|
# @param references [Array<Hash>] optional references to other schemas
|
|
47
60
|
# @param schema_type [String]
|
|
48
61
|
def register(subject, schema, references: [], schema_type: "PROTOBUF")
|
|
49
|
-
|
|
62
|
+
@mutex.synchronize do
|
|
63
|
+
key = [subject, schema]
|
|
50
64
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
65
|
+
@ids_by_schema[key] ||= @upstream.register(subject,
|
|
66
|
+
schema,
|
|
67
|
+
references: references,
|
|
68
|
+
schema_type: schema_type)
|
|
69
|
+
end
|
|
55
70
|
end
|
|
56
71
|
end
|
|
57
72
|
end
|
|
@@ -24,9 +24,6 @@ module SchemaRegistry
|
|
|
24
24
|
resolv_resolver: nil,
|
|
25
25
|
retry_limit: nil
|
|
26
26
|
)
|
|
27
|
-
if SchemaRegistry.debug
|
|
28
|
-
logger.info("Creating Confluent Schema Registry client with url: #{url}, schema_context: #{schema_context}, user: #{user}, path_prefix: #{path_prefix}")
|
|
29
|
-
end
|
|
30
27
|
@path_prefix = path_prefix
|
|
31
28
|
@schema_context_prefix = schema_context.nil? ? "" : ":.#{schema_context}:"
|
|
32
29
|
@schema_context_options = schema_context.nil? ? {} : {query: {subject: @schema_context_prefix}}
|
|
@@ -76,7 +73,6 @@ module SchemaRegistry
|
|
|
76
73
|
# @param references [Array<Hash>] optional references to other schemas
|
|
77
74
|
# @return [Integer] the ID of the registered schema
|
|
78
75
|
def register(subject, schema, references: [], schema_type: "PROTOBUF")
|
|
79
|
-
puts("schema_type #{schema_type}")
|
|
80
76
|
body = {schema: schema.to_s}
|
|
81
77
|
# Not all schema registry versions support schemaType
|
|
82
78
|
if schema_type != "AVRO"
|
|
@@ -22,12 +22,18 @@ module SchemaRegistry
|
|
|
22
22
|
@schema_store ||= SchemaRegistry::AvroSchemaStore.new(
|
|
23
23
|
path: SchemaRegistry.avro_schema_path || DEFAULT_SCHEMAS_PATH
|
|
24
24
|
)
|
|
25
|
-
@
|
|
25
|
+
unless @schemas_loaded
|
|
26
|
+
@schema_store.load_schemas!
|
|
27
|
+
@schemas_loaded = true
|
|
28
|
+
end
|
|
26
29
|
@schema_store
|
|
27
30
|
end
|
|
28
31
|
|
|
32
|
+
# Register the fully-resolved (inlined) schema. The raw .avsc text is not a
|
|
33
|
+
# valid standalone schema when it references a type defined in another file.
|
|
29
34
|
def schema_text(_message, schema_name: nil)
|
|
30
|
-
|
|
35
|
+
@registration_text ||= {}
|
|
36
|
+
@registration_text[schema_name] ||= schema_store.find(schema_name).to_avro.to_json
|
|
31
37
|
end
|
|
32
38
|
|
|
33
39
|
def encode(message, stream, schema_name: nil)
|
|
@@ -44,10 +50,10 @@ module SchemaRegistry
|
|
|
44
50
|
end
|
|
45
51
|
|
|
46
52
|
def decode(stream, schema_text)
|
|
47
|
-
#
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
53
|
+
# Cache parsed writer schemas to avoid re-parsing on every decode
|
|
54
|
+
@parsed_writers_schemas ||= {}
|
|
55
|
+
@parsed_writers_schemas[schema_text] ||= ::Avro::Schema.parse(schema_text)
|
|
56
|
+
writers_schema = @parsed_writers_schemas[schema_text]
|
|
51
57
|
decoder = ::Avro::IO::BinaryDecoder.new(stream)
|
|
52
58
|
|
|
53
59
|
# Try to find the reader schema locally, fall back to writer schema
|
|
@@ -14,8 +14,6 @@ module SchemaRegistry
|
|
|
14
14
|
|
|
15
15
|
class << self
|
|
16
16
|
attr_accessor :avro_schema_path
|
|
17
|
-
|
|
18
|
-
attr_accessor :debug
|
|
19
17
|
end
|
|
20
18
|
|
|
21
19
|
class Client
|
|
@@ -136,7 +134,12 @@ module SchemaRegistry
|
|
|
136
134
|
|
|
137
135
|
def register_schema(message, subject, schema_text: nil, schema_name: nil)
|
|
138
136
|
schema_text ||= @schema.schema_text(message, schema_name: schema_name)
|
|
139
|
-
|
|
137
|
+
|
|
138
|
+
# Fast path: if already registered, return the cached ID without
|
|
139
|
+
# resolving dependencies again.
|
|
140
|
+
if @registry.registered?(subject, schema_text)
|
|
141
|
+
return @registry.fetch_id(subject, schema_text)
|
|
142
|
+
end
|
|
140
143
|
|
|
141
144
|
# register dependencies first
|
|
142
145
|
dependencies = @schema.dependencies(message)
|
|
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
|
|
|
12
12
|
spec.summary = "Confluent Schema Registry client with support for Avro and Protobuf"
|
|
13
13
|
spec.homepage = "https://github.com/flipp-oss/schema_registry_client"
|
|
14
14
|
spec.license = "MIT"
|
|
15
|
-
spec.required_ruby_version = ">= 3.
|
|
15
|
+
spec.required_ruby_version = ">= 3.2"
|
|
16
16
|
|
|
17
17
|
spec.metadata["rubygems_mfa_required"] = "true"
|
|
18
18
|
|
data/spec/decoding_spec.rb
CHANGED
|
@@ -60,18 +60,19 @@ RSpec.describe "decoding" do
|
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
describe "with Avro" do
|
|
63
|
-
|
|
63
|
+
around(:each) do |ex|
|
|
64
64
|
SchemaRegistry.avro_schema_path = "#{__dir__}/schemas"
|
|
65
|
+
ex.run
|
|
66
|
+
SchemaRegistry.avro_schema_path = nil
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
let(:schema_registry_client) do
|
|
65
70
|
SchemaRegistry::Client.new(
|
|
66
71
|
registry_url: "http://localhost:8081",
|
|
67
72
|
schema_type: SchemaRegistry::Schema::Avro.new
|
|
68
73
|
)
|
|
69
74
|
end
|
|
70
75
|
|
|
71
|
-
after do
|
|
72
|
-
SchemaRegistry.avro_schema_path = nil
|
|
73
|
-
end
|
|
74
|
-
|
|
75
76
|
it "should decode a simple message" do
|
|
76
77
|
schema = File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc")
|
|
77
78
|
stub = stub_request(:get, "http://localhost:8081/schemas/ids/15")
|
|
@@ -180,4 +181,121 @@ RSpec.describe "decoding" do
|
|
|
180
181
|
end.to raise_error(/Schema|not found/i)
|
|
181
182
|
end
|
|
182
183
|
end
|
|
184
|
+
|
|
185
|
+
describe "caching" do
|
|
186
|
+
it "should not fetch the same schema ID twice (Protobuf)" do
|
|
187
|
+
schema = File.read("#{__dir__}/schemas/simple/simple.proto")
|
|
188
|
+
stub = stub_request(:get, "http://localhost:8081/schemas/ids/15")
|
|
189
|
+
.to_return_json(body: {schema: schema})
|
|
190
|
+
msg = Simple::V1::SimpleMessage.new(name: "my name")
|
|
191
|
+
encoded = "\u0000\u0000\u0000\u0000\u000F\u0000#{msg.to_proto}"
|
|
192
|
+
|
|
193
|
+
3.times { schema_registry_client.decode(encoded) }
|
|
194
|
+
|
|
195
|
+
expect(stub).to have_been_requested.once
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
it "should fetch different schema IDs independently" do
|
|
199
|
+
simple_schema = File.read("#{__dir__}/schemas/simple/simple.proto")
|
|
200
|
+
stub_15 = stub_request(:get, "http://localhost:8081/schemas/ids/15")
|
|
201
|
+
.to_return_json(body: {schema: simple_schema})
|
|
202
|
+
|
|
203
|
+
ref_schema = File.read("#{__dir__}/schemas/referenced/referer.proto")
|
|
204
|
+
stub_20 = stub_request(:get, "http://localhost:8081/schemas/ids/20")
|
|
205
|
+
.to_return_json(body: {schema: ref_schema})
|
|
206
|
+
|
|
207
|
+
msg1 = Simple::V1::SimpleMessage.new(name: "my name")
|
|
208
|
+
encoded1 = "\u0000\u0000\u0000\u0000\u000F\u0000#{msg1.to_proto}"
|
|
209
|
+
|
|
210
|
+
msg2 = Referenced::V1::MessageB::MessageBA.new(
|
|
211
|
+
simple: Simple::V1::SimpleMessage.new(name: "my name")
|
|
212
|
+
)
|
|
213
|
+
encoded2 = "\u0000\u0000\u0000\u0000\u0014\u0004\u0002\u0000#{msg2.to_proto}"
|
|
214
|
+
|
|
215
|
+
# Decode both messages
|
|
216
|
+
schema_registry_client.decode(encoded1)
|
|
217
|
+
schema_registry_client.decode(encoded2)
|
|
218
|
+
|
|
219
|
+
# Each schema fetched exactly once
|
|
220
|
+
expect(stub_15).to have_been_requested.once
|
|
221
|
+
expect(stub_20).to have_been_requested.once
|
|
222
|
+
|
|
223
|
+
# Decode again - no new requests
|
|
224
|
+
schema_registry_client.decode(encoded1)
|
|
225
|
+
schema_registry_client.decode(encoded2)
|
|
226
|
+
|
|
227
|
+
expect(stub_15).to have_been_requested.once
|
|
228
|
+
expect(stub_20).to have_been_requested.once
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
it "should not share cache between different client instances" do
|
|
232
|
+
schema = File.read("#{__dir__}/schemas/simple/simple.proto")
|
|
233
|
+
stub = stub_request(:get, "http://localhost:8081/schemas/ids/15")
|
|
234
|
+
.to_return_json(body: {schema: schema})
|
|
235
|
+
msg = Simple::V1::SimpleMessage.new(name: "my name")
|
|
236
|
+
encoded = "\u0000\u0000\u0000\u0000\u000F\u0000#{msg.to_proto}"
|
|
237
|
+
|
|
238
|
+
schema_registry_client.decode(encoded)
|
|
239
|
+
|
|
240
|
+
# A second client should make its own request
|
|
241
|
+
client2 = SchemaRegistry::Client.new(registry_url: "http://localhost:8081")
|
|
242
|
+
client2.decode(encoded)
|
|
243
|
+
|
|
244
|
+
expect(stub).to have_been_requested.twice
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
describe "with Avro" do
|
|
248
|
+
around(:each) do |ex|
|
|
249
|
+
SchemaRegistry.avro_schema_path = "#{__dir__}/schemas"
|
|
250
|
+
ex.run
|
|
251
|
+
SchemaRegistry.avro_schema_path = nil
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
let(:schema_registry_client) do
|
|
255
|
+
SchemaRegistry::Client.new(
|
|
256
|
+
registry_url: "http://localhost:8081",
|
|
257
|
+
schema_type: SchemaRegistry::Schema::Avro.new
|
|
258
|
+
)
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
it "should not fetch the same schema ID twice" do
|
|
262
|
+
schema = File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc")
|
|
263
|
+
stub = stub_request(:get, "http://localhost:8081/schemas/ids/15")
|
|
264
|
+
.to_return_json(body: {schema: schema})
|
|
265
|
+
|
|
266
|
+
encoded = "\u0000\u0000\u0000\u0000\u000F\u000Emy name"
|
|
267
|
+
|
|
268
|
+
3.times { schema_registry_client.decode(encoded) }
|
|
269
|
+
|
|
270
|
+
expect(stub).to have_been_requested.once
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
it "should fetch different schema IDs independently" do
|
|
274
|
+
simple_schema = File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc")
|
|
275
|
+
stub_15 = stub_request(:get, "http://localhost:8081/schemas/ids/15")
|
|
276
|
+
.to_return_json(body: {schema: simple_schema})
|
|
277
|
+
|
|
278
|
+
nested_schema = File.read("#{__dir__}/schemas/referenced/v1/MessageBA.avsc")
|
|
279
|
+
stub_20 = stub_request(:get, "http://localhost:8081/schemas/ids/20")
|
|
280
|
+
.to_return_json(body: {schema: nested_schema})
|
|
281
|
+
|
|
282
|
+
encoded1 = "\u0000\u0000\u0000\u0000\u000F\u000Emy name"
|
|
283
|
+
encoded2 = "\u0000\u0000\u0000\u0000\u0014\u000Emy name"
|
|
284
|
+
|
|
285
|
+
# Decode both messages
|
|
286
|
+
schema_registry_client.decode(encoded1)
|
|
287
|
+
schema_registry_client.decode(encoded2)
|
|
288
|
+
|
|
289
|
+
expect(stub_15).to have_been_requested.once
|
|
290
|
+
expect(stub_20).to have_been_requested.once
|
|
291
|
+
|
|
292
|
+
# Decode again - no new requests
|
|
293
|
+
schema_registry_client.decode(encoded1)
|
|
294
|
+
schema_registry_client.decode(encoded2)
|
|
295
|
+
|
|
296
|
+
expect(stub_15).to have_been_requested.once
|
|
297
|
+
expect(stub_20).to have_been_requested.once
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
end
|
|
183
301
|
end
|
data/spec/encoding_spec.rb
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
RSpec.describe "encoding" do
|
|
4
|
+
def avro_registration_text(raw)
|
|
5
|
+
Avro::Schema.parse(raw).to_avro.to_json
|
|
6
|
+
end
|
|
7
|
+
|
|
4
8
|
let(:schema_registry_client) do
|
|
5
9
|
SchemaRegistry::Client.new(
|
|
6
10
|
registry_url: "http://localhost:8081"
|
|
@@ -102,20 +106,21 @@ RSpec.describe "encoding" do
|
|
|
102
106
|
end
|
|
103
107
|
|
|
104
108
|
describe "with Avro" do
|
|
105
|
-
|
|
109
|
+
around(:each) do |ex|
|
|
106
110
|
SchemaRegistry.avro_schema_path = "#{__dir__}/schemas"
|
|
111
|
+
ex.run
|
|
112
|
+
SchemaRegistry.avro_schema_path = nil
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
let(:schema_registry_client) do
|
|
107
116
|
SchemaRegistry::Client.new(
|
|
108
117
|
registry_url: "http://localhost:8081",
|
|
109
118
|
schema_type: SchemaRegistry::Schema::Avro.new
|
|
110
119
|
)
|
|
111
120
|
end
|
|
112
121
|
|
|
113
|
-
after do
|
|
114
|
-
SchemaRegistry.avro_schema_path = nil
|
|
115
|
-
end
|
|
116
|
-
|
|
117
122
|
it "should encode a simple message" do
|
|
118
|
-
schema = File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc")
|
|
123
|
+
schema = avro_registration_text(File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc"))
|
|
119
124
|
stub = stub_request(:post, "http://localhost:8081/subjects/simple/versions")
|
|
120
125
|
.with(body: {"schema" => schema}).to_return_json(body: {id: 15})
|
|
121
126
|
msg = {"name" => "my name"}
|
|
@@ -132,7 +137,7 @@ RSpec.describe "encoding" do
|
|
|
132
137
|
end
|
|
133
138
|
|
|
134
139
|
it "should encode a complex message with nested record" do
|
|
135
|
-
schema = File.read("#{__dir__}/schemas/referenced/v1/MessageBA.avsc")
|
|
140
|
+
schema = avro_registration_text(File.read("#{__dir__}/schemas/referenced/v1/MessageBA.avsc"))
|
|
136
141
|
stub = stub_request(:post, "http://localhost:8081/subjects/referenced/versions")
|
|
137
142
|
.with(body: {"schema" => schema}).to_return_json(body: {id: 20})
|
|
138
143
|
msg = {
|
|
@@ -168,7 +173,7 @@ RSpec.describe "encoding" do
|
|
|
168
173
|
File.write("#{multi_schema_path}/MultiFieldMessage.avsc", schema_json)
|
|
169
174
|
|
|
170
175
|
stub = stub_request(:post, "http://localhost:8081/subjects/multi/versions")
|
|
171
|
-
.with(body: {"schema" => schema_json}).to_return_json(body: {id: 25})
|
|
176
|
+
.with(body: {"schema" => avro_registration_text(schema_json)}).to_return_json(body: {id: 25})
|
|
172
177
|
|
|
173
178
|
msg = {"name" => "Alice", "age" => 30}
|
|
174
179
|
encoded = schema_registry_client.encode(msg, subject: "multi", schema_name: "test.v1.MultiFieldMessage")
|
|
@@ -184,7 +189,7 @@ RSpec.describe "encoding" do
|
|
|
184
189
|
end
|
|
185
190
|
|
|
186
191
|
it "should validate schema before encoding" do
|
|
187
|
-
schema = File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc")
|
|
192
|
+
schema = avro_registration_text(File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc"))
|
|
188
193
|
stub_request(:post, "http://localhost:8081/subjects/simple/versions")
|
|
189
194
|
.with(body: {"schema" => schema}).to_return_json(body: {id: 15})
|
|
190
195
|
|
|
@@ -195,5 +200,165 @@ RSpec.describe "encoding" do
|
|
|
195
200
|
schema_registry_client.encode(msg, subject: "simple", schema_name: "simple.v1.SimpleMessage")
|
|
196
201
|
end.to raise_error(Avro::SchemaValidator::ValidationError)
|
|
197
202
|
end
|
|
203
|
+
|
|
204
|
+
it "registers a self-contained schema when the .avsc references another file" do
|
|
205
|
+
# cross_file.v1.Outer references cross_file.v1.Inner, which is defined in a
|
|
206
|
+
# *separate* .avsc file, so the raw Outer.avsc text is not a valid
|
|
207
|
+
# standalone Avro schema...
|
|
208
|
+
raw = File.read("#{__dir__}/schemas/cross_file/v1/Outer.avsc")
|
|
209
|
+
expect { Avro::Schema.parse(raw) }.to raise_error(Avro::UnknownSchemaError)
|
|
210
|
+
|
|
211
|
+
# ...and the fully-resolved (inlined) schema is registered instead.
|
|
212
|
+
resolved = SchemaRegistry::Schema::Avro.new.schema_text(nil, schema_name: "cross_file.v1.Outer")
|
|
213
|
+
expect { Avro::Schema.parse(resolved) }.not_to raise_error
|
|
214
|
+
|
|
215
|
+
stub = stub_request(:post, "http://localhost:8081/subjects/cross/versions")
|
|
216
|
+
.with(body: {"schema" => resolved}).to_return_json(body: {id: 30})
|
|
217
|
+
|
|
218
|
+
encoded = schema_registry_client.encode({"inner" => {"name" => "my name"}},
|
|
219
|
+
subject: "cross", schema_name: "cross_file.v1.Outer")
|
|
220
|
+
|
|
221
|
+
expect(encoded[1..4].unpack1("N")).to eq(30)
|
|
222
|
+
expect(stub).to have_been_requested.once
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
describe "caching" do
|
|
227
|
+
it "should not register the same schema twice (Protobuf)" do
|
|
228
|
+
schema = File.read("#{__dir__}/schemas/simple/simple.proto")
|
|
229
|
+
stub = stub_request(:post, "http://localhost:8081/subjects/simple/versions")
|
|
230
|
+
.with(body: {"schemaType" => "PROTOBUF",
|
|
231
|
+
"references" => [],
|
|
232
|
+
"schema" => schema}).to_return_json(body: {id: 15})
|
|
233
|
+
msg = Simple::V1::SimpleMessage.new(name: "my name")
|
|
234
|
+
|
|
235
|
+
3.times { schema_registry_client.encode(msg, subject: "simple") }
|
|
236
|
+
|
|
237
|
+
expect(stub).to have_been_requested.once
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
it "should register schemas for different subjects separately" do
|
|
241
|
+
schema = File.read("#{__dir__}/schemas/simple/simple.proto")
|
|
242
|
+
stub_a = stub_request(:post, "http://localhost:8081/subjects/subject-a/versions")
|
|
243
|
+
.with(body: {"schemaType" => "PROTOBUF",
|
|
244
|
+
"references" => [],
|
|
245
|
+
"schema" => schema}).to_return_json(body: {id: 15})
|
|
246
|
+
stub_b = stub_request(:post, "http://localhost:8081/subjects/subject-b/versions")
|
|
247
|
+
.with(body: {"schemaType" => "PROTOBUF",
|
|
248
|
+
"references" => [],
|
|
249
|
+
"schema" => schema}).to_return_json(body: {id: 16})
|
|
250
|
+
|
|
251
|
+
msg = Simple::V1::SimpleMessage.new(name: "my name")
|
|
252
|
+
|
|
253
|
+
schema_registry_client.encode(msg, subject: "subject-a")
|
|
254
|
+
schema_registry_client.encode(msg, subject: "subject-b")
|
|
255
|
+
|
|
256
|
+
# Both subjects should get their own registration
|
|
257
|
+
expect(stub_a).to have_been_requested.once
|
|
258
|
+
expect(stub_b).to have_been_requested.once
|
|
259
|
+
|
|
260
|
+
# Encoding again for either subject should not re-register
|
|
261
|
+
schema_registry_client.encode(msg, subject: "subject-a")
|
|
262
|
+
schema_registry_client.encode(msg, subject: "subject-b")
|
|
263
|
+
|
|
264
|
+
expect(stub_a).to have_been_requested.once
|
|
265
|
+
expect(stub_b).to have_been_requested.once
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
it "should not re-register dependencies on subsequent encodes (Protobuf)" do
|
|
269
|
+
schema = File.read("#{__dir__}/schemas/referenced/referer.proto")
|
|
270
|
+
dep_schema = File.read("#{__dir__}/schemas/simple/simple.proto")
|
|
271
|
+
dep_stub = stub_request(:post, "http://localhost:8081/subjects/simple%2Fsimple.proto/versions")
|
|
272
|
+
.with(body: {"schemaType" => "PROTOBUF",
|
|
273
|
+
"references" => [],
|
|
274
|
+
"schema" => dep_schema}).to_return_json(body: {id: 15})
|
|
275
|
+
version_stub = stub_request(:get, "http://localhost:8081/schemas/ids/15/versions")
|
|
276
|
+
.to_return_json(body: [{version: 1, subject: "simple/simple.proto"}])
|
|
277
|
+
stub = stub_request(:post, "http://localhost:8081/subjects/referenced/versions")
|
|
278
|
+
.with(body: {"schemaType" => "PROTOBUF",
|
|
279
|
+
"references" => [
|
|
280
|
+
{
|
|
281
|
+
name: "simple/simple.proto",
|
|
282
|
+
subject: "simple/simple.proto",
|
|
283
|
+
version: 1
|
|
284
|
+
}
|
|
285
|
+
],
|
|
286
|
+
"schema" => schema}).to_return_json(body: {id: 20})
|
|
287
|
+
msg = Referenced::V1::MessageB::MessageBA.new(
|
|
288
|
+
simple: Simple::V1::SimpleMessage.new(name: "my name")
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
3.times { schema_registry_client.encode(msg, subject: "referenced") }
|
|
292
|
+
|
|
293
|
+
expect(stub).to have_been_requested.once
|
|
294
|
+
expect(dep_stub).to have_been_requested.once
|
|
295
|
+
expect(version_stub).to have_been_requested.once
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
it "should not share cache between different client instances" do
|
|
299
|
+
schema = File.read("#{__dir__}/schemas/simple/simple.proto")
|
|
300
|
+
stub = stub_request(:post, "http://localhost:8081/subjects/simple/versions")
|
|
301
|
+
.with(body: {"schemaType" => "PROTOBUF",
|
|
302
|
+
"references" => [],
|
|
303
|
+
"schema" => schema}).to_return_json(body: {id: 15})
|
|
304
|
+
msg = Simple::V1::SimpleMessage.new(name: "my name")
|
|
305
|
+
|
|
306
|
+
schema_registry_client.encode(msg, subject: "simple")
|
|
307
|
+
|
|
308
|
+
# A second client should make its own registration request
|
|
309
|
+
client2 = SchemaRegistry::Client.new(registry_url: "http://localhost:8081")
|
|
310
|
+
client2.encode(msg, subject: "simple")
|
|
311
|
+
|
|
312
|
+
expect(stub).to have_been_requested.twice
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
describe "with Avro" do
|
|
316
|
+
around(:each) do |ex|
|
|
317
|
+
SchemaRegistry.avro_schema_path = "#{__dir__}/schemas"
|
|
318
|
+
ex.run
|
|
319
|
+
SchemaRegistry.avro_schema_path = nil
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
let(:schema_registry_client) do
|
|
323
|
+
SchemaRegistry::Client.new(
|
|
324
|
+
registry_url: "http://localhost:8081",
|
|
325
|
+
schema_type: SchemaRegistry::Schema::Avro.new
|
|
326
|
+
)
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
it "should not register the same schema twice" do
|
|
330
|
+
schema = avro_registration_text(File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc"))
|
|
331
|
+
stub = stub_request(:post, "http://localhost:8081/subjects/simple/versions")
|
|
332
|
+
.with(body: {"schema" => schema}).to_return_json(body: {id: 15})
|
|
333
|
+
msg = {"name" => "my name"}
|
|
334
|
+
|
|
335
|
+
3.times { schema_registry_client.encode(msg, subject: "simple", schema_name: "simple.v1.SimpleMessage") }
|
|
336
|
+
|
|
337
|
+
expect(stub).to have_been_requested.once
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
it "should register schemas for different subjects separately" do
|
|
341
|
+
schema = avro_registration_text(File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc"))
|
|
342
|
+
stub_a = stub_request(:post, "http://localhost:8081/subjects/subject-a/versions")
|
|
343
|
+
.with(body: {"schema" => schema}).to_return_json(body: {id: 15})
|
|
344
|
+
stub_b = stub_request(:post, "http://localhost:8081/subjects/subject-b/versions")
|
|
345
|
+
.with(body: {"schema" => schema}).to_return_json(body: {id: 16})
|
|
346
|
+
|
|
347
|
+
msg = {"name" => "my name"}
|
|
348
|
+
|
|
349
|
+
schema_registry_client.encode(msg, subject: "subject-a", schema_name: "simple.v1.SimpleMessage")
|
|
350
|
+
schema_registry_client.encode(msg, subject: "subject-b", schema_name: "simple.v1.SimpleMessage")
|
|
351
|
+
|
|
352
|
+
expect(stub_a).to have_been_requested.once
|
|
353
|
+
expect(stub_b).to have_been_requested.once
|
|
354
|
+
|
|
355
|
+
# Encoding again should not re-register
|
|
356
|
+
schema_registry_client.encode(msg, subject: "subject-a", schema_name: "simple.v1.SimpleMessage")
|
|
357
|
+
schema_registry_client.encode(msg, subject: "subject-b", schema_name: "simple.v1.SimpleMessage")
|
|
358
|
+
|
|
359
|
+
expect(stub_a).to have_been_requested.once
|
|
360
|
+
expect(stub_b).to have_been_requested.once
|
|
361
|
+
end
|
|
362
|
+
end
|
|
198
363
|
end
|
|
199
364
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: schema_registry_client
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Orner
|
|
@@ -173,6 +173,8 @@ files:
|
|
|
173
173
|
- spec/gen/simple/simple_pb.rb
|
|
174
174
|
- spec/json_schema_spec.rb
|
|
175
175
|
- spec/proto_text_spec.rb
|
|
176
|
+
- spec/schemas/cross_file/v1/Inner.avsc
|
|
177
|
+
- spec/schemas/cross_file/v1/Outer.avsc
|
|
176
178
|
- spec/schemas/everything/everything.json
|
|
177
179
|
- spec/schemas/everything/everything.proto
|
|
178
180
|
- spec/schemas/referenced/referenced.json
|
|
@@ -194,7 +196,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
194
196
|
requirements:
|
|
195
197
|
- - ">="
|
|
196
198
|
- !ruby/object:Gem::Version
|
|
197
|
-
version: '3.
|
|
199
|
+
version: '3.2'
|
|
198
200
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
201
|
requirements:
|
|
200
202
|
- - ">="
|