schema_registry_client 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Gemfile.lock +2 -30
- data/LICENSE +2 -0
- data/README.md +51 -6
- data/lib/schema_registry_client/confluent_schema_registry.rb +7 -3
- data/lib/schema_registry_client/schema/avro.rb +41 -37
- data/lib/schema_registry_client/schema/base.rb +27 -29
- data/lib/schema_registry_client/schema/proto_json_schema.rb +13 -15
- data/lib/schema_registry_client/schema/protobuf.rb +93 -93
- data/lib/schema_registry_client/version.rb +1 -1
- data/lib/schema_registry_client.rb +2 -4
- data/spec/decoding_spec.rb +2 -2
- data/spec/encoding_spec.rb +6 -10
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d0919f12f0457fbdf7becd62389a7ed72e34afe9d7f3ce5c819f2e0ebf145174
|
|
4
|
+
data.tar.gz: 4dc5c91669e508e89bc07763bebb6b754403ee72ccbe125ecfc888c48cf974fa
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 07d614eaceb30420a69221399fab2e360f3f3b4089e5271b42fcaca30eb9f643cebcdd59187bdfc6a33d721641b960e2855fefce4703e59f455e459088bbd996
|
|
7
|
+
data.tar.gz: 65a8cf4a6899e8f32febb2b408ebb20ed379224b7fbd0d716da33c6e0e6c10844559e5d4e1af73c3ee06a1601ecd0dc3853137aa3987837157bbd07c07655a21
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## UNRELEASED
|
|
9
9
|
|
|
10
|
+
# 0.0.9 - 2026-02-11
|
|
11
|
+
|
|
12
|
+
* Fix: Do not send `schemaType` if schema type is `AVRO`, for backwards compatibility with older schema registries.
|
|
13
|
+
|
|
14
|
+
# 0.0.8 - 2026-02-04
|
|
15
|
+
|
|
16
|
+
* Support passing a `schema_store` into the `Avro` schema backend.
|
|
17
|
+
* Move schema backend methods from class-level to instance-level and require instantiation.
|
|
18
|
+
|
|
10
19
|
# 0.0.7 - 2026-01-05
|
|
11
20
|
|
|
12
21
|
* Switch to using `SchemaRegistry::Client` instead of bare `SchemaRegistry`.
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
schema_registry_client (0.0.
|
|
4
|
+
schema_registry_client (0.0.9)
|
|
5
5
|
avro
|
|
6
6
|
excon
|
|
7
7
|
google-protobuf
|
|
@@ -22,33 +22,12 @@ GEM
|
|
|
22
22
|
docile (1.4.1)
|
|
23
23
|
excon (1.3.2)
|
|
24
24
|
logger
|
|
25
|
-
google-protobuf (4.33.2)
|
|
26
|
-
bigdecimal
|
|
27
|
-
rake (>= 13)
|
|
28
|
-
google-protobuf (4.33.2-aarch64-linux-gnu)
|
|
29
|
-
bigdecimal
|
|
30
|
-
rake (>= 13)
|
|
31
|
-
google-protobuf (4.33.2-aarch64-linux-musl)
|
|
32
|
-
bigdecimal
|
|
33
|
-
rake (>= 13)
|
|
34
25
|
google-protobuf (4.33.2-arm64-darwin)
|
|
35
26
|
bigdecimal
|
|
36
27
|
rake (>= 13)
|
|
37
|
-
google-protobuf (4.33.2-x86-linux-gnu)
|
|
38
|
-
bigdecimal
|
|
39
|
-
rake (>= 13)
|
|
40
|
-
google-protobuf (4.33.2-x86-linux-musl)
|
|
41
|
-
bigdecimal
|
|
42
|
-
rake (>= 13)
|
|
43
|
-
google-protobuf (4.33.2-x86_64-darwin)
|
|
44
|
-
bigdecimal
|
|
45
|
-
rake (>= 13)
|
|
46
28
|
google-protobuf (4.33.2-x86_64-linux-gnu)
|
|
47
29
|
bigdecimal
|
|
48
30
|
rake (>= 13)
|
|
49
|
-
google-protobuf (4.33.2-x86_64-linux-musl)
|
|
50
|
-
bigdecimal
|
|
51
|
-
rake (>= 13)
|
|
52
31
|
hashdiff (1.2.1)
|
|
53
32
|
json (2.18.0)
|
|
54
33
|
language_server-protocol (3.17.0.5)
|
|
@@ -127,15 +106,8 @@ GEM
|
|
|
127
106
|
hashdiff (>= 0.4.0, < 2.0.0)
|
|
128
107
|
|
|
129
108
|
PLATFORMS
|
|
130
|
-
aarch64-linux-gnu
|
|
131
|
-
aarch64-linux-musl
|
|
132
109
|
arm64-darwin
|
|
133
|
-
|
|
134
|
-
x86-linux-gnu
|
|
135
|
-
x86-linux-musl
|
|
136
|
-
x86_64-darwin
|
|
137
|
-
x86_64-linux-gnu
|
|
138
|
-
x86_64-linux-musl
|
|
110
|
+
x86_64-linux
|
|
139
111
|
|
|
140
112
|
DEPENDENCIES
|
|
141
113
|
bundler (~> 2.0)
|
data/LICENSE
CHANGED
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# schema_registry_client
|
|
2
2
|
|
|
3
|
-
`schema_registry_client` is a library to interact with the Confluent Schema Registry
|
|
3
|
+
`schema_registry_client` is a library to interact with the Confluent Schema Registry. It is inspired by and based off of [avro_turf](https://github.com/dasch/avro_turf).
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -24,21 +24,66 @@ SchemaRegistry interacts with the Confluent Schema Registry, and caches all resu
|
|
|
24
24
|
|
|
25
25
|
Example usage:
|
|
26
26
|
|
|
27
|
+
### Avro
|
|
28
|
+
|
|
29
|
+
Note that unlike AvroTurf, you must concatenate the namespace and schema name together when encoding.
|
|
30
|
+
|
|
27
31
|
```ruby
|
|
28
32
|
require 'schema_registry_client'
|
|
29
33
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
34
|
+
client = SchemaRegistry::Client.new(registry_url: 'http://localhost:8081', schema_type: SchemaRegistry::Schema::Avro.new)
|
|
35
|
+
SchemaRegistry.avro_schema_path = 'path/to/schemas'
|
|
36
|
+
message = {field1: 'value1', field2: 42 }
|
|
37
|
+
encoded = client.encode(message, schema_name: 'com.my-namespace.MySchema', subject: 'my-subject')
|
|
38
|
+
|
|
39
|
+
# Decoding
|
|
40
|
+
|
|
41
|
+
decoded_avro_message = client.decode(encoded_string)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
### Protobuf
|
|
45
|
+
|
|
46
|
+
```ruby
|
|
47
|
+
client = SchemaRegistry::Client.new(registry_url: 'http://localhost:8081', schema_type: SchemaRegistry::Schema::Protobuf.new)
|
|
48
|
+
message = MyProtoMessage.new(field1: 'value1', field2: 42)
|
|
49
|
+
encoded = client.encode(message, subject: 'my-proto-subject')
|
|
33
50
|
|
|
34
51
|
# Decoding
|
|
52
|
+
decoded_proto_message = client.decode(encoded_string)
|
|
53
|
+
```
|
|
35
54
|
|
|
36
|
-
|
|
55
|
+
### Protobuf JSON Schema
|
|
56
|
+
|
|
57
|
+
Since Protobuf is [not recommended to use for Kafka keys](https://protobuf.dev/programming-guides/encoding/#implications), it's instead recommended that you use the `ProtoJsonSchema` format. This will transform the Protobuf message into a JSON schema and register that with the registry instead. The encoded message will be the JSON representation of the Protobuf message, with keys sorted alphabetically to ensure consistent encoding.
|
|
58
|
+
|
|
59
|
+
Note that the algorithm to translate into JSON Schema is currently very naive (e.g. it does not handle nested messages) since keys should usually be very simple. If more complex logic is needed, pull requests are welcome.
|
|
60
|
+
|
|
61
|
+
```ruby
|
|
62
|
+
client = SchemaRegistry::Client.new(registry_url: 'http://localhost:8081', schema_type: SchemaRegistry::Schema::ProtoJsonSchema)
|
|
63
|
+
message = MyProtoMessage.new(field1: 'value1', field2: 42)
|
|
64
|
+
encoded = client.encode(message, subject: 'my-proto-subject') # will register a JSON Schema subject and encode into JSON
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
You can use JSON Schema with regular Ruby hashes as well by passing `schema_text` into the encode method:
|
|
69
|
+
|
|
70
|
+
```ruby
|
|
71
|
+
client = SchemaRegistry::Client.new(registry_url: 'http://localhost:8081', schema_type: SchemaRegistry::Schema::ProtoJsonSchema)
|
|
72
|
+
message = { field1: 'value1', field2: 42 }
|
|
73
|
+
schema_text = {
|
|
74
|
+
"type" => "object",
|
|
75
|
+
"properties" => {
|
|
76
|
+
"field1" => { "type" => "string" },
|
|
77
|
+
"field2" => { "type" => "integer" }
|
|
78
|
+
},
|
|
79
|
+
"required" => ["field1", "field2"]
|
|
80
|
+
}.to_json
|
|
81
|
+
encoded = client.encode(message, subject: 'my-proto-subject', schema_text: schema_text)
|
|
37
82
|
```
|
|
38
83
|
|
|
39
84
|
## Notes about usage
|
|
40
85
|
|
|
41
|
-
* When decoding, this library does *not* attempt to fully parse the
|
|
86
|
+
* When decoding, this library does *not* attempt to fully parse the Protobuf definition stored on the schema registry and generate dynamic classes. Instead, it simply parses out the package and message and assumes that the reader has the message available in the descriptor pool. Any compatibility issues should be detected through normal means, i.e. just by instantiating the message and seeing if any errors are raised.
|
|
42
87
|
|
|
43
88
|
### Regenerating test protos
|
|
44
89
|
Run the following to regenerate:
|
|
@@ -73,10 +73,14 @@ module SchemaRegistry
|
|
|
73
73
|
# @param references [Array<Hash>] optional references to other schemas
|
|
74
74
|
# @return [Integer] the ID of the registered schema
|
|
75
75
|
def register(subject, schema, references: [], schema_type: "PROTOBUF")
|
|
76
|
+
body = {references: references,
|
|
77
|
+
schema: schema.to_s}
|
|
78
|
+
# Not all schema registry versions support schemaType
|
|
79
|
+
if schema_type != "AVRO"
|
|
80
|
+
body[:schemaType] = schema_type
|
|
81
|
+
end
|
|
76
82
|
data = post("/subjects/#{@schema_context_prefix}#{CGI.escapeURIComponent(subject)}/versions",
|
|
77
|
-
body:
|
|
78
|
-
references: references,
|
|
79
|
-
schema: schema.to_s}.to_json)
|
|
83
|
+
body: body.to_json)
|
|
80
84
|
|
|
81
85
|
id = data.fetch("id")
|
|
82
86
|
|
|
@@ -8,53 +8,57 @@ module SchemaRegistry
|
|
|
8
8
|
class Avro < Base
|
|
9
9
|
DEFAULT_SCHEMAS_PATH = "./schemas"
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
end
|
|
11
|
+
def self.schema_type
|
|
12
|
+
"AVRO"
|
|
13
|
+
end
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
@schema_store.load_schemas!
|
|
21
|
-
@schema_store
|
|
22
|
-
end
|
|
15
|
+
# @param schema_store [SchemaRegistry::AvroSchemaStore, nil]
|
|
16
|
+
def initialize(schema_store: nil)
|
|
17
|
+
@schema_store = schema_store
|
|
18
|
+
end
|
|
23
19
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
20
|
+
# @return [SchemaRegistry::AvroSchemaStore]
|
|
21
|
+
def schema_store
|
|
22
|
+
@schema_store ||= SchemaRegistry::AvroSchemaStore.new(
|
|
23
|
+
path: SchemaRegistry.avro_schema_path || DEFAULT_SCHEMAS_PATH
|
|
24
|
+
)
|
|
25
|
+
@schema_store.load_schemas!
|
|
26
|
+
@schema_store
|
|
27
|
+
end
|
|
27
28
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
fail_on_extra_fields: true}
|
|
32
|
-
schema = schema_store.find(schema_name)
|
|
29
|
+
def schema_text(_message, schema_name: nil)
|
|
30
|
+
schema_store.find_text(schema_name)
|
|
31
|
+
end
|
|
33
32
|
|
|
34
|
-
|
|
33
|
+
def encode(message, stream, schema_name: nil)
|
|
34
|
+
validate_options = {recursive: true,
|
|
35
|
+
encoded: false,
|
|
36
|
+
fail_on_extra_fields: true}
|
|
37
|
+
schema = schema_store.find(schema_name)
|
|
35
38
|
|
|
36
|
-
|
|
37
|
-
encoder = ::Avro::IO::BinaryEncoder.new(stream)
|
|
38
|
-
writer.write(message, encoder)
|
|
39
|
-
end
|
|
39
|
+
::Avro::SchemaValidator.validate!(schema, message, **validate_options)
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
writer = ::Avro::IO::DatumWriter.new(schema)
|
|
42
|
+
encoder = ::Avro::IO::BinaryEncoder.new(stream)
|
|
43
|
+
writer.write(message, encoder)
|
|
44
|
+
end
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
def decode(stream, schema_text)
|
|
47
|
+
# Parse the schema text from the registry into an Avro schema object
|
|
48
|
+
JSON.parse(schema_text)
|
|
49
|
+
writers_schema = ::Avro::Schema.parse(schema_text)
|
|
47
50
|
|
|
48
|
-
|
|
49
|
-
readers_schema = begin
|
|
50
|
-
schema_store.find(writers_schema.fullname)
|
|
51
|
-
rescue
|
|
52
|
-
writers_schema
|
|
53
|
-
end
|
|
51
|
+
decoder = ::Avro::IO::BinaryDecoder.new(stream)
|
|
54
52
|
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
# Try to find the reader schema locally, fall back to writer schema
|
|
54
|
+
readers_schema = begin
|
|
55
|
+
schema_store.find(writers_schema.fullname)
|
|
56
|
+
rescue
|
|
57
|
+
writers_schema
|
|
57
58
|
end
|
|
59
|
+
|
|
60
|
+
reader = ::Avro::IO::DatumReader.new(writers_schema, readers_schema)
|
|
61
|
+
reader.read(decoder)
|
|
58
62
|
end
|
|
59
63
|
end
|
|
60
64
|
end
|
|
@@ -5,39 +5,37 @@ module SchemaRegistry
|
|
|
5
5
|
class MissingImplementationError < StandardError; end
|
|
6
6
|
|
|
7
7
|
class Base
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
end
|
|
8
|
+
# @param message [Object]
|
|
9
|
+
# @param schema_name [String]
|
|
10
|
+
# @return [String]
|
|
11
|
+
def schema_text(_message, schema_name: nil)
|
|
12
|
+
raise MissingImplementationError, "Subclasses must implement schema_text"
|
|
13
|
+
end
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
# @return [String]
|
|
16
|
+
def self.schema_type
|
|
17
|
+
raise MissingImplementationError, "Subclasses must implement schema_type"
|
|
18
|
+
end
|
|
20
19
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
20
|
+
# @param message [Object]
|
|
21
|
+
# @param stream [StringIO]
|
|
22
|
+
# @param schema_name [String]
|
|
23
|
+
def encode(_message, _stream, schema_name: nil)
|
|
24
|
+
raise MissingImplementationError, "Subclasses must implement encode"
|
|
25
|
+
end
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
27
|
+
# @param stream [StringIO]
|
|
28
|
+
# @param schema_text [String]
|
|
29
|
+
# @param registry [Object]
|
|
30
|
+
# @return [Object]
|
|
31
|
+
def decode(_stream, _schema_text)
|
|
32
|
+
raise MissingImplementationError, "Subclasses must implement decode"
|
|
33
|
+
end
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
end
|
|
35
|
+
# @param message [Object]
|
|
36
|
+
# @return [Hash<String, String>]
|
|
37
|
+
def dependencies(_message)
|
|
38
|
+
{}
|
|
41
39
|
end
|
|
42
40
|
end
|
|
43
41
|
end
|
|
@@ -6,24 +6,22 @@ require "schema_registry_client/output/json_schema"
|
|
|
6
6
|
module SchemaRegistry
|
|
7
7
|
module Schema
|
|
8
8
|
class ProtoJsonSchema < Base
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
end
|
|
9
|
+
def self.schema_type
|
|
10
|
+
"JSON"
|
|
11
|
+
end
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
def schema_text(message, schema_name: nil)
|
|
14
|
+
SchemaRegistry::Output::JsonSchema.output(message.class.descriptor.to_proto)
|
|
15
|
+
end
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
17
|
+
def encode(message, stream, schema_name: nil)
|
|
18
|
+
json = message.to_h.sort.to_h.to_json
|
|
19
|
+
stream.write(json)
|
|
20
|
+
end
|
|
22
21
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
end
|
|
22
|
+
def decode(stream, _schema_text)
|
|
23
|
+
json = stream.read
|
|
24
|
+
JSON.parse(json)
|
|
27
25
|
end
|
|
28
26
|
end
|
|
29
27
|
end
|
|
@@ -10,121 +10,121 @@ require "schema_registry_client/wire"
|
|
|
10
10
|
module SchemaRegistry
|
|
11
11
|
module Schema
|
|
12
12
|
class Protobuf < Base
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
end
|
|
13
|
+
def self.schema_type
|
|
14
|
+
"PROTOBUF"
|
|
15
|
+
end
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
end
|
|
24
|
-
SchemaRegistry::Output::ProtoText.output(file_descriptor.to_proto)
|
|
17
|
+
def schema_text(message, schema_name: nil)
|
|
18
|
+
file_descriptor = if message.is_a?(Google::Protobuf::FileDescriptor)
|
|
19
|
+
message
|
|
20
|
+
else
|
|
21
|
+
message.class.descriptor.file_descriptor
|
|
25
22
|
end
|
|
23
|
+
SchemaRegistry::Output::ProtoText.output(file_descriptor.to_proto)
|
|
24
|
+
end
|
|
26
25
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if indexes == [0]
|
|
32
|
-
SchemaRegistry::Wire.write_int(stream, 0)
|
|
33
|
-
else
|
|
34
|
-
SchemaRegistry::Wire.write_int(stream, indexes.length)
|
|
35
|
-
indexes.each { |i| SchemaRegistry::Wire.write_int(stream, i) }
|
|
36
|
-
end
|
|
26
|
+
def encode(message, stream, schema_name: nil)
|
|
27
|
+
_, indexes = find_index(message.class.descriptor.to_proto,
|
|
28
|
+
message.class.descriptor.file_descriptor.to_proto.message_type)
|
|
37
29
|
|
|
38
|
-
|
|
39
|
-
|
|
30
|
+
if indexes == [0]
|
|
31
|
+
SchemaRegistry::Wire.write_int(stream, 0)
|
|
32
|
+
else
|
|
33
|
+
SchemaRegistry::Wire.write_int(stream, indexes.length)
|
|
34
|
+
indexes.each { |i| SchemaRegistry::Wire.write_int(stream, i) }
|
|
40
35
|
end
|
|
41
36
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
indexes = []
|
|
46
|
-
if index_length.zero?
|
|
47
|
-
indexes.push(0)
|
|
48
|
-
else
|
|
49
|
-
index_length.times do
|
|
50
|
-
indexes.push(SchemaRegistry::Wire.read_int(stream))
|
|
51
|
-
end
|
|
52
|
-
end
|
|
37
|
+
# Now we write the actual message.
|
|
38
|
+
stream.write(message.to_proto)
|
|
39
|
+
end
|
|
53
40
|
|
|
54
|
-
|
|
55
|
-
|
|
41
|
+
def decode(stream, schema_text)
|
|
42
|
+
# See https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format
|
|
43
|
+
index_length = SchemaRegistry::Wire.read_int(stream)
|
|
44
|
+
indexes = []
|
|
45
|
+
if index_length.zero?
|
|
46
|
+
indexes.push(0)
|
|
47
|
+
else
|
|
48
|
+
index_length.times do
|
|
49
|
+
indexes.push(SchemaRegistry::Wire.read_int(stream))
|
|
50
|
+
end
|
|
56
51
|
end
|
|
57
52
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
all_files.each do |file_desc|
|
|
62
|
-
file_path = file_desc.name
|
|
63
|
-
next if file_path.start_with?("google/protobuf/") # skip built-in protos
|
|
53
|
+
encoded = stream.read
|
|
54
|
+
decode_protobuf(schema_text, encoded, indexes)
|
|
55
|
+
end
|
|
64
56
|
|
|
65
|
-
|
|
66
|
-
|
|
57
|
+
def load_schemas!
|
|
58
|
+
@all_schemas = {}
|
|
59
|
+
all_files = ObjectSpace.each_object(Google::Protobuf::FileDescriptor).to_a
|
|
60
|
+
all_files.each do |file_desc|
|
|
61
|
+
file_path = file_desc.name
|
|
62
|
+
next if file_path.start_with?("google/protobuf/") # skip built-in protos
|
|
63
|
+
|
|
64
|
+
@all_schemas[file_path] = file_desc
|
|
67
65
|
end
|
|
66
|
+
end
|
|
68
67
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
file_descriptor = if message.is_a?(Google::Protobuf::FileDescriptor)
|
|
72
|
-
message
|
|
73
|
-
else
|
|
74
|
-
message.class.descriptor.file_descriptor
|
|
75
|
-
end
|
|
68
|
+
def dependencies(message)
|
|
69
|
+
return [] if message.nil?
|
|
76
70
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
end
|
|
71
|
+
load_schemas! unless @all_schemas&.any?
|
|
72
|
+
file_descriptor = if message.is_a?(Google::Protobuf::FileDescriptor)
|
|
73
|
+
message
|
|
74
|
+
else
|
|
75
|
+
message.class.descriptor.file_descriptor
|
|
83
76
|
end
|
|
84
77
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
else
|
|
91
|
-
found, found_indexes = find_index(descriptor, sub_descriptor.nested_type, indexes + [i])
|
|
92
|
-
return [true, found_indexes] if found
|
|
93
|
-
end
|
|
94
|
-
end
|
|
95
|
-
[]
|
|
78
|
+
deps = file_descriptor.to_proto.dependency.to_a
|
|
79
|
+
.reject { |d| d.start_with?("google/protobuf/") }
|
|
80
|
+
deps.to_h do |dep|
|
|
81
|
+
dependency_schema = @all_schemas[dep]
|
|
82
|
+
[dependency_schema.name, dependency_schema]
|
|
96
83
|
end
|
|
84
|
+
end
|
|
97
85
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
86
|
+
def find_index(descriptor, messages, indexes = [])
|
|
87
|
+
messages.each_with_index do |sub_descriptor, i|
|
|
88
|
+
if sub_descriptor == descriptor
|
|
89
|
+
indexes.push(i)
|
|
90
|
+
return [true, indexes]
|
|
91
|
+
else
|
|
92
|
+
found, found_indexes = find_index(descriptor, sub_descriptor.nested_type, indexes + [i])
|
|
93
|
+
return [true, found_indexes] if found
|
|
105
94
|
end
|
|
106
|
-
path
|
|
107
95
|
end
|
|
96
|
+
[]
|
|
97
|
+
end
|
|
108
98
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
msg = "Could not find schema for #{full_name}. " \
|
|
120
|
-
"Make sure the corresponding .proto file has been compiled and loaded."
|
|
121
|
-
raise msg
|
|
122
|
-
end
|
|
99
|
+
def find_descriptor(indexes, messages)
|
|
100
|
+
first_index = indexes.shift
|
|
101
|
+
message = messages[first_index]
|
|
102
|
+
path = [message.name]
|
|
103
|
+
while indexes.length.positive?
|
|
104
|
+
message = message.nested_type[indexes.shift]
|
|
105
|
+
path.push(message.name)
|
|
106
|
+
end
|
|
107
|
+
path
|
|
108
|
+
end
|
|
123
109
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
110
|
+
def decode_protobuf(schema, encoded, indexes)
|
|
111
|
+
# get the package
|
|
112
|
+
package = schema.match(/package (\S+);/)[1]
|
|
113
|
+
# get the first message in the protobuf text
|
|
114
|
+
# TODO - get the correct message based on schema index
|
|
115
|
+
message_name = schema.match(/message (\w+) {/)[1]
|
|
116
|
+
# look up the descriptor
|
|
117
|
+
full_name = "#{package}.#{message_name}"
|
|
118
|
+
descriptor = Google::Protobuf::DescriptorPool.generated_pool.lookup(full_name)
|
|
119
|
+
unless descriptor
|
|
120
|
+
msg = "Could not find schema for #{full_name}. " \
|
|
121
|
+
"Make sure the corresponding .proto file has been compiled and loaded."
|
|
122
|
+
raise msg
|
|
127
123
|
end
|
|
124
|
+
|
|
125
|
+
path = find_descriptor(indexes, descriptor.file_descriptor.to_proto.message_type)
|
|
126
|
+
correct_message = Google::Protobuf::DescriptorPool.generated_pool.lookup("#{package}.#{path.join(".")}")
|
|
127
|
+
correct_message.msgclass.decode(encoded)
|
|
128
128
|
end
|
|
129
129
|
end
|
|
130
130
|
end
|
|
@@ -66,7 +66,7 @@ module SchemaRegistry
|
|
|
66
66
|
client_key_data: nil,
|
|
67
67
|
connect_timeout: nil,
|
|
68
68
|
resolv_resolver: nil,
|
|
69
|
-
schema_type: SchemaRegistry::Schema::Protobuf
|
|
69
|
+
schema_type: SchemaRegistry::Schema::Protobuf.new
|
|
70
70
|
)
|
|
71
71
|
@logger = logger || Logger.new($stderr)
|
|
72
72
|
@registry = registry || SchemaRegistry::CachedConfluentSchemaRegistry.new(
|
|
@@ -152,9 +152,7 @@ module SchemaRegistry
|
|
|
152
152
|
version: versions[i]
|
|
153
153
|
}
|
|
154
154
|
end,
|
|
155
|
-
schema_type: @schema.schema_type)
|
|
155
|
+
schema_type: @schema.class.schema_type)
|
|
156
156
|
end
|
|
157
|
-
|
|
158
157
|
end
|
|
159
|
-
|
|
160
158
|
end
|
data/spec/decoding_spec.rb
CHANGED
|
@@ -40,7 +40,7 @@ RSpec.describe "decoding" do
|
|
|
40
40
|
let(:schema_registry_client) do
|
|
41
41
|
SchemaRegistry::Client.new(
|
|
42
42
|
registry_url: "http://localhost:8081",
|
|
43
|
-
schema_type: SchemaRegistry::Schema::ProtoJsonSchema
|
|
43
|
+
schema_type: SchemaRegistry::Schema::ProtoJsonSchema.new
|
|
44
44
|
)
|
|
45
45
|
end
|
|
46
46
|
|
|
@@ -64,7 +64,7 @@ RSpec.describe "decoding" do
|
|
|
64
64
|
SchemaRegistry.avro_schema_path = "#{__dir__}/schemas"
|
|
65
65
|
SchemaRegistry::Client.new(
|
|
66
66
|
registry_url: "http://localhost:8081",
|
|
67
|
-
schema_type: SchemaRegistry::Schema::Avro
|
|
67
|
+
schema_type: SchemaRegistry::Schema::Avro.new
|
|
68
68
|
)
|
|
69
69
|
end
|
|
70
70
|
|
data/spec/encoding_spec.rb
CHANGED
|
@@ -61,7 +61,7 @@ RSpec.describe "encoding" do
|
|
|
61
61
|
let(:schema_registry_client) do
|
|
62
62
|
SchemaRegistry::Client.new(
|
|
63
63
|
registry_url: "http://localhost:8081",
|
|
64
|
-
schema_type: SchemaRegistry::Schema::ProtoJsonSchema
|
|
64
|
+
schema_type: SchemaRegistry::Schema::ProtoJsonSchema.new
|
|
65
65
|
)
|
|
66
66
|
end
|
|
67
67
|
|
|
@@ -106,7 +106,7 @@ RSpec.describe "encoding" do
|
|
|
106
106
|
SchemaRegistry.avro_schema_path = "#{__dir__}/schemas"
|
|
107
107
|
SchemaRegistry::Client.new(
|
|
108
108
|
registry_url: "http://localhost:8081",
|
|
109
|
-
schema_type: SchemaRegistry::Schema::Avro
|
|
109
|
+
schema_type: SchemaRegistry::Schema::Avro.new
|
|
110
110
|
)
|
|
111
111
|
end
|
|
112
112
|
|
|
@@ -117,8 +117,7 @@ RSpec.describe "encoding" do
|
|
|
117
117
|
it "should encode a simple message" do
|
|
118
118
|
schema = File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc")
|
|
119
119
|
stub = stub_request(:post, "http://localhost:8081/subjects/simple/versions")
|
|
120
|
-
.with(body: {"
|
|
121
|
-
"references" => [],
|
|
120
|
+
.with(body: {"references" => [],
|
|
122
121
|
"schema" => schema}).to_return_json(body: {id: 15})
|
|
123
122
|
msg = {"name" => "my name"}
|
|
124
123
|
encoded = schema_registry_client.encode(msg, subject: "simple", schema_name: "simple.v1.SimpleMessage")
|
|
@@ -136,8 +135,7 @@ RSpec.describe "encoding" do
|
|
|
136
135
|
it "should encode a complex message with nested record" do
|
|
137
136
|
schema = File.read("#{__dir__}/schemas/referenced/v1/MessageBA.avsc")
|
|
138
137
|
stub = stub_request(:post, "http://localhost:8081/subjects/referenced/versions")
|
|
139
|
-
.with(body: {"
|
|
140
|
-
"references" => [],
|
|
138
|
+
.with(body: {"references" => [],
|
|
141
139
|
"schema" => schema}).to_return_json(body: {id: 20})
|
|
142
140
|
msg = {
|
|
143
141
|
"simple" => {
|
|
@@ -172,8 +170,7 @@ RSpec.describe "encoding" do
|
|
|
172
170
|
File.write("#{multi_schema_path}/MultiFieldMessage.avsc", schema_json)
|
|
173
171
|
|
|
174
172
|
stub = stub_request(:post, "http://localhost:8081/subjects/multi/versions")
|
|
175
|
-
.with(body: {"
|
|
176
|
-
"references" => [],
|
|
173
|
+
.with(body: {"references" => [],
|
|
177
174
|
"schema" => schema_json}).to_return_json(body: {id: 25})
|
|
178
175
|
|
|
179
176
|
msg = {"name" => "Alice", "age" => 30}
|
|
@@ -192,8 +189,7 @@ RSpec.describe "encoding" do
|
|
|
192
189
|
it "should validate schema before encoding" do
|
|
193
190
|
schema = File.read("#{__dir__}/schemas/simple/v1/SimpleMessage.avsc")
|
|
194
191
|
stub_request(:post, "http://localhost:8081/subjects/simple/versions")
|
|
195
|
-
.with(body: {"
|
|
196
|
-
"references" => [],
|
|
192
|
+
.with(body: {"references" => [],
|
|
197
193
|
"schema" => schema}).to_return_json(body: {id: 15})
|
|
198
194
|
|
|
199
195
|
# Invalid message - missing required field
|