avro_turf 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +2 -1
- data/.github/workflows/ruby.yml +20 -0
- data/CHANGELOG.md +10 -0
- data/Gemfile +0 -3
- data/README.md +15 -1
- data/avro_turf.gemspec +4 -4
- data/lib/avro_turf/confluent_schema_registry.rb +23 -4
- data/lib/avro_turf/disk_cache.rb +45 -0
- data/lib/avro_turf/messaging.rb +77 -14
- data/lib/avro_turf/version.rb +1 -1
- data/spec/confluent_schema_registry_spec.rb +13 -1
- data/spec/disk_cached_confluent_schema_registry_spec.rb +47 -0
- data/spec/messaging_spec.rb +154 -3
- metadata +13 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1e2ee4d9598bcaa2ec5738a1130fae0b19be7b5e7250f27540313b7471f97e23
|
4
|
+
data.tar.gz: 0b441cb30a153958c2ea283300a1a05c26218e2a97cb807fce36f8ad9d0240da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 12779eac5c325752cfa1be34da94ef5f332490cda4ff0aef29529a00557008cecf39592396a3f3525a2a12cb67a46744800781e0da69d4bf02511f5a2284e5e7
|
7
|
+
data.tar.gz: a2e4c84fb338d62296aefb8ae8c206c262d756475ec11bd9cead17bfe6015ea069ff36a891df3381385fa650cb4b5ea1584855fccc5294f5910ab34563ad973a
|
data/.circleci/config.yml
CHANGED
@@ -17,6 +17,7 @@ jobs:
|
|
17
17
|
- v1-dep-master-
|
18
18
|
# Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
|
19
19
|
- v1-dep-
|
20
|
+
- run: gem install bundler --no-document
|
20
21
|
- run: 'bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3'
|
21
22
|
# Save dependency cache
|
22
23
|
- save_cache:
|
@@ -26,7 +27,7 @@ jobs:
|
|
26
27
|
- ~/.bundle
|
27
28
|
- run: mkdir -p $CIRCLE_TEST_REPORTS/rspec
|
28
29
|
- run:
|
29
|
-
command: bundle exec rspec --color --require spec_helper --format
|
30
|
+
command: bundle exec rspec --color --require spec_helper --format progress
|
30
31
|
- store_test_results:
|
31
32
|
path: /tmp/circleci-test-results
|
32
33
|
- store_artifacts:
|
@@ -0,0 +1,20 @@
|
|
1
|
+
name: Ruby
|
2
|
+
|
3
|
+
on: [push]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
|
10
|
+
steps:
|
11
|
+
- uses: actions/checkout@v1
|
12
|
+
- name: Set up Ruby 2.6
|
13
|
+
uses: actions/setup-ruby@v1
|
14
|
+
with:
|
15
|
+
ruby-version: 2.6.x
|
16
|
+
- name: Build and test with RSpec
|
17
|
+
run: |
|
18
|
+
gem install bundler
|
19
|
+
bundle install --jobs 4 --retry 3
|
20
|
+
bundle exec rspec
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,16 @@
|
|
2
2
|
|
3
3
|
## Unreleased
|
4
4
|
|
5
|
+
## v0.11.0
|
6
|
+
|
7
|
+
- Add proxy support (#107)
|
8
|
+
- Adding support for client certs (#109)
|
9
|
+
|
10
|
+
## v0.10.0
|
11
|
+
|
12
|
+
- Add more disk caching (#103)
|
13
|
+
- Include schema information when decoding (#100, #101, #104)
|
14
|
+
|
5
15
|
## v0.9.0
|
6
16
|
|
7
17
|
- Compatibility with Avro v1.9.0 (#94)
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -126,13 +126,27 @@ data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
|
|
126
126
|
|
127
127
|
# If you don't want to automatically register new schemas, you can pass explicitly
|
128
128
|
# subject and version to specify which schema should be used for encoding.
|
129
|
-
# It will fetch that schema from the registry and cache it. Subsequent instances
|
129
|
+
# It will fetch that schema from the registry and cache it. Subsequent instances
|
130
130
|
# of the same schema version will be served by the cache.
|
131
131
|
data = avro.encode({ "title" => "hello, world" }, subject: 'greeting', version: 1)
|
132
132
|
|
133
|
+
# You can also pass explicitly schema_id to specify which schema
|
134
|
+
# should be used for encoding.
|
135
|
+
# It will fetch that schema from the registry and cache it. Subsequent instances
|
136
|
+
# of the same schema version will be served by the cache.
|
137
|
+
data = avro.encode({ "title" => "hello, world" }, schema_id: 2)
|
138
|
+
|
133
139
|
# When decoding, the schema will be fetched from the registry and cached. Subsequent
|
134
140
|
# instances of the same schema id will be served by the cache.
|
135
141
|
avro.decode(data) #=> { "title" => "hello, world" }
|
142
|
+
|
143
|
+
# If you want to get decoded message as well as the schema used to encode the message,
|
144
|
+
# you can use `#decode_message` method.
|
145
|
+
result = avro.decode_message(data)
|
146
|
+
result.message #=> { "title" => "hello, world" }
|
147
|
+
result.schema_id #=> 3
|
148
|
+
result.writer_schema #=> #<Avro::Schema: ...>
|
149
|
+
result.reader_schema #=> nil
|
136
150
|
```
|
137
151
|
|
138
152
|
### Confluent Schema Registry Client
|
data/avro_turf.gemspec
CHANGED
@@ -20,10 +20,10 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_dependency "avro", ">= 1.7.7", "< 1.10"
|
21
21
|
spec.add_dependency "excon", "~> 0.45"
|
22
22
|
|
23
|
-
spec.add_development_dependency "bundler", "~>
|
24
|
-
spec.add_development_dependency "rake", "~>
|
25
|
-
spec.add_development_dependency "rspec", "~> 3.2
|
26
|
-
spec.add_development_dependency "fakefs", "~> 0.
|
23
|
+
spec.add_development_dependency "bundler", "~> 2.0"
|
24
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
25
|
+
spec.add_development_dependency "rspec", "~> 3.2"
|
26
|
+
spec.add_development_dependency "fakefs", "~> 0.20.0"
|
27
27
|
spec.add_development_dependency "webmock"
|
28
28
|
spec.add_development_dependency "sinatra"
|
29
29
|
spec.add_development_dependency "json_spec"
|
@@ -3,11 +3,30 @@ require 'excon'
|
|
3
3
|
class AvroTurf::ConfluentSchemaRegistry
|
4
4
|
CONTENT_TYPE = "application/vnd.schemaregistry.v1+json".freeze
|
5
5
|
|
6
|
-
def initialize(
|
6
|
+
def initialize(
|
7
|
+
url,
|
8
|
+
logger: Logger.new($stdout),
|
9
|
+
proxy: nil,
|
10
|
+
client_cert: nil,
|
11
|
+
client_key: nil,
|
12
|
+
client_key_pass: nil,
|
13
|
+
client_cert_data: nil,
|
14
|
+
client_key_data: nil
|
15
|
+
)
|
7
16
|
@logger = logger
|
8
|
-
|
9
|
-
"Content-Type" => CONTENT_TYPE
|
10
|
-
}
|
17
|
+
headers = {
|
18
|
+
"Content-Type" => CONTENT_TYPE
|
19
|
+
}
|
20
|
+
headers[:proxy] = proxy if proxy&.present?
|
21
|
+
@connection = Excon.new(
|
22
|
+
url,
|
23
|
+
headers: headers,
|
24
|
+
client_cert: client_cert,
|
25
|
+
client_key: client_key,
|
26
|
+
client_key_pass: client_key_pass,
|
27
|
+
client_cert_data: client_cert_data,
|
28
|
+
client_key_data: client_key_data
|
29
|
+
)
|
11
30
|
end
|
12
31
|
|
13
32
|
def fetch(id)
|
data/lib/avro_turf/disk_cache.rb
CHANGED
@@ -11,6 +11,9 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
11
11
|
|
12
12
|
@ids_by_schema_path = File.join(disk_path, 'ids_by_schema.json')
|
13
13
|
@ids_by_schema = JSON.parse(File.read(@ids_by_schema_path)) if File.exist?(@ids_by_schema_path)
|
14
|
+
|
15
|
+
@schemas_by_subject_version_path = File.join(disk_path, 'schemas_by_subject_version.json')
|
16
|
+
@schemas_by_subject_version = {}
|
14
17
|
end
|
15
18
|
|
16
19
|
# override
|
@@ -35,4 +38,46 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
35
38
|
File.write(@ids_by_schema_path, JSON.pretty_generate(@ids_by_schema))
|
36
39
|
return value
|
37
40
|
end
|
41
|
+
|
42
|
+
# checks instance var (in-memory cache) for schema
|
43
|
+
# checks disk cache if in-memory cache doesn't exists
|
44
|
+
# if file exists but no in-memory cache, read from file and sync in-memory cache
|
45
|
+
# finally, if file doesn't exist return nil
|
46
|
+
def lookup_by_version(subject, version)
|
47
|
+
key = "#{subject}#{version}"
|
48
|
+
schema = @schemas_by_subject_version[key]
|
49
|
+
|
50
|
+
return schema unless schema.nil?
|
51
|
+
|
52
|
+
hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
|
53
|
+
if hash
|
54
|
+
@schemas_by_subject_version = hash
|
55
|
+
@schemas_by_subject_version[key]
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# check if file exists and parse json into a hash
|
60
|
+
# if file exists take json and overwite/insert schema at key
|
61
|
+
# if file doesn't exist create new hash
|
62
|
+
# write the new/updated hash to file
|
63
|
+
# update instance var (in memory-cache) to match
|
64
|
+
def store_by_version(subject, version, schema)
|
65
|
+
key = "#{subject}#{version}"
|
66
|
+
hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
|
67
|
+
hash = if hash
|
68
|
+
hash[key] = schema
|
69
|
+
hash
|
70
|
+
else
|
71
|
+
{key => schema}
|
72
|
+
end
|
73
|
+
|
74
|
+
write_to_disk_cache(@schemas_by_subject_version_path, hash)
|
75
|
+
|
76
|
+
@schemas_by_subject_version = hash
|
77
|
+
@schemas_by_subject_version[key]
|
78
|
+
end
|
79
|
+
|
80
|
+
private def write_to_disk_cache(path, hash)
|
81
|
+
File.write(path, JSON.pretty_generate(hash))
|
82
|
+
end
|
38
83
|
end
|
data/lib/avro_turf/messaging.rb
CHANGED
@@ -21,21 +21,53 @@ class AvroTurf
|
|
21
21
|
# 1: https://github.com/confluentinc/schema-registry
|
22
22
|
class Messaging
|
23
23
|
MAGIC_BYTE = [0].pack("C").freeze
|
24
|
+
DecodedMessage = Struct.new(:schema_id, :writer_schema, :reader_schema, :message)
|
25
|
+
private_constant(:DecodedMessage)
|
24
26
|
|
25
27
|
# Instantiate a new Messaging instance with the given configuration.
|
26
28
|
#
|
27
|
-
# registry
|
28
|
-
#
|
29
|
-
# registry_url
|
30
|
-
# schema_store
|
31
|
-
# schemas_path
|
32
|
-
# namespace
|
33
|
-
# logger
|
34
|
-
|
29
|
+
# registry - A schema registry object that responds to all methods in the
|
30
|
+
# AvroTurf::ConfluentSchemaRegistry interface.
|
31
|
+
# registry_url - The String URL of the schema registry that should be used.
|
32
|
+
# schema_store - A schema store object that responds to #find(schema_name, namespace).
|
33
|
+
# schemas_path - The String file system path where local schemas are stored.
|
34
|
+
# namespace - The String default schema namespace.
|
35
|
+
# logger - The Logger that should be used to log information (optional).
|
36
|
+
# proxy - Forward the request via proxy (optional).
|
37
|
+
# client_cert - Name of file containing client certificate (optional).
|
38
|
+
# client_key - Name of file containing client private key to go with client_cert (optional).
|
39
|
+
# client_key_pass - Password to go with client_key (optional).
|
40
|
+
# client_cert_data - In-memory client certificate (optional).
|
41
|
+
# client_key_data - In-memory client private key to go with client_cert_data (optional).
|
42
|
+
def initialize(
|
43
|
+
registry: nil,
|
44
|
+
registry_url: nil,
|
45
|
+
schema_store: nil,
|
46
|
+
schemas_path: nil,
|
47
|
+
namespace: nil,
|
48
|
+
logger: nil,
|
49
|
+
proxy: nil,
|
50
|
+
client_cert: nil,
|
51
|
+
client_key: nil,
|
52
|
+
client_key_pass: nil,
|
53
|
+
client_cert_data: nil,
|
54
|
+
client_key_data: nil
|
55
|
+
)
|
35
56
|
@logger = logger || Logger.new($stderr)
|
36
57
|
@namespace = namespace
|
37
58
|
@schema_store = schema_store || SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
|
38
|
-
@registry = registry || CachedConfluentSchemaRegistry.new(
|
59
|
+
@registry = registry || CachedConfluentSchemaRegistry.new(
|
60
|
+
ConfluentSchemaRegistry.new(
|
61
|
+
registry_url,
|
62
|
+
logger: @logger,
|
63
|
+
proxy: proxy,
|
64
|
+
client_cert: client_cert,
|
65
|
+
client_key: client_key,
|
66
|
+
client_key_pass: client_key_pass,
|
67
|
+
client_cert_data: client_cert_data,
|
68
|
+
client_key_data: client_key_data
|
69
|
+
)
|
70
|
+
)
|
39
71
|
@schemas_by_id = {}
|
40
72
|
end
|
41
73
|
|
@@ -50,15 +82,19 @@ class AvroTurf
|
|
50
82
|
# the schema registry (optional).
|
51
83
|
# version - The integer version of the schema that should be used to decode
|
52
84
|
# the data. Must match the schema used when encoding (optional).
|
85
|
+
# schema_id - The integer id of the schema that should be used to encode
|
86
|
+
# the data.
|
53
87
|
#
|
54
88
|
# Returns the encoded data as a String.
|
55
|
-
def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil)
|
56
|
-
schema_id, schema = if
|
89
|
+
def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil)
|
90
|
+
schema_id, schema = if schema_id
|
91
|
+
fetch_schema_by_id(schema_id)
|
92
|
+
elsif subject && version
|
57
93
|
fetch_schema(subject, version)
|
58
94
|
elsif schema_name
|
59
95
|
register_schema(subject, schema_name, namespace)
|
60
96
|
else
|
61
|
-
raise ArgumentError.new('Neither schema_name nor subject + version provided to determine the schema.')
|
97
|
+
raise ArgumentError.new('Neither schema_name nor schema_id nor subject + version provided to determine the schema.')
|
62
98
|
end
|
63
99
|
|
64
100
|
stream = StringIO.new
|
@@ -76,7 +112,11 @@ class AvroTurf
|
|
76
112
|
|
77
113
|
stream.string
|
78
114
|
rescue Excon::Error::NotFound
|
79
|
-
|
115
|
+
if schema_id
|
116
|
+
raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
|
117
|
+
else
|
118
|
+
raise SchemaNotFoundError.new("Schema with subject: `#{subject}` version: `#{version}` is not found on registry")
|
119
|
+
end
|
80
120
|
end
|
81
121
|
|
82
122
|
# Decodes data into the original message.
|
@@ -88,6 +128,20 @@ class AvroTurf
|
|
88
128
|
#
|
89
129
|
# Returns the decoded message.
|
90
130
|
def decode(data, schema_name: nil, namespace: @namespace)
|
131
|
+
decode_message(data, schema_name: schema_name, namespace: namespace).message
|
132
|
+
end
|
133
|
+
|
134
|
+
# Decodes data into the original message.
|
135
|
+
#
|
136
|
+
# data - A String containing encoded data.
|
137
|
+
# schema_name - The String name of the schema that should be used to decode
|
138
|
+
# the data. Must match the schema used when encoding (optional).
|
139
|
+
# namespace - The namespace of the schema (optional).
|
140
|
+
#
|
141
|
+
# Returns Struct with the next attributes:
|
142
|
+
# schema_id - The integer id of schema used to encode the message
|
143
|
+
# message - The decoded message
|
144
|
+
def decode_message(data, schema_name: nil, namespace: @namespace)
|
91
145
|
readers_schema = schema_name && @schema_store.find(schema_name, namespace)
|
92
146
|
stream = StringIO.new(data)
|
93
147
|
decoder = Avro::IO::BinaryDecoder.new(stream)
|
@@ -108,7 +162,9 @@ class AvroTurf
|
|
108
162
|
end
|
109
163
|
|
110
164
|
reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
|
111
|
-
reader.read(decoder)
|
165
|
+
message = reader.read(decoder)
|
166
|
+
|
167
|
+
DecodedMessage.new(schema_id, writers_schema, readers_schema, message)
|
112
168
|
rescue Excon::Error::NotFound
|
113
169
|
raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
|
114
170
|
end
|
@@ -125,6 +181,13 @@ class AvroTurf
|
|
125
181
|
[schema_id, schema]
|
126
182
|
end
|
127
183
|
|
184
|
+
# Fetch the schema from registry with the provided schema_id.
|
185
|
+
def fetch_schema_by_id(schema_id)
|
186
|
+
schema_json = @registry.fetch(schema_id)
|
187
|
+
schema = Avro::Schema.parse(schema_json)
|
188
|
+
[schema_id, schema]
|
189
|
+
end
|
190
|
+
|
128
191
|
# Schemas are registered under the full name of the top level Avro record
|
129
192
|
# type, or `subject` if it's provided.
|
130
193
|
def register_schema(subject, schema_name, namespace)
|
data/lib/avro_turf/version.rb
CHANGED
@@ -3,7 +3,19 @@ require 'avro_turf/confluent_schema_registry'
|
|
3
3
|
require 'avro_turf/test/fake_confluent_schema_registry_server'
|
4
4
|
|
5
5
|
describe AvroTurf::ConfluentSchemaRegistry do
|
6
|
+
let(:client_cert) { "test client cert" }
|
7
|
+
let(:client_key) { "test client key" }
|
8
|
+
let(:client_key_pass) { "test client key password" }
|
9
|
+
|
6
10
|
it_behaves_like "a confluent schema registry client" do
|
7
|
-
let(:registry) {
|
11
|
+
let(:registry) {
|
12
|
+
described_class.new(
|
13
|
+
registry_url,
|
14
|
+
logger: logger,
|
15
|
+
client_cert: client_cert,
|
16
|
+
client_key: client_key,
|
17
|
+
client_key_pass: client_key_pass
|
18
|
+
)
|
19
|
+
}
|
8
20
|
end
|
9
21
|
end
|
@@ -24,6 +24,21 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
|
|
24
24
|
}.to_json
|
25
25
|
end
|
26
26
|
|
27
|
+
let(:subject) { 'subject' }
|
28
|
+
let(:version) { rand(999) }
|
29
|
+
let(:subject_version_schema) do
|
30
|
+
{
|
31
|
+
subject: subject,
|
32
|
+
version: version,
|
33
|
+
id: id,
|
34
|
+
schema: {
|
35
|
+
type: "record",
|
36
|
+
name: "city",
|
37
|
+
fields: { name: "name", type: "string" }
|
38
|
+
}
|
39
|
+
}.to_json
|
40
|
+
end
|
41
|
+
|
27
42
|
before do
|
28
43
|
FileUtils.mkdir_p("spec/cache")
|
29
44
|
end
|
@@ -105,6 +120,38 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
|
|
105
120
|
end
|
106
121
|
end
|
107
122
|
|
123
|
+
describe "#subject_version" do
|
124
|
+
it "writes thru to disk cache" do
|
125
|
+
# multiple calls return same result, with zero upstream calls
|
126
|
+
allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
|
127
|
+
expect(File).not_to exist("./spec/cache/schemas_by_subject_version.json")
|
128
|
+
|
129
|
+
expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
|
130
|
+
|
131
|
+
json = JSON.parse(File.read("./spec/cache/schemas_by_subject_version.json"))["#{subject}#{version}"]
|
132
|
+
expect(json).to eq(subject_version_schema)
|
133
|
+
|
134
|
+
expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
|
135
|
+
expect(upstream).to have_received(:subject_version).exactly(1).times
|
136
|
+
end
|
137
|
+
|
138
|
+
it "reads from disk cache and populates mem cache" do
|
139
|
+
allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
|
140
|
+
key = "#{subject}#{version}"
|
141
|
+
hash = {key => subject_version_schema}
|
142
|
+
cache.send(:write_to_disk_cache, "./spec/cache/schemas_by_subject_version.json", hash)
|
143
|
+
|
144
|
+
cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
|
145
|
+
expect(cached_schema).to eq({})
|
146
|
+
|
147
|
+
expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
|
148
|
+
expect(upstream).to have_received(:subject_version).exactly(0).times
|
149
|
+
|
150
|
+
cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
|
151
|
+
expect(cached_schema).to eq({key => subject_version_schema})
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
108
155
|
it_behaves_like "a confluent schema registry client" do
|
109
156
|
let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
110
157
|
let(:registry) { described_class.new(upstream) }
|
data/spec/messaging_spec.rb
CHANGED
@@ -4,13 +4,19 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
|
|
4
4
|
|
5
5
|
describe AvroTurf::Messaging do
|
6
6
|
let(:registry_url) { "http://registry.example.com" }
|
7
|
+
let(:client_cert) { "test client cert" }
|
8
|
+
let(:client_key) { "test client key" }
|
9
|
+
let(:client_key_pass) { "test client key password" }
|
7
10
|
let(:logger) { Logger.new(StringIO.new) }
|
8
11
|
|
9
12
|
let(:avro) {
|
10
13
|
AvroTurf::Messaging.new(
|
11
14
|
registry_url: registry_url,
|
12
15
|
schemas_path: "spec/schemas",
|
13
|
-
logger: logger
|
16
|
+
logger: logger,
|
17
|
+
client_cert: client_cert,
|
18
|
+
client_key: client_key,
|
19
|
+
client_key_pass: client_key_pass
|
14
20
|
)
|
15
21
|
}
|
16
22
|
|
@@ -29,6 +35,7 @@ describe AvroTurf::Messaging do
|
|
29
35
|
}
|
30
36
|
AVSC
|
31
37
|
end
|
38
|
+
let(:schema) { Avro::Schema.parse(schema_json) }
|
32
39
|
|
33
40
|
before do
|
34
41
|
FileUtils.mkdir_p("spec/schemas")
|
@@ -66,8 +73,8 @@ describe AvroTurf::Messaging do
|
|
66
73
|
shared_examples_for 'encoding and decoding with the schema from registry' do
|
67
74
|
before do
|
68
75
|
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
69
|
-
registry.register('person',
|
70
|
-
registry.register('people',
|
76
|
+
registry.register('person', schema)
|
77
|
+
registry.register('people', schema)
|
71
78
|
end
|
72
79
|
|
73
80
|
it 'encodes and decodes messages' do
|
@@ -93,10 +100,37 @@ describe AvroTurf::Messaging do
|
|
93
100
|
end
|
94
101
|
end
|
95
102
|
|
103
|
+
shared_examples_for 'encoding and decoding with the schema_id from registry' do
|
104
|
+
before do
|
105
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
106
|
+
registry.register('person', schema)
|
107
|
+
registry.register('people', schema)
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'encodes and decodes messages' do
|
111
|
+
data = avro.encode(message, schema_id: 1)
|
112
|
+
expect(avro.decode(data)).to eq message
|
113
|
+
end
|
114
|
+
|
115
|
+
it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
|
116
|
+
expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
|
117
|
+
end
|
118
|
+
|
119
|
+
it 'caches parsed schemas for decoding' do
|
120
|
+
data = avro.encode(message, schema_id: 1)
|
121
|
+
avro.decode(data)
|
122
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
123
|
+
expect(avro.decode(data)).to eq message
|
124
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
96
128
|
it_behaves_like "encoding and decoding with the schema from schema store"
|
97
129
|
|
98
130
|
it_behaves_like 'encoding and decoding with the schema from registry'
|
99
131
|
|
132
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
133
|
+
|
100
134
|
context "with a provided registry" do
|
101
135
|
let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
102
136
|
|
@@ -112,6 +146,8 @@ describe AvroTurf::Messaging do
|
|
112
146
|
|
113
147
|
it_behaves_like 'encoding and decoding with the schema from registry'
|
114
148
|
|
149
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
150
|
+
|
115
151
|
it "uses the provided registry" do
|
116
152
|
allow(registry).to receive(:register).and_call_original
|
117
153
|
message = { "full_name" => "John Doe" }
|
@@ -146,4 +182,119 @@ describe AvroTurf::Messaging do
|
|
146
182
|
expect(schema_store).to have_received(:find).with("person", nil)
|
147
183
|
end
|
148
184
|
end
|
185
|
+
|
186
|
+
describe 'decoding with #decode_message' do
|
187
|
+
shared_examples_for "encoding and decoding with the schema from schema store" do
|
188
|
+
it "encodes and decodes messages" do
|
189
|
+
data = avro.encode(message, schema_name: "person")
|
190
|
+
result = avro.decode_message(data)
|
191
|
+
expect(result.message).to eq message
|
192
|
+
expect(result.schema_id).to eq 0
|
193
|
+
expect(result.writer_schema).to eq schema
|
194
|
+
expect(result.reader_schema).to eq nil
|
195
|
+
end
|
196
|
+
|
197
|
+
it "allows specifying a reader's schema" do
|
198
|
+
data = avro.encode(message, schema_name: "person")
|
199
|
+
result = avro.decode_message(data, schema_name: "person")
|
200
|
+
expect(result.message).to eq message
|
201
|
+
expect(result.writer_schema).to eq schema
|
202
|
+
expect(result.reader_schema).to eq schema
|
203
|
+
end
|
204
|
+
|
205
|
+
it "caches parsed schemas for decoding" do
|
206
|
+
data = avro.encode(message, schema_name: "person")
|
207
|
+
avro.decode_message(data)
|
208
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
209
|
+
expect(avro.decode_message(data).message).to eq message
|
210
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
shared_examples_for 'encoding and decoding with the schema from registry' do
|
215
|
+
before do
|
216
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
217
|
+
registry.register('person', schema)
|
218
|
+
registry.register('people', schema)
|
219
|
+
end
|
220
|
+
|
221
|
+
it 'encodes and decodes messages' do
|
222
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
223
|
+
result = avro.decode_message(data)
|
224
|
+
expect(result.message).to eq message
|
225
|
+
expect(result.schema_id).to eq 0
|
226
|
+
end
|
227
|
+
|
228
|
+
it "allows specifying a reader's schema by subject and version" do
|
229
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
230
|
+
expect(avro.decode_message(data, schema_name: 'person').message).to eq message
|
231
|
+
end
|
232
|
+
|
233
|
+
it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
|
234
|
+
expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
|
235
|
+
end
|
236
|
+
|
237
|
+
it 'caches parsed schemas for decoding' do
|
238
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
239
|
+
avro.decode_message(data)
|
240
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
241
|
+
expect(avro.decode_message(data).message).to eq message
|
242
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
247
|
+
|
248
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
249
|
+
|
250
|
+
context "with a provided registry" do
|
251
|
+
let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
252
|
+
|
253
|
+
let(:avro) do
|
254
|
+
AvroTurf::Messaging.new(
|
255
|
+
registry: registry,
|
256
|
+
schemas_path: "spec/schemas",
|
257
|
+
logger: logger
|
258
|
+
)
|
259
|
+
end
|
260
|
+
|
261
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
262
|
+
|
263
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
264
|
+
|
265
|
+
it "uses the provided registry" do
|
266
|
+
allow(registry).to receive(:register).and_call_original
|
267
|
+
message = { "full_name" => "John Doe" }
|
268
|
+
avro.encode(message, schema_name: "person")
|
269
|
+
expect(registry).to have_received(:register).with("person", anything)
|
270
|
+
end
|
271
|
+
|
272
|
+
it "allows specifying a schema registry subject" do
|
273
|
+
allow(registry).to receive(:register).and_call_original
|
274
|
+
message = { "full_name" => "John Doe" }
|
275
|
+
avro.encode(message, schema_name: "person", subject: "people")
|
276
|
+
expect(registry).to have_received(:register).with("people", anything)
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
context "with a provided schema store" do
|
281
|
+
let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
|
282
|
+
|
283
|
+
let(:avro) do
|
284
|
+
AvroTurf::Messaging.new(
|
285
|
+
registry_url: registry_url,
|
286
|
+
schema_store: schema_store,
|
287
|
+
logger: logger
|
288
|
+
)
|
289
|
+
end
|
290
|
+
|
291
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
292
|
+
|
293
|
+
it "uses the provided schema store" do
|
294
|
+
allow(schema_store).to receive(:find).and_call_original
|
295
|
+
avro.encode(message, schema_name: "person")
|
296
|
+
expect(schema_store).to have_received(:find).with("person", nil)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
149
300
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro_turf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -50,56 +50,56 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
53
|
+
version: '2.0'
|
54
54
|
type: :development
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
60
|
+
version: '2.0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: rake
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
64
64
|
requirements:
|
65
65
|
- - "~>"
|
66
66
|
- !ruby/object:Gem::Version
|
67
|
-
version: '
|
67
|
+
version: '13.0'
|
68
68
|
type: :development
|
69
69
|
prerelease: false
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
72
|
- - "~>"
|
73
73
|
- !ruby/object:Gem::Version
|
74
|
-
version: '
|
74
|
+
version: '13.0'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
76
|
name: rspec
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
79
|
- - "~>"
|
80
80
|
- !ruby/object:Gem::Version
|
81
|
-
version: 3.2
|
81
|
+
version: '3.2'
|
82
82
|
type: :development
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
86
|
- - "~>"
|
87
87
|
- !ruby/object:Gem::Version
|
88
|
-
version: 3.2
|
88
|
+
version: '3.2'
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
90
|
name: fakefs
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
93
|
- - "~>"
|
94
94
|
- !ruby/object:Gem::Version
|
95
|
-
version: 0.
|
95
|
+
version: 0.20.0
|
96
96
|
type: :development
|
97
97
|
prerelease: false
|
98
98
|
version_requirements: !ruby/object:Gem::Requirement
|
99
99
|
requirements:
|
100
100
|
- - "~>"
|
101
101
|
- !ruby/object:Gem::Version
|
102
|
-
version: 0.
|
102
|
+
version: 0.20.0
|
103
103
|
- !ruby/object:Gem::Dependency
|
104
104
|
name: webmock
|
105
105
|
requirement: !ruby/object:Gem::Requirement
|
@@ -164,6 +164,7 @@ extensions: []
|
|
164
164
|
extra_rdoc_files: []
|
165
165
|
files:
|
166
166
|
- ".circleci/config.yml"
|
167
|
+
- ".github/workflows/ruby.yml"
|
167
168
|
- ".gitignore"
|
168
169
|
- ".rspec"
|
169
170
|
- CHANGELOG.md
|
@@ -248,7 +249,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
248
249
|
- !ruby/object:Gem::Version
|
249
250
|
version: '0'
|
250
251
|
requirements: []
|
251
|
-
|
252
|
+
rubyforge_project:
|
253
|
+
rubygems_version: 2.7.6
|
252
254
|
signing_key:
|
253
255
|
specification_version: 4
|
254
256
|
summary: A library that makes it easier to use the Avro serialization format from
|