avro_turf 0.9.0 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +2 -1
- data/.github/workflows/ruby.yml +20 -0
- data/CHANGELOG.md +10 -0
- data/Gemfile +0 -3
- data/README.md +15 -1
- data/avro_turf.gemspec +4 -4
- data/lib/avro_turf/confluent_schema_registry.rb +23 -4
- data/lib/avro_turf/disk_cache.rb +45 -0
- data/lib/avro_turf/messaging.rb +77 -14
- data/lib/avro_turf/version.rb +1 -1
- data/spec/confluent_schema_registry_spec.rb +13 -1
- data/spec/disk_cached_confluent_schema_registry_spec.rb +47 -0
- data/spec/messaging_spec.rb +154 -3
- metadata +13 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1e2ee4d9598bcaa2ec5738a1130fae0b19be7b5e7250f27540313b7471f97e23
|
4
|
+
data.tar.gz: 0b441cb30a153958c2ea283300a1a05c26218e2a97cb807fce36f8ad9d0240da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 12779eac5c325752cfa1be34da94ef5f332490cda4ff0aef29529a00557008cecf39592396a3f3525a2a12cb67a46744800781e0da69d4bf02511f5a2284e5e7
|
7
|
+
data.tar.gz: a2e4c84fb338d62296aefb8ae8c206c262d756475ec11bd9cead17bfe6015ea069ff36a891df3381385fa650cb4b5ea1584855fccc5294f5910ab34563ad973a
|
data/.circleci/config.yml
CHANGED
@@ -17,6 +17,7 @@ jobs:
|
|
17
17
|
- v1-dep-master-
|
18
18
|
# Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
|
19
19
|
- v1-dep-
|
20
|
+
- run: gem install bundler --no-document
|
20
21
|
- run: 'bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3'
|
21
22
|
# Save dependency cache
|
22
23
|
- save_cache:
|
@@ -26,7 +27,7 @@ jobs:
|
|
26
27
|
- ~/.bundle
|
27
28
|
- run: mkdir -p $CIRCLE_TEST_REPORTS/rspec
|
28
29
|
- run:
|
29
|
-
command: bundle exec rspec --color --require spec_helper --format
|
30
|
+
command: bundle exec rspec --color --require spec_helper --format progress
|
30
31
|
- store_test_results:
|
31
32
|
path: /tmp/circleci-test-results
|
32
33
|
- store_artifacts:
|
@@ -0,0 +1,20 @@
|
|
1
|
+
name: Ruby
|
2
|
+
|
3
|
+
on: [push]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
|
10
|
+
steps:
|
11
|
+
- uses: actions/checkout@v1
|
12
|
+
- name: Set up Ruby 2.6
|
13
|
+
uses: actions/setup-ruby@v1
|
14
|
+
with:
|
15
|
+
ruby-version: 2.6.x
|
16
|
+
- name: Build and test with RSpec
|
17
|
+
run: |
|
18
|
+
gem install bundler
|
19
|
+
bundle install --jobs 4 --retry 3
|
20
|
+
bundle exec rspec
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,16 @@
|
|
2
2
|
|
3
3
|
## Unreleased
|
4
4
|
|
5
|
+
## v0.11.0
|
6
|
+
|
7
|
+
- Add proxy support (#107)
|
8
|
+
- Adding support for client certs (#109)
|
9
|
+
|
10
|
+
## v0.10.0
|
11
|
+
|
12
|
+
- Add more disk caching (#103)
|
13
|
+
- Include schema information when decoding (#100, #101, #104)
|
14
|
+
|
5
15
|
## v0.9.0
|
6
16
|
|
7
17
|
- Compatibility with Avro v1.9.0 (#94)
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -126,13 +126,27 @@ data = avro.encode({ "title" => "hello, world" }, schema_name: "greeting")
|
|
126
126
|
|
127
127
|
# If you don't want to automatically register new schemas, you can pass explicitly
|
128
128
|
# subject and version to specify which schema should be used for encoding.
|
129
|
-
# It will fetch that schema from the registry and cache it. Subsequent instances
|
129
|
+
# It will fetch that schema from the registry and cache it. Subsequent instances
|
130
130
|
# of the same schema version will be served by the cache.
|
131
131
|
data = avro.encode({ "title" => "hello, world" }, subject: 'greeting', version: 1)
|
132
132
|
|
133
|
+
# You can also pass explicitly schema_id to specify which schema
|
134
|
+
# should be used for encoding.
|
135
|
+
# It will fetch that schema from the registry and cache it. Subsequent instances
|
136
|
+
# of the same schema version will be served by the cache.
|
137
|
+
data = avro.encode({ "title" => "hello, world" }, schema_id: 2)
|
138
|
+
|
133
139
|
# When decoding, the schema will be fetched from the registry and cached. Subsequent
|
134
140
|
# instances of the same schema id will be served by the cache.
|
135
141
|
avro.decode(data) #=> { "title" => "hello, world" }
|
142
|
+
|
143
|
+
# If you want to get decoded message as well as the schema used to encode the message,
|
144
|
+
# you can use `#decode_message` method.
|
145
|
+
result = avro.decode_message(data)
|
146
|
+
result.message #=> { "title" => "hello, world" }
|
147
|
+
result.schema_id #=> 3
|
148
|
+
result.writer_schema #=> #<Avro::Schema: ...>
|
149
|
+
result.reader_schema #=> nil
|
136
150
|
```
|
137
151
|
|
138
152
|
### Confluent Schema Registry Client
|
data/avro_turf.gemspec
CHANGED
@@ -20,10 +20,10 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_dependency "avro", ">= 1.7.7", "< 1.10"
|
21
21
|
spec.add_dependency "excon", "~> 0.45"
|
22
22
|
|
23
|
-
spec.add_development_dependency "bundler", "~>
|
24
|
-
spec.add_development_dependency "rake", "~>
|
25
|
-
spec.add_development_dependency "rspec", "~> 3.2
|
26
|
-
spec.add_development_dependency "fakefs", "~> 0.
|
23
|
+
spec.add_development_dependency "bundler", "~> 2.0"
|
24
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
25
|
+
spec.add_development_dependency "rspec", "~> 3.2"
|
26
|
+
spec.add_development_dependency "fakefs", "~> 0.20.0"
|
27
27
|
spec.add_development_dependency "webmock"
|
28
28
|
spec.add_development_dependency "sinatra"
|
29
29
|
spec.add_development_dependency "json_spec"
|
@@ -3,11 +3,30 @@ require 'excon'
|
|
3
3
|
class AvroTurf::ConfluentSchemaRegistry
|
4
4
|
CONTENT_TYPE = "application/vnd.schemaregistry.v1+json".freeze
|
5
5
|
|
6
|
-
def initialize(
|
6
|
+
def initialize(
|
7
|
+
url,
|
8
|
+
logger: Logger.new($stdout),
|
9
|
+
proxy: nil,
|
10
|
+
client_cert: nil,
|
11
|
+
client_key: nil,
|
12
|
+
client_key_pass: nil,
|
13
|
+
client_cert_data: nil,
|
14
|
+
client_key_data: nil
|
15
|
+
)
|
7
16
|
@logger = logger
|
8
|
-
|
9
|
-
"Content-Type" => CONTENT_TYPE
|
10
|
-
}
|
17
|
+
headers = {
|
18
|
+
"Content-Type" => CONTENT_TYPE
|
19
|
+
}
|
20
|
+
headers[:proxy] = proxy if proxy&.present?
|
21
|
+
@connection = Excon.new(
|
22
|
+
url,
|
23
|
+
headers: headers,
|
24
|
+
client_cert: client_cert,
|
25
|
+
client_key: client_key,
|
26
|
+
client_key_pass: client_key_pass,
|
27
|
+
client_cert_data: client_cert_data,
|
28
|
+
client_key_data: client_key_data
|
29
|
+
)
|
11
30
|
end
|
12
31
|
|
13
32
|
def fetch(id)
|
data/lib/avro_turf/disk_cache.rb
CHANGED
@@ -11,6 +11,9 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
11
11
|
|
12
12
|
@ids_by_schema_path = File.join(disk_path, 'ids_by_schema.json')
|
13
13
|
@ids_by_schema = JSON.parse(File.read(@ids_by_schema_path)) if File.exist?(@ids_by_schema_path)
|
14
|
+
|
15
|
+
@schemas_by_subject_version_path = File.join(disk_path, 'schemas_by_subject_version.json')
|
16
|
+
@schemas_by_subject_version = {}
|
14
17
|
end
|
15
18
|
|
16
19
|
# override
|
@@ -35,4 +38,46 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
35
38
|
File.write(@ids_by_schema_path, JSON.pretty_generate(@ids_by_schema))
|
36
39
|
return value
|
37
40
|
end
|
41
|
+
|
42
|
+
# checks instance var (in-memory cache) for schema
|
43
|
+
# checks disk cache if in-memory cache doesn't exists
|
44
|
+
# if file exists but no in-memory cache, read from file and sync in-memory cache
|
45
|
+
# finally, if file doesn't exist return nil
|
46
|
+
def lookup_by_version(subject, version)
|
47
|
+
key = "#{subject}#{version}"
|
48
|
+
schema = @schemas_by_subject_version[key]
|
49
|
+
|
50
|
+
return schema unless schema.nil?
|
51
|
+
|
52
|
+
hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
|
53
|
+
if hash
|
54
|
+
@schemas_by_subject_version = hash
|
55
|
+
@schemas_by_subject_version[key]
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# check if file exists and parse json into a hash
|
60
|
+
# if file exists take json and overwite/insert schema at key
|
61
|
+
# if file doesn't exist create new hash
|
62
|
+
# write the new/updated hash to file
|
63
|
+
# update instance var (in memory-cache) to match
|
64
|
+
def store_by_version(subject, version, schema)
|
65
|
+
key = "#{subject}#{version}"
|
66
|
+
hash = JSON.parse(File.read(@schemas_by_subject_version_path)) if File.exist?(@schemas_by_subject_version_path)
|
67
|
+
hash = if hash
|
68
|
+
hash[key] = schema
|
69
|
+
hash
|
70
|
+
else
|
71
|
+
{key => schema}
|
72
|
+
end
|
73
|
+
|
74
|
+
write_to_disk_cache(@schemas_by_subject_version_path, hash)
|
75
|
+
|
76
|
+
@schemas_by_subject_version = hash
|
77
|
+
@schemas_by_subject_version[key]
|
78
|
+
end
|
79
|
+
|
80
|
+
private def write_to_disk_cache(path, hash)
|
81
|
+
File.write(path, JSON.pretty_generate(hash))
|
82
|
+
end
|
38
83
|
end
|
data/lib/avro_turf/messaging.rb
CHANGED
@@ -21,21 +21,53 @@ class AvroTurf
|
|
21
21
|
# 1: https://github.com/confluentinc/schema-registry
|
22
22
|
class Messaging
|
23
23
|
MAGIC_BYTE = [0].pack("C").freeze
|
24
|
+
DecodedMessage = Struct.new(:schema_id, :writer_schema, :reader_schema, :message)
|
25
|
+
private_constant(:DecodedMessage)
|
24
26
|
|
25
27
|
# Instantiate a new Messaging instance with the given configuration.
|
26
28
|
#
|
27
|
-
# registry
|
28
|
-
#
|
29
|
-
# registry_url
|
30
|
-
# schema_store
|
31
|
-
# schemas_path
|
32
|
-
# namespace
|
33
|
-
# logger
|
34
|
-
|
29
|
+
# registry - A schema registry object that responds to all methods in the
|
30
|
+
# AvroTurf::ConfluentSchemaRegistry interface.
|
31
|
+
# registry_url - The String URL of the schema registry that should be used.
|
32
|
+
# schema_store - A schema store object that responds to #find(schema_name, namespace).
|
33
|
+
# schemas_path - The String file system path where local schemas are stored.
|
34
|
+
# namespace - The String default schema namespace.
|
35
|
+
# logger - The Logger that should be used to log information (optional).
|
36
|
+
# proxy - Forward the request via proxy (optional).
|
37
|
+
# client_cert - Name of file containing client certificate (optional).
|
38
|
+
# client_key - Name of file containing client private key to go with client_cert (optional).
|
39
|
+
# client_key_pass - Password to go with client_key (optional).
|
40
|
+
# client_cert_data - In-memory client certificate (optional).
|
41
|
+
# client_key_data - In-memory client private key to go with client_cert_data (optional).
|
42
|
+
def initialize(
|
43
|
+
registry: nil,
|
44
|
+
registry_url: nil,
|
45
|
+
schema_store: nil,
|
46
|
+
schemas_path: nil,
|
47
|
+
namespace: nil,
|
48
|
+
logger: nil,
|
49
|
+
proxy: nil,
|
50
|
+
client_cert: nil,
|
51
|
+
client_key: nil,
|
52
|
+
client_key_pass: nil,
|
53
|
+
client_cert_data: nil,
|
54
|
+
client_key_data: nil
|
55
|
+
)
|
35
56
|
@logger = logger || Logger.new($stderr)
|
36
57
|
@namespace = namespace
|
37
58
|
@schema_store = schema_store || SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
|
38
|
-
@registry = registry || CachedConfluentSchemaRegistry.new(
|
59
|
+
@registry = registry || CachedConfluentSchemaRegistry.new(
|
60
|
+
ConfluentSchemaRegistry.new(
|
61
|
+
registry_url,
|
62
|
+
logger: @logger,
|
63
|
+
proxy: proxy,
|
64
|
+
client_cert: client_cert,
|
65
|
+
client_key: client_key,
|
66
|
+
client_key_pass: client_key_pass,
|
67
|
+
client_cert_data: client_cert_data,
|
68
|
+
client_key_data: client_key_data
|
69
|
+
)
|
70
|
+
)
|
39
71
|
@schemas_by_id = {}
|
40
72
|
end
|
41
73
|
|
@@ -50,15 +82,19 @@ class AvroTurf
|
|
50
82
|
# the schema registry (optional).
|
51
83
|
# version - The integer version of the schema that should be used to decode
|
52
84
|
# the data. Must match the schema used when encoding (optional).
|
85
|
+
# schema_id - The integer id of the schema that should be used to encode
|
86
|
+
# the data.
|
53
87
|
#
|
54
88
|
# Returns the encoded data as a String.
|
55
|
-
def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil)
|
56
|
-
schema_id, schema = if
|
89
|
+
def encode(message, schema_name: nil, namespace: @namespace, subject: nil, version: nil, schema_id: nil)
|
90
|
+
schema_id, schema = if schema_id
|
91
|
+
fetch_schema_by_id(schema_id)
|
92
|
+
elsif subject && version
|
57
93
|
fetch_schema(subject, version)
|
58
94
|
elsif schema_name
|
59
95
|
register_schema(subject, schema_name, namespace)
|
60
96
|
else
|
61
|
-
raise ArgumentError.new('Neither schema_name nor subject + version provided to determine the schema.')
|
97
|
+
raise ArgumentError.new('Neither schema_name nor schema_id nor subject + version provided to determine the schema.')
|
62
98
|
end
|
63
99
|
|
64
100
|
stream = StringIO.new
|
@@ -76,7 +112,11 @@ class AvroTurf
|
|
76
112
|
|
77
113
|
stream.string
|
78
114
|
rescue Excon::Error::NotFound
|
79
|
-
|
115
|
+
if schema_id
|
116
|
+
raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
|
117
|
+
else
|
118
|
+
raise SchemaNotFoundError.new("Schema with subject: `#{subject}` version: `#{version}` is not found on registry")
|
119
|
+
end
|
80
120
|
end
|
81
121
|
|
82
122
|
# Decodes data into the original message.
|
@@ -88,6 +128,20 @@ class AvroTurf
|
|
88
128
|
#
|
89
129
|
# Returns the decoded message.
|
90
130
|
def decode(data, schema_name: nil, namespace: @namespace)
|
131
|
+
decode_message(data, schema_name: schema_name, namespace: namespace).message
|
132
|
+
end
|
133
|
+
|
134
|
+
# Decodes data into the original message.
|
135
|
+
#
|
136
|
+
# data - A String containing encoded data.
|
137
|
+
# schema_name - The String name of the schema that should be used to decode
|
138
|
+
# the data. Must match the schema used when encoding (optional).
|
139
|
+
# namespace - The namespace of the schema (optional).
|
140
|
+
#
|
141
|
+
# Returns Struct with the next attributes:
|
142
|
+
# schema_id - The integer id of schema used to encode the message
|
143
|
+
# message - The decoded message
|
144
|
+
def decode_message(data, schema_name: nil, namespace: @namespace)
|
91
145
|
readers_schema = schema_name && @schema_store.find(schema_name, namespace)
|
92
146
|
stream = StringIO.new(data)
|
93
147
|
decoder = Avro::IO::BinaryDecoder.new(stream)
|
@@ -108,7 +162,9 @@ class AvroTurf
|
|
108
162
|
end
|
109
163
|
|
110
164
|
reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
|
111
|
-
reader.read(decoder)
|
165
|
+
message = reader.read(decoder)
|
166
|
+
|
167
|
+
DecodedMessage.new(schema_id, writers_schema, readers_schema, message)
|
112
168
|
rescue Excon::Error::NotFound
|
113
169
|
raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
|
114
170
|
end
|
@@ -125,6 +181,13 @@ class AvroTurf
|
|
125
181
|
[schema_id, schema]
|
126
182
|
end
|
127
183
|
|
184
|
+
# Fetch the schema from registry with the provided schema_id.
|
185
|
+
def fetch_schema_by_id(schema_id)
|
186
|
+
schema_json = @registry.fetch(schema_id)
|
187
|
+
schema = Avro::Schema.parse(schema_json)
|
188
|
+
[schema_id, schema]
|
189
|
+
end
|
190
|
+
|
128
191
|
# Schemas are registered under the full name of the top level Avro record
|
129
192
|
# type, or `subject` if it's provided.
|
130
193
|
def register_schema(subject, schema_name, namespace)
|
data/lib/avro_turf/version.rb
CHANGED
@@ -3,7 +3,19 @@ require 'avro_turf/confluent_schema_registry'
|
|
3
3
|
require 'avro_turf/test/fake_confluent_schema_registry_server'
|
4
4
|
|
5
5
|
describe AvroTurf::ConfluentSchemaRegistry do
|
6
|
+
let(:client_cert) { "test client cert" }
|
7
|
+
let(:client_key) { "test client key" }
|
8
|
+
let(:client_key_pass) { "test client key password" }
|
9
|
+
|
6
10
|
it_behaves_like "a confluent schema registry client" do
|
7
|
-
let(:registry) {
|
11
|
+
let(:registry) {
|
12
|
+
described_class.new(
|
13
|
+
registry_url,
|
14
|
+
logger: logger,
|
15
|
+
client_cert: client_cert,
|
16
|
+
client_key: client_key,
|
17
|
+
client_key_pass: client_key_pass
|
18
|
+
)
|
19
|
+
}
|
8
20
|
end
|
9
21
|
end
|
@@ -24,6 +24,21 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
|
|
24
24
|
}.to_json
|
25
25
|
end
|
26
26
|
|
27
|
+
let(:subject) { 'subject' }
|
28
|
+
let(:version) { rand(999) }
|
29
|
+
let(:subject_version_schema) do
|
30
|
+
{
|
31
|
+
subject: subject,
|
32
|
+
version: version,
|
33
|
+
id: id,
|
34
|
+
schema: {
|
35
|
+
type: "record",
|
36
|
+
name: "city",
|
37
|
+
fields: { name: "name", type: "string" }
|
38
|
+
}
|
39
|
+
}.to_json
|
40
|
+
end
|
41
|
+
|
27
42
|
before do
|
28
43
|
FileUtils.mkdir_p("spec/cache")
|
29
44
|
end
|
@@ -105,6 +120,38 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
|
|
105
120
|
end
|
106
121
|
end
|
107
122
|
|
123
|
+
describe "#subject_version" do
|
124
|
+
it "writes thru to disk cache" do
|
125
|
+
# multiple calls return same result, with zero upstream calls
|
126
|
+
allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
|
127
|
+
expect(File).not_to exist("./spec/cache/schemas_by_subject_version.json")
|
128
|
+
|
129
|
+
expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
|
130
|
+
|
131
|
+
json = JSON.parse(File.read("./spec/cache/schemas_by_subject_version.json"))["#{subject}#{version}"]
|
132
|
+
expect(json).to eq(subject_version_schema)
|
133
|
+
|
134
|
+
expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
|
135
|
+
expect(upstream).to have_received(:subject_version).exactly(1).times
|
136
|
+
end
|
137
|
+
|
138
|
+
it "reads from disk cache and populates mem cache" do
|
139
|
+
allow(upstream).to receive(:subject_version).with(subject, version).and_return(subject_version_schema)
|
140
|
+
key = "#{subject}#{version}"
|
141
|
+
hash = {key => subject_version_schema}
|
142
|
+
cache.send(:write_to_disk_cache, "./spec/cache/schemas_by_subject_version.json", hash)
|
143
|
+
|
144
|
+
cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
|
145
|
+
expect(cached_schema).to eq({})
|
146
|
+
|
147
|
+
expect(registry.subject_version(subject, version)).to eq(subject_version_schema)
|
148
|
+
expect(upstream).to have_received(:subject_version).exactly(0).times
|
149
|
+
|
150
|
+
cached_schema = cache.instance_variable_get(:@schemas_by_subject_version)
|
151
|
+
expect(cached_schema).to eq({key => subject_version_schema})
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
108
155
|
it_behaves_like "a confluent schema registry client" do
|
109
156
|
let(:upstream) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
110
157
|
let(:registry) { described_class.new(upstream) }
|
data/spec/messaging_spec.rb
CHANGED
@@ -4,13 +4,19 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
|
|
4
4
|
|
5
5
|
describe AvroTurf::Messaging do
|
6
6
|
let(:registry_url) { "http://registry.example.com" }
|
7
|
+
let(:client_cert) { "test client cert" }
|
8
|
+
let(:client_key) { "test client key" }
|
9
|
+
let(:client_key_pass) { "test client key password" }
|
7
10
|
let(:logger) { Logger.new(StringIO.new) }
|
8
11
|
|
9
12
|
let(:avro) {
|
10
13
|
AvroTurf::Messaging.new(
|
11
14
|
registry_url: registry_url,
|
12
15
|
schemas_path: "spec/schemas",
|
13
|
-
logger: logger
|
16
|
+
logger: logger,
|
17
|
+
client_cert: client_cert,
|
18
|
+
client_key: client_key,
|
19
|
+
client_key_pass: client_key_pass
|
14
20
|
)
|
15
21
|
}
|
16
22
|
|
@@ -29,6 +35,7 @@ describe AvroTurf::Messaging do
|
|
29
35
|
}
|
30
36
|
AVSC
|
31
37
|
end
|
38
|
+
let(:schema) { Avro::Schema.parse(schema_json) }
|
32
39
|
|
33
40
|
before do
|
34
41
|
FileUtils.mkdir_p("spec/schemas")
|
@@ -66,8 +73,8 @@ describe AvroTurf::Messaging do
|
|
66
73
|
shared_examples_for 'encoding and decoding with the schema from registry' do
|
67
74
|
before do
|
68
75
|
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
69
|
-
registry.register('person',
|
70
|
-
registry.register('people',
|
76
|
+
registry.register('person', schema)
|
77
|
+
registry.register('people', schema)
|
71
78
|
end
|
72
79
|
|
73
80
|
it 'encodes and decodes messages' do
|
@@ -93,10 +100,37 @@ describe AvroTurf::Messaging do
|
|
93
100
|
end
|
94
101
|
end
|
95
102
|
|
103
|
+
shared_examples_for 'encoding and decoding with the schema_id from registry' do
|
104
|
+
before do
|
105
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
106
|
+
registry.register('person', schema)
|
107
|
+
registry.register('people', schema)
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'encodes and decodes messages' do
|
111
|
+
data = avro.encode(message, schema_id: 1)
|
112
|
+
expect(avro.decode(data)).to eq message
|
113
|
+
end
|
114
|
+
|
115
|
+
it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
|
116
|
+
expect { avro.encode(message, schema_id: 5) }.to raise_error(AvroTurf::SchemaNotFoundError)
|
117
|
+
end
|
118
|
+
|
119
|
+
it 'caches parsed schemas for decoding' do
|
120
|
+
data = avro.encode(message, schema_id: 1)
|
121
|
+
avro.decode(data)
|
122
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
123
|
+
expect(avro.decode(data)).to eq message
|
124
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
96
128
|
it_behaves_like "encoding and decoding with the schema from schema store"
|
97
129
|
|
98
130
|
it_behaves_like 'encoding and decoding with the schema from registry'
|
99
131
|
|
132
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
133
|
+
|
100
134
|
context "with a provided registry" do
|
101
135
|
let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
102
136
|
|
@@ -112,6 +146,8 @@ describe AvroTurf::Messaging do
|
|
112
146
|
|
113
147
|
it_behaves_like 'encoding and decoding with the schema from registry'
|
114
148
|
|
149
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
150
|
+
|
115
151
|
it "uses the provided registry" do
|
116
152
|
allow(registry).to receive(:register).and_call_original
|
117
153
|
message = { "full_name" => "John Doe" }
|
@@ -146,4 +182,119 @@ describe AvroTurf::Messaging do
|
|
146
182
|
expect(schema_store).to have_received(:find).with("person", nil)
|
147
183
|
end
|
148
184
|
end
|
185
|
+
|
186
|
+
describe 'decoding with #decode_message' do
|
187
|
+
shared_examples_for "encoding and decoding with the schema from schema store" do
|
188
|
+
it "encodes and decodes messages" do
|
189
|
+
data = avro.encode(message, schema_name: "person")
|
190
|
+
result = avro.decode_message(data)
|
191
|
+
expect(result.message).to eq message
|
192
|
+
expect(result.schema_id).to eq 0
|
193
|
+
expect(result.writer_schema).to eq schema
|
194
|
+
expect(result.reader_schema).to eq nil
|
195
|
+
end
|
196
|
+
|
197
|
+
it "allows specifying a reader's schema" do
|
198
|
+
data = avro.encode(message, schema_name: "person")
|
199
|
+
result = avro.decode_message(data, schema_name: "person")
|
200
|
+
expect(result.message).to eq message
|
201
|
+
expect(result.writer_schema).to eq schema
|
202
|
+
expect(result.reader_schema).to eq schema
|
203
|
+
end
|
204
|
+
|
205
|
+
it "caches parsed schemas for decoding" do
|
206
|
+
data = avro.encode(message, schema_name: "person")
|
207
|
+
avro.decode_message(data)
|
208
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
209
|
+
expect(avro.decode_message(data).message).to eq message
|
210
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
shared_examples_for 'encoding and decoding with the schema from registry' do
|
215
|
+
before do
|
216
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
217
|
+
registry.register('person', schema)
|
218
|
+
registry.register('people', schema)
|
219
|
+
end
|
220
|
+
|
221
|
+
it 'encodes and decodes messages' do
|
222
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
223
|
+
result = avro.decode_message(data)
|
224
|
+
expect(result.message).to eq message
|
225
|
+
expect(result.schema_id).to eq 0
|
226
|
+
end
|
227
|
+
|
228
|
+
it "allows specifying a reader's schema by subject and version" do
|
229
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
230
|
+
expect(avro.decode_message(data, schema_name: 'person').message).to eq message
|
231
|
+
end
|
232
|
+
|
233
|
+
it 'raises AvroTurf::SchemaNotFoundError when the schema does not exist on registry' do
|
234
|
+
expect { avro.encode(message, subject: 'missing', version: 1) }.to raise_error(AvroTurf::SchemaNotFoundError)
|
235
|
+
end
|
236
|
+
|
237
|
+
it 'caches parsed schemas for decoding' do
|
238
|
+
data = avro.encode(message, subject: 'person', version: 1)
|
239
|
+
avro.decode_message(data)
|
240
|
+
allow(Avro::Schema).to receive(:parse).and_call_original
|
241
|
+
expect(avro.decode_message(data).message).to eq message
|
242
|
+
expect(Avro::Schema).not_to have_received(:parse)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
247
|
+
|
248
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
249
|
+
|
250
|
+
context "with a provided registry" do
|
251
|
+
let(:registry) { AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) }
|
252
|
+
|
253
|
+
let(:avro) do
|
254
|
+
AvroTurf::Messaging.new(
|
255
|
+
registry: registry,
|
256
|
+
schemas_path: "spec/schemas",
|
257
|
+
logger: logger
|
258
|
+
)
|
259
|
+
end
|
260
|
+
|
261
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
262
|
+
|
263
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
264
|
+
|
265
|
+
it "uses the provided registry" do
|
266
|
+
allow(registry).to receive(:register).and_call_original
|
267
|
+
message = { "full_name" => "John Doe" }
|
268
|
+
avro.encode(message, schema_name: "person")
|
269
|
+
expect(registry).to have_received(:register).with("person", anything)
|
270
|
+
end
|
271
|
+
|
272
|
+
it "allows specifying a schema registry subject" do
|
273
|
+
allow(registry).to receive(:register).and_call_original
|
274
|
+
message = { "full_name" => "John Doe" }
|
275
|
+
avro.encode(message, schema_name: "person", subject: "people")
|
276
|
+
expect(registry).to have_received(:register).with("people", anything)
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
context "with a provided schema store" do
|
281
|
+
let(:schema_store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
|
282
|
+
|
283
|
+
let(:avro) do
|
284
|
+
AvroTurf::Messaging.new(
|
285
|
+
registry_url: registry_url,
|
286
|
+
schema_store: schema_store,
|
287
|
+
logger: logger
|
288
|
+
)
|
289
|
+
end
|
290
|
+
|
291
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
292
|
+
|
293
|
+
it "uses the provided schema store" do
|
294
|
+
allow(schema_store).to receive(:find).and_call_original
|
295
|
+
avro.encode(message, schema_name: "person")
|
296
|
+
expect(schema_store).to have_received(:find).with("person", nil)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
149
300
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro_turf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -50,56 +50,56 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
53
|
+
version: '2.0'
|
54
54
|
type: :development
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
60
|
+
version: '2.0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: rake
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
64
64
|
requirements:
|
65
65
|
- - "~>"
|
66
66
|
- !ruby/object:Gem::Version
|
67
|
-
version: '
|
67
|
+
version: '13.0'
|
68
68
|
type: :development
|
69
69
|
prerelease: false
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
72
|
- - "~>"
|
73
73
|
- !ruby/object:Gem::Version
|
74
|
-
version: '
|
74
|
+
version: '13.0'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
76
|
name: rspec
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
79
|
- - "~>"
|
80
80
|
- !ruby/object:Gem::Version
|
81
|
-
version: 3.2
|
81
|
+
version: '3.2'
|
82
82
|
type: :development
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
86
|
- - "~>"
|
87
87
|
- !ruby/object:Gem::Version
|
88
|
-
version: 3.2
|
88
|
+
version: '3.2'
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
90
|
name: fakefs
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
93
|
- - "~>"
|
94
94
|
- !ruby/object:Gem::Version
|
95
|
-
version: 0.
|
95
|
+
version: 0.20.0
|
96
96
|
type: :development
|
97
97
|
prerelease: false
|
98
98
|
version_requirements: !ruby/object:Gem::Requirement
|
99
99
|
requirements:
|
100
100
|
- - "~>"
|
101
101
|
- !ruby/object:Gem::Version
|
102
|
-
version: 0.
|
102
|
+
version: 0.20.0
|
103
103
|
- !ruby/object:Gem::Dependency
|
104
104
|
name: webmock
|
105
105
|
requirement: !ruby/object:Gem::Requirement
|
@@ -164,6 +164,7 @@ extensions: []
|
|
164
164
|
extra_rdoc_files: []
|
165
165
|
files:
|
166
166
|
- ".circleci/config.yml"
|
167
|
+
- ".github/workflows/ruby.yml"
|
167
168
|
- ".gitignore"
|
168
169
|
- ".rspec"
|
169
170
|
- CHANGELOG.md
|
@@ -248,7 +249,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
248
249
|
- !ruby/object:Gem::Version
|
249
250
|
version: '0'
|
250
251
|
requirements: []
|
251
|
-
|
252
|
+
rubyforge_project:
|
253
|
+
rubygems_version: 2.7.6
|
252
254
|
signing_key:
|
253
255
|
specification_version: 4
|
254
256
|
summary: A library that makes it easier to use the Avro serialization format from
|