avro_turf 1.4.0 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/avro_turf.gemspec +3 -1
- data/lib/avro_turf/confluent_schema_registry.rb +6 -1
- data/lib/avro_turf/messaging.rb +22 -16
- data/lib/avro_turf/schema_store.rb +2 -2
- data/lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb +102 -0
- data/lib/avro_turf/version.rb +1 -1
- data/spec/avro_turf_spec.rb +21 -3
- data/spec/messaging_spec.rb +30 -4
- metadata +7 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: db17e2aed8677c0288cd1c817b2e4ecae50da47255663c2b0a75d64a43e91152
|
4
|
+
data.tar.gz: e88f6481337b05b80db9710283daf20818fcad9621c09ca9ebd75e533ca5135c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3c226b186752b4feed4519d329e7789dc02ad29793d3573ca2832a182074010652c16746308f785320018bed528d00b241c9b6354b2a3271a72278812d552d56
|
7
|
+
data.tar.gz: c07515285fbd182752764d43c350d529f9a60bce1ad7d7aa6992cb1cda9fb958c189092d78a432a6e141eca380c8e9459857b29e2b038f03e32b645f5b4f0461
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,19 @@
|
|
2
2
|
|
3
3
|
## Unreleased
|
4
4
|
|
5
|
+
## v1.6.0
|
6
|
+
|
7
|
+
- Schema registry path prefix (#162)
|
8
|
+
|
9
|
+
## v1.5.0
|
10
|
+
|
11
|
+
- Add CA cert file option (#157)
|
12
|
+
- Add compatibility with Avro v1.11.x.
|
13
|
+
|
14
|
+
## v1.4.1
|
15
|
+
|
16
|
+
- Purge sub-schemas from cache before re-parsing schema (#151)
|
17
|
+
|
5
18
|
## v1.4.0
|
6
19
|
|
7
20
|
- Add support for Ruby 3 (#146)
|
data/avro_turf.gemspec
CHANGED
@@ -12,12 +12,14 @@ Gem::Specification.new do |spec|
|
|
12
12
|
spec.homepage = "https://github.com/dasch/avro_turf"
|
13
13
|
spec.license = "MIT"
|
14
14
|
|
15
|
+
spec.metadata["rubygems_mfa_required"] = "true"
|
16
|
+
|
15
17
|
spec.files = `git ls-files -z`.split("\x0")
|
16
18
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
20
|
spec.require_paths = ["lib"]
|
19
21
|
|
20
|
-
spec.add_dependency "avro", ">= 1.7.7", "< 1.
|
22
|
+
spec.add_dependency "avro", ">= 1.7.7", "< 1.12"
|
21
23
|
spec.add_dependency "excon", "~> 0.71"
|
22
24
|
|
23
25
|
spec.add_development_dependency "bundler", "~> 2.0"
|
@@ -9,12 +9,15 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
9
9
|
proxy: nil,
|
10
10
|
user: nil,
|
11
11
|
password: nil,
|
12
|
+
ssl_ca_file: nil,
|
12
13
|
client_cert: nil,
|
13
14
|
client_key: nil,
|
14
15
|
client_key_pass: nil,
|
15
16
|
client_cert_data: nil,
|
16
|
-
client_key_data: nil
|
17
|
+
client_key_data: nil,
|
18
|
+
path_prefix: nil
|
17
19
|
)
|
20
|
+
@path_prefix = path_prefix
|
18
21
|
@logger = logger
|
19
22
|
headers = {
|
20
23
|
"Content-Type" => CONTENT_TYPE
|
@@ -25,6 +28,7 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
25
28
|
headers: headers,
|
26
29
|
user: user,
|
27
30
|
password: password,
|
31
|
+
ssl_ca_file: ssl_ca_file,
|
28
32
|
client_cert: client_cert,
|
29
33
|
client_key: client_key,
|
30
34
|
client_key_pass: client_key_pass,
|
@@ -120,6 +124,7 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
120
124
|
|
121
125
|
def request(path, **options)
|
122
126
|
options = { expects: 200 }.merge!(options)
|
127
|
+
path = File.join(@path_prefix, path) unless @path_prefix.nil?
|
123
128
|
response = @connection.request(path: path, **options)
|
124
129
|
JSON.parse(response.body)
|
125
130
|
end
|
data/lib/avro_turf/messaging.rb
CHANGED
@@ -26,31 +26,35 @@ class AvroTurf
|
|
26
26
|
|
27
27
|
# Instantiate a new Messaging instance with the given configuration.
|
28
28
|
#
|
29
|
-
# registry
|
30
|
-
#
|
31
|
-
# registry_url
|
32
|
-
# schema_store
|
33
|
-
# schemas_path
|
34
|
-
# namespace
|
35
|
-
#
|
36
|
-
#
|
37
|
-
#
|
38
|
-
#
|
39
|
-
#
|
40
|
-
#
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
29
|
+
# registry - A schema registry object that responds to all methods in the
|
30
|
+
# AvroTurf::ConfluentSchemaRegistry interface.
|
31
|
+
# registry_url - The String URL of the schema registry that should be used.
|
32
|
+
# schema_store - A schema store object that responds to #find(schema_name, namespace).
|
33
|
+
# schemas_path - The String file system path where local schemas are stored.
|
34
|
+
# namespace - The String default schema namespace.
|
35
|
+
# registry_path_prefix - The String URL path prefix used to namespace schema registry requests (optional).
|
36
|
+
# logger - The Logger that should be used to log information (optional).
|
37
|
+
# proxy - Forward the request via proxy (optional).
|
38
|
+
# user - User for basic auth (optional).
|
39
|
+
# password - Password for basic auth (optional).
|
40
|
+
# ssl_ca_file - Name of file containing CA certificate (optional).
|
41
|
+
# client_cert - Name of file containing client certificate (optional).
|
42
|
+
# client_key - Name of file containing client private key to go with client_cert (optional).
|
43
|
+
# client_key_pass - Password to go with client_key (optional).
|
44
|
+
# client_cert_data - In-memory client certificate (optional).
|
45
|
+
# client_key_data - In-memory client private key to go with client_cert_data (optional).
|
44
46
|
def initialize(
|
45
47
|
registry: nil,
|
46
48
|
registry_url: nil,
|
47
49
|
schema_store: nil,
|
48
50
|
schemas_path: nil,
|
49
51
|
namespace: nil,
|
52
|
+
registry_path_prefix: nil,
|
50
53
|
logger: nil,
|
51
54
|
proxy: nil,
|
52
55
|
user: nil,
|
53
56
|
password: nil,
|
57
|
+
ssl_ca_file: nil,
|
54
58
|
client_cert: nil,
|
55
59
|
client_key: nil,
|
56
60
|
client_key_pass: nil,
|
@@ -67,11 +71,13 @@ class AvroTurf
|
|
67
71
|
proxy: proxy,
|
68
72
|
user: user,
|
69
73
|
password: password,
|
74
|
+
ssl_ca_file: ssl_ca_file,
|
70
75
|
client_cert: client_cert,
|
71
76
|
client_key: client_key,
|
72
77
|
client_key_pass: client_key_pass,
|
73
78
|
client_cert_data: client_cert_data,
|
74
|
-
client_key_data: client_key_data
|
79
|
+
client_key_data: client_key_data,
|
80
|
+
path_prefix: registry_path_prefix
|
75
81
|
)
|
76
82
|
)
|
77
83
|
@schemas_by_id = {}
|
@@ -83,10 +83,10 @@ class AvroTurf::SchemaStore
|
|
83
83
|
# has been resolved and use the now-updated local_schemas_cache to
|
84
84
|
# pick up where we left off.
|
85
85
|
local_schemas_cache.delete(fullname)
|
86
|
-
# Ensure
|
86
|
+
# Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
|
87
87
|
# schema.
|
88
88
|
local_schemas_cache.each do |schema_name, schema|
|
89
|
-
local_schemas_cache.delete(schema_name)
|
89
|
+
local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
|
90
90
|
end
|
91
91
|
load_schema!(fullname, local_schemas_cache)
|
92
92
|
else
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'sinatra/base'
|
2
|
+
|
3
|
+
class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistryServer
|
4
|
+
post "/prefix/subjects/:subject/versions" do
|
5
|
+
schema = parse_schema
|
6
|
+
ids_for_subject = SUBJECTS[params[:subject]]
|
7
|
+
|
8
|
+
schemas_for_subject =
|
9
|
+
SCHEMAS.select
|
10
|
+
.with_index { |_, i| ids_for_subject.include?(i) }
|
11
|
+
|
12
|
+
if schemas_for_subject.include?(schema)
|
13
|
+
schema_id = SCHEMAS.index(schema)
|
14
|
+
else
|
15
|
+
SCHEMAS << schema
|
16
|
+
schema_id = SCHEMAS.size - 1
|
17
|
+
SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
|
18
|
+
end
|
19
|
+
|
20
|
+
{ id: schema_id }.to_json
|
21
|
+
end
|
22
|
+
|
23
|
+
get "/prefix/schemas/ids/:schema_id" do
|
24
|
+
schema = SCHEMAS.at(params[:schema_id].to_i)
|
25
|
+
halt(404, SCHEMA_NOT_FOUND) unless schema
|
26
|
+
{ schema: schema }.to_json
|
27
|
+
end
|
28
|
+
|
29
|
+
get "/prefix/subjects" do
|
30
|
+
SUBJECTS.keys.to_json
|
31
|
+
end
|
32
|
+
|
33
|
+
get "/prefix/subjects/:subject/versions" do
|
34
|
+
schema_ids = SUBJECTS[params[:subject]]
|
35
|
+
halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
|
36
|
+
(1..schema_ids.size).to_a.to_json
|
37
|
+
end
|
38
|
+
|
39
|
+
get "/prefix/subjects/:subject/versions/:version" do
|
40
|
+
schema_ids = SUBJECTS[params[:subject]]
|
41
|
+
halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
|
42
|
+
|
43
|
+
schema_id = if params[:version] == 'latest'
|
44
|
+
schema_ids.last
|
45
|
+
else
|
46
|
+
schema_ids.at(Integer(params[:version]) - 1)
|
47
|
+
end
|
48
|
+
halt(404, VERSION_NOT_FOUND) unless schema_id
|
49
|
+
|
50
|
+
schema = SCHEMAS.at(schema_id)
|
51
|
+
|
52
|
+
{
|
53
|
+
name: params[:subject],
|
54
|
+
version: schema_ids.index(schema_id) + 1,
|
55
|
+
id: schema_id,
|
56
|
+
schema: schema
|
57
|
+
}.to_json
|
58
|
+
end
|
59
|
+
|
60
|
+
post "/prefix/subjects/:subject" do
|
61
|
+
schema = parse_schema
|
62
|
+
|
63
|
+
# Note: this does not actually handle the same schema registered under
|
64
|
+
# multiple subjects
|
65
|
+
schema_id = SCHEMAS.index(schema)
|
66
|
+
|
67
|
+
halt(404, SCHEMA_NOT_FOUND) unless schema_id
|
68
|
+
|
69
|
+
{
|
70
|
+
subject: params[:subject],
|
71
|
+
id: schema_id,
|
72
|
+
version: SUBJECTS[params[:subject]].index(schema_id) + 1,
|
73
|
+
schema: schema
|
74
|
+
}.to_json
|
75
|
+
end
|
76
|
+
|
77
|
+
post "/prefix/compatibility/subjects/:subject/versions/:version" do
|
78
|
+
# The ruby avro gem does not yet include a compatibility check between schemas.
|
79
|
+
# See https://github.com/apache/avro/pull/170
|
80
|
+
raise NotImplementedError
|
81
|
+
end
|
82
|
+
|
83
|
+
get "/prefix/config" do
|
84
|
+
global_config.to_json
|
85
|
+
end
|
86
|
+
|
87
|
+
put "/prefix/config" do
|
88
|
+
global_config.merge!(parse_config).to_json
|
89
|
+
end
|
90
|
+
|
91
|
+
get "/prefix/config/:subject" do
|
92
|
+
CONFIGS.fetch(params[:subject], global_config).to_json
|
93
|
+
end
|
94
|
+
|
95
|
+
put "/prefix/config/:subject" do
|
96
|
+
config = parse_config
|
97
|
+
subject = params[:subject]
|
98
|
+
CONFIGS.fetch(subject) do
|
99
|
+
CONFIGS[subject] = {}
|
100
|
+
end.merge!(config).to_json
|
101
|
+
end
|
102
|
+
end
|
data/lib/avro_turf/version.rb
CHANGED
data/spec/avro_turf_spec.rb
CHANGED
@@ -68,11 +68,23 @@ describe AvroTurf do
|
|
68
68
|
"type": "array",
|
69
69
|
"items": "message"
|
70
70
|
}
|
71
|
+
},
|
72
|
+
{
|
73
|
+
"name": "status",
|
74
|
+
"type": "publishing_status"
|
71
75
|
}
|
72
76
|
]
|
73
77
|
}
|
74
78
|
AVSC
|
75
79
|
|
80
|
+
define_schema "publishing_status.avsc", <<-AVSC
|
81
|
+
{
|
82
|
+
"name": "publishing_status",
|
83
|
+
"type": "enum",
|
84
|
+
"symbols": ["draft", "published", "archived"]
|
85
|
+
}
|
86
|
+
AVSC
|
87
|
+
|
76
88
|
define_schema "message.avsc", <<-AVSC
|
77
89
|
{
|
78
90
|
"name": "message",
|
@@ -89,6 +101,10 @@ describe AvroTurf do
|
|
89
101
|
"name": "label",
|
90
102
|
"symbols": ["foo", "bar"]
|
91
103
|
}
|
104
|
+
},
|
105
|
+
{
|
106
|
+
"name": "status",
|
107
|
+
"type": "publishing_status"
|
92
108
|
}
|
93
109
|
]
|
94
110
|
}
|
@@ -101,9 +117,11 @@ describe AvroTurf do
|
|
101
117
|
"messages" => [
|
102
118
|
{
|
103
119
|
"content" => "hello",
|
104
|
-
"label" => "bar"
|
120
|
+
"label" => "bar",
|
121
|
+
"status" => "draft"
|
105
122
|
}
|
106
|
-
]
|
123
|
+
],
|
124
|
+
"status" => "published"
|
107
125
|
}
|
108
126
|
|
109
127
|
encoded_data = avro.encode(data, schema_name: "post")
|
@@ -177,7 +195,7 @@ describe AvroTurf do
|
|
177
195
|
context "validating" do
|
178
196
|
subject(:encode_to_stream) do
|
179
197
|
stream = StringIO.new
|
180
|
-
avro.encode_to_stream(message, stream: stream, schema_name: "message", validate: true)
|
198
|
+
avro.encode_to_stream(message, stream: stream, schema_name: "message", validate: true)
|
181
199
|
end
|
182
200
|
|
183
201
|
context "with a valid message" do
|
data/spec/messaging_spec.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'webmock/rspec'
|
2
2
|
require 'avro_turf/messaging'
|
3
3
|
require 'avro_turf/test/fake_confluent_schema_registry_server'
|
4
|
+
require 'avro_turf/test/fake_prefixed_confluent_schema_registry_server'
|
4
5
|
|
5
6
|
describe AvroTurf::Messaging do
|
6
7
|
let(:registry_url) { "http://registry.example.com" }
|
@@ -8,7 +9,7 @@ describe AvroTurf::Messaging do
|
|
8
9
|
let(:client_key) { "test client key" }
|
9
10
|
let(:client_key_pass) { "test client key password" }
|
10
11
|
let(:logger) { Logger.new(StringIO.new) }
|
11
|
-
|
12
|
+
let(:path_prefix) { nil }
|
12
13
|
let(:avro) {
|
13
14
|
AvroTurf::Messaging.new(
|
14
15
|
registry_url: registry_url,
|
@@ -72,7 +73,7 @@ describe AvroTurf::Messaging do
|
|
72
73
|
|
73
74
|
shared_examples_for 'encoding and decoding with the schema from registry' do
|
74
75
|
before do
|
75
|
-
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
76
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger, path_prefix: path_prefix)
|
76
77
|
registry.register('person', schema)
|
77
78
|
registry.register('people', schema)
|
78
79
|
end
|
@@ -102,7 +103,7 @@ describe AvroTurf::Messaging do
|
|
102
103
|
|
103
104
|
shared_examples_for 'encoding and decoding with the schema_id from registry' do
|
104
105
|
before do
|
105
|
-
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
106
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger, path_prefix: path_prefix)
|
106
107
|
registry.register('person', schema)
|
107
108
|
registry.register('people', schema)
|
108
109
|
end
|
@@ -399,4 +400,29 @@ describe AvroTurf::Messaging do
|
|
399
400
|
end
|
400
401
|
end
|
401
402
|
end
|
402
|
-
|
403
|
+
|
404
|
+
context 'with a registry path prefix' do
|
405
|
+
let(:path_prefix) { '/prefix' }
|
406
|
+
|
407
|
+
let(:avro) {
|
408
|
+
AvroTurf::Messaging.new(
|
409
|
+
registry_path_prefix: path_prefix,
|
410
|
+
registry_url: registry_url,
|
411
|
+
schemas_path: "spec/schemas",
|
412
|
+
logger: logger,
|
413
|
+
client_cert: client_cert,
|
414
|
+
client_key: client_key,
|
415
|
+
client_key_pass: client_key_pass
|
416
|
+
)
|
417
|
+
}
|
418
|
+
|
419
|
+
before do
|
420
|
+
stub_request(:any, /^#{registry_url}/).to_rack(FakePrefixedConfluentSchemaRegistryServer)
|
421
|
+
FakePrefixedConfluentSchemaRegistryServer.clear
|
422
|
+
end
|
423
|
+
|
424
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
425
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
426
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
427
|
+
end
|
428
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro_turf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-06-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -19,7 +19,7 @@ dependencies:
|
|
19
19
|
version: 1.7.7
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version: '1.
|
22
|
+
version: '1.12'
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -29,7 +29,7 @@ dependencies:
|
|
29
29
|
version: 1.7.7
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: '1.
|
32
|
+
version: '1.12'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
name: excon
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -196,6 +196,7 @@ files:
|
|
196
196
|
- lib/avro_turf/schema_store.rb
|
197
197
|
- lib/avro_turf/schema_to_avro_patch.rb
|
198
198
|
- lib/avro_turf/test/fake_confluent_schema_registry_server.rb
|
199
|
+
- lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb
|
199
200
|
- lib/avro_turf/test/fake_schema_registry_server.rb
|
200
201
|
- lib/avro_turf/version.rb
|
201
202
|
- perf/address.avsc
|
@@ -225,7 +226,8 @@ files:
|
|
225
226
|
homepage: https://github.com/dasch/avro_turf
|
226
227
|
licenses:
|
227
228
|
- MIT
|
228
|
-
metadata:
|
229
|
+
metadata:
|
230
|
+
rubygems_mfa_required: 'true'
|
229
231
|
post_install_message: |2
|
230
232
|
|
231
233
|
avro_turf v0.8.0 deprecates the names AvroTurf::SchemaRegistry,
|