avro_turf 1.5.0 → 1.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -1
- data/lib/avro_turf/cached_confluent_schema_registry.rb +2 -2
- data/lib/avro_turf/confluent_schema_registry.rb +4 -1
- data/lib/avro_turf/core_ext/date.rb +1 -1
- data/lib/avro_turf/disk_cache.rb +25 -15
- data/lib/avro_turf/messaging.rb +35 -20
- data/lib/avro_turf/schema_store.rb +1 -1
- data/lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb +102 -0
- data/lib/avro_turf/version.rb +1 -1
- data/lib/avro_turf.rb +11 -3
- data/spec/avro_turf_spec.rb +64 -2
- data/spec/core_ext/date_spec.rb +2 -2
- data/spec/messaging_spec.rb +30 -4
- data/spec/schema_store_spec.rb +27 -0
- data/spec/spec_helper.rb +10 -0
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 67ff8ee9b578227cd54a33d76a95fed8bd2c9a396aa360ad9df9f52d01e5c25a
|
4
|
+
data.tar.gz: 8bca24236ce42f19f6db457ad13d76d33e89b18703dc0eb74a8adca53b2c7b6e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 859e8fa6938679007704d35b1b4a54616da7637c83f41c58cbd10828216e0af198846e755334f465613ceb9086ec0c63a823dc2cbf3ddbdc191a97423d2e3147
|
7
|
+
data.tar.gz: 7900a85abaf9dd55ac3b89bba2d04935ddbafc850a92fb1941ff8984df864cfaa4bb5527ca4d48a74ef3715915d598044b7c486363909f2705ece83b0ffca50f
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,20 @@
|
|
2
2
|
|
3
3
|
## Unreleased
|
4
4
|
|
5
|
+
## v1.8.0
|
6
|
+
|
7
|
+
- add support for `Date` via appropriate logicalType defintion. This is a backwards incompatible change (#177)
|
8
|
+
- Fixed schema file cache truncation on multiple running instances and parallel access to the cache files.
|
9
|
+
|
10
|
+
## v1.7.0
|
11
|
+
|
12
|
+
- Added extra params for the validation message schema before encode (#169)
|
13
|
+
- Fix infinite retry when loading schema with nested primary type in separate file (#165)
|
14
|
+
|
15
|
+
## v1.6.0
|
16
|
+
|
17
|
+
- Schema registry path prefix (#162)
|
18
|
+
|
5
19
|
## v1.5.0
|
6
20
|
|
7
21
|
- Add CA cert file option (#157)
|
@@ -51,7 +65,7 @@
|
|
51
65
|
## v0.9.0
|
52
66
|
|
53
67
|
- Compatibility with Avro v1.9.0 (#94)
|
54
|
-
- Disable the auto
|
68
|
+
- Disable the auto registration of schema (#95)
|
55
69
|
- abstracted caching from CachedConfluentSchemaRegistry (#74)
|
56
70
|
- Load avro-patches if installed to silence deprecation errors (#85)
|
57
71
|
- Make schema store to be thread safe (#92)
|
@@ -13,7 +13,7 @@ class AvroTurf::CachedConfluentSchemaRegistry
|
|
13
13
|
# cache - Optional user provided Cache object that responds to all methods in the AvroTurf::InMemoryCache interface.
|
14
14
|
def initialize(upstream, cache: nil)
|
15
15
|
@upstream = upstream
|
16
|
-
@cache = cache || AvroTurf::InMemoryCache.new
|
16
|
+
@cache = cache || AvroTurf::InMemoryCache.new
|
17
17
|
end
|
18
18
|
|
19
19
|
# Delegate the following methods to the upstream
|
@@ -34,7 +34,7 @@ class AvroTurf::CachedConfluentSchemaRegistry
|
|
34
34
|
|
35
35
|
def subject_version(subject, version = 'latest')
|
36
36
|
return @upstream.subject_version(subject, version) if version == 'latest'
|
37
|
-
|
37
|
+
|
38
38
|
@cache.lookup_by_version(subject, version) ||
|
39
39
|
@cache.store_by_version(subject, version, @upstream.subject_version(subject, version))
|
40
40
|
end
|
@@ -14,8 +14,10 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
14
14
|
client_key: nil,
|
15
15
|
client_key_pass: nil,
|
16
16
|
client_cert_data: nil,
|
17
|
-
client_key_data: nil
|
17
|
+
client_key_data: nil,
|
18
|
+
path_prefix: nil
|
18
19
|
)
|
20
|
+
@path_prefix = path_prefix
|
19
21
|
@logger = logger
|
20
22
|
headers = {
|
21
23
|
"Content-Type" => CONTENT_TYPE
|
@@ -122,6 +124,7 @@ class AvroTurf::ConfluentSchemaRegistry
|
|
122
124
|
|
123
125
|
def request(path, **options)
|
124
126
|
options = { expects: 200 }.merge!(options)
|
127
|
+
path = File.join(@path_prefix, path) unless @path_prefix.nil?
|
125
128
|
response = @connection.request(path: path, **options)
|
126
129
|
JSON.parse(response.body)
|
127
130
|
end
|
data/lib/avro_turf/disk_cache.rb
CHANGED
@@ -1,20 +1,18 @@
|
|
1
1
|
# A cache for the CachedConfluentSchemaRegistry.
|
2
2
|
# Extends the InMemoryCache to provide a write-thru to disk for persistent cache.
|
3
|
-
class AvroTurf::DiskCache
|
3
|
+
class AvroTurf::DiskCache
|
4
4
|
|
5
5
|
def initialize(disk_path, logger: Logger.new($stdout))
|
6
|
-
super()
|
7
|
-
|
8
6
|
@logger = logger
|
9
7
|
|
10
8
|
# load the write-thru cache on startup, if it exists
|
11
9
|
@schemas_by_id_path = File.join(disk_path, 'schemas_by_id.json')
|
12
10
|
hash = read_from_disk_cache(@schemas_by_id_path)
|
13
|
-
@schemas_by_id = hash
|
11
|
+
@schemas_by_id = hash || {}
|
14
12
|
|
15
13
|
@ids_by_schema_path = File.join(disk_path, 'ids_by_schema.json')
|
16
14
|
hash = read_from_disk_cache(@ids_by_schema_path)
|
17
|
-
@ids_by_schema = hash
|
15
|
+
@ids_by_schema = hash || {}
|
18
16
|
|
19
17
|
@schemas_by_subject_version_path = File.join(disk_path, 'schemas_by_subject_version.json')
|
20
18
|
@schemas_by_subject_version = {}
|
@@ -24,15 +22,16 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
24
22
|
# the write-thru cache (json) does not store keys in numeric format
|
25
23
|
# so, convert id to a string for caching purposes
|
26
24
|
def lookup_by_id(id)
|
27
|
-
|
25
|
+
@schemas_by_id[id.to_s]
|
28
26
|
end
|
29
27
|
|
30
28
|
# override to include write-thru cache after storing result from upstream
|
31
29
|
def store_by_id(id, schema)
|
32
30
|
# must return the value from storing the result (i.e. do not return result from file write)
|
33
|
-
|
34
|
-
|
35
|
-
|
31
|
+
@schemas_by_id[id.to_s] = schema
|
32
|
+
write_to_disk_cache(@schemas_by_id_path, @schemas_by_id)
|
33
|
+
|
34
|
+
schema
|
36
35
|
end
|
37
36
|
|
38
37
|
# override to use a json serializable cache key
|
@@ -45,7 +44,8 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
45
44
|
def store_by_schema(subject, schema, id)
|
46
45
|
key = "#{subject}#{schema}"
|
47
46
|
@ids_by_schema[key] = id
|
48
|
-
|
47
|
+
|
48
|
+
write_to_disk_cache(@ids_by_schema_path, @ids_by_schema)
|
49
49
|
id
|
50
50
|
end
|
51
51
|
|
@@ -78,7 +78,7 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
78
78
|
hash[key] = schema
|
79
79
|
hash
|
80
80
|
else
|
81
|
-
{key => schema}
|
81
|
+
{ key => schema }
|
82
82
|
end
|
83
83
|
|
84
84
|
write_to_disk_cache(@schemas_by_subject_version_path, hash)
|
@@ -90,17 +90,27 @@ class AvroTurf::DiskCache < AvroTurf::InMemoryCache
|
|
90
90
|
# Parse the file from disk, if it exists and is not zero length
|
91
91
|
private def read_from_disk_cache(path)
|
92
92
|
if File.exist?(path)
|
93
|
-
if File.size(path)!=0
|
94
|
-
|
93
|
+
if File.size(path) != 0
|
94
|
+
json_data = File.open(path, 'r') do |file|
|
95
|
+
file.flock(File::LOCK_SH)
|
96
|
+
file.read
|
97
|
+
end
|
98
|
+
|
99
|
+
return JSON.parse(json_data)
|
95
100
|
else
|
96
101
|
# just log a message if skipping zero length file
|
97
102
|
@logger.warn "skipping JSON.parse of zero length file at #{path}"
|
98
103
|
end
|
99
104
|
end
|
100
|
-
|
105
|
+
|
106
|
+
nil
|
101
107
|
end
|
102
108
|
|
103
109
|
private def write_to_disk_cache(path, hash)
|
104
|
-
|
110
|
+
# don't use "w" because it truncates the file before lock
|
111
|
+
File.open(path, File::RDWR | File::CREAT, 0644) do |file|
|
112
|
+
file.flock(File::LOCK_EX)
|
113
|
+
file.write(JSON.pretty_generate(hash))
|
114
|
+
end
|
105
115
|
end
|
106
116
|
end
|
data/lib/avro_turf/messaging.rb
CHANGED
@@ -21,33 +21,47 @@ class AvroTurf
|
|
21
21
|
# 1: https://github.com/confluentinc/schema-registry
|
22
22
|
class Messaging
|
23
23
|
MAGIC_BYTE = [0].pack("C").freeze
|
24
|
-
|
25
|
-
|
24
|
+
|
25
|
+
class DecodedMessage
|
26
|
+
attr_reader :schema_id
|
27
|
+
attr_reader :writer_schema
|
28
|
+
attr_reader :reader_schema
|
29
|
+
attr_reader :message
|
30
|
+
|
31
|
+
def initialize(schema_id, writer_schema, reader_schema, message)
|
32
|
+
@schema_id = schema_id
|
33
|
+
@writer_schema = writer_schema
|
34
|
+
@reader_schema = reader_schema
|
35
|
+
@message = message
|
36
|
+
end
|
37
|
+
end
|
26
38
|
|
27
39
|
# Instantiate a new Messaging instance with the given configuration.
|
28
40
|
#
|
29
|
-
# registry
|
30
|
-
#
|
31
|
-
# registry_url
|
32
|
-
# schema_store
|
33
|
-
# schemas_path
|
34
|
-
# namespace
|
35
|
-
#
|
36
|
-
#
|
37
|
-
#
|
38
|
-
#
|
39
|
-
#
|
40
|
-
#
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
#
|
41
|
+
# registry - A schema registry object that responds to all methods in the
|
42
|
+
# AvroTurf::ConfluentSchemaRegistry interface.
|
43
|
+
# registry_url - The String URL of the schema registry that should be used.
|
44
|
+
# schema_store - A schema store object that responds to #find(schema_name, namespace).
|
45
|
+
# schemas_path - The String file system path where local schemas are stored.
|
46
|
+
# namespace - The String default schema namespace.
|
47
|
+
# registry_path_prefix - The String URL path prefix used to namespace schema registry requests (optional).
|
48
|
+
# logger - The Logger that should be used to log information (optional).
|
49
|
+
# proxy - Forward the request via proxy (optional).
|
50
|
+
# user - User for basic auth (optional).
|
51
|
+
# password - Password for basic auth (optional).
|
52
|
+
# ssl_ca_file - Name of file containing CA certificate (optional).
|
53
|
+
# client_cert - Name of file containing client certificate (optional).
|
54
|
+
# client_key - Name of file containing client private key to go with client_cert (optional).
|
55
|
+
# client_key_pass - Password to go with client_key (optional).
|
56
|
+
# client_cert_data - In-memory client certificate (optional).
|
57
|
+
# client_key_data - In-memory client private key to go with client_cert_data (optional).
|
45
58
|
def initialize(
|
46
59
|
registry: nil,
|
47
60
|
registry_url: nil,
|
48
61
|
schema_store: nil,
|
49
62
|
schemas_path: nil,
|
50
63
|
namespace: nil,
|
64
|
+
registry_path_prefix: nil,
|
51
65
|
logger: nil,
|
52
66
|
proxy: nil,
|
53
67
|
user: nil,
|
@@ -74,7 +88,8 @@ class AvroTurf
|
|
74
88
|
client_key: client_key,
|
75
89
|
client_key_pass: client_key_pass,
|
76
90
|
client_cert_data: client_cert_data,
|
77
|
-
client_key_data: client_key_data
|
91
|
+
client_key_data: client_key_data,
|
92
|
+
path_prefix: registry_path_prefix
|
78
93
|
)
|
79
94
|
)
|
80
95
|
@schemas_by_id = {}
|
@@ -127,7 +142,7 @@ class AvroTurf
|
|
127
142
|
writer.write(message, encoder)
|
128
143
|
|
129
144
|
stream.string
|
130
|
-
rescue Excon::
|
145
|
+
rescue Excon::Error::NotFound
|
131
146
|
if schema_id
|
132
147
|
raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
|
133
148
|
else
|
@@ -88,7 +88,7 @@ class AvroTurf::SchemaStore
|
|
88
88
|
local_schemas_cache.each do |schema_name, schema|
|
89
89
|
local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
|
90
90
|
end
|
91
|
-
load_schema!(fullname,
|
91
|
+
load_schema!(fullname, @schemas.dup)
|
92
92
|
else
|
93
93
|
raise
|
94
94
|
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'sinatra/base'
|
2
|
+
|
3
|
+
class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistryServer
|
4
|
+
post "/prefix/subjects/:subject/versions" do
|
5
|
+
schema = parse_schema
|
6
|
+
ids_for_subject = SUBJECTS[params[:subject]]
|
7
|
+
|
8
|
+
schemas_for_subject =
|
9
|
+
SCHEMAS.select
|
10
|
+
.with_index { |_, i| ids_for_subject.include?(i) }
|
11
|
+
|
12
|
+
if schemas_for_subject.include?(schema)
|
13
|
+
schema_id = SCHEMAS.index(schema)
|
14
|
+
else
|
15
|
+
SCHEMAS << schema
|
16
|
+
schema_id = SCHEMAS.size - 1
|
17
|
+
SUBJECTS[params[:subject]] = SUBJECTS[params[:subject]] << schema_id
|
18
|
+
end
|
19
|
+
|
20
|
+
{ id: schema_id }.to_json
|
21
|
+
end
|
22
|
+
|
23
|
+
get "/prefix/schemas/ids/:schema_id" do
|
24
|
+
schema = SCHEMAS.at(params[:schema_id].to_i)
|
25
|
+
halt(404, SCHEMA_NOT_FOUND) unless schema
|
26
|
+
{ schema: schema }.to_json
|
27
|
+
end
|
28
|
+
|
29
|
+
get "/prefix/subjects" do
|
30
|
+
SUBJECTS.keys.to_json
|
31
|
+
end
|
32
|
+
|
33
|
+
get "/prefix/subjects/:subject/versions" do
|
34
|
+
schema_ids = SUBJECTS[params[:subject]]
|
35
|
+
halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
|
36
|
+
(1..schema_ids.size).to_a.to_json
|
37
|
+
end
|
38
|
+
|
39
|
+
get "/prefix/subjects/:subject/versions/:version" do
|
40
|
+
schema_ids = SUBJECTS[params[:subject]]
|
41
|
+
halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
|
42
|
+
|
43
|
+
schema_id = if params[:version] == 'latest'
|
44
|
+
schema_ids.last
|
45
|
+
else
|
46
|
+
schema_ids.at(Integer(params[:version]) - 1)
|
47
|
+
end
|
48
|
+
halt(404, VERSION_NOT_FOUND) unless schema_id
|
49
|
+
|
50
|
+
schema = SCHEMAS.at(schema_id)
|
51
|
+
|
52
|
+
{
|
53
|
+
name: params[:subject],
|
54
|
+
version: schema_ids.index(schema_id) + 1,
|
55
|
+
id: schema_id,
|
56
|
+
schema: schema
|
57
|
+
}.to_json
|
58
|
+
end
|
59
|
+
|
60
|
+
post "/prefix/subjects/:subject" do
|
61
|
+
schema = parse_schema
|
62
|
+
|
63
|
+
# Note: this does not actually handle the same schema registered under
|
64
|
+
# multiple subjects
|
65
|
+
schema_id = SCHEMAS.index(schema)
|
66
|
+
|
67
|
+
halt(404, SCHEMA_NOT_FOUND) unless schema_id
|
68
|
+
|
69
|
+
{
|
70
|
+
subject: params[:subject],
|
71
|
+
id: schema_id,
|
72
|
+
version: SUBJECTS[params[:subject]].index(schema_id) + 1,
|
73
|
+
schema: schema
|
74
|
+
}.to_json
|
75
|
+
end
|
76
|
+
|
77
|
+
post "/prefix/compatibility/subjects/:subject/versions/:version" do
|
78
|
+
# The ruby avro gem does not yet include a compatibility check between schemas.
|
79
|
+
# See https://github.com/apache/avro/pull/170
|
80
|
+
raise NotImplementedError
|
81
|
+
end
|
82
|
+
|
83
|
+
get "/prefix/config" do
|
84
|
+
global_config.to_json
|
85
|
+
end
|
86
|
+
|
87
|
+
put "/prefix/config" do
|
88
|
+
global_config.merge!(parse_config).to_json
|
89
|
+
end
|
90
|
+
|
91
|
+
get "/prefix/config/:subject" do
|
92
|
+
CONFIGS.fetch(params[:subject], global_config).to_json
|
93
|
+
end
|
94
|
+
|
95
|
+
put "/prefix/config/:subject" do
|
96
|
+
config = parse_config
|
97
|
+
subject = params[:subject]
|
98
|
+
CONFIGS.fetch(subject) do
|
99
|
+
CONFIGS[subject] = {}
|
100
|
+
end.merge!(config).to_json
|
101
|
+
end
|
102
|
+
end
|
data/lib/avro_turf/version.rb
CHANGED
data/lib/avro_turf.rb
CHANGED
@@ -16,7 +16,9 @@ end
|
|
16
16
|
|
17
17
|
class AvroTurf
|
18
18
|
class Error < StandardError; end
|
19
|
+
|
19
20
|
class SchemaError < Error; end
|
21
|
+
|
20
22
|
class SchemaNotFoundError < Error; end
|
21
23
|
|
22
24
|
DEFAULT_SCHEMAS_PATH = "./schemas"
|
@@ -31,7 +33,7 @@ class AvroTurf
|
|
31
33
|
# Currently, the only valid codec name is `deflate`.
|
32
34
|
def initialize(schemas_path: nil, schema_store: nil, namespace: nil, codec: nil)
|
33
35
|
@namespace = namespace
|
34
|
-
@schema_store = schema_store ||
|
36
|
+
@schema_store = schema_store ||
|
35
37
|
SchemaStore.new(path: schemas_path || DEFAULT_SCHEMAS_PATH)
|
36
38
|
@codec = codec
|
37
39
|
end
|
@@ -62,14 +64,20 @@ class AvroTurf
|
|
62
64
|
# validate - The boolean for performing complete data validation before
|
63
65
|
# encoding it, Avro::SchemaValidator::ValidationError with
|
64
66
|
# a descriptive message will be raised in case of invalid message.
|
67
|
+
# validate_options - Hash for the Avro::SchemaValidator, default
|
68
|
+
# {recursive: true, encoded: false, fail_on_extra_fields: true}
|
65
69
|
#
|
66
70
|
# Returns nothing.
|
67
|
-
def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace,
|
71
|
+
def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace,
|
72
|
+
validate: false,
|
73
|
+
validate_options: { recursive: true,
|
74
|
+
encoded: false,
|
75
|
+
fail_on_extra_fields: true })
|
68
76
|
schema = @schema_store.find(schema_name, namespace)
|
69
77
|
writer = Avro::IO::DatumWriter.new(schema)
|
70
78
|
|
71
79
|
if validate
|
72
|
-
Avro::SchemaValidator.validate!(schema, data,
|
80
|
+
Avro::SchemaValidator.validate!(schema, data, **validate_options)
|
73
81
|
end
|
74
82
|
|
75
83
|
dw = Avro::DataFile::Writer.new(stream, writer, schema, @codec)
|
data/spec/avro_turf_spec.rb
CHANGED
@@ -16,6 +16,13 @@ describe AvroTurf do
|
|
16
16
|
{
|
17
17
|
"type": "string",
|
18
18
|
"name": "full_name"
|
19
|
+
},
|
20
|
+
{
|
21
|
+
"name": "birth_date",
|
22
|
+
"type": {
|
23
|
+
"type": "int",
|
24
|
+
"logicalType": "date"
|
25
|
+
}
|
19
26
|
}
|
20
27
|
]
|
21
28
|
}
|
@@ -24,7 +31,8 @@ describe AvroTurf do
|
|
24
31
|
|
25
32
|
it "encodes data with Avro" do
|
26
33
|
data = {
|
27
|
-
"full_name" => "John Doe"
|
34
|
+
"full_name" => "John Doe",
|
35
|
+
"birth_date" => Date.new(1934, 1, 2)
|
28
36
|
}
|
29
37
|
|
30
38
|
encoded_data = avro.encode(data, schema_name: "person")
|
@@ -36,7 +44,8 @@ describe AvroTurf do
|
|
36
44
|
compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
|
37
45
|
|
38
46
|
data = {
|
39
|
-
"full_name" => "John Doe" * 100
|
47
|
+
"full_name" => "John Doe" * 100,
|
48
|
+
"birth_date" => Date.new(1934, 1, 2)
|
40
49
|
}
|
41
50
|
|
42
51
|
uncompressed_data = avro.encode(data, schema_name: "person")
|
@@ -251,6 +260,31 @@ describe AvroTurf do
|
|
251
260
|
expect { encode_to_stream }.to raise_error(Avro::SchemaValidator::ValidationError, /extra field 'fulll_name'/)
|
252
261
|
end
|
253
262
|
end
|
263
|
+
|
264
|
+
context "when the `fail_on_extra_fields` validation option is disabled" do
|
265
|
+
let(:message) { { "full_name" => "John Doe", "first_name" => "John", "last_name" => "Doe" } }
|
266
|
+
subject(:encode_to_stream) do
|
267
|
+
stream = StringIO.new
|
268
|
+
avro.encode_to_stream(message, stream: stream, schema_name: "message",
|
269
|
+
validate: true,
|
270
|
+
validate_options: { recursive: true, encoded: false, fail_on_extra_fields: false }
|
271
|
+
)
|
272
|
+
end
|
273
|
+
|
274
|
+
it "should not raise Avro::SchemaValidator::ValidationError with a message about extra field" do
|
275
|
+
define_schema "message.avsc", <<-AVSC
|
276
|
+
{
|
277
|
+
"name": "message",
|
278
|
+
"type": "record",
|
279
|
+
"fields": [
|
280
|
+
{ "name": "full_name", "type": "string" }
|
281
|
+
]
|
282
|
+
}
|
283
|
+
AVSC
|
284
|
+
|
285
|
+
expect { encode_to_stream }.not_to raise_error
|
286
|
+
end
|
287
|
+
end
|
254
288
|
end
|
255
289
|
end
|
256
290
|
|
@@ -304,5 +338,33 @@ describe AvroTurf do
|
|
304
338
|
datum = { message: "hello" }
|
305
339
|
expect(avro.valid?(datum, schema_name: "postcard")).to eq true
|
306
340
|
end
|
341
|
+
|
342
|
+
it "handles logicalType of date in schema" do
|
343
|
+
define_schema "postcard.avsc", <<-AVSC
|
344
|
+
{
|
345
|
+
"name": "postcard",
|
346
|
+
"type": "record",
|
347
|
+
"fields": [
|
348
|
+
{
|
349
|
+
"name": "message",
|
350
|
+
"type": "string"
|
351
|
+
},
|
352
|
+
{
|
353
|
+
"name": "sent_date",
|
354
|
+
"type": {
|
355
|
+
"type": "int",
|
356
|
+
"logicalType": "date"
|
357
|
+
}
|
358
|
+
}
|
359
|
+
]
|
360
|
+
}
|
361
|
+
AVSC
|
362
|
+
|
363
|
+
datum = {
|
364
|
+
message: "hello",
|
365
|
+
sent_date: Date.new(2022, 9, 11)
|
366
|
+
}
|
367
|
+
expect(avro.valid?(datum, schema_name: "postcard")).to eq true
|
368
|
+
end
|
307
369
|
end
|
308
370
|
end
|
data/spec/core_ext/date_spec.rb
CHANGED
data/spec/messaging_spec.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'webmock/rspec'
|
2
2
|
require 'avro_turf/messaging'
|
3
3
|
require 'avro_turf/test/fake_confluent_schema_registry_server'
|
4
|
+
require 'avro_turf/test/fake_prefixed_confluent_schema_registry_server'
|
4
5
|
|
5
6
|
describe AvroTurf::Messaging do
|
6
7
|
let(:registry_url) { "http://registry.example.com" }
|
@@ -8,7 +9,7 @@ describe AvroTurf::Messaging do
|
|
8
9
|
let(:client_key) { "test client key" }
|
9
10
|
let(:client_key_pass) { "test client key password" }
|
10
11
|
let(:logger) { Logger.new(StringIO.new) }
|
11
|
-
|
12
|
+
let(:path_prefix) { nil }
|
12
13
|
let(:avro) {
|
13
14
|
AvroTurf::Messaging.new(
|
14
15
|
registry_url: registry_url,
|
@@ -72,7 +73,7 @@ describe AvroTurf::Messaging do
|
|
72
73
|
|
73
74
|
shared_examples_for 'encoding and decoding with the schema from registry' do
|
74
75
|
before do
|
75
|
-
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
76
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger, path_prefix: path_prefix)
|
76
77
|
registry.register('person', schema)
|
77
78
|
registry.register('people', schema)
|
78
79
|
end
|
@@ -102,7 +103,7 @@ describe AvroTurf::Messaging do
|
|
102
103
|
|
103
104
|
shared_examples_for 'encoding and decoding with the schema_id from registry' do
|
104
105
|
before do
|
105
|
-
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger)
|
106
|
+
registry = AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger, path_prefix: path_prefix)
|
106
107
|
registry.register('person', schema)
|
107
108
|
registry.register('people', schema)
|
108
109
|
end
|
@@ -399,4 +400,29 @@ describe AvroTurf::Messaging do
|
|
399
400
|
end
|
400
401
|
end
|
401
402
|
end
|
402
|
-
|
403
|
+
|
404
|
+
context 'with a registry path prefix' do
|
405
|
+
let(:path_prefix) { '/prefix' }
|
406
|
+
|
407
|
+
let(:avro) {
|
408
|
+
AvroTurf::Messaging.new(
|
409
|
+
registry_path_prefix: path_prefix,
|
410
|
+
registry_url: registry_url,
|
411
|
+
schemas_path: "spec/schemas",
|
412
|
+
logger: logger,
|
413
|
+
client_cert: client_cert,
|
414
|
+
client_key: client_key,
|
415
|
+
client_key_pass: client_key_pass
|
416
|
+
)
|
417
|
+
}
|
418
|
+
|
419
|
+
before do
|
420
|
+
stub_request(:any, /^#{registry_url}/).to_rack(FakePrefixedConfluentSchemaRegistryServer)
|
421
|
+
FakePrefixedConfluentSchemaRegistryServer.clear
|
422
|
+
end
|
423
|
+
|
424
|
+
it_behaves_like "encoding and decoding with the schema from schema store"
|
425
|
+
it_behaves_like 'encoding and decoding with the schema from registry'
|
426
|
+
it_behaves_like 'encoding and decoding with the schema_id from registry'
|
427
|
+
end
|
428
|
+
end
|
data/spec/schema_store_spec.rb
CHANGED
@@ -26,6 +26,33 @@ describe AvroTurf::SchemaStore do
|
|
26
26
|
expect(schema.fullname).to eq "message"
|
27
27
|
end
|
28
28
|
|
29
|
+
it "resolves missing references when nested schema is not a named type" do
|
30
|
+
define_schema "root.avsc", <<-AVSC
|
31
|
+
{
|
32
|
+
"type": "record",
|
33
|
+
"name": "root",
|
34
|
+
"fields": [
|
35
|
+
{
|
36
|
+
"type": "nested",
|
37
|
+
"name": "nested_value"
|
38
|
+
}
|
39
|
+
]
|
40
|
+
}
|
41
|
+
AVSC
|
42
|
+
|
43
|
+
define_schema "nested.avsc", <<-AVSC
|
44
|
+
{
|
45
|
+
"name": "nested",
|
46
|
+
"type": "string",
|
47
|
+
"logicalType": "uuid"
|
48
|
+
}
|
49
|
+
AVSC
|
50
|
+
|
51
|
+
schema = store.find("root")
|
52
|
+
|
53
|
+
expect(schema.fullname).to eq "root"
|
54
|
+
end
|
55
|
+
|
29
56
|
it "resolves missing references" do
|
30
57
|
define_schema "person.avsc", <<-AVSC
|
31
58
|
{
|
data/spec/spec_helper.rb
CHANGED
@@ -22,6 +22,16 @@ module Helpers
|
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
+
# gem `fakefs` does not support flock for the file, and require patch
|
26
|
+
# https://github.com/fakefs/fakefs/issues/433
|
27
|
+
module FakeFS
|
28
|
+
class File < StringIO
|
29
|
+
def flock(*)
|
30
|
+
true
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
25
35
|
RSpec.configure do |config|
|
26
36
|
config.include FakeFS::SpecHelpers
|
27
37
|
config.include Helpers
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro_turf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-11-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -156,7 +156,7 @@ dependencies:
|
|
156
156
|
- - ">="
|
157
157
|
- !ruby/object:Gem::Version
|
158
158
|
version: '0'
|
159
|
-
description:
|
159
|
+
description:
|
160
160
|
email:
|
161
161
|
- dasch@zendesk.com
|
162
162
|
executables: []
|
@@ -196,6 +196,7 @@ files:
|
|
196
196
|
- lib/avro_turf/schema_store.rb
|
197
197
|
- lib/avro_turf/schema_to_avro_patch.rb
|
198
198
|
- lib/avro_turf/test/fake_confluent_schema_registry_server.rb
|
199
|
+
- lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb
|
199
200
|
- lib/avro_turf/test/fake_schema_registry_server.rb
|
200
201
|
- lib/avro_turf/version.rb
|
201
202
|
- perf/address.avsc
|
@@ -250,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
250
251
|
- !ruby/object:Gem::Version
|
251
252
|
version: '0'
|
252
253
|
requirements: []
|
253
|
-
rubygems_version: 3.
|
254
|
-
signing_key:
|
254
|
+
rubygems_version: 3.3.3
|
255
|
+
signing_key:
|
255
256
|
specification_version: 4
|
256
257
|
summary: A library that makes it easier to use the Avro serialization format from
|
257
258
|
Ruby
|