avro 1.10.1 → 1.10.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/avro.gemspec +3 -3
- data/lib/avro/VERSION.txt +1 -1
- data/lib/avro/io.rb +10 -5
- data/lib/avro/schema.rb +3 -1
- data/lib/avro/schema_compatibility.rb +8 -2
- data/lib/avro/schema_validator.rb +35 -30
- data/test/test_schema_compatibility.rb +2 -0
- data/test/test_schema_validator.rb +7 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 38a2f2d15367014464b4613e0fc80724b345d3b3
|
4
|
+
data.tar.gz: 823aa3ebb6ea719cbd4c94060757a545c7d626a9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: da4d7d3ecf02bb18c30c044635188ba2b8a34ff13f9450330e850198610ee87f4e9116a79a95366bc23b5e3fb7f70906736d1ff7834e9293825c5c3eba072b9b
|
7
|
+
data.tar.gz: 3194b9379e1ae396acd0a120f583ade7127628d5adf78ac382fcf7215643c69c5341ee0a5f234db56eec3e62070683340b4a21bcff4ba0e2208a832c571f8dc9
|
data/avro.gemspec
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
# stub: avro 1.10.
|
2
|
+
# stub: avro 1.10.2 ruby lib
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "avro".freeze
|
6
|
-
s.version = "1.10.
|
6
|
+
s.version = "1.10.2"
|
7
7
|
|
8
8
|
s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
|
9
9
|
s.require_paths = ["lib".freeze]
|
10
10
|
s.authors = ["Apache Software Foundation".freeze]
|
11
|
-
s.date = "
|
11
|
+
s.date = "2021-03-09"
|
12
12
|
s.description = "Avro is a data serialization and RPC format".freeze
|
13
13
|
s.email = "dev@avro.apache.org".freeze
|
14
14
|
s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
|
data/lib/avro/VERSION.txt
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.10.
|
1
|
+
1.10.2
|
data/lib/avro/io.rb
CHANGED
@@ -510,6 +510,8 @@ module Avro
|
|
510
510
|
|
511
511
|
# DatumWriter for generic ruby objects
|
512
512
|
class DatumWriter
|
513
|
+
VALIDATION_OPTIONS = { recursive: false, encoded: true }.freeze
|
514
|
+
|
513
515
|
attr_accessor :writers_schema
|
514
516
|
def initialize(writers_schema=nil)
|
515
517
|
@writers_schema = writers_schema
|
@@ -522,7 +524,7 @@ module Avro
|
|
522
524
|
def write_data(writers_schema, logical_datum, encoder)
|
523
525
|
datum = writers_schema.type_adapter.encode(logical_datum)
|
524
526
|
|
525
|
-
unless Schema.validate(writers_schema, datum,
|
527
|
+
unless Schema.validate(writers_schema, datum, VALIDATION_OPTIONS)
|
526
528
|
raise AvroTypeError.new(writers_schema, datum)
|
527
529
|
end
|
528
530
|
|
@@ -580,12 +582,15 @@ module Avro
|
|
580
582
|
end
|
581
583
|
|
582
584
|
def write_union(writers_schema, datum, encoder)
|
583
|
-
index_of_schema =
|
584
|
-
|
585
|
-
|
586
|
-
|
585
|
+
index_of_schema = writers_schema.schemas.find_index do |schema|
|
586
|
+
# Optimize away expensive validation calls for the common null type
|
587
|
+
schema.type_sym == :null ? datum.nil? : Schema.validate(schema, datum)
|
588
|
+
end
|
589
|
+
|
590
|
+
unless index_of_schema
|
587
591
|
raise AvroTypeError.new(writers_schema, datum)
|
588
592
|
end
|
593
|
+
|
589
594
|
encoder.write_long(index_of_schema)
|
590
595
|
write_data(writers_schema.schemas[index_of_schema], datum, encoder)
|
591
596
|
end
|
data/lib/avro/schema.rb
CHANGED
@@ -36,6 +36,8 @@ module Avro
|
|
36
36
|
LONG_MIN_VALUE = -(1 << 63)
|
37
37
|
LONG_MAX_VALUE = (1 << 63) - 1
|
38
38
|
|
39
|
+
DEFAULT_VALIDATE_OPTIONS = { recursive: true, encoded: false }.freeze
|
40
|
+
|
39
41
|
def self.parse(json_string)
|
40
42
|
real_parse(MultiJson.load(json_string), {})
|
41
43
|
end
|
@@ -109,7 +111,7 @@ module Avro
|
|
109
111
|
end
|
110
112
|
|
111
113
|
# Determine if a ruby datum is an instance of a schema
|
112
|
-
def self.validate(expected_schema, logical_datum, options =
|
114
|
+
def self.validate(expected_schema, logical_datum, options = DEFAULT_VALIDATE_OPTIONS)
|
113
115
|
SchemaValidator.validate!(expected_schema, logical_datum, options)
|
114
116
|
true
|
115
117
|
rescue SchemaValidator::ValidationError
|
@@ -15,6 +15,9 @@
|
|
15
15
|
# limitations under the License.
|
16
16
|
module Avro
|
17
17
|
module SchemaCompatibility
|
18
|
+
INT_COERCIBLE_TYPES_SYM = [:long, :float, :double].freeze
|
19
|
+
LONG_COERCIBLE_TYPES_SYM = [:float, :double].freeze
|
20
|
+
|
18
21
|
# Perform a full, recursive check that a datum written using the writers_schema
|
19
22
|
# can be read using the readers_schema.
|
20
23
|
def self.can_read?(writers_schema, readers_schema)
|
@@ -31,6 +34,9 @@ module Avro
|
|
31
34
|
# be read using the readers_schema. This check includes matching the types,
|
32
35
|
# including schema promotion, and matching the full name (including aliases) for named types.
|
33
36
|
def self.match_schemas(writers_schema, readers_schema)
|
37
|
+
# Bypass deeper checks if the schemas are the same Ruby objects
|
38
|
+
return true if writers_schema.equal?(readers_schema)
|
39
|
+
|
34
40
|
w_type = writers_schema.type_sym
|
35
41
|
r_type = readers_schema.type_sym
|
36
42
|
|
@@ -62,9 +68,9 @@ module Avro
|
|
62
68
|
end
|
63
69
|
|
64
70
|
# Handle schema promotion
|
65
|
-
if w_type == :int &&
|
71
|
+
if w_type == :int && INT_COERCIBLE_TYPES_SYM.include?(r_type)
|
66
72
|
return true
|
67
|
-
elsif w_type == :long &&
|
73
|
+
elsif w_type == :long && LONG_COERCIBLE_TYPES_SYM.include?(r_type)
|
68
74
|
return true
|
69
75
|
elsif w_type == :float && r_type == :double
|
70
76
|
return true
|
@@ -22,16 +22,18 @@ module Avro
|
|
22
22
|
LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
|
23
23
|
COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
|
24
24
|
BOOLEAN_VALUES = [true, false].freeze
|
25
|
+
DEFAULT_VALIDATION_OPTIONS = { recursive: true, encoded: false, fail_on_extra_fields: false }.freeze
|
26
|
+
RECURSIVE_SIMPLE_VALIDATION_OPTIONS = { encoded: true }.freeze
|
27
|
+
RUBY_CLASS_TO_AVRO_TYPE = {
|
28
|
+
NilClass => 'null'.freeze,
|
29
|
+
String => 'string'.freeze,
|
30
|
+
Float => 'float'.freeze,
|
31
|
+
Hash => 'record'.freeze
|
32
|
+
}.freeze
|
25
33
|
|
26
34
|
class Result
|
27
|
-
attr_reader :errors
|
28
|
-
|
29
|
-
def initialize
|
30
|
-
@errors = []
|
31
|
-
end
|
32
|
-
|
33
35
|
def <<(error)
|
34
|
-
|
36
|
+
errors << error
|
35
37
|
end
|
36
38
|
|
37
39
|
def add_error(path, message)
|
@@ -39,11 +41,16 @@ module Avro
|
|
39
41
|
end
|
40
42
|
|
41
43
|
def failure?
|
42
|
-
@errors.any?
|
44
|
+
defined?(@errors) && errors.any?
|
43
45
|
end
|
44
46
|
|
45
47
|
def to_s
|
46
|
-
errors.join("\n")
|
48
|
+
failure? ? errors.join("\n") : ''
|
49
|
+
end
|
50
|
+
|
51
|
+
def errors
|
52
|
+
# Use less memory for success results by lazily creating the errors array
|
53
|
+
@errors ||= []
|
47
54
|
end
|
48
55
|
end
|
49
56
|
|
@@ -63,12 +70,9 @@ module Avro
|
|
63
70
|
TypeMismatchError = Class.new(ValidationError)
|
64
71
|
|
65
72
|
class << self
|
66
|
-
def validate!(expected_schema, logical_datum, options =
|
67
|
-
options ||= {}
|
68
|
-
options[:recursive] = true unless options.key?(:recursive)
|
69
|
-
|
73
|
+
def validate!(expected_schema, logical_datum, options = DEFAULT_VALIDATION_OPTIONS)
|
70
74
|
result = Result.new
|
71
|
-
if options
|
75
|
+
if options.fetch(:recursive, true)
|
72
76
|
validate_recursive(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
|
73
77
|
else
|
74
78
|
validate_simple(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
|
@@ -79,10 +83,10 @@ module Avro
|
|
79
83
|
|
80
84
|
private
|
81
85
|
|
82
|
-
def validate_recursive(expected_schema, logical_datum, path, result, options
|
86
|
+
def validate_recursive(expected_schema, logical_datum, path, result, options)
|
83
87
|
datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
|
84
88
|
|
85
|
-
validate_simple(expected_schema, datum, path, result,
|
89
|
+
validate_simple(expected_schema, datum, path, result, RECURSIVE_SIMPLE_VALIDATION_OPTIONS)
|
86
90
|
|
87
91
|
case expected_schema.type_sym
|
88
92
|
when :array
|
@@ -95,7 +99,8 @@ module Avro
|
|
95
99
|
fail TypeMismatchError unless datum.is_a?(Hash)
|
96
100
|
expected_schema.fields.each do |field|
|
97
101
|
deeper_path = deeper_path_for_hash(field.name, path)
|
98
|
-
|
102
|
+
nested_value = datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym]
|
103
|
+
validate_recursive(field.type, nested_value, deeper_path, result, options)
|
99
104
|
end
|
100
105
|
if options[:fail_on_extra_fields]
|
101
106
|
datum_fields = datum.keys.map(&:to_s)
|
@@ -109,7 +114,7 @@ module Avro
|
|
109
114
|
result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
|
110
115
|
end
|
111
116
|
|
112
|
-
def validate_simple(expected_schema, logical_datum, path, result, options
|
117
|
+
def validate_simple(expected_schema, logical_datum, path, result, options)
|
113
118
|
datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
|
114
119
|
validate_type(expected_schema)
|
115
120
|
|
@@ -163,14 +168,14 @@ module Avro
|
|
163
168
|
"expected enum with values #{symbols}, got #{actual_value_message(datum)}"
|
164
169
|
end
|
165
170
|
|
166
|
-
def validate_array(expected_schema, datum, path, result, options
|
171
|
+
def validate_array(expected_schema, datum, path, result, options)
|
167
172
|
fail TypeMismatchError unless datum.is_a?(Array)
|
168
173
|
datum.each_with_index do |d, i|
|
169
|
-
validate_recursive(expected_schema.items, d, path
|
174
|
+
validate_recursive(expected_schema.items, d, "#{path}[#{i}]", result, options)
|
170
175
|
end
|
171
176
|
end
|
172
177
|
|
173
|
-
def validate_map(expected_schema, datum, path, result, options
|
178
|
+
def validate_map(expected_schema, datum, path, result, options)
|
174
179
|
fail TypeMismatchError unless datum.is_a?(Hash)
|
175
180
|
datum.keys.each do |k|
|
176
181
|
result.add_error(path, "unexpected key type '#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String)
|
@@ -181,7 +186,7 @@ module Avro
|
|
181
186
|
end
|
182
187
|
end
|
183
188
|
|
184
|
-
def validate_union(expected_schema, datum, path, result, options
|
189
|
+
def validate_union(expected_schema, datum, path, result, options)
|
185
190
|
if expected_schema.schemas.size == 1
|
186
191
|
validate_recursive(expected_schema.schemas.first, datum, path, result, options)
|
187
192
|
return
|
@@ -201,6 +206,9 @@ module Avro
|
|
201
206
|
|
202
207
|
def first_compatible_type(datum, expected_schema, path, failures, options = {})
|
203
208
|
expected_schema.schemas.find do |schema|
|
209
|
+
# Avoid expensive validation if we're just validating a nil
|
210
|
+
next datum.nil? if schema.type_sym == :null
|
211
|
+
|
204
212
|
result = Result.new
|
205
213
|
validate_recursive(schema, datum, path, result, options)
|
206
214
|
failures << { type: schema.type_sym, result: result } if result.failure?
|
@@ -209,7 +217,9 @@ module Avro
|
|
209
217
|
end
|
210
218
|
|
211
219
|
def deeper_path_for_hash(sub_key, path)
|
212
|
-
"#{path}#{PATH_SEPARATOR}#{sub_key}"
|
220
|
+
deeper_path = "#{path}#{PATH_SEPARATOR}#{sub_key}"
|
221
|
+
deeper_path.squeeze!(PATH_SEPARATOR)
|
222
|
+
deeper_path
|
213
223
|
end
|
214
224
|
|
215
225
|
def actual_value_message(value)
|
@@ -226,16 +236,11 @@ module Avro
|
|
226
236
|
end
|
227
237
|
|
228
238
|
def ruby_to_avro_type(ruby_class)
|
229
|
-
|
230
|
-
NilClass => 'null',
|
231
|
-
String => 'string',
|
232
|
-
Float => 'float',
|
233
|
-
Hash => 'record'
|
234
|
-
}.fetch(ruby_class, ruby_class)
|
239
|
+
RUBY_CLASS_TO_AVRO_TYPE.fetch(ruby_class, ruby_class)
|
235
240
|
end
|
236
241
|
|
237
242
|
def ruby_integer_to_avro_type(value)
|
238
|
-
INT_RANGE.cover?(value) ? 'int' : 'long'
|
243
|
+
INT_RANGE.cover?(value) ? 'int'.freeze : 'long'.freeze
|
239
244
|
end
|
240
245
|
end
|
241
246
|
end
|
@@ -25,7 +25,9 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def test_compatible_reader_writer_pairs
|
28
|
+
cached_schema = a_int_record1_schema
|
28
29
|
[
|
30
|
+
cached_schema, cached_schema,
|
29
31
|
long_schema, int_schema,
|
30
32
|
float_schema, int_schema,
|
31
33
|
float_schema, long_schema,
|
@@ -17,7 +17,7 @@
|
|
17
17
|
require 'test_help'
|
18
18
|
|
19
19
|
class TestSchemaValidator < Test::Unit::TestCase
|
20
|
-
def validate!(schema, value, options=
|
20
|
+
def validate!(schema, value, options = {})
|
21
21
|
Avro::SchemaValidator.validate!(schema, value, options)
|
22
22
|
end
|
23
23
|
|
@@ -196,6 +196,12 @@ class TestSchemaValidator < Test::Unit::TestCase
|
|
196
196
|
assert_valid_schema(schema, [{ 'sub' => nil }], [{ 'sub' => 1 }])
|
197
197
|
end
|
198
198
|
|
199
|
+
def test_validate_record_with_symbol_keys
|
200
|
+
schema = hash_to_schema(type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }])
|
201
|
+
|
202
|
+
assert_valid_schema(schema, [{ sub: 1 }], [{ sub: '1' }])
|
203
|
+
end
|
204
|
+
|
199
205
|
def test_validate_shallow_record
|
200
206
|
schema = hash_to_schema(
|
201
207
|
type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }]
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.10.
|
4
|
+
version: 1.10.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Apache Software Foundation
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: multi_json
|