avro 1.10.2 → 1.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Manifest +0 -1
- data/NOTICE +1 -1
- data/Rakefile +13 -19
- data/avro.gemspec +36 -29
- data/interop/test_interop.rb +2 -1
- data/lib/avro/VERSION.txt +1 -1
- data/lib/avro/data_file.rb +5 -4
- data/lib/avro/io.rb +8 -9
- data/lib/avro/ipc.rb +9 -5
- data/lib/avro/logical_types.rb +186 -2
- data/lib/avro/protocol.rb +1 -0
- data/lib/avro/schema.rb +68 -11
- data/lib/avro/schema_compatibility.rb +11 -11
- data/lib/avro/schema_normalization.rb +1 -0
- data/lib/avro/schema_validator.rb +13 -12
- data/lib/avro.rb +1 -0
- data/test/case_finder.rb +1 -0
- data/test/random_data.rb +5 -4
- data/test/sample_ipc_client.rb +1 -0
- data/test/sample_ipc_http_client.rb +1 -0
- data/test/sample_ipc_http_server.rb +1 -0
- data/test/sample_ipc_server.rb +1 -0
- data/test/test_datafile.rb +1 -0
- data/test/test_fingerprints.rb +1 -0
- data/test/test_help.rb +1 -0
- data/test/test_io.rb +31 -16
- data/test/test_logical_types.rb +138 -1
- data/test/test_protocol.rb +2 -1
- data/test/test_schema.rb +134 -4
- data/test/test_schema_compatibility.rb +106 -0
- data/test/test_schema_normalization.rb +1 -0
- data/test/test_schema_validator.rb +17 -2
- data/test/test_socket_transport.rb +1 -0
- data/test/tool.rb +8 -7
- metadata +26 -30
- data/CHANGELOG +0 -1
data/lib/avro/schema.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -29,7 +30,7 @@ module Avro
|
|
29
30
|
NAMED_TYPES_SYM = Set.new(NAMED_TYPES.map(&:to_sym))
|
30
31
|
VALID_TYPES_SYM = Set.new(VALID_TYPES.map(&:to_sym))
|
31
32
|
|
32
|
-
NAME_REGEX = /^([A-Za-z_][A-Za-z0-9_]*)(\.([A-Za-z_][A-Za-z0-9_]*))
|
33
|
+
NAME_REGEX = /^([A-Za-z_][A-Za-z0-9_]*)(\.([A-Za-z_][A-Za-z0-9_]*))*$/.freeze
|
33
34
|
|
34
35
|
INT_MIN_VALUE = -(1 << 31)
|
35
36
|
INT_MAX_VALUE = (1 << 31) - 1
|
@@ -38,6 +39,8 @@ module Avro
|
|
38
39
|
|
39
40
|
DEFAULT_VALIDATE_OPTIONS = { recursive: true, encoded: false }.freeze
|
40
41
|
|
42
|
+
DECIMAL_LOGICAL_TYPE = 'decimal'
|
43
|
+
|
41
44
|
def self.parse(json_string)
|
42
45
|
real_parse(MultiJson.load(json_string), {})
|
43
46
|
end
|
@@ -75,7 +78,9 @@ module Avro
|
|
75
78
|
case type_sym
|
76
79
|
when :fixed
|
77
80
|
size = json_obj['size']
|
78
|
-
|
81
|
+
precision = json_obj['precision']
|
82
|
+
scale = json_obj['scale']
|
83
|
+
return FixedSchema.new(name, namespace, size, names, logical_type, aliases, precision, scale)
|
79
84
|
when :enum
|
80
85
|
symbols = json_obj['symbols']
|
81
86
|
doc = json_obj['doc']
|
@@ -131,7 +136,7 @@ module Avro
|
|
131
136
|
def type; @type_sym.to_s; end
|
132
137
|
|
133
138
|
def type_adapter
|
134
|
-
@type_adapter ||= LogicalTypes.type_adapter(type, logical_type) || LogicalTypes::Identity
|
139
|
+
@type_adapter ||= LogicalTypes.type_adapter(type, logical_type, self) || LogicalTypes::Identity
|
135
140
|
end
|
136
141
|
|
137
142
|
# Returns the MD5 fingerprint of the schema as an Integer.
|
@@ -175,7 +180,7 @@ module Avro
|
|
175
180
|
fp
|
176
181
|
end
|
177
182
|
|
178
|
-
SINGLE_OBJECT_MAGIC_NUMBER = [0xC3, 0x01]
|
183
|
+
SINGLE_OBJECT_MAGIC_NUMBER = [0xC3, 0x01].freeze
|
179
184
|
def single_object_encoding_header
|
180
185
|
[SINGLE_OBJECT_MAGIC_NUMBER, single_object_schema_fingerprint].flatten
|
181
186
|
end
|
@@ -283,6 +288,10 @@ module Avro
|
|
283
288
|
def match_fullname?(name)
|
284
289
|
name == fullname || fullname_aliases.include?(name)
|
285
290
|
end
|
291
|
+
|
292
|
+
def match_schema?(schema)
|
293
|
+
type_sym == schema.type_sym && match_fullname?(schema.fullname)
|
294
|
+
end
|
286
295
|
end
|
287
296
|
|
288
297
|
class RecordSchema < NamedSchema
|
@@ -413,7 +422,7 @@ module Avro
|
|
413
422
|
end
|
414
423
|
|
415
424
|
class EnumSchema < NamedSchema
|
416
|
-
SYMBOL_REGEX = /^[A-Za-z_][A-Za-z0-9_]
|
425
|
+
SYMBOL_REGEX = /^[A-Za-z_][A-Za-z0-9_]*$/.freeze
|
417
426
|
|
418
427
|
attr_reader :symbols, :doc, :default
|
419
428
|
|
@@ -467,14 +476,27 @@ module Avro
|
|
467
476
|
hsh = super
|
468
477
|
hsh.size == 1 ? type : hsh
|
469
478
|
end
|
479
|
+
|
480
|
+
def match_schema?(schema)
|
481
|
+
return type_sym == schema.type_sym
|
482
|
+
# TODO: eventually this could handle schema promotion for primitive schemas too
|
483
|
+
end
|
470
484
|
end
|
471
485
|
|
472
486
|
class BytesSchema < PrimitiveSchema
|
487
|
+
ERROR_INVALID_SCALE = 'Scale must be greater than or equal to 0'
|
488
|
+
ERROR_INVALID_PRECISION = 'Precision must be positive'
|
489
|
+
ERROR_PRECISION_TOO_SMALL = 'Precision must be greater than scale'
|
490
|
+
|
473
491
|
attr_reader :precision, :scale
|
492
|
+
|
474
493
|
def initialize(type, logical_type=nil, precision=nil, scale=nil)
|
475
494
|
super(type.to_sym, logical_type)
|
476
|
-
|
477
|
-
@
|
495
|
+
|
496
|
+
@precision = precision.to_i if precision
|
497
|
+
@scale = scale.to_i if scale
|
498
|
+
|
499
|
+
validate_decimal! if logical_type == DECIMAL_LOGICAL_TYPE
|
478
500
|
end
|
479
501
|
|
480
502
|
def to_avro(names=nil)
|
@@ -485,29 +507,64 @@ module Avro
|
|
485
507
|
avro['scale'] = scale if scale
|
486
508
|
avro
|
487
509
|
end
|
510
|
+
|
511
|
+
def match_schema?(schema)
|
512
|
+
return true if super
|
513
|
+
|
514
|
+
if logical_type == DECIMAL_LOGICAL_TYPE && schema.logical_type == DECIMAL_LOGICAL_TYPE
|
515
|
+
return precision == schema.precision && (scale || 0) == (schema.scale || 0)
|
516
|
+
end
|
517
|
+
|
518
|
+
false
|
519
|
+
end
|
520
|
+
|
521
|
+
private
|
522
|
+
|
523
|
+
def validate_decimal!
|
524
|
+
raise Avro::SchemaParseError, ERROR_INVALID_PRECISION unless precision.to_i.positive?
|
525
|
+
raise Avro::SchemaParseError, ERROR_INVALID_SCALE if scale.to_i.negative?
|
526
|
+
raise Avro::SchemaParseError, ERROR_PRECISION_TOO_SMALL if precision < scale.to_i
|
527
|
+
end
|
488
528
|
end
|
489
529
|
|
490
530
|
class FixedSchema < NamedSchema
|
491
|
-
attr_reader :size
|
492
|
-
def initialize(name, space, size, names=nil, logical_type=nil, aliases=nil)
|
531
|
+
attr_reader :size, :precision, :scale
|
532
|
+
def initialize(name, space, size, names=nil, logical_type=nil, aliases=nil, precision=nil, scale=nil)
|
493
533
|
# Ensure valid cto args
|
494
534
|
unless size.is_a?(Integer)
|
495
535
|
raise AvroError, 'Fixed Schema requires a valid integer for size property.'
|
496
536
|
end
|
497
537
|
super(:fixed, name, space, names, nil, logical_type, aliases)
|
498
538
|
@size = size
|
539
|
+
@precision = precision
|
540
|
+
@scale = scale
|
499
541
|
end
|
500
542
|
|
501
543
|
def to_avro(names=Set.new)
|
502
544
|
avro = super
|
503
|
-
avro
|
545
|
+
return avro if avro.is_a?(String)
|
546
|
+
|
547
|
+
avro['size'] = size
|
548
|
+
avro['precision'] = precision if precision
|
549
|
+
avro['scale'] = scale if scale
|
550
|
+
avro
|
551
|
+
end
|
552
|
+
|
553
|
+
def match_schema?(schema)
|
554
|
+
return true if super && size == schema.size
|
555
|
+
|
556
|
+
if logical_type == DECIMAL_LOGICAL_TYPE && schema.logical_type == DECIMAL_LOGICAL_TYPE
|
557
|
+
return precision == schema.precision && (scale || 0) == (schema.scale || 0)
|
558
|
+
end
|
559
|
+
|
560
|
+
false
|
504
561
|
end
|
505
562
|
end
|
506
563
|
|
507
564
|
class Field < Schema
|
508
565
|
attr_reader :type, :name, :default, :order, :doc, :aliases
|
509
566
|
|
510
|
-
def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil, doc=nil, aliases=nil)
|
567
|
+
def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil, doc=nil, aliases=nil) # rubocop:disable Lint/MissingSuper
|
511
568
|
@type = subparse(type, names, namespace)
|
512
569
|
@name = name
|
513
570
|
@default = default
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -46,28 +47,22 @@ module Avro
|
|
46
47
|
end
|
47
48
|
|
48
49
|
if w_type == r_type
|
49
|
-
return
|
50
|
+
return readers_schema.match_schema?(writers_schema) if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
|
50
51
|
|
51
52
|
case r_type
|
52
|
-
when :record
|
53
|
-
return readers_schema.match_fullname?(writers_schema.fullname)
|
54
|
-
when :error
|
55
|
-
return readers_schema.match_fullname?(writers_schema.fullname)
|
56
53
|
when :request
|
57
54
|
return true
|
58
|
-
when :fixed
|
59
|
-
return readers_schema.match_fullname?(writers_schema.fullname) &&
|
60
|
-
writers_schema.size == readers_schema.size
|
61
|
-
when :enum
|
62
|
-
return readers_schema.match_fullname?(writers_schema.fullname)
|
63
55
|
when :map
|
64
56
|
return match_schemas(writers_schema.values, readers_schema.values)
|
65
57
|
when :array
|
66
58
|
return match_schemas(writers_schema.items, readers_schema.items)
|
59
|
+
else
|
60
|
+
return readers_schema.match_schema?(writers_schema)
|
67
61
|
end
|
68
62
|
end
|
69
63
|
|
70
64
|
# Handle schema promotion
|
65
|
+
# rubocop:disable Lint/DuplicateBranch
|
71
66
|
if w_type == :int && INT_COERCIBLE_TYPES_SYM.include?(r_type)
|
72
67
|
return true
|
73
68
|
elsif w_type == :long && LONG_COERCIBLE_TYPES_SYM.include?(r_type)
|
@@ -79,8 +74,13 @@ module Avro
|
|
79
74
|
elsif w_type == :bytes && r_type == :string
|
80
75
|
return true
|
81
76
|
end
|
77
|
+
# rubocop:enable Lint/DuplicateBranch
|
82
78
|
|
83
|
-
|
79
|
+
if readers_schema.respond_to?(:match_schema?)
|
80
|
+
readers_schema.match_schema?(writers_schema)
|
81
|
+
else
|
82
|
+
false
|
83
|
+
end
|
84
84
|
end
|
85
85
|
|
86
86
|
class Checker
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -16,19 +17,19 @@
|
|
16
17
|
|
17
18
|
module Avro
|
18
19
|
class SchemaValidator
|
19
|
-
ROOT_IDENTIFIER = '.'
|
20
|
-
PATH_SEPARATOR = '.'
|
21
|
-
INT_RANGE = Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE
|
22
|
-
LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
|
20
|
+
ROOT_IDENTIFIER = '.'
|
21
|
+
PATH_SEPARATOR = '.'
|
22
|
+
INT_RANGE = (Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE).freeze
|
23
|
+
LONG_RANGE = (Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE).freeze
|
23
24
|
COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
|
24
25
|
BOOLEAN_VALUES = [true, false].freeze
|
25
26
|
DEFAULT_VALIDATION_OPTIONS = { recursive: true, encoded: false, fail_on_extra_fields: false }.freeze
|
26
27
|
RECURSIVE_SIMPLE_VALIDATION_OPTIONS = { encoded: true }.freeze
|
27
28
|
RUBY_CLASS_TO_AVRO_TYPE = {
|
28
|
-
NilClass => 'null'
|
29
|
-
String => 'string'
|
30
|
-
Float => 'float'
|
31
|
-
Hash => 'record'
|
29
|
+
NilClass => 'null',
|
30
|
+
String => 'string',
|
31
|
+
Float => 'float',
|
32
|
+
Hash => 'record'
|
32
33
|
}.freeze
|
33
34
|
|
34
35
|
class Result
|
@@ -132,7 +133,7 @@ module Avro
|
|
132
133
|
fail TypeMismatchError unless datum.is_a?(Integer)
|
133
134
|
result.add_error(path, "out of bound value #{datum}") unless LONG_RANGE.cover?(datum)
|
134
135
|
when :float, :double
|
135
|
-
fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer)
|
136
|
+
fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer) || datum.is_a?(BigDecimal)
|
136
137
|
when :fixed
|
137
138
|
if datum.is_a? String
|
138
139
|
result.add_error(path, fixed_string_message(expected_schema.size, datum)) unless datum.bytesize == expected_schema.size
|
@@ -217,9 +218,9 @@ module Avro
|
|
217
218
|
end
|
218
219
|
|
219
220
|
def deeper_path_for_hash(sub_key, path)
|
220
|
-
deeper_path = "#{path}#{PATH_SEPARATOR}#{sub_key}"
|
221
|
+
deeper_path = +"#{path}#{PATH_SEPARATOR}#{sub_key}"
|
221
222
|
deeper_path.squeeze!(PATH_SEPARATOR)
|
222
|
-
deeper_path
|
223
|
+
deeper_path.freeze
|
223
224
|
end
|
224
225
|
|
225
226
|
def actual_value_message(value)
|
@@ -240,7 +241,7 @@ module Avro
|
|
240
241
|
end
|
241
242
|
|
242
243
|
def ruby_integer_to_avro_type(value)
|
243
|
-
INT_RANGE.cover?(value) ? 'int'
|
244
|
+
INT_RANGE.cover?(value) ? 'int' : 'long'
|
244
245
|
end
|
245
246
|
end
|
246
247
|
end
|
data/lib/avro.rb
CHANGED
data/test/case_finder.rb
CHANGED
data/test/random_data.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -5,9 +6,9 @@
|
|
5
6
|
# to you under the Apache License, Version 2.0 (the
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
+
#
|
9
10
|
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
+
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
14
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
@@ -74,7 +75,7 @@ class RandomData
|
|
74
75
|
return nil if len == 0
|
75
76
|
symbols[rand(len)]
|
76
77
|
when :fixed
|
77
|
-
f = ""
|
78
|
+
f = +""
|
78
79
|
schm.size.times { f << BYTEPOOL[rand(BYTEPOOL.size), 1] }
|
79
80
|
f
|
80
81
|
end
|
@@ -95,7 +96,7 @@ class RandomData
|
|
95
96
|
BYTEPOOL = '12345abcd'
|
96
97
|
|
97
98
|
def randstr(chars=CHARPOOL, length=20)
|
98
|
-
str = ''
|
99
|
+
str = +''
|
99
100
|
rand(length+1).times { str << chars[rand(chars.size)] }
|
100
101
|
str
|
101
102
|
end
|
data/test/sample_ipc_client.rb
CHANGED
data/test/sample_ipc_server.rb
CHANGED
data/test/test_datafile.rb
CHANGED
data/test/test_fingerprints.rb
CHANGED
data/test/test_help.rb
CHANGED
data/test/test_io.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -217,7 +218,7 @@ EOS
|
|
217
218
|
[64, '80 01'],
|
218
219
|
[8192, '80 80 01'],
|
219
220
|
[-8193, '81 80 01'],
|
220
|
-
]
|
221
|
+
].freeze
|
221
222
|
|
222
223
|
def avro_hexlify(reader)
|
223
224
|
bytes = []
|
@@ -266,46 +267,46 @@ EOS
|
|
266
267
|
|
267
268
|
def test_utf8_string_encoding
|
268
269
|
[
|
269
|
-
"\xC3"
|
270
|
-
"\xC3\x83"
|
270
|
+
String.new("\xC3", encoding: 'ISO-8859-1'),
|
271
|
+
String.new("\xC3\x83", encoding: 'UTF-8')
|
271
272
|
].each do |value|
|
272
|
-
output =
|
273
|
+
output = String.new('', encoding: 'BINARY')
|
273
274
|
encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
|
274
275
|
datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"string"'))
|
275
276
|
datum_writer.write(value, encoder)
|
276
277
|
|
277
|
-
assert_equal "\x04\xc3\x83"
|
278
|
+
assert_equal String.new("\x04\xc3\x83", encoding: 'BINARY'), output
|
278
279
|
end
|
279
280
|
end
|
280
281
|
|
281
282
|
def test_bytes_encoding
|
282
283
|
[
|
283
|
-
"\xC3\x83"
|
284
|
-
"\xC3\x83"
|
285
|
-
"\xC3\x83"
|
284
|
+
String.new("\xC3\x83", encoding: 'BINARY'),
|
285
|
+
String.new("\xC3\x83", encoding: 'ISO-8859-1'),
|
286
|
+
String.new("\xC3\x83", encoding: 'UTF-8')
|
286
287
|
].each do |value|
|
287
|
-
output =
|
288
|
+
output = String.new('', encoding: 'BINARY')
|
288
289
|
encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
|
289
290
|
datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"bytes"'))
|
290
291
|
datum_writer.write(value, encoder)
|
291
292
|
|
292
|
-
assert_equal "\x04\xc3\x83"
|
293
|
+
assert_equal String.new("\x04\xc3\x83", encoding: 'BINARY'), output
|
293
294
|
end
|
294
295
|
end
|
295
296
|
|
296
297
|
def test_fixed_encoding
|
297
298
|
[
|
298
|
-
"\xC3\x83"
|
299
|
-
"\xC3\x83"
|
300
|
-
"\xC3\x83"
|
299
|
+
String.new("\xC3\x83", encoding: 'BINARY'),
|
300
|
+
String.new("\xC3\x83", encoding: 'ISO-8859-1'),
|
301
|
+
String.new("\xC3\x83", encoding: 'UTF-8')
|
301
302
|
].each do |value|
|
302
|
-
output =
|
303
|
+
output = String.new('', encoding: 'BINARY')
|
303
304
|
encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
|
304
305
|
schema = '{"type": "fixed", "name": "TwoBytes", "size": 2}'
|
305
306
|
datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse(schema))
|
306
307
|
datum_writer.write(value, encoder)
|
307
308
|
|
308
|
-
assert_equal "\
|
309
|
+
assert_equal String.new("\xC3\x83", encoding: 'BINARY'), output
|
309
310
|
end
|
310
311
|
end
|
311
312
|
|
@@ -489,6 +490,20 @@ EOS
|
|
489
490
|
assert_equal(datum_read, { 'field2' => 1 })
|
490
491
|
end
|
491
492
|
|
493
|
+
def test_big_decimal_datum_for_float
|
494
|
+
writers_schema = Avro::Schema.parse('"float"')
|
495
|
+
writer, * = write_datum(BigDecimal('1.2'), writers_schema)
|
496
|
+
datum_read = read_datum(writer, writers_schema)
|
497
|
+
assert_in_delta(1.2, datum_read)
|
498
|
+
end
|
499
|
+
|
500
|
+
def test_big_decimal_datum_for_double
|
501
|
+
writers_schema = Avro::Schema.parse('"double"')
|
502
|
+
writer, * = write_datum(BigDecimal("1.2"), writers_schema)
|
503
|
+
datum_read = read_datum(writer, writers_schema)
|
504
|
+
assert_in_delta(1.2, datum_read)
|
505
|
+
end
|
506
|
+
|
492
507
|
def test_snappy_backward_compat
|
493
508
|
# a snappy-compressed block payload without the checksum
|
494
509
|
# this has no back-references, just one literal so the last 9
|
@@ -567,7 +582,7 @@ EOS
|
|
567
582
|
datum = randomdata.next
|
568
583
|
assert validate(schm, datum), 'datum is not valid for schema'
|
569
584
|
w = Avro::IO::DatumWriter.new(schm)
|
570
|
-
writer = StringIO.new
|
585
|
+
writer = StringIO.new(+"", "w")
|
571
586
|
w.write(datum, Avro::IO::BinaryEncoder.new(writer))
|
572
587
|
r = datum_reader(schm)
|
573
588
|
reader = StringIO.new(writer.string)
|
data/test/test_logical_types.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
4
5
|
# distributed with this work for additional information
|
@@ -16,6 +17,7 @@
|
|
16
17
|
# limitations under the License.
|
17
18
|
|
18
19
|
require 'test_help'
|
20
|
+
require 'memory_profiler'
|
19
21
|
|
20
22
|
class TestLogicalTypes < Test::Unit::TestCase
|
21
23
|
def test_int_date
|
@@ -98,8 +100,143 @@ class TestLogicalTypes < Test::Unit::TestCase
|
|
98
100
|
assert_equal 'duration', schema.logical_type
|
99
101
|
end
|
100
102
|
|
103
|
+
def test_bytes_decimal
|
104
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
105
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 6 }
|
106
|
+
SCHEMA
|
107
|
+
|
108
|
+
assert_equal 'decimal', schema.logical_type
|
109
|
+
assert_equal 9, schema.precision
|
110
|
+
assert_equal 6, schema.scale
|
111
|
+
|
112
|
+
assert_encode_and_decode BigDecimal('-3.4562'), schema
|
113
|
+
assert_encode_and_decode BigDecimal('3.4562'), schema
|
114
|
+
assert_encode_and_decode 15.123, schema
|
115
|
+
assert_encode_and_decode 15, schema
|
116
|
+
assert_encode_and_decode BigDecimal('0.123456'), schema
|
117
|
+
assert_encode_and_decode BigDecimal('0'), schema
|
118
|
+
assert_encode_and_decode BigDecimal('1'), schema
|
119
|
+
assert_encode_and_decode BigDecimal('-1'), schema
|
120
|
+
|
121
|
+
assert_raise ArgumentError do
|
122
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
123
|
+
type.encode('1.23')
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def test_bytes_decimal_range_errors
|
128
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
129
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
130
|
+
SCHEMA
|
131
|
+
|
132
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
133
|
+
|
134
|
+
assert_raises RangeError do
|
135
|
+
type.encode(BigDecimal('345'))
|
136
|
+
end
|
137
|
+
|
138
|
+
assert_raises RangeError do
|
139
|
+
type.encode(BigDecimal('1.5342'))
|
140
|
+
end
|
141
|
+
|
142
|
+
assert_raises RangeError do
|
143
|
+
type.encode(BigDecimal('-1.5342'))
|
144
|
+
end
|
145
|
+
|
146
|
+
assert_raises RangeError do
|
147
|
+
type.encode(BigDecimal('-100.2'))
|
148
|
+
end
|
149
|
+
|
150
|
+
assert_raises RangeError do
|
151
|
+
type.encode(BigDecimal('-99.991'))
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_bytes_decimal_conversion
|
156
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
157
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
158
|
+
SCHEMA
|
159
|
+
|
160
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
161
|
+
|
162
|
+
enc = "\xcb\x43\x38".dup.force_encoding('BINARY')
|
163
|
+
assert_equal enc, type.encode(BigDecimal('-3.4562'))
|
164
|
+
assert_equal BigDecimal('-3.4562'), type.decode(enc)
|
165
|
+
|
166
|
+
assert_equal "\x34\xbc\xc8".dup.force_encoding('BINARY'), type.encode(BigDecimal('3.4562'))
|
167
|
+
assert_equal BigDecimal('3.4562'), type.decode("\x34\xbc\xc8".dup.force_encoding('BINARY'))
|
168
|
+
|
169
|
+
assert_equal "\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'), type.encode(BigDecimal('456123.123456'))
|
170
|
+
assert_equal BigDecimal('456123.123456'), type.decode("\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'))
|
171
|
+
end
|
172
|
+
|
173
|
+
def test_logical_type_with_schema
|
174
|
+
exception = assert_raises(ArgumentError) do
|
175
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(nil)
|
176
|
+
end
|
177
|
+
assert_equal exception.to_s, 'schema is required'
|
178
|
+
|
179
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
180
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
181
|
+
SCHEMA
|
182
|
+
|
183
|
+
assert_nothing_raised do
|
184
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema)
|
185
|
+
end
|
186
|
+
|
187
|
+
assert_raises NotImplementedError do
|
188
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).encode(BigDecimal('2'))
|
189
|
+
end
|
190
|
+
|
191
|
+
assert_raises NotImplementedError do
|
192
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).decode('foo')
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def test_bytes_decimal_object_allocations_encode
|
197
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
198
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
199
|
+
SCHEMA
|
200
|
+
|
201
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
202
|
+
|
203
|
+
positive_value = BigDecimal('5.2')
|
204
|
+
negative_value = BigDecimal('-5.2')
|
205
|
+
|
206
|
+
[positive_value, negative_value].each do |value|
|
207
|
+
report = MemoryProfiler.report do
|
208
|
+
type.encode(value)
|
209
|
+
end
|
210
|
+
|
211
|
+
assert_equal 5, report.total_allocated
|
212
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
213
|
+
assert_operator 1, :>=, report.total_retained
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def test_bytes_decimal_object_allocations_decode
|
218
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
219
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
220
|
+
SCHEMA
|
221
|
+
|
222
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
223
|
+
|
224
|
+
positive_enc = "\x02\b".dup.force_encoding('BINARY')
|
225
|
+
negative_enc = "\xFD\xF8".dup.force_encoding('BINARY')
|
226
|
+
|
227
|
+
[positive_enc, negative_enc].each do |encoded|
|
228
|
+
report = MemoryProfiler.report do
|
229
|
+
type.decode(encoded)
|
230
|
+
end
|
231
|
+
|
232
|
+
assert_equal 5, report.total_allocated
|
233
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
234
|
+
assert_operator 1, :>=, report.total_retained
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
101
238
|
def encode(datum, schema)
|
102
|
-
buffer = StringIO.new
|
239
|
+
buffer = StringIO.new
|
103
240
|
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
104
241
|
|
105
242
|
datum_writer = Avro::IO::DatumWriter.new(schema)
|
data/test/test_protocol.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -184,7 +185,7 @@ EOS
|
|
184
185
|
}
|
185
186
|
}
|
186
187
|
EOS
|
187
|
-
]
|
188
|
+
].freeze
|
188
189
|
|
189
190
|
Protocol = Avro::Protocol
|
190
191
|
def test_parse
|