avro 1.10.2 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Manifest +0 -1
- data/NOTICE +1 -1
- data/Rakefile +13 -19
- data/avro.gemspec +36 -29
- data/interop/test_interop.rb +2 -1
- data/lib/avro/VERSION.txt +1 -1
- data/lib/avro/data_file.rb +5 -4
- data/lib/avro/io.rb +8 -9
- data/lib/avro/ipc.rb +9 -5
- data/lib/avro/logical_types.rb +186 -2
- data/lib/avro/protocol.rb +1 -0
- data/lib/avro/schema.rb +68 -11
- data/lib/avro/schema_compatibility.rb +11 -11
- data/lib/avro/schema_normalization.rb +1 -0
- data/lib/avro/schema_validator.rb +13 -12
- data/lib/avro.rb +1 -0
- data/test/case_finder.rb +1 -0
- data/test/random_data.rb +5 -4
- data/test/sample_ipc_client.rb +1 -0
- data/test/sample_ipc_http_client.rb +1 -0
- data/test/sample_ipc_http_server.rb +1 -0
- data/test/sample_ipc_server.rb +1 -0
- data/test/test_datafile.rb +1 -0
- data/test/test_fingerprints.rb +1 -0
- data/test/test_help.rb +1 -0
- data/test/test_io.rb +31 -16
- data/test/test_logical_types.rb +138 -1
- data/test/test_protocol.rb +2 -1
- data/test/test_schema.rb +134 -4
- data/test/test_schema_compatibility.rb +106 -0
- data/test/test_schema_normalization.rb +1 -0
- data/test/test_schema_validator.rb +17 -2
- data/test/test_socket_transport.rb +1 -0
- data/test/tool.rb +8 -7
- metadata +26 -30
- data/CHANGELOG +0 -1
data/lib/avro/schema.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -29,7 +30,7 @@ module Avro
|
|
|
29
30
|
NAMED_TYPES_SYM = Set.new(NAMED_TYPES.map(&:to_sym))
|
|
30
31
|
VALID_TYPES_SYM = Set.new(VALID_TYPES.map(&:to_sym))
|
|
31
32
|
|
|
32
|
-
NAME_REGEX = /^([A-Za-z_][A-Za-z0-9_]*)(\.([A-Za-z_][A-Za-z0-9_]*))
|
|
33
|
+
NAME_REGEX = /^([A-Za-z_][A-Za-z0-9_]*)(\.([A-Za-z_][A-Za-z0-9_]*))*$/.freeze
|
|
33
34
|
|
|
34
35
|
INT_MIN_VALUE = -(1 << 31)
|
|
35
36
|
INT_MAX_VALUE = (1 << 31) - 1
|
|
@@ -38,6 +39,8 @@ module Avro
|
|
|
38
39
|
|
|
39
40
|
DEFAULT_VALIDATE_OPTIONS = { recursive: true, encoded: false }.freeze
|
|
40
41
|
|
|
42
|
+
DECIMAL_LOGICAL_TYPE = 'decimal'
|
|
43
|
+
|
|
41
44
|
def self.parse(json_string)
|
|
42
45
|
real_parse(MultiJson.load(json_string), {})
|
|
43
46
|
end
|
|
@@ -75,7 +78,9 @@ module Avro
|
|
|
75
78
|
case type_sym
|
|
76
79
|
when :fixed
|
|
77
80
|
size = json_obj['size']
|
|
78
|
-
|
|
81
|
+
precision = json_obj['precision']
|
|
82
|
+
scale = json_obj['scale']
|
|
83
|
+
return FixedSchema.new(name, namespace, size, names, logical_type, aliases, precision, scale)
|
|
79
84
|
when :enum
|
|
80
85
|
symbols = json_obj['symbols']
|
|
81
86
|
doc = json_obj['doc']
|
|
@@ -131,7 +136,7 @@ module Avro
|
|
|
131
136
|
def type; @type_sym.to_s; end
|
|
132
137
|
|
|
133
138
|
def type_adapter
|
|
134
|
-
@type_adapter ||= LogicalTypes.type_adapter(type, logical_type) || LogicalTypes::Identity
|
|
139
|
+
@type_adapter ||= LogicalTypes.type_adapter(type, logical_type, self) || LogicalTypes::Identity
|
|
135
140
|
end
|
|
136
141
|
|
|
137
142
|
# Returns the MD5 fingerprint of the schema as an Integer.
|
|
@@ -175,7 +180,7 @@ module Avro
|
|
|
175
180
|
fp
|
|
176
181
|
end
|
|
177
182
|
|
|
178
|
-
SINGLE_OBJECT_MAGIC_NUMBER = [0xC3, 0x01]
|
|
183
|
+
SINGLE_OBJECT_MAGIC_NUMBER = [0xC3, 0x01].freeze
|
|
179
184
|
def single_object_encoding_header
|
|
180
185
|
[SINGLE_OBJECT_MAGIC_NUMBER, single_object_schema_fingerprint].flatten
|
|
181
186
|
end
|
|
@@ -283,6 +288,10 @@ module Avro
|
|
|
283
288
|
def match_fullname?(name)
|
|
284
289
|
name == fullname || fullname_aliases.include?(name)
|
|
285
290
|
end
|
|
291
|
+
|
|
292
|
+
def match_schema?(schema)
|
|
293
|
+
type_sym == schema.type_sym && match_fullname?(schema.fullname)
|
|
294
|
+
end
|
|
286
295
|
end
|
|
287
296
|
|
|
288
297
|
class RecordSchema < NamedSchema
|
|
@@ -413,7 +422,7 @@ module Avro
|
|
|
413
422
|
end
|
|
414
423
|
|
|
415
424
|
class EnumSchema < NamedSchema
|
|
416
|
-
SYMBOL_REGEX = /^[A-Za-z_][A-Za-z0-9_]
|
|
425
|
+
SYMBOL_REGEX = /^[A-Za-z_][A-Za-z0-9_]*$/.freeze
|
|
417
426
|
|
|
418
427
|
attr_reader :symbols, :doc, :default
|
|
419
428
|
|
|
@@ -467,14 +476,27 @@ module Avro
|
|
|
467
476
|
hsh = super
|
|
468
477
|
hsh.size == 1 ? type : hsh
|
|
469
478
|
end
|
|
479
|
+
|
|
480
|
+
def match_schema?(schema)
|
|
481
|
+
return type_sym == schema.type_sym
|
|
482
|
+
# TODO: eventually this could handle schema promotion for primitive schemas too
|
|
483
|
+
end
|
|
470
484
|
end
|
|
471
485
|
|
|
472
486
|
class BytesSchema < PrimitiveSchema
|
|
487
|
+
ERROR_INVALID_SCALE = 'Scale must be greater than or equal to 0'
|
|
488
|
+
ERROR_INVALID_PRECISION = 'Precision must be positive'
|
|
489
|
+
ERROR_PRECISION_TOO_SMALL = 'Precision must be greater than scale'
|
|
490
|
+
|
|
473
491
|
attr_reader :precision, :scale
|
|
492
|
+
|
|
474
493
|
def initialize(type, logical_type=nil, precision=nil, scale=nil)
|
|
475
494
|
super(type.to_sym, logical_type)
|
|
476
|
-
|
|
477
|
-
@
|
|
495
|
+
|
|
496
|
+
@precision = precision.to_i if precision
|
|
497
|
+
@scale = scale.to_i if scale
|
|
498
|
+
|
|
499
|
+
validate_decimal! if logical_type == DECIMAL_LOGICAL_TYPE
|
|
478
500
|
end
|
|
479
501
|
|
|
480
502
|
def to_avro(names=nil)
|
|
@@ -485,29 +507,64 @@ module Avro
|
|
|
485
507
|
avro['scale'] = scale if scale
|
|
486
508
|
avro
|
|
487
509
|
end
|
|
510
|
+
|
|
511
|
+
def match_schema?(schema)
|
|
512
|
+
return true if super
|
|
513
|
+
|
|
514
|
+
if logical_type == DECIMAL_LOGICAL_TYPE && schema.logical_type == DECIMAL_LOGICAL_TYPE
|
|
515
|
+
return precision == schema.precision && (scale || 0) == (schema.scale || 0)
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
false
|
|
519
|
+
end
|
|
520
|
+
|
|
521
|
+
private
|
|
522
|
+
|
|
523
|
+
def validate_decimal!
|
|
524
|
+
raise Avro::SchemaParseError, ERROR_INVALID_PRECISION unless precision.to_i.positive?
|
|
525
|
+
raise Avro::SchemaParseError, ERROR_INVALID_SCALE if scale.to_i.negative?
|
|
526
|
+
raise Avro::SchemaParseError, ERROR_PRECISION_TOO_SMALL if precision < scale.to_i
|
|
527
|
+
end
|
|
488
528
|
end
|
|
489
529
|
|
|
490
530
|
class FixedSchema < NamedSchema
|
|
491
|
-
attr_reader :size
|
|
492
|
-
def initialize(name, space, size, names=nil, logical_type=nil, aliases=nil)
|
|
531
|
+
attr_reader :size, :precision, :scale
|
|
532
|
+
def initialize(name, space, size, names=nil, logical_type=nil, aliases=nil, precision=nil, scale=nil)
|
|
493
533
|
# Ensure valid cto args
|
|
494
534
|
unless size.is_a?(Integer)
|
|
495
535
|
raise AvroError, 'Fixed Schema requires a valid integer for size property.'
|
|
496
536
|
end
|
|
497
537
|
super(:fixed, name, space, names, nil, logical_type, aliases)
|
|
498
538
|
@size = size
|
|
539
|
+
@precision = precision
|
|
540
|
+
@scale = scale
|
|
499
541
|
end
|
|
500
542
|
|
|
501
543
|
def to_avro(names=Set.new)
|
|
502
544
|
avro = super
|
|
503
|
-
avro
|
|
545
|
+
return avro if avro.is_a?(String)
|
|
546
|
+
|
|
547
|
+
avro['size'] = size
|
|
548
|
+
avro['precision'] = precision if precision
|
|
549
|
+
avro['scale'] = scale if scale
|
|
550
|
+
avro
|
|
551
|
+
end
|
|
552
|
+
|
|
553
|
+
def match_schema?(schema)
|
|
554
|
+
return true if super && size == schema.size
|
|
555
|
+
|
|
556
|
+
if logical_type == DECIMAL_LOGICAL_TYPE && schema.logical_type == DECIMAL_LOGICAL_TYPE
|
|
557
|
+
return precision == schema.precision && (scale || 0) == (schema.scale || 0)
|
|
558
|
+
end
|
|
559
|
+
|
|
560
|
+
false
|
|
504
561
|
end
|
|
505
562
|
end
|
|
506
563
|
|
|
507
564
|
class Field < Schema
|
|
508
565
|
attr_reader :type, :name, :default, :order, :doc, :aliases
|
|
509
566
|
|
|
510
|
-
def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil, doc=nil, aliases=nil)
|
|
567
|
+
def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil, doc=nil, aliases=nil) # rubocop:disable Lint/MissingSuper
|
|
511
568
|
@type = subparse(type, names, namespace)
|
|
512
569
|
@name = name
|
|
513
570
|
@default = default
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -46,28 +47,22 @@ module Avro
|
|
|
46
47
|
end
|
|
47
48
|
|
|
48
49
|
if w_type == r_type
|
|
49
|
-
return
|
|
50
|
+
return readers_schema.match_schema?(writers_schema) if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
|
|
50
51
|
|
|
51
52
|
case r_type
|
|
52
|
-
when :record
|
|
53
|
-
return readers_schema.match_fullname?(writers_schema.fullname)
|
|
54
|
-
when :error
|
|
55
|
-
return readers_schema.match_fullname?(writers_schema.fullname)
|
|
56
53
|
when :request
|
|
57
54
|
return true
|
|
58
|
-
when :fixed
|
|
59
|
-
return readers_schema.match_fullname?(writers_schema.fullname) &&
|
|
60
|
-
writers_schema.size == readers_schema.size
|
|
61
|
-
when :enum
|
|
62
|
-
return readers_schema.match_fullname?(writers_schema.fullname)
|
|
63
55
|
when :map
|
|
64
56
|
return match_schemas(writers_schema.values, readers_schema.values)
|
|
65
57
|
when :array
|
|
66
58
|
return match_schemas(writers_schema.items, readers_schema.items)
|
|
59
|
+
else
|
|
60
|
+
return readers_schema.match_schema?(writers_schema)
|
|
67
61
|
end
|
|
68
62
|
end
|
|
69
63
|
|
|
70
64
|
# Handle schema promotion
|
|
65
|
+
# rubocop:disable Lint/DuplicateBranch
|
|
71
66
|
if w_type == :int && INT_COERCIBLE_TYPES_SYM.include?(r_type)
|
|
72
67
|
return true
|
|
73
68
|
elsif w_type == :long && LONG_COERCIBLE_TYPES_SYM.include?(r_type)
|
|
@@ -79,8 +74,13 @@ module Avro
|
|
|
79
74
|
elsif w_type == :bytes && r_type == :string
|
|
80
75
|
return true
|
|
81
76
|
end
|
|
77
|
+
# rubocop:enable Lint/DuplicateBranch
|
|
82
78
|
|
|
83
|
-
|
|
79
|
+
if readers_schema.respond_to?(:match_schema?)
|
|
80
|
+
readers_schema.match_schema?(writers_schema)
|
|
81
|
+
else
|
|
82
|
+
false
|
|
83
|
+
end
|
|
84
84
|
end
|
|
85
85
|
|
|
86
86
|
class Checker
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -16,19 +17,19 @@
|
|
|
16
17
|
|
|
17
18
|
module Avro
|
|
18
19
|
class SchemaValidator
|
|
19
|
-
ROOT_IDENTIFIER = '.'
|
|
20
|
-
PATH_SEPARATOR = '.'
|
|
21
|
-
INT_RANGE = Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE
|
|
22
|
-
LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
|
|
20
|
+
ROOT_IDENTIFIER = '.'
|
|
21
|
+
PATH_SEPARATOR = '.'
|
|
22
|
+
INT_RANGE = (Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE).freeze
|
|
23
|
+
LONG_RANGE = (Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE).freeze
|
|
23
24
|
COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
|
|
24
25
|
BOOLEAN_VALUES = [true, false].freeze
|
|
25
26
|
DEFAULT_VALIDATION_OPTIONS = { recursive: true, encoded: false, fail_on_extra_fields: false }.freeze
|
|
26
27
|
RECURSIVE_SIMPLE_VALIDATION_OPTIONS = { encoded: true }.freeze
|
|
27
28
|
RUBY_CLASS_TO_AVRO_TYPE = {
|
|
28
|
-
NilClass => 'null'
|
|
29
|
-
String => 'string'
|
|
30
|
-
Float => 'float'
|
|
31
|
-
Hash => 'record'
|
|
29
|
+
NilClass => 'null',
|
|
30
|
+
String => 'string',
|
|
31
|
+
Float => 'float',
|
|
32
|
+
Hash => 'record'
|
|
32
33
|
}.freeze
|
|
33
34
|
|
|
34
35
|
class Result
|
|
@@ -132,7 +133,7 @@ module Avro
|
|
|
132
133
|
fail TypeMismatchError unless datum.is_a?(Integer)
|
|
133
134
|
result.add_error(path, "out of bound value #{datum}") unless LONG_RANGE.cover?(datum)
|
|
134
135
|
when :float, :double
|
|
135
|
-
fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer)
|
|
136
|
+
fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer) || datum.is_a?(BigDecimal)
|
|
136
137
|
when :fixed
|
|
137
138
|
if datum.is_a? String
|
|
138
139
|
result.add_error(path, fixed_string_message(expected_schema.size, datum)) unless datum.bytesize == expected_schema.size
|
|
@@ -217,9 +218,9 @@ module Avro
|
|
|
217
218
|
end
|
|
218
219
|
|
|
219
220
|
def deeper_path_for_hash(sub_key, path)
|
|
220
|
-
deeper_path = "#{path}#{PATH_SEPARATOR}#{sub_key}"
|
|
221
|
+
deeper_path = +"#{path}#{PATH_SEPARATOR}#{sub_key}"
|
|
221
222
|
deeper_path.squeeze!(PATH_SEPARATOR)
|
|
222
|
-
deeper_path
|
|
223
|
+
deeper_path.freeze
|
|
223
224
|
end
|
|
224
225
|
|
|
225
226
|
def actual_value_message(value)
|
|
@@ -240,7 +241,7 @@ module Avro
|
|
|
240
241
|
end
|
|
241
242
|
|
|
242
243
|
def ruby_integer_to_avro_type(value)
|
|
243
|
-
INT_RANGE.cover?(value) ? 'int'
|
|
244
|
+
INT_RANGE.cover?(value) ? 'int' : 'long'
|
|
244
245
|
end
|
|
245
246
|
end
|
|
246
247
|
end
|
data/lib/avro.rb
CHANGED
data/test/case_finder.rb
CHANGED
data/test/random_data.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -5,9 +6,9 @@
|
|
|
5
6
|
# to you under the Apache License, Version 2.0 (the
|
|
6
7
|
# "License"); you may not use this file except in compliance
|
|
7
8
|
# with the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
+
#
|
|
9
10
|
# https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
+
#
|
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
14
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
@@ -74,7 +75,7 @@ class RandomData
|
|
|
74
75
|
return nil if len == 0
|
|
75
76
|
symbols[rand(len)]
|
|
76
77
|
when :fixed
|
|
77
|
-
f = ""
|
|
78
|
+
f = +""
|
|
78
79
|
schm.size.times { f << BYTEPOOL[rand(BYTEPOOL.size), 1] }
|
|
79
80
|
f
|
|
80
81
|
end
|
|
@@ -95,7 +96,7 @@ class RandomData
|
|
|
95
96
|
BYTEPOOL = '12345abcd'
|
|
96
97
|
|
|
97
98
|
def randstr(chars=CHARPOOL, length=20)
|
|
98
|
-
str = ''
|
|
99
|
+
str = +''
|
|
99
100
|
rand(length+1).times { str << chars[rand(chars.size)] }
|
|
100
101
|
str
|
|
101
102
|
end
|
data/test/sample_ipc_client.rb
CHANGED
data/test/sample_ipc_server.rb
CHANGED
data/test/test_datafile.rb
CHANGED
data/test/test_fingerprints.rb
CHANGED
data/test/test_help.rb
CHANGED
data/test/test_io.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -217,7 +218,7 @@ EOS
|
|
|
217
218
|
[64, '80 01'],
|
|
218
219
|
[8192, '80 80 01'],
|
|
219
220
|
[-8193, '81 80 01'],
|
|
220
|
-
]
|
|
221
|
+
].freeze
|
|
221
222
|
|
|
222
223
|
def avro_hexlify(reader)
|
|
223
224
|
bytes = []
|
|
@@ -266,46 +267,46 @@ EOS
|
|
|
266
267
|
|
|
267
268
|
def test_utf8_string_encoding
|
|
268
269
|
[
|
|
269
|
-
"\xC3"
|
|
270
|
-
"\xC3\x83"
|
|
270
|
+
String.new("\xC3", encoding: 'ISO-8859-1'),
|
|
271
|
+
String.new("\xC3\x83", encoding: 'UTF-8')
|
|
271
272
|
].each do |value|
|
|
272
|
-
output =
|
|
273
|
+
output = String.new('', encoding: 'BINARY')
|
|
273
274
|
encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
|
|
274
275
|
datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"string"'))
|
|
275
276
|
datum_writer.write(value, encoder)
|
|
276
277
|
|
|
277
|
-
assert_equal "\x04\xc3\x83"
|
|
278
|
+
assert_equal String.new("\x04\xc3\x83", encoding: 'BINARY'), output
|
|
278
279
|
end
|
|
279
280
|
end
|
|
280
281
|
|
|
281
282
|
def test_bytes_encoding
|
|
282
283
|
[
|
|
283
|
-
"\xC3\x83"
|
|
284
|
-
"\xC3\x83"
|
|
285
|
-
"\xC3\x83"
|
|
284
|
+
String.new("\xC3\x83", encoding: 'BINARY'),
|
|
285
|
+
String.new("\xC3\x83", encoding: 'ISO-8859-1'),
|
|
286
|
+
String.new("\xC3\x83", encoding: 'UTF-8')
|
|
286
287
|
].each do |value|
|
|
287
|
-
output =
|
|
288
|
+
output = String.new('', encoding: 'BINARY')
|
|
288
289
|
encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
|
|
289
290
|
datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"bytes"'))
|
|
290
291
|
datum_writer.write(value, encoder)
|
|
291
292
|
|
|
292
|
-
assert_equal "\x04\xc3\x83"
|
|
293
|
+
assert_equal String.new("\x04\xc3\x83", encoding: 'BINARY'), output
|
|
293
294
|
end
|
|
294
295
|
end
|
|
295
296
|
|
|
296
297
|
def test_fixed_encoding
|
|
297
298
|
[
|
|
298
|
-
"\xC3\x83"
|
|
299
|
-
"\xC3\x83"
|
|
300
|
-
"\xC3\x83"
|
|
299
|
+
String.new("\xC3\x83", encoding: 'BINARY'),
|
|
300
|
+
String.new("\xC3\x83", encoding: 'ISO-8859-1'),
|
|
301
|
+
String.new("\xC3\x83", encoding: 'UTF-8')
|
|
301
302
|
].each do |value|
|
|
302
|
-
output =
|
|
303
|
+
output = String.new('', encoding: 'BINARY')
|
|
303
304
|
encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
|
|
304
305
|
schema = '{"type": "fixed", "name": "TwoBytes", "size": 2}'
|
|
305
306
|
datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse(schema))
|
|
306
307
|
datum_writer.write(value, encoder)
|
|
307
308
|
|
|
308
|
-
assert_equal "\
|
|
309
|
+
assert_equal String.new("\xC3\x83", encoding: 'BINARY'), output
|
|
309
310
|
end
|
|
310
311
|
end
|
|
311
312
|
|
|
@@ -489,6 +490,20 @@ EOS
|
|
|
489
490
|
assert_equal(datum_read, { 'field2' => 1 })
|
|
490
491
|
end
|
|
491
492
|
|
|
493
|
+
def test_big_decimal_datum_for_float
|
|
494
|
+
writers_schema = Avro::Schema.parse('"float"')
|
|
495
|
+
writer, * = write_datum(BigDecimal('1.2'), writers_schema)
|
|
496
|
+
datum_read = read_datum(writer, writers_schema)
|
|
497
|
+
assert_in_delta(1.2, datum_read)
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def test_big_decimal_datum_for_double
|
|
501
|
+
writers_schema = Avro::Schema.parse('"double"')
|
|
502
|
+
writer, * = write_datum(BigDecimal("1.2"), writers_schema)
|
|
503
|
+
datum_read = read_datum(writer, writers_schema)
|
|
504
|
+
assert_in_delta(1.2, datum_read)
|
|
505
|
+
end
|
|
506
|
+
|
|
492
507
|
def test_snappy_backward_compat
|
|
493
508
|
# a snappy-compressed block payload without the checksum
|
|
494
509
|
# this has no back-references, just one literal so the last 9
|
|
@@ -567,7 +582,7 @@ EOS
|
|
|
567
582
|
datum = randomdata.next
|
|
568
583
|
assert validate(schm, datum), 'datum is not valid for schema'
|
|
569
584
|
w = Avro::IO::DatumWriter.new(schm)
|
|
570
|
-
writer = StringIO.new
|
|
585
|
+
writer = StringIO.new(+"", "w")
|
|
571
586
|
w.write(datum, Avro::IO::BinaryEncoder.new(writer))
|
|
572
587
|
r = datum_reader(schm)
|
|
573
588
|
reader = StringIO.new(writer.string)
|
data/test/test_logical_types.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
|
4
5
|
# distributed with this work for additional information
|
|
@@ -16,6 +17,7 @@
|
|
|
16
17
|
# limitations under the License.
|
|
17
18
|
|
|
18
19
|
require 'test_help'
|
|
20
|
+
require 'memory_profiler'
|
|
19
21
|
|
|
20
22
|
class TestLogicalTypes < Test::Unit::TestCase
|
|
21
23
|
def test_int_date
|
|
@@ -98,8 +100,143 @@ class TestLogicalTypes < Test::Unit::TestCase
|
|
|
98
100
|
assert_equal 'duration', schema.logical_type
|
|
99
101
|
end
|
|
100
102
|
|
|
103
|
+
def test_bytes_decimal
|
|
104
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
|
105
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 6 }
|
|
106
|
+
SCHEMA
|
|
107
|
+
|
|
108
|
+
assert_equal 'decimal', schema.logical_type
|
|
109
|
+
assert_equal 9, schema.precision
|
|
110
|
+
assert_equal 6, schema.scale
|
|
111
|
+
|
|
112
|
+
assert_encode_and_decode BigDecimal('-3.4562'), schema
|
|
113
|
+
assert_encode_and_decode BigDecimal('3.4562'), schema
|
|
114
|
+
assert_encode_and_decode 15.123, schema
|
|
115
|
+
assert_encode_and_decode 15, schema
|
|
116
|
+
assert_encode_and_decode BigDecimal('0.123456'), schema
|
|
117
|
+
assert_encode_and_decode BigDecimal('0'), schema
|
|
118
|
+
assert_encode_and_decode BigDecimal('1'), schema
|
|
119
|
+
assert_encode_and_decode BigDecimal('-1'), schema
|
|
120
|
+
|
|
121
|
+
assert_raise ArgumentError do
|
|
122
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
|
123
|
+
type.encode('1.23')
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def test_bytes_decimal_range_errors
|
|
128
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
|
129
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
|
130
|
+
SCHEMA
|
|
131
|
+
|
|
132
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
|
133
|
+
|
|
134
|
+
assert_raises RangeError do
|
|
135
|
+
type.encode(BigDecimal('345'))
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
assert_raises RangeError do
|
|
139
|
+
type.encode(BigDecimal('1.5342'))
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
assert_raises RangeError do
|
|
143
|
+
type.encode(BigDecimal('-1.5342'))
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
assert_raises RangeError do
|
|
147
|
+
type.encode(BigDecimal('-100.2'))
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
assert_raises RangeError do
|
|
151
|
+
type.encode(BigDecimal('-99.991'))
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def test_bytes_decimal_conversion
|
|
156
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
|
157
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
|
158
|
+
SCHEMA
|
|
159
|
+
|
|
160
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
|
161
|
+
|
|
162
|
+
enc = "\xcb\x43\x38".dup.force_encoding('BINARY')
|
|
163
|
+
assert_equal enc, type.encode(BigDecimal('-3.4562'))
|
|
164
|
+
assert_equal BigDecimal('-3.4562'), type.decode(enc)
|
|
165
|
+
|
|
166
|
+
assert_equal "\x34\xbc\xc8".dup.force_encoding('BINARY'), type.encode(BigDecimal('3.4562'))
|
|
167
|
+
assert_equal BigDecimal('3.4562'), type.decode("\x34\xbc\xc8".dup.force_encoding('BINARY'))
|
|
168
|
+
|
|
169
|
+
assert_equal "\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'), type.encode(BigDecimal('456123.123456'))
|
|
170
|
+
assert_equal BigDecimal('456123.123456'), type.decode("\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'))
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def test_logical_type_with_schema
|
|
174
|
+
exception = assert_raises(ArgumentError) do
|
|
175
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(nil)
|
|
176
|
+
end
|
|
177
|
+
assert_equal exception.to_s, 'schema is required'
|
|
178
|
+
|
|
179
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
|
180
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
|
181
|
+
SCHEMA
|
|
182
|
+
|
|
183
|
+
assert_nothing_raised do
|
|
184
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
assert_raises NotImplementedError do
|
|
188
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).encode(BigDecimal('2'))
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
assert_raises NotImplementedError do
|
|
192
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).decode('foo')
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def test_bytes_decimal_object_allocations_encode
|
|
197
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
|
198
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
|
199
|
+
SCHEMA
|
|
200
|
+
|
|
201
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
|
202
|
+
|
|
203
|
+
positive_value = BigDecimal('5.2')
|
|
204
|
+
negative_value = BigDecimal('-5.2')
|
|
205
|
+
|
|
206
|
+
[positive_value, negative_value].each do |value|
|
|
207
|
+
report = MemoryProfiler.report do
|
|
208
|
+
type.encode(value)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
assert_equal 5, report.total_allocated
|
|
212
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
|
213
|
+
assert_operator 1, :>=, report.total_retained
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def test_bytes_decimal_object_allocations_decode
|
|
218
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
|
219
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
|
220
|
+
SCHEMA
|
|
221
|
+
|
|
222
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
|
223
|
+
|
|
224
|
+
positive_enc = "\x02\b".dup.force_encoding('BINARY')
|
|
225
|
+
negative_enc = "\xFD\xF8".dup.force_encoding('BINARY')
|
|
226
|
+
|
|
227
|
+
[positive_enc, negative_enc].each do |encoded|
|
|
228
|
+
report = MemoryProfiler.report do
|
|
229
|
+
type.decode(encoded)
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
assert_equal 5, report.total_allocated
|
|
233
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
|
234
|
+
assert_operator 1, :>=, report.total_retained
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
101
238
|
def encode(datum, schema)
|
|
102
|
-
buffer = StringIO.new
|
|
239
|
+
buffer = StringIO.new
|
|
103
240
|
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
|
104
241
|
|
|
105
242
|
datum_writer = Avro::IO::DatumWriter.new(schema)
|
data/test/test_protocol.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -184,7 +185,7 @@ EOS
|
|
|
184
185
|
}
|
|
185
186
|
}
|
|
186
187
|
EOS
|
|
187
|
-
]
|
|
188
|
+
].freeze
|
|
188
189
|
|
|
189
190
|
Protocol = Avro::Protocol
|
|
190
191
|
def test_parse
|