avro 1.10.1 → 1.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Manifest +0 -1
- data/NOTICE +1 -1
- data/Rakefile +13 -19
- data/avro.gemspec +37 -29
- data/interop/test_interop.rb +2 -1
- data/lib/avro/VERSION.txt +1 -1
- data/lib/avro/data_file.rb +5 -4
- data/lib/avro/io.rb +18 -14
- data/lib/avro/ipc.rb +9 -5
- data/lib/avro/logical_types.rb +186 -2
- data/lib/avro/protocol.rb +1 -0
- data/lib/avro/schema.rb +71 -12
- data/lib/avro/schema_compatibility.rb +19 -13
- data/lib/avro/schema_normalization.rb +1 -0
- data/lib/avro/schema_validator.rb +40 -34
- data/lib/avro.rb +1 -0
- data/test/case_finder.rb +1 -0
- data/test/random_data.rb +5 -4
- data/test/sample_ipc_client.rb +1 -0
- data/test/sample_ipc_http_client.rb +1 -0
- data/test/sample_ipc_http_server.rb +1 -0
- data/test/sample_ipc_server.rb +1 -0
- data/test/test_datafile.rb +1 -0
- data/test/test_fingerprints.rb +1 -0
- data/test/test_help.rb +1 -0
- data/test/test_io.rb +31 -16
- data/test/test_logical_types.rb +138 -1
- data/test/test_protocol.rb +2 -1
- data/test/test_schema.rb +134 -4
- data/test/test_schema_compatibility.rb +108 -0
- data/test/test_schema_normalization.rb +1 -0
- data/test/test_schema_validator.rb +24 -3
- data/test/test_socket_transport.rb +1 -0
- data/test/tool.rb +8 -7
- metadata +30 -33
- data/CHANGELOG +0 -1
data/test/test_logical_types.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
4
5
|
# distributed with this work for additional information
|
@@ -16,6 +17,7 @@
|
|
16
17
|
# limitations under the License.
|
17
18
|
|
18
19
|
require 'test_help'
|
20
|
+
require 'memory_profiler'
|
19
21
|
|
20
22
|
class TestLogicalTypes < Test::Unit::TestCase
|
21
23
|
def test_int_date
|
@@ -98,8 +100,143 @@ class TestLogicalTypes < Test::Unit::TestCase
|
|
98
100
|
assert_equal 'duration', schema.logical_type
|
99
101
|
end
|
100
102
|
|
103
|
+
def test_bytes_decimal
|
104
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
105
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 6 }
|
106
|
+
SCHEMA
|
107
|
+
|
108
|
+
assert_equal 'decimal', schema.logical_type
|
109
|
+
assert_equal 9, schema.precision
|
110
|
+
assert_equal 6, schema.scale
|
111
|
+
|
112
|
+
assert_encode_and_decode BigDecimal('-3.4562'), schema
|
113
|
+
assert_encode_and_decode BigDecimal('3.4562'), schema
|
114
|
+
assert_encode_and_decode 15.123, schema
|
115
|
+
assert_encode_and_decode 15, schema
|
116
|
+
assert_encode_and_decode BigDecimal('0.123456'), schema
|
117
|
+
assert_encode_and_decode BigDecimal('0'), schema
|
118
|
+
assert_encode_and_decode BigDecimal('1'), schema
|
119
|
+
assert_encode_and_decode BigDecimal('-1'), schema
|
120
|
+
|
121
|
+
assert_raise ArgumentError do
|
122
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
123
|
+
type.encode('1.23')
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def test_bytes_decimal_range_errors
|
128
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
129
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
130
|
+
SCHEMA
|
131
|
+
|
132
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
133
|
+
|
134
|
+
assert_raises RangeError do
|
135
|
+
type.encode(BigDecimal('345'))
|
136
|
+
end
|
137
|
+
|
138
|
+
assert_raises RangeError do
|
139
|
+
type.encode(BigDecimal('1.5342'))
|
140
|
+
end
|
141
|
+
|
142
|
+
assert_raises RangeError do
|
143
|
+
type.encode(BigDecimal('-1.5342'))
|
144
|
+
end
|
145
|
+
|
146
|
+
assert_raises RangeError do
|
147
|
+
type.encode(BigDecimal('-100.2'))
|
148
|
+
end
|
149
|
+
|
150
|
+
assert_raises RangeError do
|
151
|
+
type.encode(BigDecimal('-99.991'))
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_bytes_decimal_conversion
|
156
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
157
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
158
|
+
SCHEMA
|
159
|
+
|
160
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
161
|
+
|
162
|
+
enc = "\xcb\x43\x38".dup.force_encoding('BINARY')
|
163
|
+
assert_equal enc, type.encode(BigDecimal('-3.4562'))
|
164
|
+
assert_equal BigDecimal('-3.4562'), type.decode(enc)
|
165
|
+
|
166
|
+
assert_equal "\x34\xbc\xc8".dup.force_encoding('BINARY'), type.encode(BigDecimal('3.4562'))
|
167
|
+
assert_equal BigDecimal('3.4562'), type.decode("\x34\xbc\xc8".dup.force_encoding('BINARY'))
|
168
|
+
|
169
|
+
assert_equal "\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'), type.encode(BigDecimal('456123.123456'))
|
170
|
+
assert_equal BigDecimal('456123.123456'), type.decode("\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'))
|
171
|
+
end
|
172
|
+
|
173
|
+
def test_logical_type_with_schema
|
174
|
+
exception = assert_raises(ArgumentError) do
|
175
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(nil)
|
176
|
+
end
|
177
|
+
assert_equal exception.to_s, 'schema is required'
|
178
|
+
|
179
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
180
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
181
|
+
SCHEMA
|
182
|
+
|
183
|
+
assert_nothing_raised do
|
184
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema)
|
185
|
+
end
|
186
|
+
|
187
|
+
assert_raises NotImplementedError do
|
188
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).encode(BigDecimal('2'))
|
189
|
+
end
|
190
|
+
|
191
|
+
assert_raises NotImplementedError do
|
192
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).decode('foo')
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def test_bytes_decimal_object_allocations_encode
|
197
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
198
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
199
|
+
SCHEMA
|
200
|
+
|
201
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
202
|
+
|
203
|
+
positive_value = BigDecimal('5.2')
|
204
|
+
negative_value = BigDecimal('-5.2')
|
205
|
+
|
206
|
+
[positive_value, negative_value].each do |value|
|
207
|
+
report = MemoryProfiler.report do
|
208
|
+
type.encode(value)
|
209
|
+
end
|
210
|
+
|
211
|
+
assert_equal 5, report.total_allocated
|
212
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
213
|
+
assert_operator 1, :>=, report.total_retained
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def test_bytes_decimal_object_allocations_decode
|
218
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
219
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
220
|
+
SCHEMA
|
221
|
+
|
222
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
223
|
+
|
224
|
+
positive_enc = "\x02\b".dup.force_encoding('BINARY')
|
225
|
+
negative_enc = "\xFD\xF8".dup.force_encoding('BINARY')
|
226
|
+
|
227
|
+
[positive_enc, negative_enc].each do |encoded|
|
228
|
+
report = MemoryProfiler.report do
|
229
|
+
type.decode(encoded)
|
230
|
+
end
|
231
|
+
|
232
|
+
assert_equal 5, report.total_allocated
|
233
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
234
|
+
assert_operator 1, :>=, report.total_retained
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
101
238
|
def encode(datum, schema)
|
102
|
-
buffer = StringIO.new
|
239
|
+
buffer = StringIO.new
|
103
240
|
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
104
241
|
|
105
242
|
datum_writer = Avro::IO::DatumWriter.new(schema)
|
data/test/test_protocol.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -184,7 +185,7 @@ EOS
|
|
184
185
|
}
|
185
186
|
}
|
186
187
|
EOS
|
187
|
-
]
|
188
|
+
].freeze
|
188
189
|
|
189
190
|
Protocol = Avro::Protocol
|
190
191
|
def test_parse
|
data/test/test_schema.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -541,6 +542,76 @@ class TestSchema < Test::Unit::TestCase
|
|
541
542
|
exception.to_s)
|
542
543
|
end
|
543
544
|
|
545
|
+
def test_fixed_decimal_to_include_precision_scale
|
546
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
547
|
+
{
|
548
|
+
"type": "fixed",
|
549
|
+
"name": "aFixed",
|
550
|
+
"logicalType": "decimal",
|
551
|
+
"size": 4,
|
552
|
+
"precision": 9,
|
553
|
+
"scale": 2
|
554
|
+
}
|
555
|
+
SCHEMA
|
556
|
+
|
557
|
+
schema_hash =
|
558
|
+
{
|
559
|
+
'type' => 'fixed',
|
560
|
+
'name' => 'aFixed',
|
561
|
+
'logicalType' => 'decimal',
|
562
|
+
'size' => 4,
|
563
|
+
'precision' => 9,
|
564
|
+
'scale' => 2
|
565
|
+
}
|
566
|
+
|
567
|
+
assert_equal schema_hash, schema.to_avro
|
568
|
+
end
|
569
|
+
|
570
|
+
def test_fixed_decimal_to_include_precision_no_scale
|
571
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
572
|
+
{
|
573
|
+
"type": "fixed",
|
574
|
+
"name": "aFixed",
|
575
|
+
"logicalType": "decimal",
|
576
|
+
"size": 4,
|
577
|
+
"precision": 9
|
578
|
+
}
|
579
|
+
SCHEMA
|
580
|
+
|
581
|
+
schema_hash =
|
582
|
+
{
|
583
|
+
'type' => 'fixed',
|
584
|
+
'name' => 'aFixed',
|
585
|
+
'logicalType' => 'decimal',
|
586
|
+
'size' => 4,
|
587
|
+
'precision' => 9
|
588
|
+
}
|
589
|
+
|
590
|
+
assert_equal schema_hash, schema.to_avro
|
591
|
+
end
|
592
|
+
|
593
|
+
# Note: this is not valid but validation is not yet implemented
|
594
|
+
def test_fixed_decimal_to_without_precision_scale
|
595
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
596
|
+
{
|
597
|
+
"type": "fixed",
|
598
|
+
"size": 4,
|
599
|
+
"name": "aFixed",
|
600
|
+
"logicalType": "decimal"
|
601
|
+
}
|
602
|
+
SCHEMA
|
603
|
+
|
604
|
+
schema_hash =
|
605
|
+
{
|
606
|
+
'type' => 'fixed',
|
607
|
+
'name' => 'aFixed',
|
608
|
+
'logicalType' => 'decimal',
|
609
|
+
'size' => 4
|
610
|
+
}
|
611
|
+
|
612
|
+
assert_equal schema_hash, schema.to_avro
|
613
|
+
end
|
614
|
+
|
544
615
|
def test_bytes_decimal_to_include_precision_scale
|
545
616
|
schema = Avro::Schema.parse <<-SCHEMA
|
546
617
|
{
|
@@ -562,23 +633,82 @@ class TestSchema < Test::Unit::TestCase
|
|
562
633
|
assert_equal schema_hash, schema.to_avro
|
563
634
|
end
|
564
635
|
|
565
|
-
def
|
636
|
+
def test_bytes_decimal_with_string_precision_no_scale
|
566
637
|
schema = Avro::Schema.parse <<-SCHEMA
|
567
638
|
{
|
568
639
|
"type": "bytes",
|
569
|
-
"logicalType": "decimal"
|
640
|
+
"logicalType": "decimal",
|
641
|
+
"precision": "7"
|
570
642
|
}
|
571
643
|
SCHEMA
|
572
644
|
|
573
645
|
schema_hash =
|
574
646
|
{
|
575
647
|
'type' => 'bytes',
|
576
|
-
'logicalType' => 'decimal'
|
648
|
+
'logicalType' => 'decimal',
|
649
|
+
'precision' => 7
|
577
650
|
}
|
578
651
|
|
579
652
|
assert_equal schema_hash, schema.to_avro
|
580
653
|
end
|
581
654
|
|
655
|
+
def test_bytes_decimal_without_precision_or_scale
|
656
|
+
error = assert_raise Avro::SchemaParseError do
|
657
|
+
Avro::Schema.parse <<-SCHEMA
|
658
|
+
{
|
659
|
+
"type": "bytes",
|
660
|
+
"logicalType": "decimal"
|
661
|
+
}
|
662
|
+
SCHEMA
|
663
|
+
end
|
664
|
+
|
665
|
+
assert_equal 'Precision must be positive', error.message
|
666
|
+
end
|
667
|
+
|
668
|
+
def test_bytes_decimal_to_negative_precision
|
669
|
+
error = assert_raise Avro::SchemaParseError do
|
670
|
+
Avro::Schema.parse <<-SCHEMA
|
671
|
+
{
|
672
|
+
"type": "bytes",
|
673
|
+
"logicalType": "decimal",
|
674
|
+
"precision": -1
|
675
|
+
}
|
676
|
+
SCHEMA
|
677
|
+
end
|
678
|
+
|
679
|
+
assert_equal 'Precision must be positive', error.message
|
680
|
+
end
|
681
|
+
|
682
|
+
def test_bytes_decimal_to_negative_scale
|
683
|
+
error = assert_raise Avro::SchemaParseError do
|
684
|
+
Avro::Schema.parse <<-SCHEMA
|
685
|
+
{
|
686
|
+
"type": "bytes",
|
687
|
+
"logicalType": "decimal",
|
688
|
+
"precision": 2,
|
689
|
+
"scale": -1
|
690
|
+
}
|
691
|
+
SCHEMA
|
692
|
+
end
|
693
|
+
|
694
|
+
assert_equal 'Scale must be greater than or equal to 0', error.message
|
695
|
+
end
|
696
|
+
|
697
|
+
def test_bytes_decimal_with_precision_less_than_scale
|
698
|
+
error = assert_raise Avro::SchemaParseError do
|
699
|
+
Avro::Schema.parse <<-SCHEMA
|
700
|
+
{
|
701
|
+
"type": "bytes",
|
702
|
+
"logicalType": "decimal",
|
703
|
+
"precision": 3,
|
704
|
+
"scale": 4
|
705
|
+
}
|
706
|
+
SCHEMA
|
707
|
+
end
|
708
|
+
|
709
|
+
assert_equal 'Precision must be greater than scale', error.message
|
710
|
+
end
|
711
|
+
|
582
712
|
def test_bytes_schema
|
583
713
|
schema = Avro::Schema.parse <<-SCHEMA
|
584
714
|
{
|
@@ -644,7 +774,7 @@ class TestSchema < Test::Unit::TestCase
|
|
644
774
|
ensure
|
645
775
|
Avro.disable_enum_symbol_validation = nil
|
646
776
|
end
|
647
|
-
|
777
|
+
|
648
778
|
def test_validate_field_aliases
|
649
779
|
exception = assert_raise(Avro::SchemaParseError) do
|
650
780
|
hash_to_schema(
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -25,7 +26,9 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
25
26
|
end
|
26
27
|
|
27
28
|
def test_compatible_reader_writer_pairs
|
29
|
+
cached_schema = a_int_record1_schema
|
28
30
|
[
|
31
|
+
cached_schema, cached_schema,
|
29
32
|
long_schema, int_schema,
|
30
33
|
float_schema, int_schema,
|
31
34
|
float_schema, long_schema,
|
@@ -290,6 +293,111 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
290
293
|
assert_false(can_read?(writer_schema, reader_schema))
|
291
294
|
end
|
292
295
|
|
296
|
+
def test_bytes_decimal
|
297
|
+
bytes_decimal_schema = Avro::Schema.
|
298
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":4}')
|
299
|
+
bytes2_decimal_schema = Avro::Schema.
|
300
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":4}')
|
301
|
+
bytes_decimal_different_precision_schema = Avro::Schema.
|
302
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":5, "scale":4}')
|
303
|
+
bytes_decimal_no_scale_schema = Avro::Schema.
|
304
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
305
|
+
bytes2_decimal_no_scale_schema = Avro::Schema.
|
306
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
307
|
+
bytes_decimal_zero_scale_schema = Avro::Schema.
|
308
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":0}')
|
309
|
+
bytes_unknown_logical_type_schema = Avro::Schema.
|
310
|
+
parse('{"type":"bytes", "logicalType":"unknown"}')
|
311
|
+
|
312
|
+
# decimal bytes and non-decimal bytes can be mixed
|
313
|
+
assert_true(can_read?(bytes_schema, bytes_decimal_schema))
|
314
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_schema))
|
315
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_unknown_logical_type_schema))
|
316
|
+
|
317
|
+
# decimal bytes match even if precision and scale differ
|
318
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_different_precision_schema))
|
319
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_no_scale_schema))
|
320
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_zero_scale_schema))
|
321
|
+
# - zero and no scale are equivalent
|
322
|
+
assert_true(can_read?(bytes_decimal_zero_scale_schema, bytes_decimal_no_scale_schema))
|
323
|
+
# - different schemas with the same attributes match
|
324
|
+
assert_true(can_read?(bytes_decimal_schema, bytes2_decimal_schema))
|
325
|
+
# - different schemas with the same no scale match
|
326
|
+
assert_true(can_read?(bytes2_decimal_no_scale_schema, bytes_decimal_no_scale_schema))
|
327
|
+
end
|
328
|
+
|
329
|
+
def test_fixed_decimal
|
330
|
+
fixed_decimal_schema = Avro::Schema.
|
331
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":2}')
|
332
|
+
fixed2_decimal_schema = Avro::Schema.
|
333
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed2", "logicalType":"decimal", "precision":4, "scale":2}')
|
334
|
+
fixed_decimal_different_precision_schema = Avro::Schema.
|
335
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":3, "scale":2}')
|
336
|
+
fixed_decimal_size3_schema = Avro::Schema.
|
337
|
+
parse('{"type":"fixed", "size":3, "name":"FixedS3", "logicalType":"decimal", "precision":4, "scale":2}')
|
338
|
+
fixed_unknown_schema = Avro::Schema.
|
339
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"unknown"}')
|
340
|
+
fixed_decimal_zero_scale_schema = Avro::Schema.
|
341
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":0}')
|
342
|
+
fixed_decimal_no_scale_schema = Avro::Schema.
|
343
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4}')
|
344
|
+
|
345
|
+
# decimal fixed and non-decimal can be mixed if fixed name matches
|
346
|
+
assert_true(can_read?(fixed_decimal_schema, fixed1_schema))
|
347
|
+
assert_true(can_read?(fixed1_schema, fixed_decimal_schema))
|
348
|
+
assert_false(can_read?(fixed2_schema, fixed_decimal_schema))
|
349
|
+
|
350
|
+
# decimal logical types match even if fixed name differs
|
351
|
+
assert_true(can_read?(fixed_decimal_schema, fixed2_decimal_schema))
|
352
|
+
|
353
|
+
# fixed with the same name & size match even if decimal precision and scale differ
|
354
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_different_precision_schema))
|
355
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_size3_schema))
|
356
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_unknown_schema))
|
357
|
+
# - zero and no scale are equivalent but these match anyway due to same name & size
|
358
|
+
assert_true(can_read?(fixed_decimal_no_scale_schema, fixed_decimal_zero_scale_schema))
|
359
|
+
# - scale does not match
|
360
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_no_scale_schema))
|
361
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_zero_scale_schema))
|
362
|
+
end
|
363
|
+
|
364
|
+
def test_decimal_different_types
|
365
|
+
fixed_decimal_schema = Avro::Schema.
|
366
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":2}')
|
367
|
+
fixed_decimal_scale4_schema = Avro::Schema.
|
368
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":4}')
|
369
|
+
bytes_decimal_schema = Avro::Schema.
|
370
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":2}')
|
371
|
+
fixed_decimal_zero_scale_schema = Avro::Schema.
|
372
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":0}')
|
373
|
+
fixed_decimal_no_scale_schema = Avro::Schema.
|
374
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4}')
|
375
|
+
bytes_decimal_zero_scale_schema = Avro::Schema.
|
376
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":0}')
|
377
|
+
bytes_decimal_no_scale_schema = Avro::Schema.
|
378
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
379
|
+
|
380
|
+
# decimal logical types can be read
|
381
|
+
assert_true(can_read?(fixed_decimal_schema, bytes_decimal_schema))
|
382
|
+
assert_true(can_read?(bytes_decimal_schema, fixed_decimal_schema))
|
383
|
+
|
384
|
+
# non-decimal bytes and fixed cannot be mixed
|
385
|
+
assert_false(can_read?(fixed_decimal_schema, bytes_schema))
|
386
|
+
assert_false(can_read?(bytes_schema, fixed_decimal_schema))
|
387
|
+
assert_false(can_read?(fixed1_schema, bytes_decimal_schema))
|
388
|
+
assert_false(can_read?(bytes_decimal_schema, fixed1_schema))
|
389
|
+
|
390
|
+
# decimal precision and scale must match
|
391
|
+
assert_false(can_read?(fixed_decimal_scale4_schema, bytes_decimal_schema))
|
392
|
+
assert_false(can_read?(bytes_decimal_schema, fixed_decimal_scale4_schema))
|
393
|
+
|
394
|
+
# zero scale and no scale are equivalent
|
395
|
+
assert_true(can_read?(bytes_decimal_no_scale_schema, fixed_decimal_zero_scale_schema))
|
396
|
+
assert_true(can_read?(fixed_decimal_zero_scale_schema, bytes_decimal_no_scale_schema))
|
397
|
+
assert_true(can_read?(bytes_decimal_zero_scale_schema, fixed_decimal_no_scale_schema))
|
398
|
+
assert_true(can_read?(fixed_decimal_no_scale_schema, bytes_decimal_zero_scale_schema))
|
399
|
+
end
|
400
|
+
|
293
401
|
# Tests from lang/java/avro/src/test/java/org/apache/avro/io/parsing/TestResolvingGrammarGenerator2.java
|
294
402
|
|
295
403
|
def point_2d_schema
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -17,7 +18,7 @@
|
|
17
18
|
require 'test_help'
|
18
19
|
|
19
20
|
class TestSchemaValidator < Test::Unit::TestCase
|
20
|
-
def validate!(schema, value, options=
|
21
|
+
def validate!(schema, value, options = {})
|
21
22
|
Avro::SchemaValidator.validate!(schema, value, options)
|
22
23
|
end
|
23
24
|
|
@@ -169,13 +170,13 @@ class TestSchemaValidator < Test::Unit::TestCase
|
|
169
170
|
def test_validate_float
|
170
171
|
schema = hash_to_schema(type: 'float', name: 'name')
|
171
172
|
|
172
|
-
assert_valid_schema(schema, [1.1, 1, Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
173
|
+
assert_valid_schema(schema, [1.1, 1, BigDecimal('1.1'), Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
173
174
|
end
|
174
175
|
|
175
176
|
def test_validate_double
|
176
177
|
schema = hash_to_schema(type: 'double', name: 'name')
|
177
178
|
|
178
|
-
assert_valid_schema(schema, [1.1, 1, Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
179
|
+
assert_valid_schema(schema, [1.1, 1, BigDecimal('1.1'), Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
179
180
|
end
|
180
181
|
|
181
182
|
def test_validate_fixed
|
@@ -196,6 +197,12 @@ class TestSchemaValidator < Test::Unit::TestCase
|
|
196
197
|
assert_valid_schema(schema, [{ 'sub' => nil }], [{ 'sub' => 1 }])
|
197
198
|
end
|
198
199
|
|
200
|
+
def test_validate_record_with_symbol_keys
|
201
|
+
schema = hash_to_schema(type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }])
|
202
|
+
|
203
|
+
assert_valid_schema(schema, [{ sub: 1 }], [{ sub: '1' }])
|
204
|
+
end
|
205
|
+
|
199
206
|
def test_validate_shallow_record
|
200
207
|
schema = hash_to_schema(
|
201
208
|
type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }]
|
@@ -551,4 +558,18 @@ class TestSchemaValidator < Test::Unit::TestCase
|
|
551
558
|
assert_equal(1, exception.result.errors.size)
|
552
559
|
assert_equal("at . extra field 'color' - not in schema", exception.to_s)
|
553
560
|
end
|
561
|
+
|
562
|
+
def test_validate_bytes_decimal
|
563
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4, scale: 2)
|
564
|
+
assert_valid_schema(schema, [BigDecimal('1.23'), 4.2, 1], ['4.2', BigDecimal('233.2')], true)
|
565
|
+
|
566
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4, scale: 4)
|
567
|
+
assert_valid_schema(schema, [BigDecimal('0.2345'), 0.2, 0.1], ['4.2', BigDecimal('233.2')], true)
|
568
|
+
|
569
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4, scale: 0)
|
570
|
+
assert_valid_schema(schema, [BigDecimal('123'), 2], ['4.2', BigDecimal('233.2')], true)
|
571
|
+
|
572
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4)
|
573
|
+
assert_valid_schema(schema, [BigDecimal('123'), 2], ['4.2', BigDecimal('233.2')], true)
|
574
|
+
end
|
554
575
|
end
|
data/test/tool.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -21,7 +22,7 @@ require 'logger'
|
|
21
22
|
|
22
23
|
class GenericResponder < Avro::IPC::Responder
|
23
24
|
def initialize(proto, msg, datum)
|
24
|
-
proto_json = open(proto).read
|
25
|
+
proto_json = File.open(proto).read
|
25
26
|
super(Avro::Protocol.parse(proto_json))
|
26
27
|
@msg = msg
|
27
28
|
@datum = datum
|
@@ -62,14 +63,14 @@ end
|
|
62
63
|
def send_message(uri, proto, msg, datum)
|
63
64
|
uri = URI.parse(uri)
|
64
65
|
trans = Avro::IPC::HTTPTransceiver.new(uri.host, uri.port)
|
65
|
-
proto_json = open(proto).read
|
66
|
+
proto_json = File.open(proto).read
|
66
67
|
requestor = Avro::IPC::Requestor.new(Avro::Protocol.parse(proto_json),
|
67
68
|
trans)
|
68
69
|
p requestor.request(msg, datum)
|
69
70
|
end
|
70
71
|
|
71
72
|
def file_or_stdin(f)
|
72
|
-
f == "-" ? STDIN : open(f)
|
73
|
+
f == "-" ? STDIN : File.open(f)
|
73
74
|
end
|
74
75
|
|
75
76
|
def main
|
@@ -100,9 +101,9 @@ def main
|
|
100
101
|
if ARGV.size > 4
|
101
102
|
case ARGV[4]
|
102
103
|
when "-file"
|
103
|
-
Avro::DataFile.open(ARGV[5])
|
104
|
-
|
105
|
-
|
104
|
+
Avro::DataFile.open(ARGV[5]) do |f|
|
105
|
+
datum = f.first
|
106
|
+
end
|
106
107
|
when "-data"
|
107
108
|
puts "JSON Decoder not yet implemented."
|
108
109
|
return 1
|
@@ -124,7 +125,7 @@ def main
|
|
124
125
|
if ARGV.size > 4
|
125
126
|
case ARGV[4]
|
126
127
|
when "-file"
|
127
|
-
Avro::DataFile.open(ARGV[5]){|f|
|
128
|
+
Avro::DataFile.open(ARGV[5]){ |f| datum = f.first }
|
128
129
|
when "-data"
|
129
130
|
puts "JSON Decoder not yet implemented"
|
130
131
|
return 1
|