avro 1.10.1 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Manifest +0 -1
- data/NOTICE +1 -1
- data/Rakefile +13 -19
- data/avro.gemspec +37 -29
- data/interop/test_interop.rb +2 -1
- data/lib/avro/VERSION.txt +1 -1
- data/lib/avro/data_file.rb +5 -4
- data/lib/avro/io.rb +18 -14
- data/lib/avro/ipc.rb +9 -5
- data/lib/avro/logical_types.rb +186 -2
- data/lib/avro/protocol.rb +1 -0
- data/lib/avro/schema.rb +71 -12
- data/lib/avro/schema_compatibility.rb +19 -13
- data/lib/avro/schema_normalization.rb +1 -0
- data/lib/avro/schema_validator.rb +40 -34
- data/lib/avro.rb +1 -0
- data/test/case_finder.rb +1 -0
- data/test/random_data.rb +5 -4
- data/test/sample_ipc_client.rb +1 -0
- data/test/sample_ipc_http_client.rb +1 -0
- data/test/sample_ipc_http_server.rb +1 -0
- data/test/sample_ipc_server.rb +1 -0
- data/test/test_datafile.rb +1 -0
- data/test/test_fingerprints.rb +1 -0
- data/test/test_help.rb +1 -0
- data/test/test_io.rb +31 -16
- data/test/test_logical_types.rb +138 -1
- data/test/test_protocol.rb +2 -1
- data/test/test_schema.rb +134 -4
- data/test/test_schema_compatibility.rb +108 -0
- data/test/test_schema_normalization.rb +1 -0
- data/test/test_schema_validator.rb +24 -3
- data/test/test_socket_transport.rb +1 -0
- data/test/tool.rb +8 -7
- metadata +30 -33
- data/CHANGELOG +0 -1
data/test/test_logical_types.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
4
5
|
# distributed with this work for additional information
|
@@ -16,6 +17,7 @@
|
|
16
17
|
# limitations under the License.
|
17
18
|
|
18
19
|
require 'test_help'
|
20
|
+
require 'memory_profiler'
|
19
21
|
|
20
22
|
class TestLogicalTypes < Test::Unit::TestCase
|
21
23
|
def test_int_date
|
@@ -98,8 +100,143 @@ class TestLogicalTypes < Test::Unit::TestCase
|
|
98
100
|
assert_equal 'duration', schema.logical_type
|
99
101
|
end
|
100
102
|
|
103
|
+
def test_bytes_decimal
|
104
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
105
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 6 }
|
106
|
+
SCHEMA
|
107
|
+
|
108
|
+
assert_equal 'decimal', schema.logical_type
|
109
|
+
assert_equal 9, schema.precision
|
110
|
+
assert_equal 6, schema.scale
|
111
|
+
|
112
|
+
assert_encode_and_decode BigDecimal('-3.4562'), schema
|
113
|
+
assert_encode_and_decode BigDecimal('3.4562'), schema
|
114
|
+
assert_encode_and_decode 15.123, schema
|
115
|
+
assert_encode_and_decode 15, schema
|
116
|
+
assert_encode_and_decode BigDecimal('0.123456'), schema
|
117
|
+
assert_encode_and_decode BigDecimal('0'), schema
|
118
|
+
assert_encode_and_decode BigDecimal('1'), schema
|
119
|
+
assert_encode_and_decode BigDecimal('-1'), schema
|
120
|
+
|
121
|
+
assert_raise ArgumentError do
|
122
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
123
|
+
type.encode('1.23')
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def test_bytes_decimal_range_errors
|
128
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
129
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
130
|
+
SCHEMA
|
131
|
+
|
132
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
133
|
+
|
134
|
+
assert_raises RangeError do
|
135
|
+
type.encode(BigDecimal('345'))
|
136
|
+
end
|
137
|
+
|
138
|
+
assert_raises RangeError do
|
139
|
+
type.encode(BigDecimal('1.5342'))
|
140
|
+
end
|
141
|
+
|
142
|
+
assert_raises RangeError do
|
143
|
+
type.encode(BigDecimal('-1.5342'))
|
144
|
+
end
|
145
|
+
|
146
|
+
assert_raises RangeError do
|
147
|
+
type.encode(BigDecimal('-100.2'))
|
148
|
+
end
|
149
|
+
|
150
|
+
assert_raises RangeError do
|
151
|
+
type.encode(BigDecimal('-99.991'))
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_bytes_decimal_conversion
|
156
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
157
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
158
|
+
SCHEMA
|
159
|
+
|
160
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
161
|
+
|
162
|
+
enc = "\xcb\x43\x38".dup.force_encoding('BINARY')
|
163
|
+
assert_equal enc, type.encode(BigDecimal('-3.4562'))
|
164
|
+
assert_equal BigDecimal('-3.4562'), type.decode(enc)
|
165
|
+
|
166
|
+
assert_equal "\x34\xbc\xc8".dup.force_encoding('BINARY'), type.encode(BigDecimal('3.4562'))
|
167
|
+
assert_equal BigDecimal('3.4562'), type.decode("\x34\xbc\xc8".dup.force_encoding('BINARY'))
|
168
|
+
|
169
|
+
assert_equal "\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'), type.encode(BigDecimal('456123.123456'))
|
170
|
+
assert_equal BigDecimal('456123.123456'), type.decode("\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'))
|
171
|
+
end
|
172
|
+
|
173
|
+
def test_logical_type_with_schema
|
174
|
+
exception = assert_raises(ArgumentError) do
|
175
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(nil)
|
176
|
+
end
|
177
|
+
assert_equal exception.to_s, 'schema is required'
|
178
|
+
|
179
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
180
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
|
181
|
+
SCHEMA
|
182
|
+
|
183
|
+
assert_nothing_raised do
|
184
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema)
|
185
|
+
end
|
186
|
+
|
187
|
+
assert_raises NotImplementedError do
|
188
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).encode(BigDecimal('2'))
|
189
|
+
end
|
190
|
+
|
191
|
+
assert_raises NotImplementedError do
|
192
|
+
Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).decode('foo')
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def test_bytes_decimal_object_allocations_encode
|
197
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
198
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
199
|
+
SCHEMA
|
200
|
+
|
201
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
202
|
+
|
203
|
+
positive_value = BigDecimal('5.2')
|
204
|
+
negative_value = BigDecimal('-5.2')
|
205
|
+
|
206
|
+
[positive_value, negative_value].each do |value|
|
207
|
+
report = MemoryProfiler.report do
|
208
|
+
type.encode(value)
|
209
|
+
end
|
210
|
+
|
211
|
+
assert_equal 5, report.total_allocated
|
212
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
213
|
+
assert_operator 1, :>=, report.total_retained
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def test_bytes_decimal_object_allocations_decode
|
218
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
219
|
+
{ "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
|
220
|
+
SCHEMA
|
221
|
+
|
222
|
+
type = Avro::LogicalTypes::BytesDecimal.new(schema)
|
223
|
+
|
224
|
+
positive_enc = "\x02\b".dup.force_encoding('BINARY')
|
225
|
+
negative_enc = "\xFD\xF8".dup.force_encoding('BINARY')
|
226
|
+
|
227
|
+
[positive_enc, negative_enc].each do |encoded|
|
228
|
+
report = MemoryProfiler.report do
|
229
|
+
type.decode(encoded)
|
230
|
+
end
|
231
|
+
|
232
|
+
assert_equal 5, report.total_allocated
|
233
|
+
# Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
|
234
|
+
assert_operator 1, :>=, report.total_retained
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
101
238
|
def encode(datum, schema)
|
102
|
-
buffer = StringIO.new
|
239
|
+
buffer = StringIO.new
|
103
240
|
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
104
241
|
|
105
242
|
datum_writer = Avro::IO::DatumWriter.new(schema)
|
data/test/test_protocol.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -184,7 +185,7 @@ EOS
|
|
184
185
|
}
|
185
186
|
}
|
186
187
|
EOS
|
187
|
-
]
|
188
|
+
].freeze
|
188
189
|
|
189
190
|
Protocol = Avro::Protocol
|
190
191
|
def test_parse
|
data/test/test_schema.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -541,6 +542,76 @@ class TestSchema < Test::Unit::TestCase
|
|
541
542
|
exception.to_s)
|
542
543
|
end
|
543
544
|
|
545
|
+
def test_fixed_decimal_to_include_precision_scale
|
546
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
547
|
+
{
|
548
|
+
"type": "fixed",
|
549
|
+
"name": "aFixed",
|
550
|
+
"logicalType": "decimal",
|
551
|
+
"size": 4,
|
552
|
+
"precision": 9,
|
553
|
+
"scale": 2
|
554
|
+
}
|
555
|
+
SCHEMA
|
556
|
+
|
557
|
+
schema_hash =
|
558
|
+
{
|
559
|
+
'type' => 'fixed',
|
560
|
+
'name' => 'aFixed',
|
561
|
+
'logicalType' => 'decimal',
|
562
|
+
'size' => 4,
|
563
|
+
'precision' => 9,
|
564
|
+
'scale' => 2
|
565
|
+
}
|
566
|
+
|
567
|
+
assert_equal schema_hash, schema.to_avro
|
568
|
+
end
|
569
|
+
|
570
|
+
def test_fixed_decimal_to_include_precision_no_scale
|
571
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
572
|
+
{
|
573
|
+
"type": "fixed",
|
574
|
+
"name": "aFixed",
|
575
|
+
"logicalType": "decimal",
|
576
|
+
"size": 4,
|
577
|
+
"precision": 9
|
578
|
+
}
|
579
|
+
SCHEMA
|
580
|
+
|
581
|
+
schema_hash =
|
582
|
+
{
|
583
|
+
'type' => 'fixed',
|
584
|
+
'name' => 'aFixed',
|
585
|
+
'logicalType' => 'decimal',
|
586
|
+
'size' => 4,
|
587
|
+
'precision' => 9
|
588
|
+
}
|
589
|
+
|
590
|
+
assert_equal schema_hash, schema.to_avro
|
591
|
+
end
|
592
|
+
|
593
|
+
# Note: this is not valid but validation is not yet implemented
|
594
|
+
def test_fixed_decimal_to_without_precision_scale
|
595
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
596
|
+
{
|
597
|
+
"type": "fixed",
|
598
|
+
"size": 4,
|
599
|
+
"name": "aFixed",
|
600
|
+
"logicalType": "decimal"
|
601
|
+
}
|
602
|
+
SCHEMA
|
603
|
+
|
604
|
+
schema_hash =
|
605
|
+
{
|
606
|
+
'type' => 'fixed',
|
607
|
+
'name' => 'aFixed',
|
608
|
+
'logicalType' => 'decimal',
|
609
|
+
'size' => 4
|
610
|
+
}
|
611
|
+
|
612
|
+
assert_equal schema_hash, schema.to_avro
|
613
|
+
end
|
614
|
+
|
544
615
|
def test_bytes_decimal_to_include_precision_scale
|
545
616
|
schema = Avro::Schema.parse <<-SCHEMA
|
546
617
|
{
|
@@ -562,23 +633,82 @@ class TestSchema < Test::Unit::TestCase
|
|
562
633
|
assert_equal schema_hash, schema.to_avro
|
563
634
|
end
|
564
635
|
|
565
|
-
def
|
636
|
+
def test_bytes_decimal_with_string_precision_no_scale
|
566
637
|
schema = Avro::Schema.parse <<-SCHEMA
|
567
638
|
{
|
568
639
|
"type": "bytes",
|
569
|
-
"logicalType": "decimal"
|
640
|
+
"logicalType": "decimal",
|
641
|
+
"precision": "7"
|
570
642
|
}
|
571
643
|
SCHEMA
|
572
644
|
|
573
645
|
schema_hash =
|
574
646
|
{
|
575
647
|
'type' => 'bytes',
|
576
|
-
'logicalType' => 'decimal'
|
648
|
+
'logicalType' => 'decimal',
|
649
|
+
'precision' => 7
|
577
650
|
}
|
578
651
|
|
579
652
|
assert_equal schema_hash, schema.to_avro
|
580
653
|
end
|
581
654
|
|
655
|
+
def test_bytes_decimal_without_precision_or_scale
|
656
|
+
error = assert_raise Avro::SchemaParseError do
|
657
|
+
Avro::Schema.parse <<-SCHEMA
|
658
|
+
{
|
659
|
+
"type": "bytes",
|
660
|
+
"logicalType": "decimal"
|
661
|
+
}
|
662
|
+
SCHEMA
|
663
|
+
end
|
664
|
+
|
665
|
+
assert_equal 'Precision must be positive', error.message
|
666
|
+
end
|
667
|
+
|
668
|
+
def test_bytes_decimal_to_negative_precision
|
669
|
+
error = assert_raise Avro::SchemaParseError do
|
670
|
+
Avro::Schema.parse <<-SCHEMA
|
671
|
+
{
|
672
|
+
"type": "bytes",
|
673
|
+
"logicalType": "decimal",
|
674
|
+
"precision": -1
|
675
|
+
}
|
676
|
+
SCHEMA
|
677
|
+
end
|
678
|
+
|
679
|
+
assert_equal 'Precision must be positive', error.message
|
680
|
+
end
|
681
|
+
|
682
|
+
def test_bytes_decimal_to_negative_scale
|
683
|
+
error = assert_raise Avro::SchemaParseError do
|
684
|
+
Avro::Schema.parse <<-SCHEMA
|
685
|
+
{
|
686
|
+
"type": "bytes",
|
687
|
+
"logicalType": "decimal",
|
688
|
+
"precision": 2,
|
689
|
+
"scale": -1
|
690
|
+
}
|
691
|
+
SCHEMA
|
692
|
+
end
|
693
|
+
|
694
|
+
assert_equal 'Scale must be greater than or equal to 0', error.message
|
695
|
+
end
|
696
|
+
|
697
|
+
def test_bytes_decimal_with_precision_less_than_scale
|
698
|
+
error = assert_raise Avro::SchemaParseError do
|
699
|
+
Avro::Schema.parse <<-SCHEMA
|
700
|
+
{
|
701
|
+
"type": "bytes",
|
702
|
+
"logicalType": "decimal",
|
703
|
+
"precision": 3,
|
704
|
+
"scale": 4
|
705
|
+
}
|
706
|
+
SCHEMA
|
707
|
+
end
|
708
|
+
|
709
|
+
assert_equal 'Precision must be greater than scale', error.message
|
710
|
+
end
|
711
|
+
|
582
712
|
def test_bytes_schema
|
583
713
|
schema = Avro::Schema.parse <<-SCHEMA
|
584
714
|
{
|
@@ -644,7 +774,7 @@ class TestSchema < Test::Unit::TestCase
|
|
644
774
|
ensure
|
645
775
|
Avro.disable_enum_symbol_validation = nil
|
646
776
|
end
|
647
|
-
|
777
|
+
|
648
778
|
def test_validate_field_aliases
|
649
779
|
exception = assert_raise(Avro::SchemaParseError) do
|
650
780
|
hash_to_schema(
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -25,7 +26,9 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
25
26
|
end
|
26
27
|
|
27
28
|
def test_compatible_reader_writer_pairs
|
29
|
+
cached_schema = a_int_record1_schema
|
28
30
|
[
|
31
|
+
cached_schema, cached_schema,
|
29
32
|
long_schema, int_schema,
|
30
33
|
float_schema, int_schema,
|
31
34
|
float_schema, long_schema,
|
@@ -290,6 +293,111 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
290
293
|
assert_false(can_read?(writer_schema, reader_schema))
|
291
294
|
end
|
292
295
|
|
296
|
+
def test_bytes_decimal
|
297
|
+
bytes_decimal_schema = Avro::Schema.
|
298
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":4}')
|
299
|
+
bytes2_decimal_schema = Avro::Schema.
|
300
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":4}')
|
301
|
+
bytes_decimal_different_precision_schema = Avro::Schema.
|
302
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":5, "scale":4}')
|
303
|
+
bytes_decimal_no_scale_schema = Avro::Schema.
|
304
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
305
|
+
bytes2_decimal_no_scale_schema = Avro::Schema.
|
306
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
307
|
+
bytes_decimal_zero_scale_schema = Avro::Schema.
|
308
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":0}')
|
309
|
+
bytes_unknown_logical_type_schema = Avro::Schema.
|
310
|
+
parse('{"type":"bytes", "logicalType":"unknown"}')
|
311
|
+
|
312
|
+
# decimal bytes and non-decimal bytes can be mixed
|
313
|
+
assert_true(can_read?(bytes_schema, bytes_decimal_schema))
|
314
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_schema))
|
315
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_unknown_logical_type_schema))
|
316
|
+
|
317
|
+
# decimal bytes match even if precision and scale differ
|
318
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_different_precision_schema))
|
319
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_no_scale_schema))
|
320
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_zero_scale_schema))
|
321
|
+
# - zero and no scale are equivalent
|
322
|
+
assert_true(can_read?(bytes_decimal_zero_scale_schema, bytes_decimal_no_scale_schema))
|
323
|
+
# - different schemas with the same attributes match
|
324
|
+
assert_true(can_read?(bytes_decimal_schema, bytes2_decimal_schema))
|
325
|
+
# - different schemas with the same no scale match
|
326
|
+
assert_true(can_read?(bytes2_decimal_no_scale_schema, bytes_decimal_no_scale_schema))
|
327
|
+
end
|
328
|
+
|
329
|
+
def test_fixed_decimal
|
330
|
+
fixed_decimal_schema = Avro::Schema.
|
331
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":2}')
|
332
|
+
fixed2_decimal_schema = Avro::Schema.
|
333
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed2", "logicalType":"decimal", "precision":4, "scale":2}')
|
334
|
+
fixed_decimal_different_precision_schema = Avro::Schema.
|
335
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":3, "scale":2}')
|
336
|
+
fixed_decimal_size3_schema = Avro::Schema.
|
337
|
+
parse('{"type":"fixed", "size":3, "name":"FixedS3", "logicalType":"decimal", "precision":4, "scale":2}')
|
338
|
+
fixed_unknown_schema = Avro::Schema.
|
339
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"unknown"}')
|
340
|
+
fixed_decimal_zero_scale_schema = Avro::Schema.
|
341
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":0}')
|
342
|
+
fixed_decimal_no_scale_schema = Avro::Schema.
|
343
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4}')
|
344
|
+
|
345
|
+
# decimal fixed and non-decimal can be mixed if fixed name matches
|
346
|
+
assert_true(can_read?(fixed_decimal_schema, fixed1_schema))
|
347
|
+
assert_true(can_read?(fixed1_schema, fixed_decimal_schema))
|
348
|
+
assert_false(can_read?(fixed2_schema, fixed_decimal_schema))
|
349
|
+
|
350
|
+
# decimal logical types match even if fixed name differs
|
351
|
+
assert_true(can_read?(fixed_decimal_schema, fixed2_decimal_schema))
|
352
|
+
|
353
|
+
# fixed with the same name & size match even if decimal precision and scale differ
|
354
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_different_precision_schema))
|
355
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_size3_schema))
|
356
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_unknown_schema))
|
357
|
+
# - zero and no scale are equivalent but these match anyway due to same name & size
|
358
|
+
assert_true(can_read?(fixed_decimal_no_scale_schema, fixed_decimal_zero_scale_schema))
|
359
|
+
# - scale does not match
|
360
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_no_scale_schema))
|
361
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_zero_scale_schema))
|
362
|
+
end
|
363
|
+
|
364
|
+
def test_decimal_different_types
|
365
|
+
fixed_decimal_schema = Avro::Schema.
|
366
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":2}')
|
367
|
+
fixed_decimal_scale4_schema = Avro::Schema.
|
368
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":4}')
|
369
|
+
bytes_decimal_schema = Avro::Schema.
|
370
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":2}')
|
371
|
+
fixed_decimal_zero_scale_schema = Avro::Schema.
|
372
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":0}')
|
373
|
+
fixed_decimal_no_scale_schema = Avro::Schema.
|
374
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4}')
|
375
|
+
bytes_decimal_zero_scale_schema = Avro::Schema.
|
376
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":0}')
|
377
|
+
bytes_decimal_no_scale_schema = Avro::Schema.
|
378
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
379
|
+
|
380
|
+
# decimal logical types can be read
|
381
|
+
assert_true(can_read?(fixed_decimal_schema, bytes_decimal_schema))
|
382
|
+
assert_true(can_read?(bytes_decimal_schema, fixed_decimal_schema))
|
383
|
+
|
384
|
+
# non-decimal bytes and fixed cannot be mixed
|
385
|
+
assert_false(can_read?(fixed_decimal_schema, bytes_schema))
|
386
|
+
assert_false(can_read?(bytes_schema, fixed_decimal_schema))
|
387
|
+
assert_false(can_read?(fixed1_schema, bytes_decimal_schema))
|
388
|
+
assert_false(can_read?(bytes_decimal_schema, fixed1_schema))
|
389
|
+
|
390
|
+
# decimal precision and scale must match
|
391
|
+
assert_false(can_read?(fixed_decimal_scale4_schema, bytes_decimal_schema))
|
392
|
+
assert_false(can_read?(bytes_decimal_schema, fixed_decimal_scale4_schema))
|
393
|
+
|
394
|
+
# zero scale and no scale are equivalent
|
395
|
+
assert_true(can_read?(bytes_decimal_no_scale_schema, fixed_decimal_zero_scale_schema))
|
396
|
+
assert_true(can_read?(fixed_decimal_zero_scale_schema, bytes_decimal_no_scale_schema))
|
397
|
+
assert_true(can_read?(bytes_decimal_zero_scale_schema, fixed_decimal_no_scale_schema))
|
398
|
+
assert_true(can_read?(fixed_decimal_no_scale_schema, bytes_decimal_zero_scale_schema))
|
399
|
+
end
|
400
|
+
|
293
401
|
# Tests from lang/java/avro/src/test/java/org/apache/avro/io/parsing/TestResolvingGrammarGenerator2.java
|
294
402
|
|
295
403
|
def point_2d_schema
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -17,7 +18,7 @@
|
|
17
18
|
require 'test_help'
|
18
19
|
|
19
20
|
class TestSchemaValidator < Test::Unit::TestCase
|
20
|
-
def validate!(schema, value, options=
|
21
|
+
def validate!(schema, value, options = {})
|
21
22
|
Avro::SchemaValidator.validate!(schema, value, options)
|
22
23
|
end
|
23
24
|
|
@@ -169,13 +170,13 @@ class TestSchemaValidator < Test::Unit::TestCase
|
|
169
170
|
def test_validate_float
|
170
171
|
schema = hash_to_schema(type: 'float', name: 'name')
|
171
172
|
|
172
|
-
assert_valid_schema(schema, [1.1, 1, Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
173
|
+
assert_valid_schema(schema, [1.1, 1, BigDecimal('1.1'), Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
173
174
|
end
|
174
175
|
|
175
176
|
def test_validate_double
|
176
177
|
schema = hash_to_schema(type: 'double', name: 'name')
|
177
178
|
|
178
|
-
assert_valid_schema(schema, [1.1, 1, Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
179
|
+
assert_valid_schema(schema, [1.1, 1, BigDecimal('1.1'), Avro::Schema::LONG_MAX_VALUE], ['string'], true)
|
179
180
|
end
|
180
181
|
|
181
182
|
def test_validate_fixed
|
@@ -196,6 +197,12 @@ class TestSchemaValidator < Test::Unit::TestCase
|
|
196
197
|
assert_valid_schema(schema, [{ 'sub' => nil }], [{ 'sub' => 1 }])
|
197
198
|
end
|
198
199
|
|
200
|
+
def test_validate_record_with_symbol_keys
|
201
|
+
schema = hash_to_schema(type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }])
|
202
|
+
|
203
|
+
assert_valid_schema(schema, [{ sub: 1 }], [{ sub: '1' }])
|
204
|
+
end
|
205
|
+
|
199
206
|
def test_validate_shallow_record
|
200
207
|
schema = hash_to_schema(
|
201
208
|
type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }]
|
@@ -551,4 +558,18 @@ class TestSchemaValidator < Test::Unit::TestCase
|
|
551
558
|
assert_equal(1, exception.result.errors.size)
|
552
559
|
assert_equal("at . extra field 'color' - not in schema", exception.to_s)
|
553
560
|
end
|
561
|
+
|
562
|
+
def test_validate_bytes_decimal
|
563
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4, scale: 2)
|
564
|
+
assert_valid_schema(schema, [BigDecimal('1.23'), 4.2, 1], ['4.2', BigDecimal('233.2')], true)
|
565
|
+
|
566
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4, scale: 4)
|
567
|
+
assert_valid_schema(schema, [BigDecimal('0.2345'), 0.2, 0.1], ['4.2', BigDecimal('233.2')], true)
|
568
|
+
|
569
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4, scale: 0)
|
570
|
+
assert_valid_schema(schema, [BigDecimal('123'), 2], ['4.2', BigDecimal('233.2')], true)
|
571
|
+
|
572
|
+
schema = hash_to_schema(type: 'bytes', logicalType: 'decimal', precision: 4)
|
573
|
+
assert_valid_schema(schema, [BigDecimal('123'), 2], ['4.2', BigDecimal('233.2')], true)
|
574
|
+
end
|
554
575
|
end
|
data/test/tool.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -21,7 +22,7 @@ require 'logger'
|
|
21
22
|
|
22
23
|
class GenericResponder < Avro::IPC::Responder
|
23
24
|
def initialize(proto, msg, datum)
|
24
|
-
proto_json = open(proto).read
|
25
|
+
proto_json = File.open(proto).read
|
25
26
|
super(Avro::Protocol.parse(proto_json))
|
26
27
|
@msg = msg
|
27
28
|
@datum = datum
|
@@ -62,14 +63,14 @@ end
|
|
62
63
|
def send_message(uri, proto, msg, datum)
|
63
64
|
uri = URI.parse(uri)
|
64
65
|
trans = Avro::IPC::HTTPTransceiver.new(uri.host, uri.port)
|
65
|
-
proto_json = open(proto).read
|
66
|
+
proto_json = File.open(proto).read
|
66
67
|
requestor = Avro::IPC::Requestor.new(Avro::Protocol.parse(proto_json),
|
67
68
|
trans)
|
68
69
|
p requestor.request(msg, datum)
|
69
70
|
end
|
70
71
|
|
71
72
|
def file_or_stdin(f)
|
72
|
-
f == "-" ? STDIN : open(f)
|
73
|
+
f == "-" ? STDIN : File.open(f)
|
73
74
|
end
|
74
75
|
|
75
76
|
def main
|
@@ -100,9 +101,9 @@ def main
|
|
100
101
|
if ARGV.size > 4
|
101
102
|
case ARGV[4]
|
102
103
|
when "-file"
|
103
|
-
Avro::DataFile.open(ARGV[5])
|
104
|
-
|
105
|
-
|
104
|
+
Avro::DataFile.open(ARGV[5]) do |f|
|
105
|
+
datum = f.first
|
106
|
+
end
|
106
107
|
when "-data"
|
107
108
|
puts "JSON Decoder not yet implemented."
|
108
109
|
return 1
|
@@ -124,7 +125,7 @@ def main
|
|
124
125
|
if ARGV.size > 4
|
125
126
|
case ARGV[4]
|
126
127
|
when "-file"
|
127
|
-
Avro::DataFile.open(ARGV[5]){|f|
|
128
|
+
Avro::DataFile.open(ARGV[5]){ |f| datum = f.first }
|
128
129
|
when "-data"
|
129
130
|
puts "JSON Decoder not yet implemented"
|
130
131
|
return 1
|