avro 1.9.0 → 1.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/LICENSE +2 -2
- data/Manifest +1 -1
- data/NOTICE +2 -2
- data/Rakefile +21 -30
- data/avro.gemspec +36 -29
- data/interop/test_interop.rb +14 -3
- data/lib/avro/VERSION.txt +1 -0
- data/lib/avro/data_file.rb +30 -6
- data/lib/avro/io.rb +42 -36
- data/lib/avro/ipc.rb +12 -8
- data/lib/avro/logical_types.rb +187 -3
- data/lib/avro/protocol.rb +2 -1
- data/lib/avro/schema.rb +234 -32
- data/lib/avro/schema_compatibility.rb +32 -21
- data/lib/avro/schema_normalization.rb +2 -1
- data/lib/avro/schema_validator.rb +41 -35
- data/lib/avro.rb +16 -3
- data/test/case_finder.rb +10 -4
- data/test/random_data.rb +9 -7
- data/test/sample_ipc_client.rb +2 -1
- data/test/sample_ipc_http_client.rb +2 -1
- data/test/sample_ipc_http_server.rb +2 -1
- data/test/sample_ipc_server.rb +2 -1
- data/test/test_datafile.rb +15 -1
- data/test/test_fingerprints.rb +21 -1
- data/test/test_help.rb +2 -1
- data/test/test_io.rb +114 -18
- data/test/test_logical_types.rb +139 -2
- data/test/test_protocol.rb +3 -2
- data/test/test_schema.rb +399 -1
- data/test/test_schema_compatibility.rb +177 -1
- data/test/test_schema_normalization.rb +3 -1
- data/test/test_schema_validator.rb +27 -6
- data/test/test_socket_transport.rb +2 -1
- data/test/tool.rb +10 -9
- metadata +32 -34
- data/CHANGELOG +0 -1
data/test/test_schema.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -6,7 +7,7 @@
|
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
9
|
#
|
9
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
11
|
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -151,6 +152,20 @@ class TestSchema < Test::Unit::TestCase
|
|
151
152
|
}
|
152
153
|
end
|
153
154
|
|
155
|
+
def test_to_avro_includes_aliases
|
156
|
+
hash = {
|
157
|
+
'type' => 'record',
|
158
|
+
'name' => 'test_record',
|
159
|
+
'aliases' => %w(alt_record),
|
160
|
+
'fields' => [
|
161
|
+
{ 'name' => 'f', 'type' => { 'type' => 'fixed', 'size' => 2, 'name' => 'test_fixed', 'aliases' => %w(alt_fixed) } },
|
162
|
+
{ 'name' => 'e', 'type' => { 'type' => 'enum', 'symbols' => %w(A B), 'name' => 'test_enum', 'aliases' => %w(alt_enum) } }
|
163
|
+
]
|
164
|
+
}
|
165
|
+
schema = hash_to_schema(hash)
|
166
|
+
assert_equal(schema.to_avro, hash)
|
167
|
+
end
|
168
|
+
|
154
169
|
def test_unknown_named_type
|
155
170
|
error = assert_raise Avro::UnknownSchemaError do
|
156
171
|
Avro::Schema.parse <<-SCHEMA
|
@@ -163,6 +178,42 @@ class TestSchema < Test::Unit::TestCase
|
|
163
178
|
assert_equal '"MissingType" is not a schema we know about.', error.message
|
164
179
|
end
|
165
180
|
|
181
|
+
def test_invalid_name
|
182
|
+
error = assert_raise Avro::SchemaParseError do
|
183
|
+
Avro::Schema.parse <<-SCHEMA
|
184
|
+
{"type": "record", "name": "my-invalid-name", "fields": [
|
185
|
+
{"name": "id", "type": "int"}
|
186
|
+
]}
|
187
|
+
SCHEMA
|
188
|
+
end
|
189
|
+
|
190
|
+
assert_equal "Name my-invalid-name is invalid for type record!", error.message
|
191
|
+
end
|
192
|
+
|
193
|
+
def test_invalid_name_with_two_periods
|
194
|
+
error = assert_raise Avro::SchemaParseError do
|
195
|
+
Avro::Schema.parse <<-SCHEMA
|
196
|
+
{"type": "record", "name": "my..invalid.name", "fields": [
|
197
|
+
{"name": "id", "type": "int"}
|
198
|
+
]}
|
199
|
+
SCHEMA
|
200
|
+
end
|
201
|
+
|
202
|
+
assert_equal "Name my..invalid.name is invalid for type record!", error.message
|
203
|
+
end
|
204
|
+
|
205
|
+
def test_invalid_name_with_validation_disabled
|
206
|
+
Avro.disable_schema_name_validation = true
|
207
|
+
assert_nothing_raised do
|
208
|
+
Avro::Schema.parse <<-SCHEMA
|
209
|
+
{"type": "record", "name": "my-invalid-name", "fields": [
|
210
|
+
{"name": "id", "type": "int"}
|
211
|
+
]}
|
212
|
+
SCHEMA
|
213
|
+
end
|
214
|
+
Avro.disable_schema_name_validation = false
|
215
|
+
end
|
216
|
+
|
166
217
|
def test_to_avro_handles_falsey_defaults
|
167
218
|
schema = Avro::Schema.parse <<-SCHEMA
|
168
219
|
{"type": "record", "name": "Record", "namespace": "my.name.space",
|
@@ -278,6 +329,40 @@ class TestSchema < Test::Unit::TestCase
|
|
278
329
|
assert_equal enum_schema_hash, enum_schema_json.to_avro
|
279
330
|
end
|
280
331
|
|
332
|
+
def test_enum_default_attribute
|
333
|
+
enum_schema = Avro::Schema.parse <<-SCHEMA
|
334
|
+
{
|
335
|
+
"type": "enum",
|
336
|
+
"name": "fruit",
|
337
|
+
"default": "apples",
|
338
|
+
"symbols": ["apples", "oranges"]
|
339
|
+
}
|
340
|
+
SCHEMA
|
341
|
+
|
342
|
+
enum_schema_hash = {
|
343
|
+
'type' => 'enum',
|
344
|
+
'name' => 'fruit',
|
345
|
+
'default' => 'apples',
|
346
|
+
'symbols' => %w(apples oranges)
|
347
|
+
}
|
348
|
+
|
349
|
+
assert_equal(enum_schema.default, "apples")
|
350
|
+
assert_equal(enum_schema_hash, enum_schema.to_avro)
|
351
|
+
end
|
352
|
+
|
353
|
+
def test_validate_enum_default
|
354
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
355
|
+
hash_to_schema(
|
356
|
+
type: 'enum',
|
357
|
+
name: 'fruit',
|
358
|
+
default: 'bananas',
|
359
|
+
symbols: %w(apples oranges)
|
360
|
+
)
|
361
|
+
end
|
362
|
+
assert_equal("Default 'bananas' is not a valid symbol for enum fruit",
|
363
|
+
exception.to_s)
|
364
|
+
end
|
365
|
+
|
281
366
|
def test_empty_record
|
282
367
|
schema = Avro::Schema.parse('{"type":"record", "name":"Empty"}')
|
283
368
|
assert_empty(schema.fields)
|
@@ -455,5 +540,318 @@ class TestSchema < Test::Unit::TestCase
|
|
455
540
|
end
|
456
541
|
assert_equal('Error validating default for veggies: at . expected type null, got string with value "apple"',
|
457
542
|
exception.to_s)
|
543
|
+
end
|
544
|
+
|
545
|
+
def test_fixed_decimal_to_include_precision_scale
|
546
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
547
|
+
{
|
548
|
+
"type": "fixed",
|
549
|
+
"name": "aFixed",
|
550
|
+
"logicalType": "decimal",
|
551
|
+
"size": 4,
|
552
|
+
"precision": 9,
|
553
|
+
"scale": 2
|
554
|
+
}
|
555
|
+
SCHEMA
|
556
|
+
|
557
|
+
schema_hash =
|
558
|
+
{
|
559
|
+
'type' => 'fixed',
|
560
|
+
'name' => 'aFixed',
|
561
|
+
'logicalType' => 'decimal',
|
562
|
+
'size' => 4,
|
563
|
+
'precision' => 9,
|
564
|
+
'scale' => 2
|
565
|
+
}
|
566
|
+
|
567
|
+
assert_equal schema_hash, schema.to_avro
|
568
|
+
end
|
569
|
+
|
570
|
+
def test_fixed_decimal_to_include_precision_no_scale
|
571
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
572
|
+
{
|
573
|
+
"type": "fixed",
|
574
|
+
"name": "aFixed",
|
575
|
+
"logicalType": "decimal",
|
576
|
+
"size": 4,
|
577
|
+
"precision": 9
|
578
|
+
}
|
579
|
+
SCHEMA
|
580
|
+
|
581
|
+
schema_hash =
|
582
|
+
{
|
583
|
+
'type' => 'fixed',
|
584
|
+
'name' => 'aFixed',
|
585
|
+
'logicalType' => 'decimal',
|
586
|
+
'size' => 4,
|
587
|
+
'precision' => 9
|
588
|
+
}
|
589
|
+
|
590
|
+
assert_equal schema_hash, schema.to_avro
|
591
|
+
end
|
592
|
+
|
593
|
+
# Note: this is not valid but validation is not yet implemented
|
594
|
+
def test_fixed_decimal_to_without_precision_scale
|
595
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
596
|
+
{
|
597
|
+
"type": "fixed",
|
598
|
+
"size": 4,
|
599
|
+
"name": "aFixed",
|
600
|
+
"logicalType": "decimal"
|
601
|
+
}
|
602
|
+
SCHEMA
|
603
|
+
|
604
|
+
schema_hash =
|
605
|
+
{
|
606
|
+
'type' => 'fixed',
|
607
|
+
'name' => 'aFixed',
|
608
|
+
'logicalType' => 'decimal',
|
609
|
+
'size' => 4
|
610
|
+
}
|
611
|
+
|
612
|
+
assert_equal schema_hash, schema.to_avro
|
613
|
+
end
|
614
|
+
|
615
|
+
def test_bytes_decimal_to_include_precision_scale
|
616
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
617
|
+
{
|
618
|
+
"type": "bytes",
|
619
|
+
"logicalType": "decimal",
|
620
|
+
"precision": 9,
|
621
|
+
"scale": 2
|
622
|
+
}
|
623
|
+
SCHEMA
|
624
|
+
|
625
|
+
schema_hash =
|
626
|
+
{
|
627
|
+
'type' => 'bytes',
|
628
|
+
'logicalType' => 'decimal',
|
629
|
+
'precision' => 9,
|
630
|
+
'scale' => 2
|
631
|
+
}
|
632
|
+
|
633
|
+
assert_equal schema_hash, schema.to_avro
|
634
|
+
end
|
635
|
+
|
636
|
+
def test_bytes_decimal_with_string_precision_no_scale
|
637
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
638
|
+
{
|
639
|
+
"type": "bytes",
|
640
|
+
"logicalType": "decimal",
|
641
|
+
"precision": "7"
|
642
|
+
}
|
643
|
+
SCHEMA
|
644
|
+
|
645
|
+
schema_hash =
|
646
|
+
{
|
647
|
+
'type' => 'bytes',
|
648
|
+
'logicalType' => 'decimal',
|
649
|
+
'precision' => 7
|
650
|
+
}
|
651
|
+
|
652
|
+
assert_equal schema_hash, schema.to_avro
|
653
|
+
end
|
654
|
+
|
655
|
+
def test_bytes_decimal_without_precision_or_scale
|
656
|
+
error = assert_raise Avro::SchemaParseError do
|
657
|
+
Avro::Schema.parse <<-SCHEMA
|
658
|
+
{
|
659
|
+
"type": "bytes",
|
660
|
+
"logicalType": "decimal"
|
661
|
+
}
|
662
|
+
SCHEMA
|
458
663
|
end
|
664
|
+
|
665
|
+
assert_equal 'Precision must be positive', error.message
|
666
|
+
end
|
667
|
+
|
668
|
+
def test_bytes_decimal_to_negative_precision
|
669
|
+
error = assert_raise Avro::SchemaParseError do
|
670
|
+
Avro::Schema.parse <<-SCHEMA
|
671
|
+
{
|
672
|
+
"type": "bytes",
|
673
|
+
"logicalType": "decimal",
|
674
|
+
"precision": -1
|
675
|
+
}
|
676
|
+
SCHEMA
|
677
|
+
end
|
678
|
+
|
679
|
+
assert_equal 'Precision must be positive', error.message
|
680
|
+
end
|
681
|
+
|
682
|
+
def test_bytes_decimal_to_negative_scale
|
683
|
+
error = assert_raise Avro::SchemaParseError do
|
684
|
+
Avro::Schema.parse <<-SCHEMA
|
685
|
+
{
|
686
|
+
"type": "bytes",
|
687
|
+
"logicalType": "decimal",
|
688
|
+
"precision": 2,
|
689
|
+
"scale": -1
|
690
|
+
}
|
691
|
+
SCHEMA
|
692
|
+
end
|
693
|
+
|
694
|
+
assert_equal 'Scale must be greater than or equal to 0', error.message
|
695
|
+
end
|
696
|
+
|
697
|
+
def test_bytes_decimal_with_precision_less_than_scale
|
698
|
+
error = assert_raise Avro::SchemaParseError do
|
699
|
+
Avro::Schema.parse <<-SCHEMA
|
700
|
+
{
|
701
|
+
"type": "bytes",
|
702
|
+
"logicalType": "decimal",
|
703
|
+
"precision": 3,
|
704
|
+
"scale": 4
|
705
|
+
}
|
706
|
+
SCHEMA
|
707
|
+
end
|
708
|
+
|
709
|
+
assert_equal 'Precision must be greater than scale', error.message
|
710
|
+
end
|
711
|
+
|
712
|
+
def test_bytes_schema
|
713
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
714
|
+
{
|
715
|
+
"type": "bytes"
|
716
|
+
}
|
717
|
+
SCHEMA
|
718
|
+
|
719
|
+
schema_str = 'bytes'
|
720
|
+
assert_equal schema_str, schema.to_avro
|
721
|
+
end
|
722
|
+
|
723
|
+
def test_validate_duplicate_symbols
|
724
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
725
|
+
hash_to_schema(
|
726
|
+
type: 'enum',
|
727
|
+
name: 'name',
|
728
|
+
symbols: ['erica', 'erica']
|
729
|
+
)
|
730
|
+
end
|
731
|
+
assert_equal(
|
732
|
+
'Duplicate symbol: ["erica", "erica"]',
|
733
|
+
exception.to_s
|
734
|
+
)
|
735
|
+
end
|
736
|
+
|
737
|
+
def test_validate_enum_symbols
|
738
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
739
|
+
hash_to_schema(
|
740
|
+
type: 'enum',
|
741
|
+
name: 'things',
|
742
|
+
symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$']
|
743
|
+
)
|
744
|
+
end
|
745
|
+
|
746
|
+
assert_equal(
|
747
|
+
"Invalid symbols for things: 8ad_symbol, also-bad-symbol, >=, $ don't match #{Avro::Schema::EnumSchema::SYMBOL_REGEX.inspect}",
|
748
|
+
exception.to_s
|
749
|
+
)
|
750
|
+
end
|
751
|
+
|
752
|
+
def test_enum_symbol_validation_disabled_via_env
|
753
|
+
Avro.disable_enum_symbol_validation = nil
|
754
|
+
ENV['AVRO_DISABLE_ENUM_SYMBOL_VALIDATION'] = '1'
|
755
|
+
|
756
|
+
hash_to_schema(
|
757
|
+
type: 'enum',
|
758
|
+
name: 'things',
|
759
|
+
symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
|
760
|
+
)
|
761
|
+
ensure
|
762
|
+
ENV.delete('AVRO_DISABLE_ENUM_SYMBOL_VALIDATION')
|
763
|
+
Avro.disable_enum_symbol_validation = nil
|
764
|
+
end
|
765
|
+
|
766
|
+
def test_enum_symbol_validation_disabled_via_class_method
|
767
|
+
Avro.disable_enum_symbol_validation = true
|
768
|
+
|
769
|
+
hash_to_schema(
|
770
|
+
type: 'enum',
|
771
|
+
name: 'things',
|
772
|
+
symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
|
773
|
+
)
|
774
|
+
ensure
|
775
|
+
Avro.disable_enum_symbol_validation = nil
|
776
|
+
end
|
777
|
+
|
778
|
+
def test_validate_field_aliases
|
779
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
780
|
+
hash_to_schema(
|
781
|
+
type: 'record',
|
782
|
+
name: 'fruits',
|
783
|
+
fields: [
|
784
|
+
{ name: 'banana', type: 'string', aliases: 'banane' }
|
785
|
+
]
|
786
|
+
)
|
787
|
+
end
|
788
|
+
|
789
|
+
assert_match(/Invalid aliases value "banane" for "string" banana/, exception.to_s)
|
790
|
+
end
|
791
|
+
|
792
|
+
def test_validate_same_alias_multiple_fields
|
793
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
794
|
+
hash_to_schema(
|
795
|
+
type: 'record',
|
796
|
+
name: 'fruits',
|
797
|
+
fields: [
|
798
|
+
{ name: 'banana', type: 'string', aliases: %w(yellow) },
|
799
|
+
{ name: 'lemo', type: 'string', aliases: %w(yellow) }
|
800
|
+
]
|
801
|
+
)
|
802
|
+
end
|
803
|
+
|
804
|
+
assert_match('Alias ["yellow"] already in use', exception.to_s)
|
805
|
+
end
|
806
|
+
|
807
|
+
def test_validate_repeated_aliases
|
808
|
+
assert_nothing_raised do
|
809
|
+
hash_to_schema(
|
810
|
+
type: 'record',
|
811
|
+
name: 'fruits',
|
812
|
+
fields: [
|
813
|
+
{ name: 'banana', type: 'string', aliases: %w(yellow yellow) },
|
814
|
+
]
|
815
|
+
)
|
816
|
+
end
|
817
|
+
end
|
818
|
+
|
819
|
+
def test_validate_record_aliases
|
820
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
821
|
+
hash_to_schema(
|
822
|
+
type: 'record',
|
823
|
+
name: 'fruits',
|
824
|
+
aliases: ["foods", 2],
|
825
|
+
fields: []
|
826
|
+
)
|
827
|
+
end
|
828
|
+
|
829
|
+
assert_match(/Invalid aliases value \["foods", 2\] for record fruits/, exception.to_s)
|
830
|
+
end
|
831
|
+
|
832
|
+
def test_validate_enum_aliases
|
833
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
834
|
+
hash_to_schema(
|
835
|
+
type: 'enum',
|
836
|
+
name: 'vowels',
|
837
|
+
aliases: [1, 2],
|
838
|
+
symbols: %w(A E I O U)
|
839
|
+
)
|
840
|
+
end
|
841
|
+
|
842
|
+
assert_match(/Invalid aliases value \[1, 2\] for enum vowels/, exception.to_s)
|
843
|
+
end
|
844
|
+
|
845
|
+
def test_validate_fixed_aliases
|
846
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
847
|
+
hash_to_schema(
|
848
|
+
type: 'fixed',
|
849
|
+
name: 'uuid',
|
850
|
+
size: 36,
|
851
|
+
aliases: "unique_id"
|
852
|
+
)
|
853
|
+
end
|
854
|
+
|
855
|
+
assert_match(/Invalid aliases value "unique_id" for fixed uuid/, exception.to_s)
|
856
|
+
end
|
459
857
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -6,7 +7,7 @@
|
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
9
|
#
|
9
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
11
|
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -25,7 +26,9 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
25
26
|
end
|
26
27
|
|
27
28
|
def test_compatible_reader_writer_pairs
|
29
|
+
cached_schema = a_int_record1_schema
|
28
30
|
[
|
31
|
+
cached_schema, cached_schema,
|
29
32
|
long_schema, int_schema,
|
30
33
|
float_schema, int_schema,
|
31
34
|
float_schema, long_schema,
|
@@ -39,7 +42,12 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
39
42
|
long_map_schema, int_map_schema,
|
40
43
|
|
41
44
|
enum1_ab_schema, enum1_ab_schema,
|
45
|
+
enum1_ab_aliased_schema, enum1_ab_schema,
|
42
46
|
enum1_abc_schema, enum1_ab_schema,
|
47
|
+
enum1_ab_default_schema, enum1_abc_schema,
|
48
|
+
|
49
|
+
fixed1_schema, fixed1_schema,
|
50
|
+
fixed1_aliased_schema, fixed1_schema,
|
43
51
|
|
44
52
|
string_schema, bytes_schema,
|
45
53
|
bytes_schema, string_schema,
|
@@ -55,6 +63,7 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
55
63
|
|
56
64
|
empty_record1_schema, empty_record1_schema,
|
57
65
|
empty_record1_schema, a_int_record1_schema,
|
66
|
+
empty_record1_aliased_schema, empty_record1_schema,
|
58
67
|
|
59
68
|
a_int_record1_schema, a_int_record1_schema,
|
60
69
|
a_dint_record1_schema, a_int_record1_schema,
|
@@ -117,16 +126,22 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
117
126
|
int_map_schema, long_map_schema,
|
118
127
|
|
119
128
|
enum1_ab_schema, enum1_abc_schema,
|
129
|
+
enum1_ab_schema, enum1_ab_aliased_schema,
|
120
130
|
enum1_bc_schema, enum1_abc_schema,
|
121
131
|
|
122
132
|
enum1_ab_schema, enum2_ab_schema,
|
123
133
|
int_schema, enum2_ab_schema,
|
124
134
|
enum2_ab_schema, int_schema,
|
125
135
|
|
136
|
+
fixed1_schema, fixed2_schema,
|
137
|
+
fixed1_schema, fixed1_size3_schema,
|
138
|
+
fixed1_schema, fixed1_aliased_schema,
|
139
|
+
|
126
140
|
int_union_schema, int_string_union_schema,
|
127
141
|
string_union_schema, int_string_union_schema,
|
128
142
|
|
129
143
|
empty_record2_schema, empty_record1_schema,
|
144
|
+
empty_record1_schema, empty_record1_aliased_schema,
|
130
145
|
a_int_record1_schema, empty_record1_schema,
|
131
146
|
a_int_b_dint_record1_schema, empty_record1_schema,
|
132
147
|
|
@@ -169,6 +184,17 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
169
184
|
assert_false(can_read?(reader_schema, writer_schema))
|
170
185
|
end
|
171
186
|
|
187
|
+
def test_aliased_field
|
188
|
+
reader_schema = Avro::Schema.parse(<<-SCHEMA)
|
189
|
+
{"type":"record", "name":"Record", "fields":[
|
190
|
+
{"name":"newname1", "aliases":["oldfield1"], "type":"int"},
|
191
|
+
{"name":"oldfield2", "type":"string"}
|
192
|
+
]}
|
193
|
+
SCHEMA
|
194
|
+
assert_true(can_read?(writer_schema, reader_schema))
|
195
|
+
assert_false(can_read?(reader_schema, writer_schema))
|
196
|
+
end
|
197
|
+
|
172
198
|
def test_all_fields
|
173
199
|
reader_schema = Avro::Schema.parse <<-SCHEMA
|
174
200
|
{"type":"record", "name":"Record", "fields":[
|
@@ -250,6 +276,128 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
250
276
|
assert_true(can_read?(enum_schema1, enum_schema2))
|
251
277
|
end
|
252
278
|
|
279
|
+
def test_crossed_aliases
|
280
|
+
writer_schema = Avro::Schema.parse(<<-SCHEMA)
|
281
|
+
{"type":"record", "name":"Record", "fields":[
|
282
|
+
{"name":"field1", "type": "int"},
|
283
|
+
{"name":"field2", "type": "string"}
|
284
|
+
]}
|
285
|
+
SCHEMA
|
286
|
+
reader_schema = Avro::Schema.parse(<<-SCHEMA)
|
287
|
+
{"type":"record", "name":"Record", "fields":[
|
288
|
+
{"name":"field1", "aliases":["field2"], "type":"string"},
|
289
|
+
{"name":"field2", "aliases":["field1"], "type":"int"}
|
290
|
+
]}
|
291
|
+
SCHEMA
|
292
|
+
# Not supported; alias is not used if there is a redirect match
|
293
|
+
assert_false(can_read?(writer_schema, reader_schema))
|
294
|
+
end
|
295
|
+
|
296
|
+
def test_bytes_decimal
|
297
|
+
bytes_decimal_schema = Avro::Schema.
|
298
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":4}')
|
299
|
+
bytes2_decimal_schema = Avro::Schema.
|
300
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":4}')
|
301
|
+
bytes_decimal_different_precision_schema = Avro::Schema.
|
302
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":5, "scale":4}')
|
303
|
+
bytes_decimal_no_scale_schema = Avro::Schema.
|
304
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
305
|
+
bytes2_decimal_no_scale_schema = Avro::Schema.
|
306
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
307
|
+
bytes_decimal_zero_scale_schema = Avro::Schema.
|
308
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":0}')
|
309
|
+
bytes_unknown_logical_type_schema = Avro::Schema.
|
310
|
+
parse('{"type":"bytes", "logicalType":"unknown"}')
|
311
|
+
|
312
|
+
# decimal bytes and non-decimal bytes can be mixed
|
313
|
+
assert_true(can_read?(bytes_schema, bytes_decimal_schema))
|
314
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_schema))
|
315
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_unknown_logical_type_schema))
|
316
|
+
|
317
|
+
# decimal bytes match even if precision and scale differ
|
318
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_different_precision_schema))
|
319
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_no_scale_schema))
|
320
|
+
assert_true(can_read?(bytes_decimal_schema, bytes_decimal_zero_scale_schema))
|
321
|
+
# - zero and no scale are equivalent
|
322
|
+
assert_true(can_read?(bytes_decimal_zero_scale_schema, bytes_decimal_no_scale_schema))
|
323
|
+
# - different schemas with the same attributes match
|
324
|
+
assert_true(can_read?(bytes_decimal_schema, bytes2_decimal_schema))
|
325
|
+
# - different schemas with the same no scale match
|
326
|
+
assert_true(can_read?(bytes2_decimal_no_scale_schema, bytes_decimal_no_scale_schema))
|
327
|
+
end
|
328
|
+
|
329
|
+
def test_fixed_decimal
|
330
|
+
fixed_decimal_schema = Avro::Schema.
|
331
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":2}')
|
332
|
+
fixed2_decimal_schema = Avro::Schema.
|
333
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed2", "logicalType":"decimal", "precision":4, "scale":2}')
|
334
|
+
fixed_decimal_different_precision_schema = Avro::Schema.
|
335
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":3, "scale":2}')
|
336
|
+
fixed_decimal_size3_schema = Avro::Schema.
|
337
|
+
parse('{"type":"fixed", "size":3, "name":"FixedS3", "logicalType":"decimal", "precision":4, "scale":2}')
|
338
|
+
fixed_unknown_schema = Avro::Schema.
|
339
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"unknown"}')
|
340
|
+
fixed_decimal_zero_scale_schema = Avro::Schema.
|
341
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":0}')
|
342
|
+
fixed_decimal_no_scale_schema = Avro::Schema.
|
343
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4}')
|
344
|
+
|
345
|
+
# decimal fixed and non-decimal can be mixed if fixed name matches
|
346
|
+
assert_true(can_read?(fixed_decimal_schema, fixed1_schema))
|
347
|
+
assert_true(can_read?(fixed1_schema, fixed_decimal_schema))
|
348
|
+
assert_false(can_read?(fixed2_schema, fixed_decimal_schema))
|
349
|
+
|
350
|
+
# decimal logical types match even if fixed name differs
|
351
|
+
assert_true(can_read?(fixed_decimal_schema, fixed2_decimal_schema))
|
352
|
+
|
353
|
+
# fixed with the same name & size match even if decimal precision and scale differ
|
354
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_different_precision_schema))
|
355
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_size3_schema))
|
356
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_unknown_schema))
|
357
|
+
# - zero and no scale are equivalent but these match anyway due to same name & size
|
358
|
+
assert_true(can_read?(fixed_decimal_no_scale_schema, fixed_decimal_zero_scale_schema))
|
359
|
+
# - scale does not match
|
360
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_no_scale_schema))
|
361
|
+
assert_true(can_read?(fixed_decimal_schema, fixed_decimal_zero_scale_schema))
|
362
|
+
end
|
363
|
+
|
364
|
+
def test_decimal_different_types
|
365
|
+
fixed_decimal_schema = Avro::Schema.
|
366
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":2}')
|
367
|
+
fixed_decimal_scale4_schema = Avro::Schema.
|
368
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":4}')
|
369
|
+
bytes_decimal_schema = Avro::Schema.
|
370
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":2}')
|
371
|
+
fixed_decimal_zero_scale_schema = Avro::Schema.
|
372
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4, "scale":0}')
|
373
|
+
fixed_decimal_no_scale_schema = Avro::Schema.
|
374
|
+
parse('{"type":"fixed", "size":2, "name":"Fixed1", "logicalType":"decimal", "precision":4}')
|
375
|
+
bytes_decimal_zero_scale_schema = Avro::Schema.
|
376
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4, "scale":0}')
|
377
|
+
bytes_decimal_no_scale_schema = Avro::Schema.
|
378
|
+
parse('{"type":"bytes", "logicalType":"decimal", "precision":4}')
|
379
|
+
|
380
|
+
# decimal logical types can be read
|
381
|
+
assert_true(can_read?(fixed_decimal_schema, bytes_decimal_schema))
|
382
|
+
assert_true(can_read?(bytes_decimal_schema, fixed_decimal_schema))
|
383
|
+
|
384
|
+
# non-decimal bytes and fixed cannot be mixed
|
385
|
+
assert_false(can_read?(fixed_decimal_schema, bytes_schema))
|
386
|
+
assert_false(can_read?(bytes_schema, fixed_decimal_schema))
|
387
|
+
assert_false(can_read?(fixed1_schema, bytes_decimal_schema))
|
388
|
+
assert_false(can_read?(bytes_decimal_schema, fixed1_schema))
|
389
|
+
|
390
|
+
# decimal precision and scale must match
|
391
|
+
assert_false(can_read?(fixed_decimal_scale4_schema, bytes_decimal_schema))
|
392
|
+
assert_false(can_read?(bytes_decimal_schema, fixed_decimal_scale4_schema))
|
393
|
+
|
394
|
+
# zero scale and no scale are equivalent
|
395
|
+
assert_true(can_read?(bytes_decimal_no_scale_schema, fixed_decimal_zero_scale_schema))
|
396
|
+
assert_true(can_read?(fixed_decimal_zero_scale_schema, bytes_decimal_no_scale_schema))
|
397
|
+
assert_true(can_read?(bytes_decimal_zero_scale_schema, fixed_decimal_no_scale_schema))
|
398
|
+
assert_true(can_read?(fixed_decimal_no_scale_schema, bytes_decimal_zero_scale_schema))
|
399
|
+
end
|
400
|
+
|
253
401
|
# Tests from lang/java/avro/src/test/java/org/apache/avro/io/parsing/TestResolvingGrammarGenerator2.java
|
254
402
|
|
255
403
|
def point_2d_schema
|
@@ -373,6 +521,14 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
373
521
|
Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B"]}')
|
374
522
|
end
|
375
523
|
|
524
|
+
def enum1_ab_default_schema
|
525
|
+
Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B"], "default":"A"}')
|
526
|
+
end
|
527
|
+
|
528
|
+
def enum1_ab_aliased_schema
|
529
|
+
Avro::Schema.parse('{"type":"enum", "name":"Enum2", "aliases":["Enum1"], "symbols":["A","B"]}')
|
530
|
+
end
|
531
|
+
|
376
532
|
def enum1_abc_schema
|
377
533
|
Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}')
|
378
534
|
end
|
@@ -385,10 +541,30 @@ class TestSchemaCompatibility < Test::Unit::TestCase
|
|
385
541
|
Avro::Schema.parse('{"type":"enum", "name":"Enum2", "symbols":["A","B"]}')
|
386
542
|
end
|
387
543
|
|
544
|
+
def fixed1_schema
|
545
|
+
Avro::Schema.parse('{"type":"fixed", "name":"Fixed1", "size": 2}')
|
546
|
+
end
|
547
|
+
|
548
|
+
def fixed1_aliased_schema
|
549
|
+
Avro::Schema.parse('{"type":"fixed", "name":"Fixed2", "aliases":["Fixed1"], "size": 2}')
|
550
|
+
end
|
551
|
+
|
552
|
+
def fixed2_schema
|
553
|
+
Avro::Schema.parse('{"type":"fixed", "name":"Fixed2", "size": 2}')
|
554
|
+
end
|
555
|
+
|
556
|
+
def fixed1_size3_schema
|
557
|
+
Avro::Schema.parse('{"type":"fixed", "name":"Fixed1", "size": 3}')
|
558
|
+
end
|
559
|
+
|
388
560
|
def empty_record1_schema
|
389
561
|
Avro::Schema.parse('{"type":"record", "name":"Record1"}')
|
390
562
|
end
|
391
563
|
|
564
|
+
def empty_record1_aliased_schema
|
565
|
+
Avro::Schema.parse('{"type":"record", "name":"Record2", "aliases":["Record1"]}')
|
566
|
+
end
|
567
|
+
|
392
568
|
def empty_record2_schema
|
393
569
|
Avro::Schema.parse('{"type":"record", "name":"Record2"}')
|
394
570
|
end
|