avro 1.8.2 → 1.9.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE +2 -2
- data/Manifest +6 -0
- data/NOTICE +1 -1
- data/Rakefile +2 -2
- data/avro.gemspec +21 -21
- data/interop/test_interop.rb +10 -2
- data/lib/avro.rb +12 -1
- data/lib/avro/data_file.rb +2 -2
- data/lib/avro/io.rb +45 -65
- data/lib/avro/ipc.rb +9 -9
- data/lib/avro/logical_types.rb +90 -0
- data/lib/avro/protocol.rb +12 -8
- data/lib/avro/schema.rb +90 -64
- data/lib/avro/schema_compatibility.rb +170 -0
- data/lib/avro/schema_normalization.rb +1 -1
- data/lib/avro/schema_validator.rb +242 -0
- data/test/case_finder.rb +1 -1
- data/test/random_data.rb +22 -3
- data/test/sample_ipc_client.rb +1 -1
- data/test/sample_ipc_http_client.rb +1 -1
- data/test/sample_ipc_http_server.rb +1 -1
- data/test/sample_ipc_server.rb +1 -1
- data/test/test_datafile.rb +4 -4
- data/test/test_fingerprints.rb +1 -1
- data/test/test_help.rb +1 -1
- data/test/test_io.rb +74 -7
- data/test/test_logical_types.rb +128 -0
- data/test/test_protocol.rb +37 -4
- data/test/test_schema.rb +324 -28
- data/test/test_schema_compatibility.rb +475 -0
- data/test/test_schema_normalization.rb +1 -1
- data/test/test_schema_validator.rb +554 -0
- data/test/test_socket_transport.rb +1 -1
- data/test/tool.rb +1 -2
- metadata +21 -9
data/test/sample_ipc_client.rb
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# "License"); you may not use this file except in compliance
|
8
8
|
# with the License. You may obtain a copy of the License at
|
9
9
|
#
|
10
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
11
|
#
|
12
12
|
# Unless required by applicable law or agreed to in writing, software
|
13
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -7,7 +7,7 @@
|
|
7
7
|
# "License"); you may not use this file except in compliance
|
8
8
|
# with the License. You may obtain a copy of the License at
|
9
9
|
#
|
10
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
11
|
#
|
12
12
|
# Unless required by applicable law or agreed to in writing, software
|
13
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -6,7 +6,7 @@
|
|
6
6
|
# "License"); you may not use this file except in compliance
|
7
7
|
# with the License. You may obtain a copy of the License at
|
8
8
|
#
|
9
|
-
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
10
|
#
|
11
11
|
# Unless required by applicable law or agreed to in writing, software
|
12
12
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
data/test/sample_ipc_server.rb
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# "License"); you may not use this file except in compliance
|
8
8
|
# with the License. You may obtain a copy of the License at
|
9
9
|
#
|
10
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
11
|
#
|
12
12
|
# Unless required by applicable law or agreed to in writing, software
|
13
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
data/test/test_datafile.rb
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# "License"); you may not use this file except in compliance
|
8
8
|
# with the License. You may obtain a copy of the License at
|
9
9
|
#
|
10
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
11
|
#
|
12
12
|
# Unless required by applicable law or agreed to in writing, software
|
13
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -20,13 +20,13 @@ require 'test_help'
|
|
20
20
|
class TestDataFile < Test::Unit::TestCase
|
21
21
|
HERE = File.expand_path File.dirname(__FILE__)
|
22
22
|
def setup
|
23
|
-
if File.
|
23
|
+
if File.exist?(HERE + '/data.avr')
|
24
24
|
File.unlink(HERE + '/data.avr')
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
28
|
def teardown
|
29
|
-
if File.
|
29
|
+
if File.exist?(HERE + '/data.avr')
|
30
30
|
File.unlink(HERE + '/data.avr')
|
31
31
|
end
|
32
32
|
end
|
@@ -38,7 +38,7 @@ class TestDataFile < Test::Unit::TestCase
|
|
38
38
|
"fields" : [
|
39
39
|
{"name": "username", "type": "string"},
|
40
40
|
{"name": "age", "type": "int"},
|
41
|
-
{"name": "verified", "type": "boolean", "default":
|
41
|
+
{"name": "verified", "type": "boolean", "default": false}
|
42
42
|
]}
|
43
43
|
JSON
|
44
44
|
|
data/test/test_fingerprints.rb
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# "License"); you may not use this file except in compliance
|
7
7
|
# with the License. You may obtain a copy of the License at
|
8
8
|
#
|
9
|
-
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
10
|
#
|
11
11
|
# Unless required by applicable law or agreed to in writing, software
|
12
12
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
data/test/test_help.rb
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# "License"); you may not use this file except in compliance
|
7
7
|
# with the License. You may obtain a copy of the License at
|
8
8
|
#
|
9
|
-
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
10
|
#
|
11
11
|
# Unless required by applicable law or agreed to in writing, software
|
12
12
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
data/test/test_io.rb
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# "License"); you may not use this file except in compliance
|
7
7
|
# with the License. You may obtain a copy of the License at
|
8
8
|
#
|
9
|
-
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
10
|
#
|
11
11
|
# Unless required by applicable law or agreed to in writing, software
|
12
12
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -84,6 +84,17 @@ EOS
|
|
84
84
|
check_default(record_schema, '{"f": 11}', {"f" => 11})
|
85
85
|
end
|
86
86
|
|
87
|
+
def test_record_with_logical_type
|
88
|
+
record_schema = <<EOS
|
89
|
+
{"type": "record",
|
90
|
+
"name": "Test",
|
91
|
+
"fields": [{"name": "ts",
|
92
|
+
"type": {"type": "long",
|
93
|
+
"logicalType": "timestamp-micros"}}]}
|
94
|
+
EOS
|
95
|
+
check(record_schema)
|
96
|
+
end
|
97
|
+
|
87
98
|
def test_error
|
88
99
|
error_schema = <<EOS
|
89
100
|
{"type": "error",
|
@@ -115,6 +126,7 @@ EOS
|
|
115
126
|
def test_union
|
116
127
|
union_schema = <<EOS
|
117
128
|
["string",
|
129
|
+
{"type": "int", "logicalType": "date"},
|
118
130
|
"null",
|
119
131
|
"long",
|
120
132
|
{"type": "record",
|
@@ -146,10 +158,31 @@ EOS
|
|
146
158
|
check_default(fixed_schema, '"a"', "a")
|
147
159
|
end
|
148
160
|
|
161
|
+
def test_record_with_nil
|
162
|
+
schema = Avro::Schema.parse('{"type":"record", "name":"rec", "fields":[{"type":"int", "name":"i"}]}')
|
163
|
+
assert_raise(Avro::IO::AvroTypeError) do
|
164
|
+
write_datum(nil, schema)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def test_array_with_nil
|
169
|
+
schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
|
170
|
+
assert_raise(Avro::IO::AvroTypeError) do
|
171
|
+
write_datum(nil, schema)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def test_map_with_nil
|
176
|
+
schema = Avro::Schema.parse('{"type":"map", "values":"long"}')
|
177
|
+
assert_raise(Avro::IO::AvroTypeError) do
|
178
|
+
write_datum(nil, schema)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
149
182
|
def test_enum_with_duplicate
|
150
183
|
str = '{"type": "enum", "name": "Test","symbols" : ["AA", "AA"]}'
|
151
|
-
assert_raises(Avro::SchemaParseError) do
|
152
|
-
|
184
|
+
assert_raises(Avro::SchemaParseError.new('Duplicate symbol: ["AA", "AA"]')) do
|
185
|
+
Avro::Schema.parse str
|
153
186
|
end
|
154
187
|
end
|
155
188
|
|
@@ -256,7 +289,7 @@ EOS
|
|
256
289
|
end
|
257
290
|
|
258
291
|
def test_skip_long
|
259
|
-
for value_to_skip,
|
292
|
+
for value_to_skip, _hex_encoding in BINARY_INT_ENCODINGS
|
260
293
|
value_to_read = 6253
|
261
294
|
|
262
295
|
# write some data in binary to string buffer
|
@@ -281,7 +314,7 @@ EOS
|
|
281
314
|
end
|
282
315
|
|
283
316
|
def test_skip_int
|
284
|
-
for value_to_skip,
|
317
|
+
for value_to_skip, _hex_encoding in BINARY_INT_ENCODINGS
|
285
318
|
value_to_read = 6253
|
286
319
|
|
287
320
|
writer = StringIO.new
|
@@ -331,7 +364,7 @@ EOS
|
|
331
364
|
datum_to_write = 219
|
332
365
|
for rs in promotable_schemas[(i + 1)..-1]
|
333
366
|
readers_schema = Avro::Schema.parse(rs)
|
334
|
-
writer,
|
367
|
+
writer, _enc, _dw = write_datum(datum_to_write, writers_schema)
|
335
368
|
datum_read = read_datum(writer, writers_schema, readers_schema)
|
336
369
|
if datum_read != datum_to_write
|
337
370
|
incorrect += 1
|
@@ -341,6 +374,40 @@ EOS
|
|
341
374
|
end
|
342
375
|
end
|
343
376
|
|
377
|
+
def test_interchangeable_schemas
|
378
|
+
interchangeable_schemas = ['"string"', '"bytes"']
|
379
|
+
incorrect = 0
|
380
|
+
interchangeable_schemas.each_with_index do |ws, i|
|
381
|
+
writers_schema = Avro::Schema.parse(ws)
|
382
|
+
datum_to_write = 'foo'
|
383
|
+
readers_schema = Avro::Schema.parse(interchangeable_schemas[i == 0 ? 1 : 0])
|
384
|
+
writer, * = write_datum(datum_to_write, writers_schema)
|
385
|
+
datum_read = read_datum(writer, writers_schema, readers_schema)
|
386
|
+
if datum_read != datum_to_write
|
387
|
+
incorrect += 1
|
388
|
+
end
|
389
|
+
end
|
390
|
+
assert_equal(incorrect, 0)
|
391
|
+
end
|
392
|
+
|
393
|
+
def test_array_schema_promotion
|
394
|
+
writers_schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
|
395
|
+
readers_schema = Avro::Schema.parse('{"type":"array", "items":"long"}')
|
396
|
+
datum_to_write = [1, 2]
|
397
|
+
writer, * = write_datum(datum_to_write, writers_schema)
|
398
|
+
datum_read = read_datum(writer, writers_schema, readers_schema)
|
399
|
+
assert_equal(datum_read, datum_to_write)
|
400
|
+
end
|
401
|
+
|
402
|
+
def test_map_schema_promotion
|
403
|
+
writers_schema = Avro::Schema.parse('{"type":"map", "values":"int"}')
|
404
|
+
readers_schema = Avro::Schema.parse('{"type":"map", "values":"long"}')
|
405
|
+
datum_to_write = { 'foo' => 1, 'bar' => 2 }
|
406
|
+
writer, * = write_datum(datum_to_write, writers_schema)
|
407
|
+
datum_read = read_datum(writer, writers_schema, readers_schema)
|
408
|
+
assert_equal(datum_read, datum_to_write)
|
409
|
+
end
|
410
|
+
|
344
411
|
def test_snappy_backward_compat
|
345
412
|
# a snappy-compressed block payload without the checksum
|
346
413
|
# this has no back-references, just one literal so the last 9
|
@@ -417,7 +484,7 @@ EOS
|
|
417
484
|
|
418
485
|
def checkser(schm, randomdata)
|
419
486
|
datum = randomdata.next
|
420
|
-
assert validate(schm, datum)
|
487
|
+
assert validate(schm, datum), 'datum is not valid for schema'
|
421
488
|
w = Avro::IO::DatumWriter.new(schm)
|
422
489
|
writer = StringIO.new "", "w"
|
423
490
|
w.write(datum, Avro::IO::BinaryEncoder.new(writer))
|
@@ -0,0 +1,128 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
# or more contributor license agreements. See the NOTICE file
|
4
|
+
# distributed with this work for additional information
|
5
|
+
# regarding copyright ownership. The ASF licenses this file
|
6
|
+
# to you under the Apache License, Version 2.0 (the
|
7
|
+
# "License"); you may not use this file except in compliance
|
8
|
+
# with the License. You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
|
18
|
+
require 'test_help'
|
19
|
+
|
20
|
+
class TestLogicalTypes < Test::Unit::TestCase
|
21
|
+
def test_int_date
|
22
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
23
|
+
{ "type": "int", "logicalType": "date" }
|
24
|
+
SCHEMA
|
25
|
+
|
26
|
+
assert_equal 'date', schema.logical_type
|
27
|
+
today = Date.today
|
28
|
+
assert_encode_and_decode today, schema
|
29
|
+
assert_preencoded Avro::LogicalTypes::IntDate.encode(today), schema, today
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_int_date_conversion
|
33
|
+
type = Avro::LogicalTypes::IntDate
|
34
|
+
|
35
|
+
assert_equal 5, type.encode(Date.new(1970, 1, 6))
|
36
|
+
assert_equal 0, type.encode(Date.new(1970, 1, 1))
|
37
|
+
assert_equal(-5, type.encode(Date.new(1969, 12, 27)))
|
38
|
+
|
39
|
+
assert_equal Date.new(1970, 1, 6), type.decode(5)
|
40
|
+
assert_equal Date.new(1970, 1, 1), type.decode(0)
|
41
|
+
assert_equal Date.new(1969, 12, 27), type.decode(-5)
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_timestamp_millis_long
|
45
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
46
|
+
{ "type": "long", "logicalType": "timestamp-millis" }
|
47
|
+
SCHEMA
|
48
|
+
|
49
|
+
# The Time.at format is (seconds, microseconds) since Epoch.
|
50
|
+
time = Time.at(628232400, 12000)
|
51
|
+
|
52
|
+
assert_equal 'timestamp-millis', schema.logical_type
|
53
|
+
assert_encode_and_decode time, schema
|
54
|
+
assert_preencoded Avro::LogicalTypes::TimestampMillis.encode(time), schema, time.utc
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_timestamp_millis_long_conversion
|
58
|
+
type = Avro::LogicalTypes::TimestampMillis
|
59
|
+
|
60
|
+
now = Time.now.utc
|
61
|
+
now_millis = Time.utc(now.year, now.month, now.day, now.hour, now.min, now.sec, now.usec / 1000 * 1000)
|
62
|
+
|
63
|
+
assert_equal now_millis, type.decode(type.encode(now_millis))
|
64
|
+
assert_equal 1432849613221, type.encode(Time.utc(2015, 5, 28, 21, 46, 53, 221000))
|
65
|
+
assert_equal 1432849613221, type.encode(DateTime.new(2015, 5, 28, 21, 46, 53.221))
|
66
|
+
assert_equal Time.utc(2015, 5, 28, 21, 46, 53, 221000), type.decode(1432849613221)
|
67
|
+
end
|
68
|
+
|
69
|
+
def test_timestamp_micros_long
|
70
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
71
|
+
{ "type": "long", "logicalType": "timestamp-micros" }
|
72
|
+
SCHEMA
|
73
|
+
|
74
|
+
# The Time.at format is (seconds, microseconds) since Epoch.
|
75
|
+
time = Time.at(628232400, 12345)
|
76
|
+
|
77
|
+
assert_equal 'timestamp-micros', schema.logical_type
|
78
|
+
assert_encode_and_decode time, schema
|
79
|
+
assert_preencoded Avro::LogicalTypes::TimestampMicros.encode(time), schema, time.utc
|
80
|
+
end
|
81
|
+
|
82
|
+
def test_timestamp_micros_long_conversion
|
83
|
+
type = Avro::LogicalTypes::TimestampMicros
|
84
|
+
|
85
|
+
now = Time.now.utc
|
86
|
+
|
87
|
+
assert_equal Time.utc(now.year, now.month, now.day, now.hour, now.min, now.sec, now.usec), type.decode(type.encode(now))
|
88
|
+
assert_equal 1432849613221843, type.encode(Time.utc(2015, 5, 28, 21, 46, 53, 221843))
|
89
|
+
assert_equal 1432849613221843, type.encode(DateTime.new(2015, 5, 28, 21, 46, 53.221843))
|
90
|
+
assert_equal Time.utc(2015, 5, 28, 21, 46, 53, 221843), type.decode(1432849613221843)
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_parse_fixed_duration
|
94
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
95
|
+
{ "type": "fixed", "size": 12, "name": "fixed_dur", "logicalType": "duration" }
|
96
|
+
SCHEMA
|
97
|
+
|
98
|
+
assert_equal 'duration', schema.logical_type
|
99
|
+
end
|
100
|
+
|
101
|
+
def encode(datum, schema)
|
102
|
+
buffer = StringIO.new("")
|
103
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
104
|
+
|
105
|
+
datum_writer = Avro::IO::DatumWriter.new(schema)
|
106
|
+
datum_writer.write(datum, encoder)
|
107
|
+
|
108
|
+
buffer.string
|
109
|
+
end
|
110
|
+
|
111
|
+
def decode(encoded, schema)
|
112
|
+
buffer = StringIO.new(encoded)
|
113
|
+
decoder = Avro::IO::BinaryDecoder.new(buffer)
|
114
|
+
|
115
|
+
datum_reader = Avro::IO::DatumReader.new(schema, schema)
|
116
|
+
datum_reader.read(decoder)
|
117
|
+
end
|
118
|
+
|
119
|
+
def assert_encode_and_decode(datum, schema)
|
120
|
+
encoded = encode(datum, schema)
|
121
|
+
assert_equal datum, decode(encoded, schema)
|
122
|
+
end
|
123
|
+
|
124
|
+
def assert_preencoded(datum, schema, decoded)
|
125
|
+
encoded = encode(datum, schema)
|
126
|
+
assert_equal decoded, decode(encoded, schema)
|
127
|
+
end
|
128
|
+
end
|
data/test/test_protocol.rb
CHANGED
@@ -5,9 +5,9 @@
|
|
5
5
|
# to you under the Apache License, Version 2.0 (the
|
6
6
|
# "License"); you may not use this file except in compliance
|
7
7
|
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
#
|
10
|
-
#
|
8
|
+
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
11
|
# Unless required by applicable law or agreed to in writing, software
|
12
12
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
13
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
@@ -139,7 +139,7 @@ EOS
|
|
139
139
|
|
140
140
|
}
|
141
141
|
EOS
|
142
|
-
ExampleProtocol.new(<<-EOS, true)
|
142
|
+
ExampleProtocol.new(<<-EOS, true),
|
143
143
|
{"namespace": "org.apache.avro.test",
|
144
144
|
"protocol": "BulkData",
|
145
145
|
|
@@ -160,6 +160,29 @@ EOS
|
|
160
160
|
}
|
161
161
|
|
162
162
|
}
|
163
|
+
EOS
|
164
|
+
ExampleProtocol.new(<<-EOS, true),
|
165
|
+
{
|
166
|
+
"namespace": "com.acme",
|
167
|
+
"protocol": "HelloWorld",
|
168
|
+
"doc": "protocol_documentation",
|
169
|
+
|
170
|
+
"types": [
|
171
|
+
{"name": "Greeting", "type": "record", "fields": [
|
172
|
+
{"name": "message", "type": "string"}]},
|
173
|
+
{"name": "Curse", "type": "error", "fields": [
|
174
|
+
{"name": "message", "type": "string"}]}
|
175
|
+
],
|
176
|
+
|
177
|
+
"messages": {
|
178
|
+
"hello": {
|
179
|
+
"doc": "message_documentation",
|
180
|
+
"request": [{"name": "greeting", "type": "Greeting" }],
|
181
|
+
"response": "Greeting",
|
182
|
+
"errors": ["Curse"]
|
183
|
+
}
|
184
|
+
}
|
185
|
+
}
|
163
186
|
EOS
|
164
187
|
]
|
165
188
|
|
@@ -196,4 +219,14 @@ EOS
|
|
196
219
|
assert_equal type.namespace, 'com.acme'
|
197
220
|
end
|
198
221
|
end
|
222
|
+
|
223
|
+
def test_protocol_doc_attribute
|
224
|
+
original = Protocol.parse(EXAMPLES.last.protocol_string)
|
225
|
+
assert_equal 'protocol_documentation', original.doc
|
226
|
+
end
|
227
|
+
|
228
|
+
def test_protocol_message_doc_attribute
|
229
|
+
original = Protocol.parse(EXAMPLES.last.protocol_string)
|
230
|
+
assert_equal 'message_documentation', original.messages['hello'].doc
|
231
|
+
end
|
199
232
|
end
|
data/test/test_schema.rb
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# "License"); you may not use this file except in compliance
|
7
7
|
# with the License. You may obtain a copy of the License at
|
8
8
|
#
|
9
|
-
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
10
|
#
|
11
11
|
# Unless required by applicable law or agreed to in writing, software
|
12
12
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -17,6 +17,10 @@
|
|
17
17
|
require 'test_help'
|
18
18
|
|
19
19
|
class TestSchema < Test::Unit::TestCase
|
20
|
+
def hash_to_schema(hash)
|
21
|
+
Avro::Schema.parse(hash.to_json)
|
22
|
+
end
|
23
|
+
|
20
24
|
def test_default_namespace
|
21
25
|
schema = Avro::Schema.parse <<-SCHEMA
|
22
26
|
{"type": "record", "name": "OuterRecord", "fields": [
|
@@ -27,13 +31,13 @@ class TestSchema < Test::Unit::TestCase
|
|
27
31
|
]}
|
28
32
|
SCHEMA
|
29
33
|
|
30
|
-
assert_equal schema.name
|
31
|
-
assert_equal schema.fullname
|
34
|
+
assert_equal 'OuterRecord', schema.name
|
35
|
+
assert_equal 'OuterRecord', schema.fullname
|
32
36
|
assert_nil schema.namespace
|
33
37
|
|
34
38
|
schema.fields.each do |field|
|
35
|
-
assert_equal field.type.name
|
36
|
-
assert_equal field.type.fullname
|
39
|
+
assert_equal 'InnerRecord', field.type.name
|
40
|
+
assert_equal 'InnerRecord', field.type.fullname
|
37
41
|
assert_nil field.type.namespace
|
38
42
|
end
|
39
43
|
end
|
@@ -50,13 +54,13 @@ class TestSchema < Test::Unit::TestCase
|
|
50
54
|
]}
|
51
55
|
SCHEMA
|
52
56
|
|
53
|
-
assert_equal schema.name
|
54
|
-
assert_equal
|
55
|
-
assert_equal
|
57
|
+
assert_equal 'OuterRecord', schema.name
|
58
|
+
assert_equal 'my.name.space.OuterRecord', schema.fullname
|
59
|
+
assert_equal 'my.name.space', schema.namespace
|
56
60
|
schema.fields.each do |field|
|
57
|
-
assert_equal field.type.name
|
58
|
-
assert_equal
|
59
|
-
assert_equal
|
61
|
+
assert_equal 'InnerRecord', field.type.name
|
62
|
+
assert_equal 'my.name.space.InnerRecord', field.type.fullname
|
63
|
+
assert_equal 'my.name.space', field.type.namespace
|
60
64
|
end
|
61
65
|
end
|
62
66
|
|
@@ -71,13 +75,13 @@ class TestSchema < Test::Unit::TestCase
|
|
71
75
|
]}
|
72
76
|
SCHEMA
|
73
77
|
|
74
|
-
assert_equal schema.name
|
75
|
-
assert_equal
|
76
|
-
assert_equal
|
78
|
+
assert_equal 'OuterRecord', schema.name
|
79
|
+
assert_equal 'my.name.space.OuterRecord', schema.fullname
|
80
|
+
assert_equal 'my.name.space', schema.namespace
|
77
81
|
schema.fields.each do |field|
|
78
|
-
assert_equal field.type.name
|
79
|
-
assert_equal
|
80
|
-
assert_equal
|
82
|
+
assert_equal 'InnerEnum', field.type.name
|
83
|
+
assert_equal 'my.name.space.InnerEnum', field.type.fullname
|
84
|
+
assert_equal 'my.name.space', field.type.namespace
|
81
85
|
end
|
82
86
|
end
|
83
87
|
|
@@ -96,18 +100,18 @@ class TestSchema < Test::Unit::TestCase
|
|
96
100
|
]}
|
97
101
|
SCHEMA
|
98
102
|
|
99
|
-
assert_equal schema.name
|
100
|
-
assert_equal
|
101
|
-
assert_equal schema.namespace
|
103
|
+
assert_equal 'OuterRecord', schema.name
|
104
|
+
assert_equal 'outer.OuterRecord', schema.fullname
|
105
|
+
assert_equal 'outer', schema.namespace
|
102
106
|
middle = schema.fields.first.type
|
103
|
-
assert_equal middle.name
|
104
|
-
assert_equal middle.
|
105
|
-
assert_equal middle
|
107
|
+
assert_equal 'MiddleRecord', middle.name
|
108
|
+
assert_equal 'middle.MiddleRecord', middle.fullname
|
109
|
+
assert_equal 'middle', middle.namespace
|
106
110
|
inner = middle.fields.first.type
|
107
|
-
assert_equal inner.name
|
108
|
-
assert_equal
|
109
|
-
assert_equal inner.namespace
|
110
|
-
assert_equal inner.fields.first.type
|
111
|
+
assert_equal 'InnerRecord', inner.name
|
112
|
+
assert_equal 'middle.InnerRecord', inner.fullname
|
113
|
+
assert_equal 'middle', inner.namespace
|
114
|
+
assert_equal middle, inner.fields.first.type
|
111
115
|
end
|
112
116
|
|
113
117
|
def test_to_avro_includes_namespaces
|
@@ -120,7 +124,7 @@ class TestSchema < Test::Unit::TestCase
|
|
120
124
|
]}
|
121
125
|
SCHEMA
|
122
126
|
|
123
|
-
assert_equal
|
127
|
+
assert_equal({
|
124
128
|
'type' => 'record', 'name' => 'OuterRecord', 'namespace' => 'my.name.space',
|
125
129
|
'fields' => [
|
126
130
|
{'name' => 'definition', 'type' => {
|
@@ -129,6 +133,21 @@ class TestSchema < Test::Unit::TestCase
|
|
129
133
|
}},
|
130
134
|
{'name' => 'reference', 'type' => 'my.name.space.InnerFixed'}
|
131
135
|
]
|
136
|
+
}, schema.to_avro)
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_to_avro_includes_logical_type
|
140
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
141
|
+
{"type": "record", "name": "has_logical", "fields": [
|
142
|
+
{"name": "dt", "type": {"type": "int", "logicalType": "date"}}]
|
143
|
+
}
|
144
|
+
SCHEMA
|
145
|
+
|
146
|
+
assert_equal schema.to_avro, {
|
147
|
+
'type' => 'record', 'name' => 'has_logical',
|
148
|
+
'fields' => [
|
149
|
+
{'name' => 'dt', 'type' => {'type' => 'int', 'logicalType' => 'date'}}
|
150
|
+
]
|
132
151
|
}
|
133
152
|
end
|
134
153
|
|
@@ -160,4 +179,281 @@ class TestSchema < Test::Unit::TestCase
|
|
160
179
|
]
|
161
180
|
}
|
162
181
|
end
|
182
|
+
|
183
|
+
def test_record_field_doc_attribute
|
184
|
+
field_schema_json = Avro::Schema.parse <<-SCHEMA
|
185
|
+
{
|
186
|
+
"type": "record",
|
187
|
+
"name": "Record",
|
188
|
+
"namespace": "my.name.space",
|
189
|
+
"fields": [
|
190
|
+
{
|
191
|
+
"name": "name",
|
192
|
+
"type": "boolean",
|
193
|
+
"doc": "documentation"
|
194
|
+
}
|
195
|
+
]
|
196
|
+
}
|
197
|
+
SCHEMA
|
198
|
+
|
199
|
+
field_schema_hash =
|
200
|
+
{
|
201
|
+
'type' => 'record',
|
202
|
+
'name' => 'Record',
|
203
|
+
'namespace' => 'my.name.space',
|
204
|
+
'fields' => [
|
205
|
+
{
|
206
|
+
'name' => 'name',
|
207
|
+
'type' => 'boolean',
|
208
|
+
'doc' => 'documentation'
|
209
|
+
}
|
210
|
+
]
|
211
|
+
}
|
212
|
+
|
213
|
+
assert_equal field_schema_hash, field_schema_json.to_avro
|
214
|
+
end
|
215
|
+
|
216
|
+
def test_record_doc_attribute
|
217
|
+
record_schema_json = Avro::Schema.parse <<-SCHEMA
|
218
|
+
{
|
219
|
+
"type": "record",
|
220
|
+
"name": "Record",
|
221
|
+
"namespace": "my.name.space",
|
222
|
+
"doc": "documentation",
|
223
|
+
"fields": [
|
224
|
+
{
|
225
|
+
"name": "name",
|
226
|
+
"type": "boolean"
|
227
|
+
}
|
228
|
+
]
|
229
|
+
}
|
230
|
+
SCHEMA
|
231
|
+
|
232
|
+
record_schema_hash =
|
233
|
+
{
|
234
|
+
'type' => 'record',
|
235
|
+
'name' => 'Record',
|
236
|
+
'namespace' => 'my.name.space',
|
237
|
+
'doc' => 'documentation',
|
238
|
+
'fields' => [
|
239
|
+
{
|
240
|
+
'name' => 'name',
|
241
|
+
'type' => 'boolean'
|
242
|
+
}
|
243
|
+
]
|
244
|
+
}
|
245
|
+
|
246
|
+
assert_equal record_schema_hash, record_schema_json.to_avro
|
247
|
+
end
|
248
|
+
|
249
|
+
def test_enum_doc_attribute
|
250
|
+
enum_schema_json = Avro::Schema.parse <<-SCHEMA
|
251
|
+
{
|
252
|
+
"type": "enum",
|
253
|
+
"name": "Enum",
|
254
|
+
"namespace": "my.name.space",
|
255
|
+
"doc": "documentation",
|
256
|
+
"symbols" : [
|
257
|
+
"SPADES",
|
258
|
+
"HEARTS",
|
259
|
+
"DIAMONDS",
|
260
|
+
"CLUBS"
|
261
|
+
]
|
262
|
+
}
|
263
|
+
SCHEMA
|
264
|
+
|
265
|
+
enum_schema_hash =
|
266
|
+
{
|
267
|
+
'type' => 'enum',
|
268
|
+
'name' => 'Enum',
|
269
|
+
'namespace' => 'my.name.space',
|
270
|
+
'doc' => 'documentation',
|
271
|
+
'symbols' => [
|
272
|
+
'SPADES',
|
273
|
+
'HEARTS',
|
274
|
+
'DIAMONDS',
|
275
|
+
'CLUBS'
|
276
|
+
]
|
277
|
+
}
|
278
|
+
assert_equal enum_schema_hash, enum_schema_json.to_avro
|
279
|
+
end
|
280
|
+
|
281
|
+
def test_empty_record
|
282
|
+
schema = Avro::Schema.parse('{"type":"record", "name":"Empty"}')
|
283
|
+
assert_empty(schema.fields)
|
284
|
+
end
|
285
|
+
|
286
|
+
def test_empty_union
|
287
|
+
schema = Avro::Schema.parse('[]')
|
288
|
+
assert_equal(schema.to_s, '[]')
|
289
|
+
end
|
290
|
+
|
291
|
+
def test_read
|
292
|
+
schema = Avro::Schema.parse('"string"')
|
293
|
+
writer_schema = Avro::Schema.parse('"int"')
|
294
|
+
assert_false(schema.read?(writer_schema))
|
295
|
+
assert_true(schema.read?(schema))
|
296
|
+
end
|
297
|
+
|
298
|
+
def test_be_read
|
299
|
+
schema = Avro::Schema.parse('"string"')
|
300
|
+
writer_schema = Avro::Schema.parse('"int"')
|
301
|
+
assert_false(schema.be_read?(writer_schema))
|
302
|
+
assert_true(schema.be_read?(schema))
|
303
|
+
end
|
304
|
+
|
305
|
+
def test_mutual_read
|
306
|
+
schema = Avro::Schema.parse('"string"')
|
307
|
+
writer_schema = Avro::Schema.parse('"int"')
|
308
|
+
default1 = Avro::Schema.parse('{"type":"record", "name":"Default", "fields":[{"name":"i", "type":"int", "default": 1}]}')
|
309
|
+
default2 = Avro::Schema.parse('{"type":"record", "name":"Default", "fields":[{"name:":"s", "type":"string", "default": ""}]}')
|
310
|
+
assert_false(schema.mutual_read?(writer_schema))
|
311
|
+
assert_true(schema.mutual_read?(schema))
|
312
|
+
assert_true(default1.mutual_read?(default2))
|
313
|
+
end
|
314
|
+
|
315
|
+
def test_validate_defaults
|
316
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
317
|
+
hash_to_schema(
|
318
|
+
type: 'record',
|
319
|
+
name: 'fruits',
|
320
|
+
fields: [
|
321
|
+
{
|
322
|
+
name: 'veggies',
|
323
|
+
type: 'string',
|
324
|
+
default: nil
|
325
|
+
}
|
326
|
+
]
|
327
|
+
)
|
328
|
+
end
|
329
|
+
assert_equal('Error validating default for veggies: at . expected type string, got null',
|
330
|
+
exception.to_s)
|
331
|
+
end
|
332
|
+
|
333
|
+
def test_field_default_validation_disabled
|
334
|
+
Avro.disable_field_default_validation = true
|
335
|
+
assert_nothing_raised do
|
336
|
+
hash_to_schema(
|
337
|
+
type: 'record',
|
338
|
+
name: 'fruits',
|
339
|
+
fields: [
|
340
|
+
{
|
341
|
+
name: 'veggies',
|
342
|
+
type: 'string',
|
343
|
+
default: nil
|
344
|
+
}
|
345
|
+
]
|
346
|
+
)
|
347
|
+
end
|
348
|
+
ensure
|
349
|
+
Avro.disable_field_default_validation = false
|
350
|
+
end
|
351
|
+
|
352
|
+
def test_field_default_validation_disabled_via_env
|
353
|
+
Avro.disable_field_default_validation = false
|
354
|
+
ENV['AVRO_DISABLE_FIELD_DEFAULT_VALIDATION'] = "1"
|
355
|
+
|
356
|
+
assert_nothing_raised do
|
357
|
+
hash_to_schema(
|
358
|
+
type: 'record',
|
359
|
+
name: 'fruits',
|
360
|
+
fields: [
|
361
|
+
{
|
362
|
+
name: 'veggies',
|
363
|
+
type: 'string',
|
364
|
+
default: nil
|
365
|
+
}
|
366
|
+
]
|
367
|
+
)
|
368
|
+
end
|
369
|
+
ensure
|
370
|
+
ENV.delete('AVRO_DISABLE_FIELD_DEFAULT_VALIDATION')
|
371
|
+
Avro.disable_field_default_validation = false
|
372
|
+
end
|
373
|
+
|
374
|
+
def test_validate_record_valid_default
|
375
|
+
assert_nothing_raised(Avro::SchemaParseError) do
|
376
|
+
hash_to_schema(
|
377
|
+
type: 'record',
|
378
|
+
name: 'with_subrecord',
|
379
|
+
fields: [
|
380
|
+
{
|
381
|
+
name: 'sub',
|
382
|
+
type: {
|
383
|
+
name: 'subrecord',
|
384
|
+
type: 'record',
|
385
|
+
fields: [
|
386
|
+
{ type: 'string', name: 'x' }
|
387
|
+
]
|
388
|
+
},
|
389
|
+
default: {
|
390
|
+
x: "y"
|
391
|
+
}
|
392
|
+
}
|
393
|
+
]
|
394
|
+
)
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
def test_validate_record_invalid_default
|
399
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
400
|
+
hash_to_schema(
|
401
|
+
type: 'record',
|
402
|
+
name: 'with_subrecord',
|
403
|
+
fields: [
|
404
|
+
{
|
405
|
+
name: 'sub',
|
406
|
+
type: {
|
407
|
+
name: 'subrecord',
|
408
|
+
type: 'record',
|
409
|
+
fields: [
|
410
|
+
{ type: 'string', name: 'x' }
|
411
|
+
]
|
412
|
+
},
|
413
|
+
default: {
|
414
|
+
a: 1
|
415
|
+
}
|
416
|
+
}
|
417
|
+
]
|
418
|
+
)
|
419
|
+
end
|
420
|
+
assert_equal('Error validating default for sub: at .x expected type string, got null',
|
421
|
+
exception.to_s)
|
422
|
+
end
|
423
|
+
|
424
|
+
def test_validate_union_defaults
|
425
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
426
|
+
hash_to_schema(
|
427
|
+
type: 'record',
|
428
|
+
name: 'fruits',
|
429
|
+
fields: [
|
430
|
+
{
|
431
|
+
name: 'veggies',
|
432
|
+
type: %w(string null),
|
433
|
+
default: 5
|
434
|
+
}
|
435
|
+
]
|
436
|
+
)
|
437
|
+
end
|
438
|
+
assert_equal('Error validating default for veggies: at . expected type string, got int with value 5',
|
439
|
+
exception.to_s)
|
440
|
+
end
|
441
|
+
|
442
|
+
def test_validate_union_default_first_type
|
443
|
+
exception = assert_raise(Avro::SchemaParseError) do
|
444
|
+
hash_to_schema(
|
445
|
+
type: 'record',
|
446
|
+
name: 'fruits',
|
447
|
+
fields: [
|
448
|
+
{
|
449
|
+
name: 'veggies',
|
450
|
+
type: %w(null string),
|
451
|
+
default: 'apple'
|
452
|
+
}
|
453
|
+
]
|
454
|
+
)
|
455
|
+
end
|
456
|
+
assert_equal('Error validating default for veggies: at . expected type null, got string with value "apple"',
|
457
|
+
exception.to_s)
|
458
|
+
end
|
163
459
|
end
|