avro 1.9.2 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Manifest +1 -1
- data/NOTICE +1 -1
- data/Rakefile +20 -29
- data/avro.gemspec +36 -29
- data/interop/test_interop.rb +6 -3
- data/lib/avro/VERSION.txt +1 -0
- data/lib/avro/data_file.rb +28 -4
- data/lib/avro/io.rb +41 -35
- data/lib/avro/ipc.rb +11 -7
- data/lib/avro/logical_types.rb +186 -2
- data/lib/avro/protocol.rb +1 -0
- data/lib/avro/schema.rb +233 -31
- data/lib/avro/schema_compatibility.rb +31 -20
- data/lib/avro/schema_normalization.rb +1 -0
- data/lib/avro/schema_validator.rb +40 -34
- data/lib/avro.rb +15 -2
- data/test/case_finder.rb +9 -3
- data/test/random_data.rb +8 -6
- data/test/sample_ipc_client.rb +1 -0
- data/test/sample_ipc_http_client.rb +1 -0
- data/test/sample_ipc_http_server.rb +1 -0
- data/test/sample_ipc_server.rb +1 -0
- data/test/test_datafile.rb +14 -0
- data/test/test_fingerprints.rb +20 -0
- data/test/test_help.rb +1 -0
- data/test/test_io.rb +113 -17
- data/test/test_logical_types.rb +138 -1
- data/test/test_protocol.rb +2 -1
- data/test/test_schema.rb +398 -0
- data/test/test_schema_compatibility.rb +176 -0
- data/test/test_schema_normalization.rb +2 -0
- data/test/test_schema_validator.rb +26 -5
- data/test/test_socket_transport.rb +1 -0
- data/test/tool.rb +9 -8
- metadata +30 -32
- data/CHANGELOG +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: d08fa062d2fbd82eeb60d63bf9259103206ec27da4da2b46e1d5e406a182901f
|
|
4
|
+
data.tar.gz: 6445d1ef065286d08b4dbd325b2613fbe975cb303835cdf036429bea6ac94650
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 37e345d05baecef377495cb440dc108b1347638b0feb4cdc2d62807f307e62c4ed384e231d605fd672061b6411b9e226296e24670800cce22a0fc09f0bd09a54
|
|
7
|
+
data.tar.gz: dd48ba91caaf404b90d10e497b297122b3bd9d074ffe8dc44e5f10d582d54759572771dac086b37ab6f0f4766a4d1483c3f29342ff4eb9671c741876552475d7
|
data/Manifest
CHANGED
data/NOTICE
CHANGED
data/Rakefile
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -14,22 +15,20 @@
|
|
|
14
15
|
# See the License for the specific language governing permissions and
|
|
15
16
|
# limitations under the License.
|
|
16
17
|
|
|
17
|
-
require
|
|
18
|
-
require '
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
p.email = "dev@avro.apache.org"
|
|
23
|
-
p.summary = "Apache Avro for Ruby"
|
|
24
|
-
p.description = "Avro is a data serialization and RPC format"
|
|
25
|
-
p.url = "https://avro.apache.org/"
|
|
26
|
-
p.runtime_dependencies = %w[multi_json]
|
|
27
|
-
p.licenses = ["Apache License 2.0 (Apache-2.0)"]
|
|
18
|
+
require "bundler/gem_tasks"
|
|
19
|
+
require 'rake/testtask'
|
|
20
|
+
|
|
21
|
+
Rake::TestTask.new(:interop) do |t|
|
|
22
|
+
t.pattern = 'interop/test*.rb'
|
|
28
23
|
end
|
|
29
24
|
|
|
30
|
-
|
|
31
|
-
t.
|
|
25
|
+
Rake::TestTask.new(:test) do |t|
|
|
26
|
+
t.libs << "test"
|
|
27
|
+
t.pattern = 'test/test_*.rb'
|
|
28
|
+
t.verbose = true
|
|
29
|
+
end
|
|
32
30
|
|
|
31
|
+
desc "Generate data for interop tests"
|
|
33
32
|
task :generate_interop do
|
|
34
33
|
$:.unshift(HERE + '/lib')
|
|
35
34
|
$:.unshift(HERE + '/test')
|
|
@@ -38,27 +37,19 @@ task :generate_interop do
|
|
|
38
37
|
|
|
39
38
|
schema = Avro::Schema.parse(File.read(SCHEMAS + '/interop.avsc'))
|
|
40
39
|
r = RandomData.new(schema, ENV['SEED'])
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
Avro::DataFile.open(BUILD + '/interop/data/ruby_deflate.avro', 'w', schema.to_s, :deflate) do |writer|
|
|
51
|
-
20.times { writer << r.next }
|
|
40
|
+
Avro::DataFile.codecs.each do |name, codec|
|
|
41
|
+
next unless codec
|
|
42
|
+
filename = name == 'null' ? 'ruby.avro' : "ruby_#{name}.avro"
|
|
43
|
+
path = File.join(BUILD, 'interop/data', filename)
|
|
44
|
+
Avro::DataFile.open(path, 'w', schema.to_s, name) do |writer|
|
|
45
|
+
writer << r.next
|
|
46
|
+
end
|
|
52
47
|
end
|
|
53
48
|
end
|
|
54
49
|
|
|
55
|
-
|
|
56
50
|
HERE = File.expand_path(File.dirname(__FILE__))
|
|
57
51
|
SHARE = HERE + '/../../share'
|
|
58
52
|
SCHEMAS = SHARE + '/test/schemas'
|
|
59
53
|
BUILD = HERE + '/../../build'
|
|
60
54
|
|
|
61
|
-
task :
|
|
62
|
-
mkdir_p "../../dist/ruby"
|
|
63
|
-
cp "pkg/avro-#{VERSION}.gem", "../../dist/ruby"
|
|
64
|
-
end
|
|
55
|
+
task default: :test
|
data/avro.gemspec
CHANGED
|
@@ -1,35 +1,42 @@
|
|
|
1
|
-
#
|
|
2
|
-
#
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
+
# or more contributor license agreements. See the NOTICE file
|
|
4
|
+
# distributed with this work for additional information
|
|
5
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
+
# to you under the Apache License, Version 2.0 (the
|
|
7
|
+
# "License"); you may not use this file except in compliance
|
|
8
|
+
# with the License. You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
# See the License for the specific language governing permissions and
|
|
16
|
+
# limitations under the License.
|
|
3
17
|
|
|
4
18
|
Gem::Specification.new do |s|
|
|
5
|
-
s.name = "avro"
|
|
6
|
-
s.version = "
|
|
19
|
+
s.name = "avro"
|
|
20
|
+
s.version = File.read("lib/avro/VERSION.txt")
|
|
21
|
+
s.authors = ["Apache Software Foundation"]
|
|
22
|
+
s.email = "dev@avro.apache.org"
|
|
7
23
|
|
|
8
|
-
s.
|
|
9
|
-
s.
|
|
10
|
-
s.
|
|
11
|
-
s.
|
|
12
|
-
s.
|
|
13
|
-
s.email = "dev@avro.apache.org".freeze
|
|
14
|
-
s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
|
|
15
|
-
s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
|
|
16
|
-
s.homepage = "https://avro.apache.org/".freeze
|
|
17
|
-
s.licenses = ["Apache License 2.0 (Apache-2.0)".freeze]
|
|
18
|
-
s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro".freeze]
|
|
19
|
-
s.rubyforge_project = "avro".freeze
|
|
20
|
-
s.rubygems_version = "2.5.2.1".freeze
|
|
21
|
-
s.summary = "Apache Avro for Ruby".freeze
|
|
22
|
-
s.test_files = ["test/test_schema.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_help.rb".freeze, "test/test_datafile.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_fingerprints.rb".freeze]
|
|
24
|
+
s.summary = "Apache Avro for Ruby"
|
|
25
|
+
s.description = "Avro is a data serialization and RPC format"
|
|
26
|
+
s.homepage = "https://avro.apache.org/"
|
|
27
|
+
s.license = "Apache-2.0"
|
|
28
|
+
s.required_ruby_version = ">= 2.6"
|
|
23
29
|
|
|
24
|
-
|
|
25
|
-
|
|
30
|
+
s.metadata["homepage_uri"] = s.homepage
|
|
31
|
+
s.metadata["bug_tracker_uri"] = "https://issues.apache.org/jira/browse/AVRO"
|
|
32
|
+
s.metadata["source_code_uri"] = "https://github.com/apache/avro"
|
|
33
|
+
s.metadata["documentation_uri"] = "https://avro.apache.org/docs/#{s.version}/"
|
|
26
34
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
end
|
|
35
|
+
files = File.read("Manifest").split("\n")
|
|
36
|
+
s.files = files.reject { |f| f.start_with?("test/") }
|
|
37
|
+
s.rdoc_options = ["--line-numbers", "--title", "Avro"]
|
|
38
|
+
s.test_files = files.select { |f| f.start_with?("test/") }
|
|
39
|
+
s.require_paths = ["lib"]
|
|
40
|
+
|
|
41
|
+
s.add_dependency("multi_json", "~> 1.0")
|
|
35
42
|
end
|
data/interop/test_interop.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
|
4
5
|
# distributed with this work for additional information
|
|
@@ -19,7 +20,7 @@ require 'rubygems'
|
|
|
19
20
|
require 'test/unit'
|
|
20
21
|
require 'avro'
|
|
21
22
|
|
|
22
|
-
CODECS_TO_VALIDATE = ['deflate'] # The 'null' codec is implicitly included
|
|
23
|
+
CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard'].freeze # The 'null' codec is implicitly included
|
|
23
24
|
|
|
24
25
|
class TestInterop < Test::Unit::TestCase
|
|
25
26
|
HERE = File.expand_path(File.dirname(__FILE__))
|
|
@@ -27,12 +28,14 @@ class TestInterop < Test::Unit::TestCase
|
|
|
27
28
|
SCHEMAS = SHARE + '/test/schemas'
|
|
28
29
|
|
|
29
30
|
files = Dir[HERE + '/../../../build/interop/data/*.avro'].select do |fn|
|
|
30
|
-
sep, codec = File.basename(fn, 'avro').rpartition('_')[1, 2]
|
|
31
|
+
sep, codec = File.basename(fn, '.avro').rpartition('_')[1, 2]
|
|
31
32
|
sep.empty? || CODECS_TO_VALIDATE.include?(codec)
|
|
32
33
|
end
|
|
34
|
+
puts "The following files will be tested:"
|
|
35
|
+
puts files
|
|
33
36
|
|
|
34
37
|
files.each do |fn|
|
|
35
|
-
define_method("test_read_#{File.basename(fn, 'avro')}") do
|
|
38
|
+
define_method("test_read_#{File.basename(fn, '.avro')}") do
|
|
36
39
|
projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
|
|
37
40
|
|
|
38
41
|
File.open(fn) do |f|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
1.11.0
|
data/lib/avro/data_file.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -5,9 +6,9 @@
|
|
|
5
6
|
# to you under the Apache License, Version 2.0 (the
|
|
6
7
|
# "License"); you may not use this file except in compliance
|
|
7
8
|
# with the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
+
#
|
|
9
10
|
# https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
+
#
|
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
14
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
@@ -25,7 +26,7 @@ module Avro
|
|
|
25
26
|
SYNC_SIZE = 16
|
|
26
27
|
SYNC_INTERVAL = 4000 * SYNC_SIZE
|
|
27
28
|
META_SCHEMA = Schema.parse('{"type": "map", "values": "bytes"}')
|
|
28
|
-
VALID_ENCODINGS = ['binary'] # not used yet
|
|
29
|
+
VALID_ENCODINGS = ['binary'].freeze # not used yet
|
|
29
30
|
|
|
30
31
|
class DataFileError < AvroError; end
|
|
31
32
|
|
|
@@ -99,7 +100,7 @@ module Avro
|
|
|
99
100
|
@encoder = IO::BinaryEncoder.new(@writer)
|
|
100
101
|
@datum_writer = datum_writer
|
|
101
102
|
@meta = meta
|
|
102
|
-
@buffer_writer = StringIO.new('', 'w')
|
|
103
|
+
@buffer_writer = StringIO.new(+'', 'w')
|
|
103
104
|
@buffer_writer.set_encoding('BINARY') if @buffer_writer.respond_to?(:set_encoding)
|
|
104
105
|
@buffer_encoder = IO::BinaryEncoder.new(@buffer_writer)
|
|
105
106
|
@block_count = 0
|
|
@@ -372,9 +373,32 @@ module Avro
|
|
|
372
373
|
end
|
|
373
374
|
end
|
|
374
375
|
|
|
376
|
+
class ZstandardCodec
|
|
377
|
+
def codec_name; 'zstandard'; end
|
|
378
|
+
|
|
379
|
+
def decompress(data)
|
|
380
|
+
load_zstandard!
|
|
381
|
+
Zstd.decompress(data)
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
def compress(data)
|
|
385
|
+
load_zstandard!
|
|
386
|
+
Zstd.compress(data)
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
private
|
|
390
|
+
|
|
391
|
+
def load_zstandard!
|
|
392
|
+
require 'zstd-ruby' unless defined?(Zstd)
|
|
393
|
+
rescue LoadError
|
|
394
|
+
raise LoadError, "Zstandard compression is not available, please install the `zstd-ruby` gem."
|
|
395
|
+
end
|
|
396
|
+
end
|
|
397
|
+
|
|
375
398
|
DataFile.register_codec NullCodec
|
|
376
399
|
DataFile.register_codec DeflateCodec
|
|
377
400
|
DataFile.register_codec SnappyCodec
|
|
401
|
+
DataFile.register_codec ZstandardCodec
|
|
378
402
|
|
|
379
403
|
# TODO this constant won't be updated if you register another codec.
|
|
380
404
|
# Deprecated in favor of Avro::DataFile::codecs
|
data/lib/avro/io.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -76,7 +77,7 @@ module Avro
|
|
|
76
77
|
# The float is converted into a 32-bit integer using a method
|
|
77
78
|
# equivalent to Java's floatToIntBits and then encoded in
|
|
78
79
|
# little-endian format.
|
|
79
|
-
read_and_unpack(4, 'e'
|
|
80
|
+
read_and_unpack(4, 'e')
|
|
80
81
|
end
|
|
81
82
|
|
|
82
83
|
def read_double
|
|
@@ -84,7 +85,7 @@ module Avro
|
|
|
84
85
|
# The double is converted into a 64-bit integer using a method
|
|
85
86
|
# equivalent to Java's doubleToLongBits and then encoded in
|
|
86
87
|
# little-endian format.
|
|
87
|
-
read_and_unpack(8, 'E'
|
|
88
|
+
read_and_unpack(8, 'E')
|
|
88
89
|
end
|
|
89
90
|
|
|
90
91
|
def read_bytes
|
|
@@ -97,7 +98,7 @@ module Avro
|
|
|
97
98
|
# A string is encoded as a long followed by that many bytes of
|
|
98
99
|
# UTF-8 encoded character data.
|
|
99
100
|
read_bytes.tap do |string|
|
|
100
|
-
string.force_encoding('UTF-8'
|
|
101
|
+
string.force_encoding('UTF-8') if string.respond_to? :force_encoding
|
|
101
102
|
end
|
|
102
103
|
end
|
|
103
104
|
|
|
@@ -172,7 +173,7 @@ module Avro
|
|
|
172
173
|
end
|
|
173
174
|
|
|
174
175
|
# null is written as zero bytes
|
|
175
|
-
def write_null(
|
|
176
|
+
def write_null(_datum)
|
|
176
177
|
nil
|
|
177
178
|
end
|
|
178
179
|
|
|
@@ -205,7 +206,7 @@ module Avro
|
|
|
205
206
|
# equivalent to Java's floatToIntBits and then encoded in
|
|
206
207
|
# little-endian format.
|
|
207
208
|
def write_float(datum)
|
|
208
|
-
@writer.write([datum].pack('e'
|
|
209
|
+
@writer.write([datum].pack('e'))
|
|
209
210
|
end
|
|
210
211
|
|
|
211
212
|
# A double is written as 8 bytes.
|
|
@@ -213,7 +214,7 @@ module Avro
|
|
|
213
214
|
# equivalent to Java's doubleToLongBits and then encoded in
|
|
214
215
|
# little-endian format.
|
|
215
216
|
def write_double(datum)
|
|
216
|
-
@writer.write([datum].pack('E'
|
|
217
|
+
@writer.write([datum].pack('E'))
|
|
217
218
|
end
|
|
218
219
|
|
|
219
220
|
# Bytes are encoded as a long followed by that many bytes of data.
|
|
@@ -225,7 +226,7 @@ module Avro
|
|
|
225
226
|
# A string is encoded as a long followed by that many bytes of
|
|
226
227
|
# UTF-8 encoded character data
|
|
227
228
|
def write_string(datum)
|
|
228
|
-
datum = datum.encode('utf-8'
|
|
229
|
+
datum = datum.encode('utf-8') if datum.respond_to? :encode
|
|
229
230
|
write_bytes(datum)
|
|
230
231
|
end
|
|
231
232
|
|
|
@@ -292,7 +293,7 @@ module Avro
|
|
|
292
293
|
readers_schema.type_adapter.decode(datum)
|
|
293
294
|
end
|
|
294
295
|
|
|
295
|
-
def read_fixed(writers_schema,
|
|
296
|
+
def read_fixed(writers_schema, _readers_schema, decoder)
|
|
296
297
|
decoder.read(writers_schema.size)
|
|
297
298
|
end
|
|
298
299
|
|
|
@@ -300,12 +301,12 @@ module Avro
|
|
|
300
301
|
index_of_symbol = decoder.read_int
|
|
301
302
|
read_symbol = writers_schema.symbols[index_of_symbol]
|
|
302
303
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
unless readers_schema.symbols.include?(read_symbol)
|
|
306
|
-
# 'unset' here
|
|
304
|
+
if !readers_schema.symbols.include?(read_symbol) && readers_schema.default
|
|
305
|
+
read_symbol = readers_schema.default
|
|
307
306
|
end
|
|
308
307
|
|
|
308
|
+
# This implementation deviates from the spec by always returning
|
|
309
|
+
# a symbol.
|
|
309
310
|
read_symbol
|
|
310
311
|
end
|
|
311
312
|
|
|
@@ -359,26 +360,28 @@ module Avro
|
|
|
359
360
|
readers_fields_hash = readers_schema.fields_hash
|
|
360
361
|
read_record = {}
|
|
361
362
|
writers_schema.fields.each do |field|
|
|
362
|
-
|
|
363
|
+
readers_field = readers_fields_hash[field.name]
|
|
364
|
+
if readers_field
|
|
363
365
|
field_val = read_data(field.type, readers_field.type, decoder)
|
|
364
366
|
read_record[field.name] = field_val
|
|
367
|
+
elsif readers_schema.fields_by_alias.key?(field.name)
|
|
368
|
+
readers_field = readers_schema.fields_by_alias[field.name]
|
|
369
|
+
field_val = read_data(field.type, readers_field.type, decoder)
|
|
370
|
+
read_record[readers_field.name] = field_val
|
|
365
371
|
else
|
|
366
372
|
skip_data(field.type, decoder)
|
|
367
373
|
end
|
|
368
374
|
end
|
|
369
375
|
|
|
370
376
|
# fill in the default values
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
raise AvroError, "Missing data for #{field.type} with no default"
|
|
380
|
-
end
|
|
381
|
-
end
|
|
377
|
+
readers_fields_hash.each do |field_name, field|
|
|
378
|
+
next if read_record.key?(field_name)
|
|
379
|
+
|
|
380
|
+
if field.default?
|
|
381
|
+
field_val = read_default_value(field.type, field.default)
|
|
382
|
+
read_record[field.name] = field_val
|
|
383
|
+
else
|
|
384
|
+
raise AvroError, "Missing data for #{field.type} with no default"
|
|
382
385
|
end
|
|
383
386
|
end
|
|
384
387
|
|
|
@@ -390,13 +393,11 @@ module Avro
|
|
|
390
393
|
case field_schema.type_sym
|
|
391
394
|
when :null
|
|
392
395
|
return nil
|
|
393
|
-
when :boolean
|
|
394
|
-
return default_value
|
|
395
396
|
when :int, :long
|
|
396
397
|
return Integer(default_value)
|
|
397
398
|
when :float, :double
|
|
398
399
|
return Float(default_value)
|
|
399
|
-
when :enum, :fixed, :string, :bytes
|
|
400
|
+
when :boolean, :enum, :fixed, :string, :bytes
|
|
400
401
|
return default_value
|
|
401
402
|
when :array
|
|
402
403
|
read_array = []
|
|
@@ -468,7 +469,7 @@ module Avro
|
|
|
468
469
|
decoder.skip(writers_schema.size)
|
|
469
470
|
end
|
|
470
471
|
|
|
471
|
-
def skip_enum(
|
|
472
|
+
def skip_enum(_writers_schema, decoder)
|
|
472
473
|
decoder.skip_int
|
|
473
474
|
end
|
|
474
475
|
|
|
@@ -508,6 +509,8 @@ module Avro
|
|
|
508
509
|
|
|
509
510
|
# DatumWriter for generic ruby objects
|
|
510
511
|
class DatumWriter
|
|
512
|
+
VALIDATION_OPTIONS = { recursive: false, encoded: true }.freeze
|
|
513
|
+
|
|
511
514
|
attr_accessor :writers_schema
|
|
512
515
|
def initialize(writers_schema=nil)
|
|
513
516
|
@writers_schema = writers_schema
|
|
@@ -520,7 +523,7 @@ module Avro
|
|
|
520
523
|
def write_data(writers_schema, logical_datum, encoder)
|
|
521
524
|
datum = writers_schema.type_adapter.encode(logical_datum)
|
|
522
525
|
|
|
523
|
-
unless Schema.validate(writers_schema, datum,
|
|
526
|
+
unless Schema.validate(writers_schema, datum, VALIDATION_OPTIONS)
|
|
524
527
|
raise AvroTypeError.new(writers_schema, datum)
|
|
525
528
|
end
|
|
526
529
|
|
|
@@ -545,7 +548,7 @@ module Avro
|
|
|
545
548
|
end
|
|
546
549
|
end
|
|
547
550
|
|
|
548
|
-
def write_fixed(
|
|
551
|
+
def write_fixed(_writers_schema, datum, encoder)
|
|
549
552
|
encoder.write(datum)
|
|
550
553
|
end
|
|
551
554
|
|
|
@@ -578,12 +581,15 @@ module Avro
|
|
|
578
581
|
end
|
|
579
582
|
|
|
580
583
|
def write_union(writers_schema, datum, encoder)
|
|
581
|
-
index_of_schema =
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
584
|
+
index_of_schema = writers_schema.schemas.find_index do |schema|
|
|
585
|
+
# Optimize away expensive validation calls for the common null type
|
|
586
|
+
schema.type_sym == :null ? datum.nil? : Schema.validate(schema, datum)
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
unless index_of_schema
|
|
585
590
|
raise AvroTypeError.new(writers_schema, datum)
|
|
586
591
|
end
|
|
592
|
+
|
|
587
593
|
encoder.write_long(index_of_schema)
|
|
588
594
|
write_data(writers_schema.schemas[index_of_schema], datum, encoder)
|
|
589
595
|
end
|
|
@@ -591,7 +597,7 @@ module Avro
|
|
|
591
597
|
def write_record(writers_schema, datum, encoder)
|
|
592
598
|
raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Hash)
|
|
593
599
|
writers_schema.fields.each do |field|
|
|
594
|
-
write_data(field.type, datum[field.name], encoder)
|
|
600
|
+
write_data(field.type, datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym], encoder)
|
|
595
601
|
end
|
|
596
602
|
end
|
|
597
603
|
end # DatumWriter
|
data/lib/avro/ipc.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
|
3
4
|
# distributed with this work for additional information
|
|
@@ -63,8 +64,11 @@ module Avro::IPC
|
|
|
63
64
|
SYSTEM_ERROR_SCHEMA = Avro::Schema.parse('["string"]')
|
|
64
65
|
|
|
65
66
|
# protocol cache
|
|
67
|
+
# rubocop:disable Style/MutableConstant
|
|
66
68
|
REMOTE_HASHES = {}
|
|
67
69
|
REMOTE_PROTOCOLS = {}
|
|
70
|
+
# rubocop:enable Style/MutableConstant
|
|
71
|
+
|
|
68
72
|
|
|
69
73
|
BUFFER_HEADER_LENGTH = 4
|
|
70
74
|
BUFFER_SIZE = 8192
|
|
@@ -100,7 +104,7 @@ module Avro::IPC
|
|
|
100
104
|
def request(message_name, request_datum)
|
|
101
105
|
# Writes a request message and reads a response or error message.
|
|
102
106
|
# build handshake and call request
|
|
103
|
-
buffer_writer = StringIO.new(
|
|
107
|
+
buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
|
|
104
108
|
buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
|
|
105
109
|
write_handshake_request(buffer_encoder)
|
|
106
110
|
write_call_request(message_name, request_datum, buffer_encoder)
|
|
@@ -244,7 +248,7 @@ module Avro::IPC
|
|
|
244
248
|
# a response or error. Compare to 'handle()' in Thrift.
|
|
245
249
|
def respond(call_request, transport=nil)
|
|
246
250
|
buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_request))
|
|
247
|
-
buffer_writer = StringIO.new(
|
|
251
|
+
buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
|
|
248
252
|
buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
|
|
249
253
|
error = nil
|
|
250
254
|
response_metadata = {}
|
|
@@ -278,7 +282,7 @@ module Avro::IPC
|
|
|
278
282
|
response = call(local_message, request)
|
|
279
283
|
rescue AvroRemoteError => e
|
|
280
284
|
error = e
|
|
281
|
-
rescue Exception => e
|
|
285
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
282
286
|
error = AvroRemoteError.new(e.to_s)
|
|
283
287
|
end
|
|
284
288
|
|
|
@@ -350,7 +354,7 @@ module Avro::IPC
|
|
|
350
354
|
remote_protocol
|
|
351
355
|
end
|
|
352
356
|
|
|
353
|
-
def call(
|
|
357
|
+
def call(_local_message, _request)
|
|
354
358
|
# Actual work done by server: cf. handler in thrift.
|
|
355
359
|
raise NotImplementedError
|
|
356
360
|
end
|
|
@@ -394,7 +398,7 @@ module Avro::IPC
|
|
|
394
398
|
def read_framed_message
|
|
395
399
|
message = []
|
|
396
400
|
loop do
|
|
397
|
-
buffer = StringIO.new(
|
|
401
|
+
buffer = StringIO.new(String.new('', encoding: 'BINARY'))
|
|
398
402
|
buffer_length = read_buffer_length
|
|
399
403
|
if buffer_length == 0
|
|
400
404
|
return message.join
|
|
@@ -506,7 +510,7 @@ module Avro::IPC
|
|
|
506
510
|
def read_framed_message
|
|
507
511
|
message = []
|
|
508
512
|
loop do
|
|
509
|
-
buffer =
|
|
513
|
+
buffer = String.new('', encoding: 'BINARY')
|
|
510
514
|
buffer_size = read_buffer_size
|
|
511
515
|
|
|
512
516
|
return message.join if buffer_size == 0
|
|
@@ -542,7 +546,7 @@ module Avro::IPC
|
|
|
542
546
|
end
|
|
543
547
|
|
|
544
548
|
def transceive(message)
|
|
545
|
-
writer = FramedWriter.new(StringIO.new(
|
|
549
|
+
writer = FramedWriter.new(StringIO.new(String.new('', encoding: 'BINARY')))
|
|
546
550
|
writer.write_framed_message(message)
|
|
547
551
|
resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
|
|
548
552
|
FramedReader.new(StringIO.new(resp.body)).read_framed_message
|