avro 1.9.0 → 1.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/LICENSE +2 -2
- data/Manifest +1 -1
- data/NOTICE +2 -2
- data/Rakefile +21 -30
- data/avro.gemspec +36 -29
- data/interop/test_interop.rb +14 -3
- data/lib/avro/VERSION.txt +1 -0
- data/lib/avro/data_file.rb +30 -6
- data/lib/avro/io.rb +42 -36
- data/lib/avro/ipc.rb +12 -8
- data/lib/avro/logical_types.rb +187 -3
- data/lib/avro/protocol.rb +2 -1
- data/lib/avro/schema.rb +234 -32
- data/lib/avro/schema_compatibility.rb +32 -21
- data/lib/avro/schema_normalization.rb +2 -1
- data/lib/avro/schema_validator.rb +41 -35
- data/lib/avro.rb +16 -3
- data/test/case_finder.rb +10 -4
- data/test/random_data.rb +9 -7
- data/test/sample_ipc_client.rb +2 -1
- data/test/sample_ipc_http_client.rb +2 -1
- data/test/sample_ipc_http_server.rb +2 -1
- data/test/sample_ipc_server.rb +2 -1
- data/test/test_datafile.rb +15 -1
- data/test/test_fingerprints.rb +21 -1
- data/test/test_help.rb +2 -1
- data/test/test_io.rb +114 -18
- data/test/test_logical_types.rb +139 -2
- data/test/test_protocol.rb +3 -2
- data/test/test_schema.rb +399 -1
- data/test/test_schema_compatibility.rb +177 -1
- data/test/test_schema_normalization.rb +3 -1
- data/test/test_schema_validator.rb +27 -6
- data/test/test_socket_transport.rb +2 -1
- data/test/tool.rb +10 -9
- metadata +32 -34
- data/CHANGELOG +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d08fa062d2fbd82eeb60d63bf9259103206ec27da4da2b46e1d5e406a182901f
|
4
|
+
data.tar.gz: 6445d1ef065286d08b4dbd325b2613fbe975cb303835cdf036429bea6ac94650
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 37e345d05baecef377495cb440dc108b1347638b0feb4cdc2d62807f307e62c4ed384e231d605fd672061b6411b9e226296e24670800cce22a0fc09f0bd09a54
|
7
|
+
data.tar.gz: dd48ba91caaf404b90d10e497b297122b3bd9d074ffe8dc44e5f10d582d54759572771dac086b37ab6f0f4766a4d1483c3f29342ff4eb9671c741876552475d7
|
data/LICENSE
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Apache License
|
3
3
|
Version 2.0, January 2004
|
4
|
-
|
4
|
+
https://www.apache.org/licenses/
|
5
5
|
|
6
6
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
7
7
|
|
@@ -193,7 +193,7 @@
|
|
193
193
|
you may not use this file except in compliance with the License.
|
194
194
|
You may obtain a copy of the License at
|
195
195
|
|
196
|
-
|
196
|
+
https://www.apache.org/licenses/LICENSE-2.0
|
197
197
|
|
198
198
|
Unless required by applicable law or agreed to in writing, software
|
199
199
|
distributed under the License is distributed on an "AS IS" BASIS,
|
data/Manifest
CHANGED
data/NOTICE
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Apache Avro
|
2
|
-
Copyright 2010-
|
2
|
+
Copyright 2010-2021 The Apache Software Foundation
|
3
3
|
|
4
4
|
This product includes software developed at
|
5
|
-
The Apache Software Foundation (
|
5
|
+
The Apache Software Foundation (https://www.apache.org/).
|
6
6
|
|
data/Rakefile
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -6,7 +7,7 @@
|
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
9
|
#
|
9
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
11
|
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -14,22 +15,20 @@
|
|
14
15
|
# See the License for the specific language governing permissions and
|
15
16
|
# limitations under the License.
|
16
17
|
|
17
|
-
require
|
18
|
-
require '
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
p.email = "dev@avro.apache.org"
|
23
|
-
p.summary = "Apache Avro for Ruby"
|
24
|
-
p.description = "Avro is a data serialization and RPC format"
|
25
|
-
p.url = "http://avro.apache.org/"
|
26
|
-
p.runtime_dependencies = %w[multi_json]
|
27
|
-
p.licenses = ["Apache License 2.0 (Apache-2.0)"]
|
18
|
+
require "bundler/gem_tasks"
|
19
|
+
require 'rake/testtask'
|
20
|
+
|
21
|
+
Rake::TestTask.new(:interop) do |t|
|
22
|
+
t.pattern = 'interop/test*.rb'
|
28
23
|
end
|
29
24
|
|
30
|
-
|
31
|
-
t.
|
25
|
+
Rake::TestTask.new(:test) do |t|
|
26
|
+
t.libs << "test"
|
27
|
+
t.pattern = 'test/test_*.rb'
|
28
|
+
t.verbose = true
|
29
|
+
end
|
32
30
|
|
31
|
+
desc "Generate data for interop tests"
|
33
32
|
task :generate_interop do
|
34
33
|
$:.unshift(HERE + '/lib')
|
35
34
|
$:.unshift(HERE + '/test')
|
@@ -38,27 +37,19 @@ task :generate_interop do
|
|
38
37
|
|
39
38
|
schema = Avro::Schema.parse(File.read(SCHEMAS + '/interop.avsc'))
|
40
39
|
r = RandomData.new(schema, ENV['SEED'])
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
end
|
49
|
-
|
50
|
-
Avro::DataFile.open(BUILD + '/interop/data/ruby_deflate.avro', 'w', schema.to_s, :deflate) do |writer|
|
51
|
-
20.times { writer << r.next }
|
40
|
+
Avro::DataFile.codecs.each do |name, codec|
|
41
|
+
next unless codec
|
42
|
+
filename = name == 'null' ? 'ruby.avro' : "ruby_#{name}.avro"
|
43
|
+
path = File.join(BUILD, 'interop/data', filename)
|
44
|
+
Avro::DataFile.open(path, 'w', schema.to_s, name) do |writer|
|
45
|
+
writer << r.next
|
46
|
+
end
|
52
47
|
end
|
53
48
|
end
|
54
49
|
|
55
|
-
|
56
50
|
HERE = File.expand_path(File.dirname(__FILE__))
|
57
51
|
SHARE = HERE + '/../../share'
|
58
52
|
SCHEMAS = SHARE + '/test/schemas'
|
59
53
|
BUILD = HERE + '/../../build'
|
60
54
|
|
61
|
-
task :
|
62
|
-
mkdir_p "../../dist/ruby"
|
63
|
-
cp "pkg/avro-#{VERSION}.gem", "../../dist/ruby"
|
64
|
-
end
|
55
|
+
task default: :test
|
data/avro.gemspec
CHANGED
@@ -1,35 +1,42 @@
|
|
1
|
-
#
|
2
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
# or more contributor license agreements. See the NOTICE file
|
4
|
+
# distributed with this work for additional information
|
5
|
+
# regarding copyright ownership. The ASF licenses this file
|
6
|
+
# to you under the Apache License, Version 2.0 (the
|
7
|
+
# "License"); you may not use this file except in compliance
|
8
|
+
# with the License. You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
3
17
|
|
4
18
|
Gem::Specification.new do |s|
|
5
|
-
s.name = "avro"
|
6
|
-
s.version = "
|
19
|
+
s.name = "avro"
|
20
|
+
s.version = File.read("lib/avro/VERSION.txt")
|
21
|
+
s.authors = ["Apache Software Foundation"]
|
22
|
+
s.email = "dev@avro.apache.org"
|
7
23
|
|
8
|
-
s.
|
9
|
-
s.
|
10
|
-
s.
|
11
|
-
s.
|
12
|
-
s.
|
13
|
-
s.email = "dev@avro.apache.org".freeze
|
14
|
-
s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
|
15
|
-
s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
|
16
|
-
s.homepage = "http://avro.apache.org/".freeze
|
17
|
-
s.licenses = ["Apache License 2.0 (Apache-2.0)".freeze]
|
18
|
-
s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro".freeze]
|
19
|
-
s.rubyforge_project = "avro".freeze
|
20
|
-
s.rubygems_version = "2.5.2.1".freeze
|
21
|
-
s.summary = "Apache Avro for Ruby".freeze
|
22
|
-
s.test_files = ["test/test_schema_validator.rb".freeze, "test/test_help.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_datafile.rb".freeze, "test/test_schema.rb".freeze, "test/test_io.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_protocol.rb".freeze]
|
24
|
+
s.summary = "Apache Avro for Ruby"
|
25
|
+
s.description = "Avro is a data serialization and RPC format"
|
26
|
+
s.homepage = "https://avro.apache.org/"
|
27
|
+
s.license = "Apache-2.0"
|
28
|
+
s.required_ruby_version = ">= 2.6"
|
23
29
|
|
24
|
-
|
25
|
-
|
30
|
+
s.metadata["homepage_uri"] = s.homepage
|
31
|
+
s.metadata["bug_tracker_uri"] = "https://issues.apache.org/jira/browse/AVRO"
|
32
|
+
s.metadata["source_code_uri"] = "https://github.com/apache/avro"
|
33
|
+
s.metadata["documentation_uri"] = "https://avro.apache.org/docs/#{s.version}/"
|
26
34
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
end
|
35
|
+
files = File.read("Manifest").split("\n")
|
36
|
+
s.files = files.reject { |f| f.start_with?("test/") }
|
37
|
+
s.rdoc_options = ["--line-numbers", "--title", "Avro"]
|
38
|
+
s.test_files = files.select { |f| f.start_with?("test/") }
|
39
|
+
s.require_paths = ["lib"]
|
40
|
+
|
41
|
+
s.add_dependency("multi_json", "~> 1.0")
|
35
42
|
end
|
data/interop/test_interop.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
4
5
|
# distributed with this work for additional information
|
@@ -7,7 +8,7 @@
|
|
7
8
|
# "License"); you may not use this file except in compliance
|
8
9
|
# with the License. You may obtain a copy of the License at
|
9
10
|
#
|
10
|
-
#
|
11
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
12
|
#
|
12
13
|
# Unless required by applicable law or agreed to in writing, software
|
13
14
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -19,12 +20,22 @@ require 'rubygems'
|
|
19
20
|
require 'test/unit'
|
20
21
|
require 'avro'
|
21
22
|
|
23
|
+
CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard'].freeze # The 'null' codec is implicitly included
|
24
|
+
|
22
25
|
class TestInterop < Test::Unit::TestCase
|
23
26
|
HERE = File.expand_path(File.dirname(__FILE__))
|
24
27
|
SHARE = HERE + '/../../../share'
|
25
28
|
SCHEMAS = SHARE + '/test/schemas'
|
26
|
-
|
27
|
-
|
29
|
+
|
30
|
+
files = Dir[HERE + '/../../../build/interop/data/*.avro'].select do |fn|
|
31
|
+
sep, codec = File.basename(fn, '.avro').rpartition('_')[1, 2]
|
32
|
+
sep.empty? || CODECS_TO_VALIDATE.include?(codec)
|
33
|
+
end
|
34
|
+
puts "The following files will be tested:"
|
35
|
+
puts files
|
36
|
+
|
37
|
+
files.each do |fn|
|
38
|
+
define_method("test_read_#{File.basename(fn, '.avro')}") do
|
28
39
|
projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
|
29
40
|
|
30
41
|
File.open(fn) do |f|
|
@@ -0,0 +1 @@
|
|
1
|
+
1.11.0
|
data/lib/avro/data_file.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -5,9 +6,9 @@
|
|
5
6
|
# to you under the Apache License, Version 2.0 (the
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
#
|
10
|
-
#
|
9
|
+
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
14
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
@@ -25,7 +26,7 @@ module Avro
|
|
25
26
|
SYNC_SIZE = 16
|
26
27
|
SYNC_INTERVAL = 4000 * SYNC_SIZE
|
27
28
|
META_SCHEMA = Schema.parse('{"type": "map", "values": "bytes"}')
|
28
|
-
VALID_ENCODINGS = ['binary'] # not used yet
|
29
|
+
VALID_ENCODINGS = ['binary'].freeze # not used yet
|
29
30
|
|
30
31
|
class DataFileError < AvroError; end
|
31
32
|
|
@@ -99,7 +100,7 @@ module Avro
|
|
99
100
|
@encoder = IO::BinaryEncoder.new(@writer)
|
100
101
|
@datum_writer = datum_writer
|
101
102
|
@meta = meta
|
102
|
-
@buffer_writer = StringIO.new('', 'w')
|
103
|
+
@buffer_writer = StringIO.new(+'', 'w')
|
103
104
|
@buffer_writer.set_encoding('BINARY') if @buffer_writer.respond_to?(:set_encoding)
|
104
105
|
@buffer_encoder = IO::BinaryEncoder.new(@buffer_writer)
|
105
106
|
@block_count = 0
|
@@ -316,7 +317,7 @@ module Avro
|
|
316
317
|
def decompress(compressed)
|
317
318
|
# Passing a negative number to Inflate puts it into "raw" RFC1951 mode
|
318
319
|
# (without the RFC1950 header & checksum). See the docs for
|
319
|
-
# inflateInit2 in
|
320
|
+
# inflateInit2 in https://www.zlib.net/manual.html
|
320
321
|
zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
|
321
322
|
data = zstream.inflate(compressed)
|
322
323
|
data << zstream.finish
|
@@ -372,9 +373,32 @@ module Avro
|
|
372
373
|
end
|
373
374
|
end
|
374
375
|
|
376
|
+
class ZstandardCodec
|
377
|
+
def codec_name; 'zstandard'; end
|
378
|
+
|
379
|
+
def decompress(data)
|
380
|
+
load_zstandard!
|
381
|
+
Zstd.decompress(data)
|
382
|
+
end
|
383
|
+
|
384
|
+
def compress(data)
|
385
|
+
load_zstandard!
|
386
|
+
Zstd.compress(data)
|
387
|
+
end
|
388
|
+
|
389
|
+
private
|
390
|
+
|
391
|
+
def load_zstandard!
|
392
|
+
require 'zstd-ruby' unless defined?(Zstd)
|
393
|
+
rescue LoadError
|
394
|
+
raise LoadError, "Zstandard compression is not available, please install the `zstd-ruby` gem."
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
375
398
|
DataFile.register_codec NullCodec
|
376
399
|
DataFile.register_codec DeflateCodec
|
377
400
|
DataFile.register_codec SnappyCodec
|
401
|
+
DataFile.register_codec ZstandardCodec
|
378
402
|
|
379
403
|
# TODO this constant won't be updated if you register another codec.
|
380
404
|
# Deprecated in favor of Avro::DataFile::codecs
|
data/lib/avro/io.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -6,7 +7,7 @@
|
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
9
|
#
|
9
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
11
|
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -76,7 +77,7 @@ module Avro
|
|
76
77
|
# The float is converted into a 32-bit integer using a method
|
77
78
|
# equivalent to Java's floatToIntBits and then encoded in
|
78
79
|
# little-endian format.
|
79
|
-
read_and_unpack(4, 'e'
|
80
|
+
read_and_unpack(4, 'e')
|
80
81
|
end
|
81
82
|
|
82
83
|
def read_double
|
@@ -84,7 +85,7 @@ module Avro
|
|
84
85
|
# The double is converted into a 64-bit integer using a method
|
85
86
|
# equivalent to Java's doubleToLongBits and then encoded in
|
86
87
|
# little-endian format.
|
87
|
-
read_and_unpack(8, 'E'
|
88
|
+
read_and_unpack(8, 'E')
|
88
89
|
end
|
89
90
|
|
90
91
|
def read_bytes
|
@@ -97,7 +98,7 @@ module Avro
|
|
97
98
|
# A string is encoded as a long followed by that many bytes of
|
98
99
|
# UTF-8 encoded character data.
|
99
100
|
read_bytes.tap do |string|
|
100
|
-
string.force_encoding('UTF-8'
|
101
|
+
string.force_encoding('UTF-8') if string.respond_to? :force_encoding
|
101
102
|
end
|
102
103
|
end
|
103
104
|
|
@@ -172,7 +173,7 @@ module Avro
|
|
172
173
|
end
|
173
174
|
|
174
175
|
# null is written as zero bytes
|
175
|
-
def write_null(
|
176
|
+
def write_null(_datum)
|
176
177
|
nil
|
177
178
|
end
|
178
179
|
|
@@ -205,7 +206,7 @@ module Avro
|
|
205
206
|
# equivalent to Java's floatToIntBits and then encoded in
|
206
207
|
# little-endian format.
|
207
208
|
def write_float(datum)
|
208
|
-
@writer.write([datum].pack('e'
|
209
|
+
@writer.write([datum].pack('e'))
|
209
210
|
end
|
210
211
|
|
211
212
|
# A double is written as 8 bytes.
|
@@ -213,7 +214,7 @@ module Avro
|
|
213
214
|
# equivalent to Java's doubleToLongBits and then encoded in
|
214
215
|
# little-endian format.
|
215
216
|
def write_double(datum)
|
216
|
-
@writer.write([datum].pack('E'
|
217
|
+
@writer.write([datum].pack('E'))
|
217
218
|
end
|
218
219
|
|
219
220
|
# Bytes are encoded as a long followed by that many bytes of data.
|
@@ -225,7 +226,7 @@ module Avro
|
|
225
226
|
# A string is encoded as a long followed by that many bytes of
|
226
227
|
# UTF-8 encoded character data
|
227
228
|
def write_string(datum)
|
228
|
-
datum = datum.encode('utf-8'
|
229
|
+
datum = datum.encode('utf-8') if datum.respond_to? :encode
|
229
230
|
write_bytes(datum)
|
230
231
|
end
|
231
232
|
|
@@ -292,7 +293,7 @@ module Avro
|
|
292
293
|
readers_schema.type_adapter.decode(datum)
|
293
294
|
end
|
294
295
|
|
295
|
-
def read_fixed(writers_schema,
|
296
|
+
def read_fixed(writers_schema, _readers_schema, decoder)
|
296
297
|
decoder.read(writers_schema.size)
|
297
298
|
end
|
298
299
|
|
@@ -300,12 +301,12 @@ module Avro
|
|
300
301
|
index_of_symbol = decoder.read_int
|
301
302
|
read_symbol = writers_schema.symbols[index_of_symbol]
|
302
303
|
|
303
|
-
|
304
|
-
|
305
|
-
unless readers_schema.symbols.include?(read_symbol)
|
306
|
-
# 'unset' here
|
304
|
+
if !readers_schema.symbols.include?(read_symbol) && readers_schema.default
|
305
|
+
read_symbol = readers_schema.default
|
307
306
|
end
|
308
307
|
|
308
|
+
# This implementation deviates from the spec by always returning
|
309
|
+
# a symbol.
|
309
310
|
read_symbol
|
310
311
|
end
|
311
312
|
|
@@ -359,26 +360,28 @@ module Avro
|
|
359
360
|
readers_fields_hash = readers_schema.fields_hash
|
360
361
|
read_record = {}
|
361
362
|
writers_schema.fields.each do |field|
|
362
|
-
|
363
|
+
readers_field = readers_fields_hash[field.name]
|
364
|
+
if readers_field
|
363
365
|
field_val = read_data(field.type, readers_field.type, decoder)
|
364
366
|
read_record[field.name] = field_val
|
367
|
+
elsif readers_schema.fields_by_alias.key?(field.name)
|
368
|
+
readers_field = readers_schema.fields_by_alias[field.name]
|
369
|
+
field_val = read_data(field.type, readers_field.type, decoder)
|
370
|
+
read_record[readers_field.name] = field_val
|
365
371
|
else
|
366
372
|
skip_data(field.type, decoder)
|
367
373
|
end
|
368
374
|
end
|
369
375
|
|
370
376
|
# fill in the default values
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
raise AvroError, "Missing data for #{field.type} with no default"
|
380
|
-
end
|
381
|
-
end
|
377
|
+
readers_fields_hash.each do |field_name, field|
|
378
|
+
next if read_record.key?(field_name)
|
379
|
+
|
380
|
+
if field.default?
|
381
|
+
field_val = read_default_value(field.type, field.default)
|
382
|
+
read_record[field.name] = field_val
|
383
|
+
else
|
384
|
+
raise AvroError, "Missing data for #{field.type} with no default"
|
382
385
|
end
|
383
386
|
end
|
384
387
|
|
@@ -390,13 +393,11 @@ module Avro
|
|
390
393
|
case field_schema.type_sym
|
391
394
|
when :null
|
392
395
|
return nil
|
393
|
-
when :boolean
|
394
|
-
return default_value
|
395
396
|
when :int, :long
|
396
397
|
return Integer(default_value)
|
397
398
|
when :float, :double
|
398
399
|
return Float(default_value)
|
399
|
-
when :enum, :fixed, :string, :bytes
|
400
|
+
when :boolean, :enum, :fixed, :string, :bytes
|
400
401
|
return default_value
|
401
402
|
when :array
|
402
403
|
read_array = []
|
@@ -468,7 +469,7 @@ module Avro
|
|
468
469
|
decoder.skip(writers_schema.size)
|
469
470
|
end
|
470
471
|
|
471
|
-
def skip_enum(
|
472
|
+
def skip_enum(_writers_schema, decoder)
|
472
473
|
decoder.skip_int
|
473
474
|
end
|
474
475
|
|
@@ -508,6 +509,8 @@ module Avro
|
|
508
509
|
|
509
510
|
# DatumWriter for generic ruby objects
|
510
511
|
class DatumWriter
|
512
|
+
VALIDATION_OPTIONS = { recursive: false, encoded: true }.freeze
|
513
|
+
|
511
514
|
attr_accessor :writers_schema
|
512
515
|
def initialize(writers_schema=nil)
|
513
516
|
@writers_schema = writers_schema
|
@@ -520,7 +523,7 @@ module Avro
|
|
520
523
|
def write_data(writers_schema, logical_datum, encoder)
|
521
524
|
datum = writers_schema.type_adapter.encode(logical_datum)
|
522
525
|
|
523
|
-
unless Schema.validate(writers_schema, datum,
|
526
|
+
unless Schema.validate(writers_schema, datum, VALIDATION_OPTIONS)
|
524
527
|
raise AvroTypeError.new(writers_schema, datum)
|
525
528
|
end
|
526
529
|
|
@@ -545,7 +548,7 @@ module Avro
|
|
545
548
|
end
|
546
549
|
end
|
547
550
|
|
548
|
-
def write_fixed(
|
551
|
+
def write_fixed(_writers_schema, datum, encoder)
|
549
552
|
encoder.write(datum)
|
550
553
|
end
|
551
554
|
|
@@ -578,12 +581,15 @@ module Avro
|
|
578
581
|
end
|
579
582
|
|
580
583
|
def write_union(writers_schema, datum, encoder)
|
581
|
-
index_of_schema =
|
582
|
-
|
583
|
-
|
584
|
-
|
584
|
+
index_of_schema = writers_schema.schemas.find_index do |schema|
|
585
|
+
# Optimize away expensive validation calls for the common null type
|
586
|
+
schema.type_sym == :null ? datum.nil? : Schema.validate(schema, datum)
|
587
|
+
end
|
588
|
+
|
589
|
+
unless index_of_schema
|
585
590
|
raise AvroTypeError.new(writers_schema, datum)
|
586
591
|
end
|
592
|
+
|
587
593
|
encoder.write_long(index_of_schema)
|
588
594
|
write_data(writers_schema.schemas[index_of_schema], datum, encoder)
|
589
595
|
end
|
@@ -591,7 +597,7 @@ module Avro
|
|
591
597
|
def write_record(writers_schema, datum, encoder)
|
592
598
|
raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Hash)
|
593
599
|
writers_schema.fields.each do |field|
|
594
|
-
write_data(field.type, datum[field.name], encoder)
|
600
|
+
write_data(field.type, datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym], encoder)
|
595
601
|
end
|
596
602
|
end
|
597
603
|
end # DatumWriter
|
data/lib/avro/ipc.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -6,7 +7,7 @@
|
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
9
|
#
|
9
|
-
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
11
|
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
@@ -63,8 +64,11 @@ module Avro::IPC
|
|
63
64
|
SYSTEM_ERROR_SCHEMA = Avro::Schema.parse('["string"]')
|
64
65
|
|
65
66
|
# protocol cache
|
67
|
+
# rubocop:disable Style/MutableConstant
|
66
68
|
REMOTE_HASHES = {}
|
67
69
|
REMOTE_PROTOCOLS = {}
|
70
|
+
# rubocop:enable Style/MutableConstant
|
71
|
+
|
68
72
|
|
69
73
|
BUFFER_HEADER_LENGTH = 4
|
70
74
|
BUFFER_SIZE = 8192
|
@@ -100,7 +104,7 @@ module Avro::IPC
|
|
100
104
|
def request(message_name, request_datum)
|
101
105
|
# Writes a request message and reads a response or error message.
|
102
106
|
# build handshake and call request
|
103
|
-
buffer_writer = StringIO.new(
|
107
|
+
buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
|
104
108
|
buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
|
105
109
|
write_handshake_request(buffer_encoder)
|
106
110
|
write_call_request(message_name, request_datum, buffer_encoder)
|
@@ -244,7 +248,7 @@ module Avro::IPC
|
|
244
248
|
# a response or error. Compare to 'handle()' in Thrift.
|
245
249
|
def respond(call_request, transport=nil)
|
246
250
|
buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_request))
|
247
|
-
buffer_writer = StringIO.new(
|
251
|
+
buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
|
248
252
|
buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
|
249
253
|
error = nil
|
250
254
|
response_metadata = {}
|
@@ -278,7 +282,7 @@ module Avro::IPC
|
|
278
282
|
response = call(local_message, request)
|
279
283
|
rescue AvroRemoteError => e
|
280
284
|
error = e
|
281
|
-
rescue Exception => e
|
285
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
282
286
|
error = AvroRemoteError.new(e.to_s)
|
283
287
|
end
|
284
288
|
|
@@ -350,7 +354,7 @@ module Avro::IPC
|
|
350
354
|
remote_protocol
|
351
355
|
end
|
352
356
|
|
353
|
-
def call(
|
357
|
+
def call(_local_message, _request)
|
354
358
|
# Actual work done by server: cf. handler in thrift.
|
355
359
|
raise NotImplementedError
|
356
360
|
end
|
@@ -394,7 +398,7 @@ module Avro::IPC
|
|
394
398
|
def read_framed_message
|
395
399
|
message = []
|
396
400
|
loop do
|
397
|
-
buffer = StringIO.new(
|
401
|
+
buffer = StringIO.new(String.new('', encoding: 'BINARY'))
|
398
402
|
buffer_length = read_buffer_length
|
399
403
|
if buffer_length == 0
|
400
404
|
return message.join
|
@@ -506,7 +510,7 @@ module Avro::IPC
|
|
506
510
|
def read_framed_message
|
507
511
|
message = []
|
508
512
|
loop do
|
509
|
-
buffer =
|
513
|
+
buffer = String.new('', encoding: 'BINARY')
|
510
514
|
buffer_size = read_buffer_size
|
511
515
|
|
512
516
|
return message.join if buffer_size == 0
|
@@ -542,7 +546,7 @@ module Avro::IPC
|
|
542
546
|
end
|
543
547
|
|
544
548
|
def transceive(message)
|
545
|
-
writer = FramedWriter.new(StringIO.new(
|
549
|
+
writer = FramedWriter.new(StringIO.new(String.new('', encoding: 'BINARY')))
|
546
550
|
writer.write_framed_message(message)
|
547
551
|
resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
|
548
552
|
FramedReader.new(StringIO.new(resp.body)).read_framed_message
|