avro 1.10.1 → 1.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Manifest +0 -1
- data/NOTICE +1 -1
- data/Rakefile +13 -19
- data/avro.gemspec +37 -29
- data/interop/test_interop.rb +2 -1
- data/lib/avro/VERSION.txt +1 -1
- data/lib/avro/data_file.rb +5 -4
- data/lib/avro/io.rb +18 -14
- data/lib/avro/ipc.rb +9 -5
- data/lib/avro/logical_types.rb +186 -2
- data/lib/avro/protocol.rb +1 -0
- data/lib/avro/schema.rb +71 -12
- data/lib/avro/schema_compatibility.rb +19 -13
- data/lib/avro/schema_normalization.rb +1 -0
- data/lib/avro/schema_validator.rb +40 -34
- data/lib/avro.rb +1 -0
- data/test/case_finder.rb +1 -0
- data/test/random_data.rb +5 -4
- data/test/sample_ipc_client.rb +1 -0
- data/test/sample_ipc_http_client.rb +1 -0
- data/test/sample_ipc_http_server.rb +1 -0
- data/test/sample_ipc_server.rb +1 -0
- data/test/test_datafile.rb +1 -0
- data/test/test_fingerprints.rb +1 -0
- data/test/test_help.rb +1 -0
- data/test/test_io.rb +31 -16
- data/test/test_logical_types.rb +138 -1
- data/test/test_protocol.rb +2 -1
- data/test/test_schema.rb +134 -4
- data/test/test_schema_compatibility.rb +108 -0
- data/test/test_schema_normalization.rb +1 -0
- data/test/test_schema_validator.rb +24 -3
- data/test/test_socket_transport.rb +1 -0
- data/test/tool.rb +8 -7
- metadata +30 -33
- data/CHANGELOG +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 660434b4d31525eed0d771a26e10c3753856fa45797e55845a82b4a8d4b2361c
|
4
|
+
data.tar.gz: a44c0c7af2a5a030c648d5cc3705ae2ea092fccb3d58ccf9186f4f4619d79092
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7eb879efaea928bf6a8d39079f0b6d92381c3905fcae130c37e1e426f33f940b13b028df43ebcf5ce0bb40dc996efd4c88d7bea5b7b6088715c2677a48146d5
|
7
|
+
data.tar.gz: 9657974edde8c940074930808706554c9db9390f061f573830194cdd9167fed1c46c1811b188cf598cc2a48673874e2af41d43c73ce74ff7c620594e641f4831
|
data/Manifest
CHANGED
data/NOTICE
CHANGED
data/Rakefile
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -14,23 +15,20 @@
|
|
14
15
|
# See the License for the specific language governing permissions and
|
15
16
|
# limitations under the License.
|
16
17
|
|
17
|
-
require
|
18
|
-
require '
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
p.author = "Apache Software Foundation"
|
23
|
-
p.email = "dev@avro.apache.org"
|
24
|
-
p.summary = "Apache Avro for Ruby"
|
25
|
-
p.description = "Avro is a data serialization and RPC format"
|
26
|
-
p.url = "https://avro.apache.org/"
|
27
|
-
p.runtime_dependencies = ["multi_json ~>1"]
|
28
|
-
p.licenses = ["Apache-2.0"]
|
18
|
+
require "bundler/gem_tasks"
|
19
|
+
require 'rake/testtask'
|
20
|
+
|
21
|
+
Rake::TestTask.new(:interop) do |t|
|
22
|
+
t.pattern = 'interop/test*.rb'
|
29
23
|
end
|
30
24
|
|
31
|
-
|
32
|
-
t.
|
25
|
+
Rake::TestTask.new(:test) do |t|
|
26
|
+
t.libs << "test"
|
27
|
+
t.pattern = 'test/test_*.rb'
|
28
|
+
t.verbose = true
|
29
|
+
end
|
33
30
|
|
31
|
+
desc "Generate data for interop tests"
|
34
32
|
task :generate_interop do
|
35
33
|
$:.unshift(HERE + '/lib')
|
36
34
|
$:.unshift(HERE + '/test')
|
@@ -49,13 +47,9 @@ task :generate_interop do
|
|
49
47
|
end
|
50
48
|
end
|
51
49
|
|
52
|
-
|
53
50
|
HERE = File.expand_path(File.dirname(__FILE__))
|
54
51
|
SHARE = HERE + '/../../share'
|
55
52
|
SCHEMAS = SHARE + '/test/schemas'
|
56
53
|
BUILD = HERE + '/../../build'
|
57
54
|
|
58
|
-
task :
|
59
|
-
mkdir_p "../../dist/ruby"
|
60
|
-
cp "pkg/avro-#{VERSION}.gem", "../../dist/ruby"
|
61
|
-
end
|
55
|
+
task default: :test
|
data/avro.gemspec
CHANGED
@@ -1,35 +1,43 @@
|
|
1
|
-
#
|
2
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
# or more contributor license agreements. See the NOTICE file
|
4
|
+
# distributed with this work for additional information
|
5
|
+
# regarding copyright ownership. The ASF licenses this file
|
6
|
+
# to you under the Apache License, Version 2.0 (the
|
7
|
+
# "License"); you may not use this file except in compliance
|
8
|
+
# with the License. You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
3
17
|
|
4
18
|
Gem::Specification.new do |s|
|
5
|
-
s.name = "avro"
|
6
|
-
s.version = "
|
19
|
+
s.name = "avro"
|
20
|
+
s.version = File.read("lib/avro/VERSION.txt")
|
21
|
+
s.authors = ["Apache Software Foundation"]
|
22
|
+
s.email = "dev@avro.apache.org"
|
7
23
|
|
8
|
-
s.
|
9
|
-
s.
|
10
|
-
s.
|
11
|
-
s.
|
12
|
-
s.
|
13
|
-
s.email = "dev@avro.apache.org".freeze
|
14
|
-
s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
|
15
|
-
s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
|
16
|
-
s.homepage = "https://avro.apache.org/".freeze
|
17
|
-
s.licenses = ["Apache-2.0".freeze]
|
18
|
-
s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro".freeze]
|
19
|
-
s.rubyforge_project = "avro".freeze
|
20
|
-
s.rubygems_version = "2.5.2.1".freeze
|
21
|
-
s.summary = "Apache Avro for Ruby".freeze
|
22
|
-
s.test_files = ["test/test_schema.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_help.rb".freeze, "test/test_datafile.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_fingerprints.rb".freeze]
|
24
|
+
s.summary = "Apache Avro for Ruby"
|
25
|
+
s.description = "Avro is a data serialization and RPC format"
|
26
|
+
s.homepage = "https://avro.apache.org/"
|
27
|
+
s.license = "Apache-2.0"
|
28
|
+
s.required_ruby_version = ">= 2.6"
|
23
29
|
|
24
|
-
|
25
|
-
|
30
|
+
s.metadata["homepage_uri"] = s.homepage
|
31
|
+
s.metadata["bug_tracker_uri"] = "https://issues.apache.org/jira/browse/AVRO"
|
32
|
+
s.metadata["source_code_uri"] = "https://github.com/apache/avro"
|
33
|
+
s.metadata["documentation_uri"] = "https://avro.apache.org/docs/#{s.version}/"
|
34
|
+
s.metadata["rubygems_mfa_required"] = "true"
|
26
35
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
end
|
36
|
+
files = File.read("Manifest").split("\n")
|
37
|
+
s.files = files.reject { |f| f.start_with?("test/") }
|
38
|
+
s.rdoc_options = ["--line-numbers", "--title", "Avro"]
|
39
|
+
s.test_files = files.select { |f| f.start_with?("test/") }
|
40
|
+
s.require_paths = ["lib"]
|
41
|
+
|
42
|
+
s.add_dependency("multi_json", "~> 1.0")
|
35
43
|
end
|
data/interop/test_interop.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
4
5
|
# distributed with this work for additional information
|
@@ -19,7 +20,7 @@ require 'rubygems'
|
|
19
20
|
require 'test/unit'
|
20
21
|
require 'avro'
|
21
22
|
|
22
|
-
CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard'] # The 'null' codec is implicitly included
|
23
|
+
CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard'].freeze # The 'null' codec is implicitly included
|
23
24
|
|
24
25
|
class TestInterop < Test::Unit::TestCase
|
25
26
|
HERE = File.expand_path(File.dirname(__FILE__))
|
data/lib/avro/VERSION.txt
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.11.1
|
data/lib/avro/data_file.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -5,9 +6,9 @@
|
|
5
6
|
# to you under the Apache License, Version 2.0 (the
|
6
7
|
# "License"); you may not use this file except in compliance
|
7
8
|
# with the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
+
#
|
9
10
|
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
+
#
|
11
12
|
# Unless required by applicable law or agreed to in writing, software
|
12
13
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
14
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
@@ -25,7 +26,7 @@ module Avro
|
|
25
26
|
SYNC_SIZE = 16
|
26
27
|
SYNC_INTERVAL = 4000 * SYNC_SIZE
|
27
28
|
META_SCHEMA = Schema.parse('{"type": "map", "values": "bytes"}')
|
28
|
-
VALID_ENCODINGS = ['binary'] # not used yet
|
29
|
+
VALID_ENCODINGS = ['binary'].freeze # not used yet
|
29
30
|
|
30
31
|
class DataFileError < AvroError; end
|
31
32
|
|
@@ -99,7 +100,7 @@ module Avro
|
|
99
100
|
@encoder = IO::BinaryEncoder.new(@writer)
|
100
101
|
@datum_writer = datum_writer
|
101
102
|
@meta = meta
|
102
|
-
@buffer_writer = StringIO.new('', 'w')
|
103
|
+
@buffer_writer = StringIO.new(+'', 'w')
|
103
104
|
@buffer_writer.set_encoding('BINARY') if @buffer_writer.respond_to?(:set_encoding)
|
104
105
|
@buffer_encoder = IO::BinaryEncoder.new(@buffer_writer)
|
105
106
|
@block_count = 0
|
data/lib/avro/io.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -76,7 +77,7 @@ module Avro
|
|
76
77
|
# The float is converted into a 32-bit integer using a method
|
77
78
|
# equivalent to Java's floatToIntBits and then encoded in
|
78
79
|
# little-endian format.
|
79
|
-
read_and_unpack(4, 'e'
|
80
|
+
read_and_unpack(4, 'e')
|
80
81
|
end
|
81
82
|
|
82
83
|
def read_double
|
@@ -84,7 +85,7 @@ module Avro
|
|
84
85
|
# The double is converted into a 64-bit integer using a method
|
85
86
|
# equivalent to Java's doubleToLongBits and then encoded in
|
86
87
|
# little-endian format.
|
87
|
-
read_and_unpack(8, 'E'
|
88
|
+
read_and_unpack(8, 'E')
|
88
89
|
end
|
89
90
|
|
90
91
|
def read_bytes
|
@@ -97,7 +98,7 @@ module Avro
|
|
97
98
|
# A string is encoded as a long followed by that many bytes of
|
98
99
|
# UTF-8 encoded character data.
|
99
100
|
read_bytes.tap do |string|
|
100
|
-
string.force_encoding('UTF-8'
|
101
|
+
string.force_encoding('UTF-8') if string.respond_to? :force_encoding
|
101
102
|
end
|
102
103
|
end
|
103
104
|
|
@@ -205,7 +206,7 @@ module Avro
|
|
205
206
|
# equivalent to Java's floatToIntBits and then encoded in
|
206
207
|
# little-endian format.
|
207
208
|
def write_float(datum)
|
208
|
-
@writer.write([datum].pack('e'
|
209
|
+
@writer.write([datum].pack('e'))
|
209
210
|
end
|
210
211
|
|
211
212
|
# A double is written as 8 bytes.
|
@@ -213,7 +214,7 @@ module Avro
|
|
213
214
|
# equivalent to Java's doubleToLongBits and then encoded in
|
214
215
|
# little-endian format.
|
215
216
|
def write_double(datum)
|
216
|
-
@writer.write([datum].pack('E'
|
217
|
+
@writer.write([datum].pack('E'))
|
217
218
|
end
|
218
219
|
|
219
220
|
# Bytes are encoded as a long followed by that many bytes of data.
|
@@ -225,7 +226,7 @@ module Avro
|
|
225
226
|
# A string is encoded as a long followed by that many bytes of
|
226
227
|
# UTF-8 encoded character data
|
227
228
|
def write_string(datum)
|
228
|
-
datum = datum.encode('utf-8'
|
229
|
+
datum = datum.encode('utf-8') if datum.respond_to? :encode
|
229
230
|
write_bytes(datum)
|
230
231
|
end
|
231
232
|
|
@@ -392,13 +393,11 @@ module Avro
|
|
392
393
|
case field_schema.type_sym
|
393
394
|
when :null
|
394
395
|
return nil
|
395
|
-
when :boolean
|
396
|
-
return default_value
|
397
396
|
when :int, :long
|
398
397
|
return Integer(default_value)
|
399
398
|
when :float, :double
|
400
399
|
return Float(default_value)
|
401
|
-
when :enum, :fixed, :string, :bytes
|
400
|
+
when :boolean, :enum, :fixed, :string, :bytes
|
402
401
|
return default_value
|
403
402
|
when :array
|
404
403
|
read_array = []
|
@@ -510,6 +509,8 @@ module Avro
|
|
510
509
|
|
511
510
|
# DatumWriter for generic ruby objects
|
512
511
|
class DatumWriter
|
512
|
+
VALIDATION_OPTIONS = { recursive: false, encoded: true }.freeze
|
513
|
+
|
513
514
|
attr_accessor :writers_schema
|
514
515
|
def initialize(writers_schema=nil)
|
515
516
|
@writers_schema = writers_schema
|
@@ -522,7 +523,7 @@ module Avro
|
|
522
523
|
def write_data(writers_schema, logical_datum, encoder)
|
523
524
|
datum = writers_schema.type_adapter.encode(logical_datum)
|
524
525
|
|
525
|
-
unless Schema.validate(writers_schema, datum,
|
526
|
+
unless Schema.validate(writers_schema, datum, VALIDATION_OPTIONS)
|
526
527
|
raise AvroTypeError.new(writers_schema, datum)
|
527
528
|
end
|
528
529
|
|
@@ -580,12 +581,15 @@ module Avro
|
|
580
581
|
end
|
581
582
|
|
582
583
|
def write_union(writers_schema, datum, encoder)
|
583
|
-
index_of_schema =
|
584
|
-
|
585
|
-
|
586
|
-
|
584
|
+
index_of_schema = writers_schema.schemas.find_index do |schema|
|
585
|
+
# Optimize away expensive validation calls for the common null type
|
586
|
+
schema.type_sym == :null ? datum.nil? : Schema.validate(schema, datum)
|
587
|
+
end
|
588
|
+
|
589
|
+
unless index_of_schema
|
587
590
|
raise AvroTypeError.new(writers_schema, datum)
|
588
591
|
end
|
592
|
+
|
589
593
|
encoder.write_long(index_of_schema)
|
590
594
|
write_data(writers_schema.schemas[index_of_schema], datum, encoder)
|
591
595
|
end
|
data/lib/avro/ipc.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# Licensed to the Apache Software Foundation (ASF) under one
|
2
3
|
# or more contributor license agreements. See the NOTICE file
|
3
4
|
# distributed with this work for additional information
|
@@ -63,8 +64,11 @@ module Avro::IPC
|
|
63
64
|
SYSTEM_ERROR_SCHEMA = Avro::Schema.parse('["string"]')
|
64
65
|
|
65
66
|
# protocol cache
|
67
|
+
# rubocop:disable Style/MutableConstant
|
66
68
|
REMOTE_HASHES = {}
|
67
69
|
REMOTE_PROTOCOLS = {}
|
70
|
+
# rubocop:enable Style/MutableConstant
|
71
|
+
|
68
72
|
|
69
73
|
BUFFER_HEADER_LENGTH = 4
|
70
74
|
BUFFER_SIZE = 8192
|
@@ -100,7 +104,7 @@ module Avro::IPC
|
|
100
104
|
def request(message_name, request_datum)
|
101
105
|
# Writes a request message and reads a response or error message.
|
102
106
|
# build handshake and call request
|
103
|
-
buffer_writer = StringIO.new(
|
107
|
+
buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
|
104
108
|
buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
|
105
109
|
write_handshake_request(buffer_encoder)
|
106
110
|
write_call_request(message_name, request_datum, buffer_encoder)
|
@@ -244,7 +248,7 @@ module Avro::IPC
|
|
244
248
|
# a response or error. Compare to 'handle()' in Thrift.
|
245
249
|
def respond(call_request, transport=nil)
|
246
250
|
buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_request))
|
247
|
-
buffer_writer = StringIO.new(
|
251
|
+
buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
|
248
252
|
buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
|
249
253
|
error = nil
|
250
254
|
response_metadata = {}
|
@@ -394,7 +398,7 @@ module Avro::IPC
|
|
394
398
|
def read_framed_message
|
395
399
|
message = []
|
396
400
|
loop do
|
397
|
-
buffer = StringIO.new(
|
401
|
+
buffer = StringIO.new(String.new('', encoding: 'BINARY'))
|
398
402
|
buffer_length = read_buffer_length
|
399
403
|
if buffer_length == 0
|
400
404
|
return message.join
|
@@ -506,7 +510,7 @@ module Avro::IPC
|
|
506
510
|
def read_framed_message
|
507
511
|
message = []
|
508
512
|
loop do
|
509
|
-
buffer =
|
513
|
+
buffer = String.new('', encoding: 'BINARY')
|
510
514
|
buffer_size = read_buffer_size
|
511
515
|
|
512
516
|
return message.join if buffer_size == 0
|
@@ -542,7 +546,7 @@ module Avro::IPC
|
|
542
546
|
end
|
543
547
|
|
544
548
|
def transceive(message)
|
545
|
-
writer = FramedWriter.new(StringIO.new(
|
549
|
+
writer = FramedWriter.new(StringIO.new(String.new('', encoding: 'BINARY')))
|
546
550
|
writer.write_framed_message(message)
|
547
551
|
resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
|
548
552
|
FramedReader.new(StringIO.new(resp.body)).read_framed_message
|
data/lib/avro/logical_types.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Licensed to the Apache Software Foundation (ASF) under one
|
3
4
|
# or more contributor license agreements. See the NOTICE file
|
4
5
|
# distributed with this work for additional information
|
@@ -16,9 +17,188 @@
|
|
16
17
|
# limitations under the License.
|
17
18
|
|
18
19
|
require 'date'
|
20
|
+
require 'bigdecimal'
|
21
|
+
require 'bigdecimal/util'
|
19
22
|
|
20
23
|
module Avro
|
21
24
|
module LogicalTypes
|
25
|
+
##
|
26
|
+
# Base class for logical types requiring a schema to be present
|
27
|
+
class LogicalTypeWithSchema
|
28
|
+
##
|
29
|
+
# @return [Avro::Schema] The schema this logical type is dealing with
|
30
|
+
attr_reader :schema
|
31
|
+
|
32
|
+
##
|
33
|
+
# Build a new instance of a logical type using the provided schema
|
34
|
+
#
|
35
|
+
# @param schema [Avro::Schema]
|
36
|
+
# The schema to use with this instance
|
37
|
+
#
|
38
|
+
# @raise [ArgumentError]
|
39
|
+
# If the provided schema is nil
|
40
|
+
def initialize(schema)
|
41
|
+
raise ArgumentError, 'schema is required' if schema.nil?
|
42
|
+
|
43
|
+
@schema = schema
|
44
|
+
end
|
45
|
+
|
46
|
+
##
|
47
|
+
# Encode the provided datum
|
48
|
+
#
|
49
|
+
# @param datum [Object] The datum to encode
|
50
|
+
#
|
51
|
+
# @raise [NotImplementedError]
|
52
|
+
# Subclass will need to override this method
|
53
|
+
def encode(datum)
|
54
|
+
raise NotImplementedError
|
55
|
+
end
|
56
|
+
|
57
|
+
##
|
58
|
+
# Decode the provided datum
|
59
|
+
#
|
60
|
+
# @param datum [Object] The datum to decode
|
61
|
+
#
|
62
|
+
# @raise [NotImplementedError]
|
63
|
+
# Subclass will need to override this method
|
64
|
+
def decode(datum)
|
65
|
+
raise NotImplementedError
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# Logical type to handle arbitrary-precision decimals using byte array.
|
71
|
+
#
|
72
|
+
# The byte array contains the two's-complement representation of the unscaled integer
|
73
|
+
# value in big-endian byte order.
|
74
|
+
class BytesDecimal < LogicalTypeWithSchema
|
75
|
+
# Messages for exceptions
|
76
|
+
ERROR_INSUFFICIENT_PRECISION = 'Precision is too small'
|
77
|
+
ERROR_ROUNDING_NECESSARY = 'Rounding necessary'
|
78
|
+
ERROR_VALUE_MUST_BE_NUMERIC = 'value must be numeric'
|
79
|
+
|
80
|
+
# The pattern used to pack up the byte array (8 bit unsigned integer/char)
|
81
|
+
PACK_UNSIGNED_CHARS = 'C*'
|
82
|
+
|
83
|
+
# The number 10 as BigDecimal
|
84
|
+
TEN = BigDecimal(10).freeze
|
85
|
+
|
86
|
+
##
|
87
|
+
# @return [Integer] The number of total digits supported by the decimal
|
88
|
+
attr_reader :precision
|
89
|
+
|
90
|
+
##
|
91
|
+
# @return [Integer] The number of fractional digits
|
92
|
+
attr_reader :scale
|
93
|
+
|
94
|
+
##
|
95
|
+
# Build a new decimal logical type
|
96
|
+
#
|
97
|
+
# @param schema [Avro::Schema]
|
98
|
+
# The schema defining precision and scale for the conversion
|
99
|
+
def initialize(schema)
|
100
|
+
super
|
101
|
+
|
102
|
+
@scale = schema.scale.to_i
|
103
|
+
@precision = schema.precision.to_i
|
104
|
+
@factor = TEN ** @scale
|
105
|
+
end
|
106
|
+
|
107
|
+
##
|
108
|
+
# Encode the provided value into a byte array
|
109
|
+
#
|
110
|
+
# @param value [BigDecimal, Float, Integer]
|
111
|
+
# The numeric value to encode
|
112
|
+
#
|
113
|
+
# @raise [ArgumentError]
|
114
|
+
# If the provided value is not a numeric type
|
115
|
+
#
|
116
|
+
# @raise [RangeError]
|
117
|
+
# If the provided value has a scale higher than the schema permits,
|
118
|
+
# or does not fit into the schema's precision
|
119
|
+
def encode(value)
|
120
|
+
raise ArgumentError, ERROR_VALUE_MUST_BE_NUMERIC unless value.is_a?(Numeric)
|
121
|
+
|
122
|
+
to_byte_array(unscaled_value(value.to_d)).pack(PACK_UNSIGNED_CHARS).freeze
|
123
|
+
end
|
124
|
+
|
125
|
+
##
|
126
|
+
# Decode a byte array (in form of a string) into a BigDecimal of the
|
127
|
+
# given precision and scale
|
128
|
+
#
|
129
|
+
# @param stream [String]
|
130
|
+
# The byte array to decode
|
131
|
+
#
|
132
|
+
# @return [BigDecimal]
|
133
|
+
def decode(stream)
|
134
|
+
from_byte_array(stream) / @factor
|
135
|
+
end
|
136
|
+
|
137
|
+
private
|
138
|
+
|
139
|
+
##
|
140
|
+
# Convert the provided stream of bytes into the unscaled value
|
141
|
+
#
|
142
|
+
# @param stream [String]
|
143
|
+
# The stream of bytes to convert
|
144
|
+
#
|
145
|
+
# @return [Integer]
|
146
|
+
def from_byte_array(stream)
|
147
|
+
bytes = stream.bytes
|
148
|
+
positive = bytes.first[7].zero?
|
149
|
+
total = 0
|
150
|
+
|
151
|
+
bytes.each_with_index do |value, ix|
|
152
|
+
total += (positive ? value : (value ^ 0xff)) << (bytes.length - ix - 1) * 8
|
153
|
+
end
|
154
|
+
|
155
|
+
return total if positive
|
156
|
+
|
157
|
+
-(total + 1)
|
158
|
+
end
|
159
|
+
|
160
|
+
##
|
161
|
+
# Convert the provided number into its two's complement representation
|
162
|
+
# in network order (big endian).
|
163
|
+
#
|
164
|
+
# @param number [Integer]
|
165
|
+
# The number to convert
|
166
|
+
#
|
167
|
+
# @return [Array<Integer>]
|
168
|
+
# The byte array in network order
|
169
|
+
def to_byte_array(number)
|
170
|
+
[].tap do |result|
|
171
|
+
loop do
|
172
|
+
result.unshift(number & 0xff)
|
173
|
+
number >>= 8
|
174
|
+
|
175
|
+
break if (number == 0 || number == -1) && (result.first[7] == number[7])
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
##
|
181
|
+
# Get the unscaled value from a BigDecimal considering the schema's scale
|
182
|
+
#
|
183
|
+
# @param decimal [BigDecimal]
|
184
|
+
# The decimal to get the unscaled value from
|
185
|
+
#
|
186
|
+
# @return [Integer]
|
187
|
+
def unscaled_value(decimal)
|
188
|
+
details = decimal.split
|
189
|
+
length = details[1].length
|
190
|
+
|
191
|
+
fractional_part = length - details[3]
|
192
|
+
raise RangeError, ERROR_ROUNDING_NECESSARY if fractional_part > scale
|
193
|
+
|
194
|
+
if length > precision || (length - fractional_part) > (precision - scale)
|
195
|
+
raise RangeError, ERROR_INSUFFICIENT_PRECISION
|
196
|
+
end
|
197
|
+
|
198
|
+
(decimal * @factor).to_i
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
22
202
|
module IntDate
|
23
203
|
EPOCH_START = Date.new(1970, 1, 1)
|
24
204
|
|
@@ -72,6 +252,9 @@ module Avro
|
|
72
252
|
end
|
73
253
|
|
74
254
|
TYPES = {
|
255
|
+
"bytes" => {
|
256
|
+
"decimal" => BytesDecimal
|
257
|
+
},
|
75
258
|
"int" => {
|
76
259
|
"date" => IntDate
|
77
260
|
},
|
@@ -81,10 +264,11 @@ module Avro
|
|
81
264
|
},
|
82
265
|
}.freeze
|
83
266
|
|
84
|
-
def self.type_adapter(type, logical_type)
|
267
|
+
def self.type_adapter(type, logical_type, schema = nil)
|
85
268
|
return unless logical_type
|
86
269
|
|
87
|
-
TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
|
270
|
+
adapter = TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
|
271
|
+
adapter.is_a?(Class) ? adapter.new(schema) : adapter
|
88
272
|
end
|
89
273
|
end
|
90
274
|
end
|
data/lib/avro/protocol.rb
CHANGED