avro 1.10.2 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 38a2f2d15367014464b4613e0fc80724b345d3b3
-  data.tar.gz: 823aa3ebb6ea719cbd4c94060757a545c7d626a9
+SHA256:
+  metadata.gz: d08fa062d2fbd82eeb60d63bf9259103206ec27da4da2b46e1d5e406a182901f
+  data.tar.gz: 6445d1ef065286d08b4dbd325b2613fbe975cb303835cdf036429bea6ac94650
 SHA512:
-  metadata.gz: da4d7d3ecf02bb18c30c044635188ba2b8a34ff13f9450330e850198610ee87f4e9116a79a95366bc23b5e3fb7f70906736d1ff7834e9293825c5c3eba072b9b
-  data.tar.gz: 3194b9379e1ae396acd0a120f583ade7127628d5adf78ac382fcf7215643c69c5341ee0a5f234db56eec3e62070683340b4a21bcff4ba0e2208a832c571f8dc9
+  metadata.gz: 37e345d05baecef377495cb440dc108b1347638b0feb4cdc2d62807f307e62c4ed384e231d605fd672061b6411b9e226296e24670800cce22a0fc09f0bd09a54
+  data.tar.gz: dd48ba91caaf404b90d10e497b297122b3bd9d074ffe8dc44e5f10d582d54759572771dac086b37ab6f0f4766a4d1483c3f29342ff4eb9671c741876552475d7

data/Manifest

@@ -1,4 +1,3 @@
-CHANGELOG
 LICENSE
 NOTICE
 Manifest

data/NOTICE

@@ -1,5 +1,5 @@
 Apache Avro
-Copyright 2010-2015 The Apache Software Foundation
+Copyright 2010-2021 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (https://www.apache.org/).

data/Rakefile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,23 +15,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'rubygems'
-require 'echoe'
-VERSION = File.open('../../share/VERSION.txt').read.sub('-SNAPSHOT', '.pre1').chomp
-File.write("lib/avro/VERSION.txt", VERSION)
-Echoe.new('avro', VERSION) do |p|
-  p.author = "Apache Software Foundation"
-  p.email = "dev@avro.apache.org"
-  p.summary = "Apache Avro for Ruby"
-  p.description = "Avro is a data serialization and RPC format"
-  p.url = "https://avro.apache.org/"
-  p.runtime_dependencies = ["multi_json ~>1"]
-  p.licenses = ["Apache-2.0"]
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:interop) do |t|
+  t.pattern = 'interop/test*.rb'
 end
 
-t = Rake::TestTask.new(:interop)
-t.pattern = 'interop/test*.rb'
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.pattern = 'test/test_*.rb'
+  t.verbose = true
+end
 
+desc "Generate data for interop tests"
 task :generate_interop do
   $:.unshift(HERE + '/lib')
   $:.unshift(HERE + '/test')
@@ -49,13 +47,9 @@ task :generate_interop do
   end
 end
 
-
 HERE = File.expand_path(File.dirname(__FILE__))
 SHARE = HERE + '/../../share'
 SCHEMAS = SHARE + '/test/schemas'
 BUILD = HERE + '/../../build'
 
-task :dist => [:gem] do
-  mkdir_p "../../dist/ruby"
-  cp "pkg/avro-#{VERSION}.gem", "../../dist/ruby"
-end
+task default: :test

data/avro.gemspec

@@ -1,35 +1,42 @@
-# -*- encoding: utf-8 -*-
-# stub: avro 1.10.2 ruby lib
+# frozen_string_literal: true
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 Gem::Specification.new do |s|
-  s.name = "avro".freeze
-  s.version = "1.10.2"
+  s.name = "avro"
+  s.version = File.read("lib/avro/VERSION.txt")
+  s.authors = ["Apache Software Foundation"]
+  s.email = "dev@avro.apache.org"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib".freeze]
-  s.authors = ["Apache Software Foundation".freeze]
-  s.date = "2021-03-09"
-  s.description = "Avro is a data serialization and RPC format".freeze
-  s.email = "dev@avro.apache.org".freeze
-  s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
-  s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
-  s.homepage = "https://avro.apache.org/".freeze
-  s.licenses = ["Apache-2.0".freeze]
-  s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro".freeze]
-  s.rubyforge_project = "avro".freeze
-  s.rubygems_version = "2.5.2.1".freeze
-  s.summary = "Apache Avro for Ruby".freeze
-  s.test_files = ["test/test_schema.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_help.rb".freeze, "test/test_datafile.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_fingerprints.rb".freeze]
+  s.summary = "Apache Avro for Ruby"
+  s.description = "Avro is a data serialization and RPC format"
+  s.homepage = "https://avro.apache.org/"
+  s.license = "Apache-2.0"
+  s.required_ruby_version = ">= 2.6"
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 4
+  s.metadata["homepage_uri"] = s.homepage
+  s.metadata["bug_tracker_uri"] = "https://issues.apache.org/jira/browse/AVRO"
+  s.metadata["source_code_uri"] = "https://github.com/apache/avro"
+  s.metadata["documentation_uri"] = "https://avro.apache.org/docs/#{s.version}/"
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<multi_json>.freeze, ["~> 1"])
-    else
-      s.add_dependency(%q<multi_json>.freeze, ["~> 1"])
-    end
-  else
-    s.add_dependency(%q<multi_json>.freeze, ["~> 1"])
-  end
+  files = File.read("Manifest").split("\n")
+  s.files = files.reject { |f| f.start_with?("test/") }
+  s.rdoc_options = ["--line-numbers", "--title", "Avro"]
+  s.test_files = files.select { |f| f.start_with?("test/") }
+  s.require_paths = ["lib"]
+
+  s.add_dependency("multi_json", "~> 1.0")
 end

data/interop/test_interop.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -19,7 +20,7 @@ require 'rubygems'
 require 'test/unit'
 require 'avro'
 
-CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard']  # The 'null' codec is implicitly included
+CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard'].freeze  # The 'null' codec is implicitly included
 
 class TestInterop < Test::Unit::TestCase
   HERE = File.expand_path(File.dirname(__FILE__))

data/lib/avro/VERSION.txt

@@ -1 +1 @@
-1.10.2
+1.11.0

data/lib/avro/data_file.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -5,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 # https://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,7 +26,7 @@ module Avro
     SYNC_SIZE = 16
     SYNC_INTERVAL = 4000 * SYNC_SIZE
     META_SCHEMA = Schema.parse('{"type": "map", "values": "bytes"}')
-    VALID_ENCODINGS = ['binary'] # not used yet
+    VALID_ENCODINGS = ['binary'].freeze # not used yet
 
     class DataFileError < AvroError; end
 
@@ -99,7 +100,7 @@ module Avro
         @encoder = IO::BinaryEncoder.new(@writer)
         @datum_writer = datum_writer
         @meta = meta
-        @buffer_writer = StringIO.new('', 'w')
+        @buffer_writer = StringIO.new(+'', 'w')
         @buffer_writer.set_encoding('BINARY') if @buffer_writer.respond_to?(:set_encoding)
         @buffer_encoder = IO::BinaryEncoder.new(@buffer_writer)
         @block_count = 0

data/lib/avro/io.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -76,7 +77,7 @@ module Avro
         # The float is converted into a 32-bit integer using a method
         # equivalent to Java's floatToIntBits and then encoded in
         # little-endian format.
-        read_and_unpack(4, 'e'.freeze)
+        read_and_unpack(4, 'e')
       end
 
       def read_double
@@ -84,7 +85,7 @@ module Avro
         # The double is converted into a 64-bit integer using a method
         # equivalent to Java's doubleToLongBits and then encoded in
         # little-endian format.
-        read_and_unpack(8, 'E'.freeze)
+        read_and_unpack(8, 'E')
       end
 
       def read_bytes
@@ -97,7 +98,7 @@ module Avro
         # A string is encoded as a long followed by that many bytes of
         # UTF-8 encoded character data.
         read_bytes.tap do |string|
-          string.force_encoding('UTF-8'.freeze) if string.respond_to? :force_encoding
+          string.force_encoding('UTF-8') if string.respond_to? :force_encoding
         end
       end
 
@@ -205,7 +206,7 @@ module Avro
       # equivalent to Java's floatToIntBits and then encoded in
       # little-endian format.
       def write_float(datum)
-        @writer.write([datum].pack('e'.freeze))
+        @writer.write([datum].pack('e'))
       end
 
       # A double is written as 8 bytes.
@@ -213,7 +214,7 @@ module Avro
       # equivalent to Java's doubleToLongBits and then encoded in
       # little-endian format.
       def write_double(datum)
-        @writer.write([datum].pack('E'.freeze))
+        @writer.write([datum].pack('E'))
       end
 
       # Bytes are encoded as a long followed by that many bytes of data.
@@ -225,7 +226,7 @@ module Avro
       # A string is encoded as a long followed by that many bytes of
       # UTF-8 encoded character data
       def write_string(datum)
-        datum = datum.encode('utf-8'.freeze) if datum.respond_to? :encode
+        datum = datum.encode('utf-8') if datum.respond_to? :encode
         write_bytes(datum)
       end
 
@@ -392,13 +393,11 @@ module Avro
         case field_schema.type_sym
         when :null
           return nil
-        when :boolean
-          return default_value
         when :int, :long
           return Integer(default_value)
         when :float, :double
           return Float(default_value)
-        when :enum, :fixed, :string, :bytes
+        when :boolean, :enum, :fixed, :string, :bytes
           return default_value
         when :array
           read_array = []

data/lib/avro/ipc.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -63,8 +64,11 @@ module Avro::IPC
   SYSTEM_ERROR_SCHEMA = Avro::Schema.parse('["string"]')
 
   # protocol cache
+  # rubocop:disable Style/MutableConstant
   REMOTE_HASHES = {}
   REMOTE_PROTOCOLS = {}
+  # rubocop:enable Style/MutableConstant
+
 
   BUFFER_HEADER_LENGTH = 4
   BUFFER_SIZE = 8192
@@ -100,7 +104,7 @@ module Avro::IPC
     def request(message_name, request_datum)
       # Writes a request message and reads a response or error message.
       # build handshake and call request
-      buffer_writer = StringIO.new(''.force_encoding('BINARY'))
+      buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
       buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
       write_handshake_request(buffer_encoder)
       write_call_request(message_name, request_datum, buffer_encoder)
@@ -244,7 +248,7 @@ module Avro::IPC
     # a response or error. Compare to 'handle()' in Thrift.
     def respond(call_request, transport=nil)
       buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_request))
-      buffer_writer = StringIO.new(''.force_encoding('BINARY'))
+      buffer_writer = StringIO.new(String.new('', encoding: 'BINARY'))
       buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
       error = nil
       response_metadata = {}
@@ -394,7 +398,7 @@ module Avro::IPC
     def read_framed_message
       message = []
       loop do
-        buffer = StringIO.new(''.force_encoding('BINARY'))
+        buffer = StringIO.new(String.new('', encoding: 'BINARY'))
         buffer_length = read_buffer_length
         if buffer_length == 0
           return message.join
@@ -506,7 +510,7 @@ module Avro::IPC
     def read_framed_message
       message = []
       loop do
-        buffer = ''.force_encoding('BINARY')
+        buffer = String.new('', encoding: 'BINARY')
         buffer_size = read_buffer_size
 
         return message.join if buffer_size == 0
@@ -542,7 +546,7 @@ module Avro::IPC
     end
 
     def transceive(message)
-      writer = FramedWriter.new(StringIO.new(''.force_encoding('BINARY')))
+      writer = FramedWriter.new(StringIO.new(String.new('', encoding: 'BINARY')))
       writer.write_framed_message(message)
       resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
       FramedReader.new(StringIO.new(resp.body)).read_framed_message

data/lib/avro/logical_types.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,9 +17,188 @@
 # limitations under the License.
 
 require 'date'
+require 'bigdecimal'
+require 'bigdecimal/util'
 
 module Avro
   module LogicalTypes
+    ##
+    # Base class for logical types requiring a schema to be present
+    class LogicalTypeWithSchema
+      ##
+      # @return [Avro::Schema] The schema this logical type is dealing with
+      attr_reader :schema
+
+      ##
+      # Build a new instance of a logical type using the provided schema
+      #
+      # @param schema [Avro::Schema]
+      #     The schema to use with this instance
+      #
+      # @raise [ArgumentError]
+      #     If the provided schema is nil
+      def initialize(schema)
+        raise ArgumentError, 'schema is required' if schema.nil?
+
+        @schema = schema
+      end
+
+      ##
+      # Encode the provided datum
+      #
+      # @param datum [Object] The datum to encode
+      #
+      # @raise [NotImplementedError]
+      #     Subclass will need to override this method
+      def encode(datum)
+        raise NotImplementedError
+      end
+
+      ##
+      # Decode the provided datum
+      #
+      # @param datum [Object] The datum to decode
+      #
+      # @raise [NotImplementedError]
+      #     Subclass will need to override this method
+      def decode(datum)
+        raise NotImplementedError
+      end
+    end
+
+    ##
+    # Logical type to handle arbitrary-precision decimals using byte array.
+    #
+    # The byte array contains the two's-complement representation of the unscaled integer
+    # value in big-endian byte order.
+    class BytesDecimal < LogicalTypeWithSchema
+      # Messages for exceptions
+      ERROR_INSUFFICIENT_PRECISION = 'Precision is too small'
+      ERROR_ROUNDING_NECESSARY     = 'Rounding necessary'
+      ERROR_VALUE_MUST_BE_NUMERIC  = 'value must be numeric'
+
+      # The pattern used to pack up the byte array (8 bit unsigned integer/char)
+      PACK_UNSIGNED_CHARS = 'C*'
+
+      # The number 10 as BigDecimal
+      TEN = BigDecimal(10).freeze
+
+      ##
+      # @return [Integer] The number of total digits supported by the decimal
+      attr_reader :precision
+
+      ##
+      # @return [Integer] The number of fractional digits
+      attr_reader :scale
+
+      ##
+      # Build a new decimal logical type
+      #
+      # @param schema [Avro::Schema]
+      #     The schema defining precision and scale for the conversion
+      def initialize(schema)
+        super
+
+        @scale     = schema.scale.to_i
+        @precision = schema.precision.to_i
+        @factor    = TEN ** @scale
+      end
+
+      ##
+      # Encode the provided value into a byte array
+      #
+      # @param value [BigDecimal, Float, Integer]
+      #     The numeric value to encode
+      #
+      # @raise [ArgumentError]
+      #     If the provided value is not a numeric type
+      #
+      # @raise [RangeError]
+      #     If the provided value has a scale higher than the schema permits,
+      #     or does not fit into the schema's precision
+      def encode(value)
+        raise ArgumentError, ERROR_VALUE_MUST_BE_NUMERIC unless value.is_a?(Numeric)
+
+        to_byte_array(unscaled_value(value.to_d)).pack(PACK_UNSIGNED_CHARS).freeze
+      end
+
+      ##
+      # Decode a byte array (in form of a string) into a BigDecimal of the
+      # given precision and scale
+      #
+      # @param stream [String]
+      #     The byte array to decode
+      #
+      # @return [BigDecimal]
+      def decode(stream)
+        from_byte_array(stream) / @factor
+      end
+
+      private
+
+      ##
+      # Convert the provided stream of bytes into the unscaled value
+      #
+      # @param stream [String]
+      #     The stream of bytes to convert
+      #
+      # @return [Integer]
+      def from_byte_array(stream)
+        bytes    = stream.bytes
+        positive = bytes.first[7].zero?
+        total    = 0
+
+        bytes.each_with_index do |value, ix|
+          total += (positive ? value : (value ^ 0xff)) << (bytes.length - ix - 1) * 8
+        end
+
+        return total if positive
+
+        -(total + 1)
+      end
+
+      ##
+      # Convert the provided number into its two's complement representation
+      # in network order (big endian).
+      #
+      # @param number [Integer]
+      #     The number to convert
+      #
+      # @return [Array<Integer>]
+      #     The byte array in network order
+      def to_byte_array(number)
+        [].tap do |result|
+          loop do
+            result.unshift(number & 0xff)
+            number >>= 8
+
+            break if (number == 0 || number == -1) && (result.first[7] == number[7])
+          end
+        end
+      end
+
+      ##
+      # Get the unscaled value from a BigDecimal considering the schema's scale
+      #
+      # @param decimal [BigDecimal]
+      #     The decimal to get the unscaled value from
+      #
+      # @return [Integer]
+      def unscaled_value(decimal)
+        details = decimal.split
+        length  = details[1].length
+
+        fractional_part = length - details[3]
+        raise RangeError, ERROR_ROUNDING_NECESSARY if fractional_part > scale
+
+        if length > precision || (length - fractional_part) > (precision - scale)
+          raise RangeError, ERROR_INSUFFICIENT_PRECISION
+        end
+
+        (decimal * @factor).to_i
+      end
+    end
+
     module IntDate
       EPOCH_START = Date.new(1970, 1, 1)
 
@@ -72,6 +252,9 @@ module Avro
     end
 
     TYPES = {
+      "bytes" => {
+        "decimal" => BytesDecimal
+      },
       "int" => {
         "date" => IntDate
       },
@@ -81,10 +264,11 @@ module Avro
       },
     }.freeze
 
-    def self.type_adapter(type, logical_type)
+    def self.type_adapter(type, logical_type, schema = nil)
       return unless logical_type
 
-      TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
+      adapter = TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
+      adapter.is_a?(Class) ? adapter.new(schema) : adapter
     end
   end
 end

data/lib/avro/protocol.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information