avro 1.9.0 → 1.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e128a7db859ace5ff9b558f8c43a5a7f1c79ee5
-  data.tar.gz: dedca07a2a2f32894edded0b785d0c7954b65246
+  metadata.gz: 38a2f2d15367014464b4613e0fc80724b345d3b3
+  data.tar.gz: 823aa3ebb6ea719cbd4c94060757a545c7d626a9
 SHA512:
-  metadata.gz: 5393149d588e12974d8036cd76b63b96b2698cc02fec3608fea7d88ecec936e5c5b81444252b0e1d42932f08299ff89b81828a7e7f877e7c67faf4713b07d979
-  data.tar.gz: b0f8c39b2fdabe08e5769e05baf1036969753be9e93ecabd1c6230df9f839167b43cc1bbea523439a430ed624d5243c7926b44867961422785010412f132dd91
+  metadata.gz: da4d7d3ecf02bb18c30c044635188ba2b8a34ff13f9450330e850198610ee87f4e9116a79a95366bc23b5e3fb7f70906736d1ff7834e9293825c5c3eba072b9b
+  data.tar.gz: 3194b9379e1ae396acd0a120f583ade7127628d5adf78ac382fcf7215643c69c5341ee0a5f234db56eec3e62070683340b4a21bcff4ba0e2208a832c571f8dc9

data/LICENSE

@@ -1,7 +1,7 @@
 
                                  Apache License
                            Version 2.0, January 2004
-                        http://www.apache.org/licenses/
+                        https://www.apache.org/licenses/
 
    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
@@ -193,7 +193,7 @@
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
 
-       http://www.apache.org/licenses/LICENSE-2.0
+       https://www.apache.org/licenses/LICENSE-2.0
 
    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,

data/Manifest

@@ -6,6 +6,7 @@ Rakefile
 avro.gemspec
 interop/test_interop.rb
 lib/avro.rb
+lib/avro/VERSION.txt
 lib/avro/data_file.rb
 lib/avro/io.rb
 lib/avro/ipc.rb

data/NOTICE

@@ -2,5 +2,5 @@ Apache Avro
 Copyright 2010-2015 The Apache Software Foundation
 
 This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
+The Apache Software Foundation (https://www.apache.org/).
 

data/Rakefile

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,14 +17,15 @@
 require 'rubygems'
 require 'echoe'
 VERSION = File.open('../../share/VERSION.txt').read.sub('-SNAPSHOT', '.pre1').chomp
+File.write("lib/avro/VERSION.txt", VERSION)
 Echoe.new('avro', VERSION) do |p|
   p.author = "Apache Software Foundation"
   p.email = "dev@avro.apache.org"
   p.summary = "Apache Avro for Ruby"
   p.description = "Avro is a data serialization and RPC format"
-  p.url = "http://avro.apache.org/"
-  p.runtime_dependencies = %w[multi_json]
-  p.licenses = ["Apache License 2.0 (Apache-2.0)"]
+  p.url = "https://avro.apache.org/"
+  p.runtime_dependencies = ["multi_json ~>1"]
+  p.licenses = ["Apache-2.0"]
 end
 
 t = Rake::TestTask.new(:interop)
@@ -38,17 +39,13 @@ task :generate_interop do
 
   schema = Avro::Schema.parse(File.read(SCHEMAS + '/interop.avsc'))
   r = RandomData.new(schema, ENV['SEED'])
-  f = File.open(BUILD + '/interop/data/ruby.avro', 'w')
-  writer = Avro::DataFile::Writer.new(f, Avro::IO::DatumWriter.new(schema), schema)
-  begin
-    writer << r.next
-    writer << r.next
-  ensure
-    writer.close
-  end
-
-  Avro::DataFile.open(BUILD + '/interop/data/ruby_deflate.avro', 'w', schema.to_s, :deflate) do |writer|
-    20.times { writer << r.next }
+  Avro::DataFile.codecs.each do |name, codec|
+    next unless codec
+    filename = name == 'null' ? 'ruby.avro' : "ruby_#{name}.avro"
+    path = File.join(BUILD, 'interop/data', filename)
+    Avro::DataFile.open(path, 'w', schema.to_s, name) do |writer|
+      writer << r.next
+    end
   end
 end
 

data/avro.gemspec

@@ -1,35 +1,35 @@
 # -*- encoding: utf-8 -*-
-# stub: avro 1.9.0 ruby lib
+# stub: avro 1.10.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "avro".freeze
-  s.version = "1.9.0"
+  s.version = "1.10.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Apache Software Foundation".freeze]
-  s.date = "2019-05-21"
+  s.date = "2021-03-09"
   s.description = "Avro is a data serialization and RPC format".freeze
   s.email = "dev@avro.apache.org".freeze
-  s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
-  s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
-  s.homepage = "http://avro.apache.org/".freeze
-  s.licenses = ["Apache License 2.0 (Apache-2.0)".freeze]
+  s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
+  s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
+  s.homepage = "https://avro.apache.org/".freeze
+  s.licenses = ["Apache-2.0".freeze]
   s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro".freeze]
   s.rubyforge_project = "avro".freeze
   s.rubygems_version = "2.5.2.1".freeze
   s.summary = "Apache Avro for Ruby".freeze
-  s.test_files = ["test/test_schema_validator.rb".freeze, "test/test_help.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_datafile.rb".freeze, "test/test_schema.rb".freeze, "test/test_io.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_protocol.rb".freeze]
+  s.test_files = ["test/test_schema.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_help.rb".freeze, "test/test_datafile.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_fingerprints.rb".freeze]
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<multi_json>.freeze, [">= 0"])
+      s.add_runtime_dependency(%q<multi_json>.freeze, ["~> 1"])
     else
-      s.add_dependency(%q<multi_json>.freeze, [">= 0"])
+      s.add_dependency(%q<multi_json>.freeze, ["~> 1"])
     end
   else
-    s.add_dependency(%q<multi_json>.freeze, [">= 0"])
+    s.add_dependency(%q<multi_json>.freeze, ["~> 1"])
   end
 end

data/interop/test_interop.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,12 +19,22 @@ require 'rubygems'
 require 'test/unit'
 require 'avro'
 
+CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard']  # The 'null' codec is implicitly included
+
 class TestInterop < Test::Unit::TestCase
   HERE = File.expand_path(File.dirname(__FILE__))
   SHARE = HERE + '/../../../share'
   SCHEMAS = SHARE + '/test/schemas'
-  Dir[HERE + '/../../../build/interop/data/*'].each do |fn|
-    define_method("test_read_#{File.basename(fn, 'avro')}") do
+
+  files = Dir[HERE + '/../../../build/interop/data/*.avro'].select do |fn|
+    sep, codec = File.basename(fn, '.avro').rpartition('_')[1, 2]
+    sep.empty? || CODECS_TO_VALIDATE.include?(codec)
+  end
+  puts "The following files will be tested:"
+  puts files
+
+  files.each do |fn|
+    define_method("test_read_#{File.basename(fn, '.avro')}") do
       projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
 
       File.open(fn) do |f|

data/lib/avro.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,24 +22,36 @@ require 'stringio'
 require 'zlib'
 
 module Avro
-  VERSION = "FIXME"
+  VERSION = File.read("#{__dir__}/avro/VERSION.txt").freeze
 
   class AvroError < StandardError; end
 
   class AvroTypeError < Avro::AvroError
     def initialize(schm=nil, datum=nil, msg=nil)
-      msg ||= "Not a #{schm.to_s}: #{datum}"
+      msg ||= "Not a #{schm}: #{datum}"
       super(msg)
     end
   end
 
   class << self
+    attr_writer :disable_enum_symbol_validation
     attr_writer :disable_field_default_validation
+    attr_writer :disable_schema_name_validation
+
+    def disable_enum_symbol_validation
+      @disable_enum_symbol_validation ||=
+        ENV.fetch('AVRO_DISABLE_ENUM_SYMBOL_VALIDATION', '') != ''
+    end
 
     def disable_field_default_validation
       @disable_field_default_validation ||=
         ENV.fetch('AVRO_DISABLE_FIELD_DEFAULT_VALIDATION', '') != ''
     end
+
+    def disable_schema_name_validation
+      @disable_schema_name_validation ||=
+        ENV.fetch('AVRO_DISABLE_SCHEMA_NAME_VALIDATION', '') != ''
+    end
   end
 end
 

data/lib/avro/VERSION.txt

@@ -0,0 +1 @@
+1.10.2

data/lib/avro/data_file.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -316,7 +316,7 @@ module Avro
       def decompress(compressed)
         # Passing a negative number to Inflate puts it into "raw" RFC1951 mode
         # (without the RFC1950 header & checksum). See the docs for
-        # inflateInit2 in http://www.zlib.net/manual.html
+        # inflateInit2 in https://www.zlib.net/manual.html
         zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
         data = zstream.inflate(compressed)
         data << zstream.finish
@@ -372,9 +372,32 @@ module Avro
       end
     end
 
+    class ZstandardCodec
+      def codec_name; 'zstandard'; end
+
+      def decompress(data)
+        load_zstandard!
+        Zstd.decompress(data)
+      end
+
+      def compress(data)
+        load_zstandard!
+        Zstd.compress(data)
+      end
+
+      private
+
+      def load_zstandard!
+        require 'zstd-ruby' unless defined?(Zstd)
+      rescue LoadError
+        raise LoadError, "Zstandard compression is not available, please install the `zstd-ruby` gem."
+      end
+    end
+
     DataFile.register_codec NullCodec
     DataFile.register_codec DeflateCodec
     DataFile.register_codec SnappyCodec
+    DataFile.register_codec ZstandardCodec
 
     # TODO this constant won't be updated if you register another codec.
     # Deprecated in favor of Avro::DataFile::codecs

data/lib/avro/io.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -172,7 +172,7 @@ module Avro
       end
 
       # null is written as zero bytes
-      def write_null(datum)
+      def write_null(_datum)
         nil
       end
 
@@ -292,7 +292,7 @@ module Avro
         readers_schema.type_adapter.decode(datum)
       end
 
-      def read_fixed(writers_schema, readers_schema, decoder)
+      def read_fixed(writers_schema, _readers_schema, decoder)
         decoder.read(writers_schema.size)
       end
 
@@ -300,12 +300,12 @@ module Avro
         index_of_symbol = decoder.read_int
         read_symbol = writers_schema.symbols[index_of_symbol]
 
-        # TODO(jmhodges): figure out what unset means for resolution
-        # schema resolution
-        unless readers_schema.symbols.include?(read_symbol)
-          # 'unset' here
+        if !readers_schema.symbols.include?(read_symbol) && readers_schema.default
+          read_symbol = readers_schema.default
         end
 
+        # This implementation deviates from the spec by always returning
+        # a symbol.
         read_symbol
       end
 
@@ -359,26 +359,28 @@ module Avro
         readers_fields_hash = readers_schema.fields_hash
         read_record = {}
         writers_schema.fields.each do |field|
-          if readers_field = readers_fields_hash[field.name]
+          readers_field = readers_fields_hash[field.name]
+          if readers_field
             field_val = read_data(field.type, readers_field.type, decoder)
             read_record[field.name] = field_val
+          elsif readers_schema.fields_by_alias.key?(field.name)
+            readers_field = readers_schema.fields_by_alias[field.name]
+            field_val = read_data(field.type, readers_field.type, decoder)
+            read_record[readers_field.name] = field_val
           else
             skip_data(field.type, decoder)
           end
         end
 
         # fill in the default values
-        if readers_fields_hash.size > read_record.size
-          writers_fields_hash = writers_schema.fields_hash
-          readers_fields_hash.each do |field_name, field|
-            unless writers_fields_hash.has_key? field_name
-              if field.default?
-                field_val = read_default_value(field.type, field.default)
-                read_record[field.name] = field_val
-              else
-                raise AvroError, "Missing data for #{field.type} with no default"
-              end
-            end
+        readers_fields_hash.each do |field_name, field|
+          next if read_record.key?(field_name)
+
+          if field.default?
+            field_val = read_default_value(field.type, field.default)
+            read_record[field.name] = field_val
+          else
+            raise AvroError, "Missing data for #{field.type} with no default"
           end
         end
 
@@ -468,7 +470,7 @@ module Avro
         decoder.skip(writers_schema.size)
       end
 
-      def skip_enum(writers_schema, decoder)
+      def skip_enum(_writers_schema, decoder)
         decoder.skip_int
       end
 
@@ -508,6 +510,8 @@ module Avro
 
     # DatumWriter for generic ruby objects
     class DatumWriter
+      VALIDATION_OPTIONS = { recursive: false, encoded: true }.freeze
+
       attr_accessor :writers_schema
       def initialize(writers_schema=nil)
         @writers_schema = writers_schema
@@ -520,7 +524,7 @@ module Avro
       def write_data(writers_schema, logical_datum, encoder)
         datum = writers_schema.type_adapter.encode(logical_datum)
 
-        unless Schema.validate(writers_schema, datum, { recursive: false, encoded: true })
+        unless Schema.validate(writers_schema, datum, VALIDATION_OPTIONS)
           raise AvroTypeError.new(writers_schema, datum)
         end
 
@@ -545,7 +549,7 @@ module Avro
         end
       end
 
-      def write_fixed(writers_schema, datum, encoder)
+      def write_fixed(_writers_schema, datum, encoder)
         encoder.write(datum)
       end
 
@@ -578,12 +582,15 @@ module Avro
       end
 
       def write_union(writers_schema, datum, encoder)
-        index_of_schema = -1
-        found = writers_schema.schemas.
-          find{|e| index_of_schema += 1; found = Schema.validate(e, datum) }
-        unless found  # Because find_index doesn't exist in 1.8.6
+        index_of_schema = writers_schema.schemas.find_index do |schema|
+          # Optimize away expensive validation calls for the common null type
+          schema.type_sym == :null ? datum.nil? : Schema.validate(schema, datum)
+        end
+
+        unless index_of_schema
           raise AvroTypeError.new(writers_schema, datum)
         end
+
         encoder.write_long(index_of_schema)
         write_data(writers_schema.schemas[index_of_schema], datum, encoder)
       end
@@ -591,7 +598,7 @@ module Avro
       def write_record(writers_schema, datum, encoder)
         raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Hash)
         writers_schema.fields.each do |field|
-          write_data(field.type, datum[field.name], encoder)
+          write_data(field.type, datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym], encoder)
         end
       end
     end # DatumWriter

data/lib/avro/ipc.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -278,7 +278,7 @@ module Avro::IPC
           response = call(local_message, request)
         rescue AvroRemoteError => e
           error = e
-        rescue Exception => e
+        rescue Exception => e # rubocop:disable Lint/RescueException
           error = AvroRemoteError.new(e.to_s)
         end
 
@@ -350,7 +350,7 @@ module Avro::IPC
       remote_protocol
     end
 
-    def call(local_message, request)
+    def call(_local_message, _request)
       # Actual work done by server: cf. handler in thrift.
       raise NotImplementedError
     end