avro 1.8.1 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d432977a240aa9a9356c21483d0f1f601079131e
-  data.tar.gz: a3a7f480daee4ae503f2aa32a053efc4c387ff1f
+  metadata.gz: eab84afb7b30d99dee71a7e360dba2a36957500b
+  data.tar.gz: 8211aea9ad1e00a4f94bb3edc2aeaf9ab06edffa
 SHA512:
-  metadata.gz: 322bf3242f8e369ed7cbceb8b82ebeb526b68a00137e18a9834cb62ea14e01e3e885549fe0301721e46546f9c657eedf596bfeba39e86f3fdb8a2a3e6ceadc13
-  data.tar.gz: 95c50679ab7d3c68e47db5e8b32e323460623dffba5fc9c22a7d95cca9a11ecba60951bcbdb3b1cdb34d30d27225dede1dbb44635791b857538aa8fdd4d33c78
+  metadata.gz: c51889575e3cd689f7288bdd7a0d8250a8147a8eaf568805f726991f9f115c28f8ca9fa396262c687021859cbd9a26c5f196b7f73a174a2da9dd7f22764fe8d8
+  data.tar.gz: 0d936920e8b8ea8edd70583164bd519ac46108de4ab9f7d359d6edb7e8d1941eb3bad06ed8462cdc6bac2d3152c16faba49833d115cc6240e26d3cf51f746c08

data/LICENSE

@@ -1,7 +1,7 @@
 
                                  Apache License
                            Version 2.0, January 2004
-                        http://www.apache.org/licenses/
+                        https://www.apache.org/licenses/
 
    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
@@ -193,7 +193,7 @@
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
 
-       http://www.apache.org/licenses/LICENSE-2.0
+       https://www.apache.org/licenses/LICENSE-2.0
 
    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,

data/Manifest

@@ -9,9 +9,12 @@ lib/avro.rb
 lib/avro/data_file.rb
 lib/avro/io.rb
 lib/avro/ipc.rb
+lib/avro/logical_types.rb
 lib/avro/protocol.rb
 lib/avro/schema.rb
+lib/avro/schema_compatibility.rb
 lib/avro/schema_normalization.rb
+lib/avro/schema_validator.rb
 test/case_finder.rb
 test/random_data.rb
 test/sample_ipc_client.rb
@@ -22,8 +25,11 @@ test/test_datafile.rb
 test/test_fingerprints.rb
 test/test_help.rb
 test/test_io.rb
+test/test_logical_types.rb
 test/test_protocol.rb
 test/test_schema.rb
+test/test_schema_compatibility.rb
 test/test_schema_normalization.rb
+test/test_schema_validator.rb
 test/test_socket_transport.rb
 test/tool.rb

data/NOTICE

@@ -2,5 +2,5 @@ Apache Avro
 Copyright 2010-2015 The Apache Software Foundation
 
 This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
+The Apache Software Foundation (https://www.apache.org/).
 

data/Rakefile

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,9 +22,9 @@ Echoe.new('avro', VERSION) do |p|
   p.email = "dev@avro.apache.org"
   p.summary = "Apache Avro for Ruby"
   p.description = "Avro is a data serialization and RPC format"
-  p.url = "http://avro.apache.org/"
-  p.runtime_dependencies = %w[multi_json]
-  p.licenses = ["Apache License 2.0 (Apache-2.0)"]
+  p.url = "https://avro.apache.org/"
+  p.runtime_dependencies = ["multi_json ~>1"]
+  p.licenses = ["Apache-2.0"]
 end
 
 t = Rake::TestTask.new(:interop)
@@ -38,17 +38,13 @@ task :generate_interop do
 
   schema = Avro::Schema.parse(File.read(SCHEMAS + '/interop.avsc'))
   r = RandomData.new(schema, ENV['SEED'])
-  f = File.open(BUILD + '/interop/data/ruby.avro', 'w')
-  writer = Avro::DataFile::Writer.new(f, Avro::IO::DatumWriter.new(schema), schema)
-  begin
-    writer << r.next
-    writer << r.next
-  ensure
-    writer.close
-  end
-
-  Avro::DataFile.open(BUILD + '/interop/data/ruby_deflate.avro', 'w', schema.to_s, :deflate) do |writer|
-    20.times { writer << r.next }
+  Avro::DataFile.codecs.each do |name, codec|
+    next unless codec
+    filename = name == 'null' ? 'ruby.avro' : "ruby_#{name}.avro"
+    path = File.join(BUILD, 'interop/data', filename)
+    Avro::DataFile.open(path, 'w', schema.to_s, name) do |writer|
+      writer << r.next
+    end
   end
 end
 

data/avro.gemspec

@@ -1,35 +1,35 @@
 # -*- encoding: utf-8 -*-
-# stub: avro 1.8.1 ruby lib
+# stub: avro 1.10.0 ruby lib
 
 Gem::Specification.new do |s|
-  s.name = "avro"
-  s.version = "1.8.1"
+  s.name = "avro".freeze
+  s.version = "1.10.0"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
-  s.authors = ["Apache Software Foundation"]
-  s.date = "2016-05-15"
-  s.description = "Avro is a data serialization and RPC format"
-  s.email = "dev@avro.apache.org"
-  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb"]
-  s.files = ["CHANGELOG", "LICENSE", "Manifest", "NOTICE", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb", "test/case_finder.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_schema.rb", "test/test_schema_normalization.rb", "test/test_socket_transport.rb", "test/tool.rb"]
-  s.homepage = "http://avro.apache.org/"
-  s.licenses = ["Apache License 2.0 (Apache-2.0)"]
-  s.rdoc_options = ["--line-numbers", "--title", "Avro"]
-  s.rubyforge_project = "avro"
-  s.rubygems_version = "2.2.2"
-  s.summary = "Apache Avro for Ruby"
-  s.test_files = ["test/test_schema_normalization.rb", "test/test_io.rb", "test/test_socket_transport.rb", "test/test_schema.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_datafile.rb", "test/test_protocol.rb"]
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib".freeze]
+  s.authors = ["Apache Software Foundation".freeze]
+  s.date = "2020-06-22"
+  s.description = "Avro is a data serialization and RPC format".freeze
+  s.email = "dev@avro.apache.org".freeze
+  s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
+  s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
+  s.homepage = "https://avro.apache.org/".freeze
+  s.licenses = ["Apache-2.0".freeze]
+  s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro".freeze]
+  s.rubyforge_project = "avro".freeze
+  s.rubygems_version = "2.5.2.1".freeze
+  s.summary = "Apache Avro for Ruby".freeze
+  s.test_files = ["test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_protocol.rb".freeze, "test/test_datafile.rb".freeze, "test/test_logical_types.rb".freeze]
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<multi_json>, [">= 0"])
+      s.add_runtime_dependency(%q<multi_json>.freeze, ["~> 1"])
     else
-      s.add_dependency(%q<multi_json>, [">= 0"])
+      s.add_dependency(%q<multi_json>.freeze, ["~> 1"])
     end
   else
-    s.add_dependency(%q<multi_json>, [">= 0"])
+    s.add_dependency(%q<multi_json>.freeze, ["~> 1"])
   end
 end

data/interop/test_interop.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,12 +19,22 @@ require 'rubygems'
 require 'test/unit'
 require 'avro'
 
+CODECS_TO_VALIDATE = ['deflate', 'snappy', 'zstandard']  # The 'null' codec is implicitly included
+
 class TestInterop < Test::Unit::TestCase
   HERE = File.expand_path(File.dirname(__FILE__))
   SHARE = HERE + '/../../../share'
   SCHEMAS = SHARE + '/test/schemas'
-  Dir[HERE + '/../../../build/interop/data/*'].each do |fn|
-    define_method("test_read_#{File.basename(fn, 'avro')}") do
+
+  files = Dir[HERE + '/../../../build/interop/data/*.avro'].select do |fn|
+    sep, codec = File.basename(fn, '.avro').rpartition('_')[1, 2]
+    sep.empty? || CODECS_TO_VALIDATE.include?(codec)
+  end
+  puts "The following files will be tested:"
+  puts files
+
+  files.each do |fn|
+    define_method("test_read_#{File.basename(fn, '.avro')}") do
       projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
 
       File.open(fn) do |f|

data/lib/avro.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -28,10 +28,31 @@ module Avro
 
   class AvroTypeError < Avro::AvroError
     def initialize(schm=nil, datum=nil, msg=nil)
-      msg ||= "Not a #{schm.to_s}: #{datum}"
+      msg ||= "Not a #{schm}: #{datum}"
       super(msg)
     end
   end
+
+  class << self
+    attr_writer :disable_enum_symbol_validation
+    attr_writer :disable_field_default_validation
+    attr_writer :disable_schema_name_validation
+
+    def disable_enum_symbol_validation
+      @disable_enum_symbol_validation ||=
+        ENV.fetch('AVRO_DISABLE_ENUM_SYMBOL_VALIDATION', '') != ''
+    end
+
+    def disable_field_default_validation
+      @disable_field_default_validation ||=
+        ENV.fetch('AVRO_DISABLE_FIELD_DEFAULT_VALIDATION', '') != ''
+    end
+
+    def disable_schema_name_validation
+      @disable_schema_name_validation ||=
+        ENV.fetch('AVRO_DISABLE_SCHEMA_NAME_VALIDATION', '') != ''
+    end
+  end
 end
 
 require 'avro/schema'
@@ -40,3 +61,5 @@ require 'avro/data_file'
 require 'avro/protocol'
 require 'avro/ipc'
 require 'avro/schema_normalization'
+require 'avro/schema_validator'
+require 'avro/schema_compatibility'

data/lib/avro/data_file.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -316,7 +316,7 @@ module Avro
       def decompress(compressed)
         # Passing a negative number to Inflate puts it into "raw" RFC1951 mode
         # (without the RFC1950 header & checksum). See the docs for
-        # inflateInit2 in http://www.zlib.net/manual.html
+        # inflateInit2 in https://www.zlib.net/manual.html
         zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
         data = zstream.inflate(compressed)
         data << zstream.finish
@@ -338,12 +338,29 @@ module Avro
 
       def decompress(data)
         load_snappy!
+        crc32 = data.slice(-4..-1).unpack('N').first
+        uncompressed = Snappy.inflate(data.slice(0..-5))
+
+        if crc32 == Zlib.crc32(uncompressed)
+          uncompressed
+        else
+          # older versions of avro-ruby didn't write the checksum, so if it
+          # doesn't match this must assume that it wasn't there and return
+          # the entire payload uncompressed.
+          Snappy.inflate(data)
+        end
+      rescue Snappy::Error
+        # older versions of avro-ruby didn't write the checksum, so removing
+        # the last 4 bytes may cause Snappy to fail. recover by assuming the
+        # payload is from an older file and uncompress the entire buffer.
         Snappy.inflate(data)
       end
 
       def compress(data)
         load_snappy!
-        Snappy.deflate(data)
+        crc32 = Zlib.crc32(data)
+        compressed = Snappy.deflate(data)
+        [compressed, crc32].pack('a*N')
       end
 
       private
@@ -355,9 +372,32 @@ module Avro
       end
     end
 
+    class ZstandardCodec
+      def codec_name; 'zstandard'; end
+
+      def decompress(data)
+        load_zstandard!
+        Zstd.decompress(data)
+      end
+
+      def compress(data)
+        load_zstandard!
+        Zstd.compress(data)
+      end
+
+      private
+
+      def load_zstandard!
+        require 'zstd-ruby' unless defined?(Zstd)
+      rescue LoadError
+        raise LoadError, "Zstandard compression is not available, please install the `zstd-ruby` gem."
+      end
+    end
+
     DataFile.register_codec NullCodec
     DataFile.register_codec DeflateCodec
     DataFile.register_codec SnappyCodec
+    DataFile.register_codec ZstandardCodec
 
     # TODO this constant won't be updated if you register another codec.
     # Deprecated in favor of Avro::DataFile::codecs

data/lib/avro/io.rb

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
-# http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -43,9 +43,9 @@ module Avro
       end
 
       def byte!
-        @reader.read(1).unpack('C').first
+        @reader.readbyte
       end
-      
+
       def read_null
         # null is written as zero byte's
         nil
@@ -76,7 +76,7 @@ module Avro
         # The float is converted into a 32-bit integer using a method
         # equivalent to Java's floatToIntBits and then encoded in
         # little-endian format.
-        @reader.read(4).unpack('e')[0]
+        read_and_unpack(4, 'e'.freeze)
       end
 
       def read_double
@@ -84,7 +84,7 @@ module Avro
         # The double is converted into a 64-bit integer using a method
         # equivalent to Java's doubleToLongBits and then encoded in
         # little-endian format.
-        @reader.read(8).unpack('E')[0]
+        read_and_unpack(8, 'E'.freeze)
       end
 
       def read_bytes
@@ -97,7 +97,7 @@ module Avro
         # A string is encoded as a long followed by that many bytes of
         # UTF-8 encoded character data.
         read_bytes.tap do |string|
-          string.force_encoding("UTF-8") if string.respond_to? :force_encoding
+          string.force_encoding('UTF-8'.freeze) if string.respond_to? :force_encoding
         end
       end
 
@@ -144,6 +144,23 @@ module Avro
       def skip(n)
         reader.seek(reader.tell() + n)
       end
+
+      private
+
+      # Optimize unpacking strings when `unpack1` is available (ruby >= 2.4)
+      if String.instance_methods.include?(:unpack1)
+
+        def read_and_unpack(byte_count, format)
+          @reader.read(byte_count).unpack1(format)
+        end
+
+      else
+
+        def read_and_unpack(byte_count, format)
+          @reader.read(byte_count).unpack(format)[0]
+        end
+
+      end
     end
 
     # Write leaf values
@@ -155,11 +172,11 @@ module Avro
       end
 
       # null is written as zero bytes
-      def write_null(datum)
+      def write_null(_datum)
         nil
       end
 
-      # a boolean is written as a single byte 
+      # a boolean is written as a single byte
       # whose value is either 0 (false) or 1 (true).
       def write_boolean(datum)
         on_disk = datum ? 1.chr : 0.chr
@@ -175,7 +192,6 @@ module Avro
       # int and long values are written using variable-length,
       # zig-zag coding.
       def write_long(n)
-        foo = n
         n = (n << 1) ^ (n >> 63)
         while (n & ~0x7F) != 0
           @writer.write(((n & 0x7f) | 0x80).chr)
@@ -189,7 +205,7 @@ module Avro
       # equivalent to Java's floatToIntBits and then encoded in
       # little-endian format.
       def write_float(datum)
-        @writer.write([datum].pack('e'))
+        @writer.write([datum].pack('e'.freeze))
       end
 
       # A double is written as 8 bytes.
@@ -197,7 +213,7 @@ module Avro
       # equivalent to Java's doubleToLongBits and then encoded in
       # little-endian format.
       def write_double(datum)
-        @writer.write([datum].pack('E'))
+        @writer.write([datum].pack('E'.freeze))
       end
 
       # Bytes are encoded as a long followed by that many bytes of data.
@@ -209,7 +225,7 @@ module Avro
       # A string is encoded as a long followed by that many bytes of
       # UTF-8 encoded character data
       def write_string(datum)
-        datum = datum.encode('utf-8') if datum.respond_to? :encode
+        datum = datum.encode('utf-8'.freeze) if datum.respond_to? :encode
         write_bytes(datum)
       end
 
@@ -221,46 +237,7 @@ module Avro
 
     class DatumReader
       def self.match_schemas(writers_schema, readers_schema)
-        w_type = writers_schema.type_sym
-        r_type = readers_schema.type_sym
-
-        # This conditional is begging for some OO love.
-        if w_type == :union || r_type == :union
-          return true
-        end
-
-        if w_type == r_type
-          return true if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
-
-          case r_type
-          when :record
-            return writers_schema.fullname == readers_schema.fullname
-          when :error
-            return writers_schema.fullname == readers_schema.fullname
-          when :request
-            return true
-          when :fixed
-            return writers_schema.fullname == readers_schema.fullname &&
-                   writers_schema.size == readers_schema.size
-          when :enum
-            return writers_schema.fullname == readers_schema.fullname
-          when :map
-            return writers_schema.values.type == readers_schema.values.type
-          when :array
-            return writers_schema.items.type == readers_schema.items.type
-          end
-        end
-
-        # Handle schema promotion
-        if w_type == :int && [:long, :float, :double].include?(r_type)
-          return true
-        elsif w_type == :long && [:float, :double].include?(r_type)
-          return true
-        elsif w_type == :float && r_type == :double
-          return true
-        end
-
-        return false
+        Avro::SchemaCompatibility.match_schemas(writers_schema, readers_schema)
       end
 
       attr_accessor :writers_schema, :readers_schema
@@ -293,7 +270,7 @@ module Avro
 
         # function dispatch for reading data based on type of writer's
         # schema
-        case writers_schema.type_sym
+        datum = case writers_schema.type_sym
         when :null;    decoder.read_null
         when :boolean; decoder.read_boolean
         when :string;  decoder.read_string
@@ -311,9 +288,11 @@ module Avro
         else
           raise AvroError, "Cannot read unknown schema type: #{writers_schema.type}"
         end
+
+        readers_schema.type_adapter.decode(datum)
       end
 
-      def read_fixed(writers_schema, readers_schema, decoder)
+      def read_fixed(writers_schema, _readers_schema, decoder)
         decoder.read(writers_schema.size)
       end
 
@@ -321,12 +300,12 @@ module Avro
         index_of_symbol = decoder.read_int
         read_symbol = writers_schema.symbols[index_of_symbol]
 
-        # TODO(jmhodges): figure out what unset means for resolution
-        # schema resolution
-        unless readers_schema.symbols.include?(read_symbol)
-          # 'unset' here
+        if !readers_schema.symbols.include?(read_symbol) && readers_schema.default
+          read_symbol = readers_schema.default
         end
 
+        # This implementation deviates from the spec by always returning
+        # a symbol.
         read_symbol
       end
 
@@ -336,7 +315,7 @@ module Avro
         while block_count != 0
           if block_count < 0
             block_count = -block_count
-            block_size = decoder.read_long
+            _block_size = decoder.read_long
           end
           block_count.times do
             read_items << read_data(writers_schema.items,
@@ -355,7 +334,7 @@ module Avro
         while block_count != 0
           if block_count < 0
             block_count = -block_count
-            block_size = decoder.read_long
+            _block_size = decoder.read_long
           end
           block_count.times do
             key = decoder.read_string
@@ -380,26 +359,28 @@ module Avro
         readers_fields_hash = readers_schema.fields_hash
         read_record = {}
         writers_schema.fields.each do |field|
-          if readers_field = readers_fields_hash[field.name]
+          readers_field = readers_fields_hash[field.name]
+          if readers_field
             field_val = read_data(field.type, readers_field.type, decoder)
             read_record[field.name] = field_val
+          elsif readers_schema.fields_by_alias.key?(field.name)
+            readers_field = readers_schema.fields_by_alias[field.name]
+            field_val = read_data(field.type, readers_field.type, decoder)
+            read_record[readers_field.name] = field_val
           else
             skip_data(field.type, decoder)
           end
         end
 
         # fill in the default values
-        if readers_fields_hash.size > read_record.size
-          writers_fields_hash = writers_schema.fields_hash
-          readers_fields_hash.each do |field_name, field|
-            unless writers_fields_hash.has_key? field_name
-              if !field.default.nil?
-                field_val = read_default_value(field.type, field.default)
-                read_record[field.name] = field_val
-              else
-                # FIXME(jmhodges) another 'unset' here
-              end
-            end
+        readers_fields_hash.each do |field_name, field|
+          next if read_record.key?(field_name)
+
+          if field.default?
+            field_val = read_default_value(field.type, field.default)
+            read_record[field.name] = field_val
+          else
+            raise AvroError, "Missing data for #{field.type} with no default"
           end
         end
 
@@ -489,7 +470,7 @@ module Avro
         decoder.skip(writers_schema.size)
       end
 
-      def skip_enum(writers_schema, decoder)
+      def skip_enum(_writers_schema, decoder)
         decoder.skip_int
       end
 
@@ -520,7 +501,7 @@ module Avro
           if block_count < 0
             decoder.skip(decoder.read_long)
           else
-            block_count.times &blk
+            block_count.times(&blk)
           end
           block_count = decoder.read_long
         end
@@ -538,8 +519,10 @@ module Avro
         write_data(writers_schema, datum, encoder)
       end
 
-      def write_data(writers_schema, datum, encoder)
-        unless Schema.validate(writers_schema, datum)
+      def write_data(writers_schema, logical_datum, encoder)
+        datum = writers_schema.type_adapter.encode(logical_datum)
+
+        unless Schema.validate(writers_schema, datum, { recursive: false, encoded: true })
           raise AvroTypeError.new(writers_schema, datum)
         end
 
@@ -564,7 +547,7 @@ module Avro
         end
       end
 
-      def write_fixed(writers_schema, datum, encoder)
+      def write_fixed(_writers_schema, datum, encoder)
         encoder.write(datum)
       end
 
@@ -574,6 +557,7 @@ module Avro
       end
 
       def write_array(writers_schema, datum, encoder)
+        raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Array)
         if datum.size > 0
           encoder.write_long(datum.size)
           datum.each do |item|
@@ -584,6 +568,7 @@ module Avro
       end
 
       def write_map(writers_schema, datum, encoder)
+        raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Hash)
         if datum.size > 0
           encoder.write_long(datum.size)
           datum.each do |k,v|
@@ -606,8 +591,9 @@ module Avro
       end
 
       def write_record(writers_schema, datum, encoder)
+        raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Hash)
         writers_schema.fields.each do |field|
-          write_data(field.type, datum[field.name], encoder)
+          write_data(field.type, datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym], encoder)
         end
       end
     end # DatumWriter