avro-salsify-fork 1.9.0.2 → 1.9.0.3.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 525f50ff0c98925e120b4ab2517c0e1769f88a8c
-  data.tar.gz: d3332be0b5b397baaa6fa83c9b0d979470d849e7
+  metadata.gz: f26367215fdb88d9e6a8acebb159d7f30f226d4e
+  data.tar.gz: cef6d73aed57d1731a58ccfb5793759150b81fc1
 SHA512:
-  metadata.gz: 4c44254ba038292e519c4129802061eac7b1ba7282503972420e3869adf39b43dcef3d65ebc19b9696528e9720ed53dc0834071e57406868628fab160b6eedc1
-  data.tar.gz: 102a76c34941d746945ec12c12646e0500f0a6f8a7d50fdc9014fcac1316c202721f5ff19532e9a99519edb0045552d176d288f09d55c6cc4fa613458b8a819a
+  metadata.gz: 8b2419d791ef4e26b27aa61017c0f71867af272480d612d8da0e244a28767af71e55d1a4a5746af982645c79825d31731ac98ad9819f8cbde594b6c20e456502
+  data.tar.gz: 104f099f07aea4dc5aa19d36e14f536ea6620896d5039ba4f8bffa1bdc872a4b48853a951a1f59af492eefa46f96a5089e2649a966bbca0737c9e6b83e141ca0

data/Rakefile

@@ -16,7 +16,7 @@
 
 require 'rubygems'
 require 'echoe'
-VERSION = '1.9.0.2'
+VERSION = '1.9.0.3.pre1'
 Echoe.new('avro-salsify-fork', VERSION) do |p|
   p.author = "Apache Software Foundation / Salsify Engineering"
   p.email = "engineering@salsify.com"

data/avro-salsify-fork.gemspec

@@ -1,25 +1,25 @@
 # -*- encoding: utf-8 -*-
-# stub: avro-salsify-fork 1.9.0.2 ruby lib
+# stub: avro-salsify-fork 1.9.0.3.pre1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "avro-salsify-fork".freeze
-  s.version = "1.9.0.2"
+  s.version = "1.9.0.3.pre1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Apache Software Foundation / Salsify Engineering".freeze]
-  s.date = "2016-09-06"
+  s.date = "2016-12-15"
   s.description = "Avro is a data serialization and RPC format.\nThis release contains the changes submitted in https://github.com/apache/avro/pull/116\nto support logical types in the Ruby gem.".freeze
   s.email = "engineering@salsify.com".freeze
   s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_normalization.rb".freeze]
-  s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro-salsify-fork.gemspec".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
+  s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro-salsify-fork.gemspec".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
   s.homepage = "https://github.com/salsify/avro".freeze
   s.licenses = ["Apache License 2.0 (Apache-2.0)".freeze]
   s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro-salsify-fork".freeze]
   s.rubyforge_project = "avro-salsify-fork".freeze
   s.rubygems_version = "2.6.6".freeze
   s.summary = "Apache Avro for Ruby with logical types patch".freeze
-  s.test_files = ["test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_socket_transport.rb".freeze]
+  s.test_files = ["test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_socket_transport.rb".freeze]
 
   if s.respond_to? :specification_version then
     s.specification_version = 4

data/avro.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.rubyforge_project = "avro".freeze
   s.rubygems_version = "2.6.6".freeze
   s.summary = "Apache Avro for Ruby".freeze
-  s.test_files = ["test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_socket_transport.rb".freeze]
+  s.test_files = ["test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_socket_transport.rb".freeze]
 
   if s.respond_to? :specification_version then
     s.specification_version = 4

data/lib/avro.rb

@@ -40,3 +40,4 @@ require 'avro/data_file'
 require 'avro/protocol'
 require 'avro/ipc'
 require 'avro/schema_normalization'
+require 'avro/schema_compatibility'

data/lib/avro/data_file.rb

@@ -338,12 +338,29 @@ module Avro
 
       def decompress(data)
         load_snappy!
+        crc32 = data.slice(-4..-1).unpack('N').first
+        uncompressed = Snappy.inflate(data.slice(0..-5))
+
+        if crc32 == Zlib.crc32(uncompressed)
+          uncompressed
+        else
+          # older versions of avro-ruby didn't write the checksum, so if it
+          # doesn't match this must assume that it wasn't there and return
+          # the entire payload uncompressed.
+          Snappy.inflate(data)
+        end
+      rescue Snappy::Error
+        # older versions of avro-ruby didn't write the checksum, so removing
+        # the last 4 bytes may cause Snappy to fail. recover by assuming the
+        # payload is from an older file and uncompress the entire buffer.
         Snappy.inflate(data)
       end
 
       def compress(data)
         load_snappy!
-        Snappy.deflate(data)
+        crc32 = Zlib.crc32(data)
+        compressed = Snappy.deflate(data)
+        [compressed, crc32].pack('a*N')
       end
 
       private

data/lib/avro/io.rb

@@ -221,46 +221,7 @@ module Avro
 
     class DatumReader
       def self.match_schemas(writers_schema, readers_schema)
-        w_type = writers_schema.type_sym
-        r_type = readers_schema.type_sym
-
-        # This conditional is begging for some OO love.
-        if w_type == :union || r_type == :union
-          return true
-        end
-
-        if w_type == r_type
-          return true if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
-
-          case r_type
-          when :record
-            return writers_schema.fullname == readers_schema.fullname
-          when :error
-            return writers_schema.fullname == readers_schema.fullname
-          when :request
-            return true
-          when :fixed
-            return writers_schema.fullname == readers_schema.fullname &&
-                   writers_schema.size == readers_schema.size
-          when :enum
-            return writers_schema.fullname == readers_schema.fullname
-          when :map
-            return writers_schema.values.type == readers_schema.values.type
-          when :array
-            return writers_schema.items.type == readers_schema.items.type
-          end
-        end
-
-        # Handle schema promotion
-        if w_type == :int && [:long, :float, :double].include?(r_type)
-          return true
-        elsif w_type == :long && [:float, :double].include?(r_type)
-          return true
-        elsif w_type == :float && r_type == :double
-          return true
-        end
-
-        return false
+        Avro::SchemaCompatibility.match_schemas(writers_schema, readers_schema)
       end
 
       attr_accessor :writers_schema, :readers_schema
@@ -395,11 +356,11 @@ module Avro
           writers_fields_hash = writers_schema.fields_hash
           readers_fields_hash.each do |field_name, field|
             unless writers_fields_hash.has_key? field_name
-              if !field.default.nil?
+              if field.default?
                 field_val = read_default_value(field.type, field.default)
                 read_record[field.name] = field_val
               else
-                # FIXME(jmhodges) another 'unset' here
+                raise AvroError, "Missing data for #{field.type} with no default"
               end
             end
           end
@@ -409,6 +370,10 @@ module Avro
       end
 
       def read_default_value(field_schema, default_value)
+        if default_value == :no_default
+          raise AvroError, "Missing data for #{field_schema} with no default"
+        end
+
         # Basically a JSON Decoder?
         case field_schema.type_sym
         when :null

data/lib/avro/schema.rb

@@ -164,6 +164,18 @@ module Avro
       Digest::SHA256.hexdigest(parsing_form).to_i(16)
     end
 
+    def read?(writers_schema)
+      SchemaCompatibility.can_read?(writers_schema, self)
+    end
+
+    def be_read?(other_schema)
+      other_schema.read?(self)
+    end
+
+    def mutual_read?(other_schema)
+      SchemaCompatibility.mutual_read?(other_schema, self)
+    end
+
     def ==(other, seen=nil)
       other.is_a?(Schema) && type_sym == other.type_sym
     end
@@ -228,7 +240,7 @@ module Avro
           if field.respond_to?(:[]) # TODO(jmhodges) wtffffff
             type = field['type']
             name = field['name']
-            default = field['default']
+            default = field.key?('default') ? field['default'] : :no_default
             order = field['order']
             new_field = Field.new(type, name, default, order, names, namespace)
             # make sure field name has not been used yet
@@ -251,7 +263,11 @@ module Avro
         else
           super(schema_type, name, namespace, names)
         end
-        @fields = RecordSchema.make_field_objects(fields, names, self.namespace)
+        @fields = if fields
+                    RecordSchema.make_field_objects(fields, names, self.namespace)
+                  else
+                    {}
+                  end
       end
 
       def fields_hash
@@ -302,8 +318,7 @@ module Avro
       def initialize(schemas, names=nil, default_namespace=nil)
         super(:union)
 
-        schema_objects = []
-        schemas.each_with_index do |schema, i|
+        @schemas = schemas.each_with_object([]) do |schema, schema_objects|
           new_schema = subparse(schema, names, default_namespace)
           ns_type = new_schema.type_sym
 
@@ -316,7 +331,6 @@ module Avro
           else
             schema_objects << new_schema
           end
-          @schemas = schema_objects
         end
       end
 
@@ -380,16 +394,20 @@ module Avro
     class Field < Schema
       attr_reader :type, :name, :default, :order
 
-      def initialize(type, name, default=nil, order=nil, names=nil, namespace=nil)
+      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
         @type = subparse(type, names, namespace)
         @name = name
         @default = default
         @order = order
       end
 
+      def default?
+        @default != :no_default
+      end
+
       def to_avro(names=Set.new)
         {'name' => name, 'type' => type.to_avro(names)}.tap do |avro|
-          avro['default'] = default if default
+          avro['default'] = default if default?
           avro['order'] = order if order
         end
       end

data/test/test_io.rb

@@ -352,8 +352,71 @@ EOS
       assert_equal(incorrect, 0)
     end
   end
+
+  def test_interchangeable_schemas
+    interchangeable_schemas = ['"string"', '"bytes"']
+    incorrect = 0
+    interchangeable_schemas.each_with_index do |ws, i|
+      writers_schema = Avro::Schema.parse(ws)
+      datum_to_write = 'foo'
+      readers_schema = Avro::Schema.parse(interchangeable_schemas[i == 0 ? 1 : 0])
+      writer, * = write_datum(datum_to_write, writers_schema)
+      datum_read = read_datum(writer, writers_schema, readers_schema)
+      if datum_read != datum_to_write
+        incorrect += 1
+      end
+    end
+    assert_equal(incorrect, 0)
+  end
+
+  def test_array_schema_promotion
+    writers_schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
+    readers_schema = Avro::Schema.parse('{"type":"array", "items":"long"}')
+    datum_to_write = [1, 2]
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_map_schema_promotion
+    writers_schema = Avro::Schema.parse('{"type":"map", "values":"int"}')
+    readers_schema = Avro::Schema.parse('{"type":"map", "values":"long"}')
+    datum_to_write = { 'foo' => 1, 'bar' => 2 }
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_snappy_backward_compat
+    # a snappy-compressed block payload without the checksum
+    # this has no back-references, just one literal so the last 9
+    # bytes are the uncompressed payload.
+    old_snappy_bytes = "\x09\x20\x02\x06\x02\x0a\x67\x72\x65\x65\x6e"
+    uncompressed_bytes = "\x02\x06\x02\x0a\x67\x72\x65\x65\x6e"
+    snappy = Avro::DataFile::SnappyCodec.new
+    assert_equal(uncompressed_bytes, snappy.decompress(old_snappy_bytes))
+  end
+
   private
 
+  def check_no_default(schema_json)
+    actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
+    actual = Avro::Schema.parse(actual_schema)
+
+    expected_schema = <<EOS
+      {"type": "record",
+       "name": "Foo",
+       "fields": [{"name": "f", "type": #{schema_json}}]}
+EOS
+    expected = Avro::Schema.parse(expected_schema)
+
+    reader = Avro::IO::DatumReader.new(actual, expected)
+    assert_raise Avro::AvroError do
+      value = reader.read(Avro::IO::BinaryDecoder.new(StringIO.new))
+      assert_not_equal(value, :no_default) # should never return this
+    end
+  end
+
   def check_default(schema_json, default_json, default_value)
     actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
     actual = Avro::Schema.parse(actual_schema)
@@ -393,6 +456,9 @@ EOS
 
     # test writing of data to file
     check_datafile(schema)
+
+    # check that AvroError is raised when there is no default
+    check_no_default(str)
   end
 
   def checkser(schm, randomdata)

data/test/test_schema.rb

@@ -158,4 +158,55 @@ class TestSchema < Test::Unit::TestCase
 
     assert_equal '"MissingType" is not a schema we know about.', error.message
   end
+
+  def test_to_avro_handles_falsey_defaults
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "Record", "namespace": "my.name.space",
+        "fields": [
+          {"name": "is_usable", "type": "boolean", "default": false}
+        ]
+      }
+    SCHEMA
+
+    assert_equal schema.to_avro, {
+      'type' => 'record', 'name' => 'Record', 'namespace' => 'my.name.space',
+      'fields' => [
+        {'name' => 'is_usable', 'type' => 'boolean', 'default' => false}
+      ]
+    }
+  end
+
+  def test_empty_record
+    schema = Avro::Schema.parse('{"type":"record", "name":"Empty"}')
+    assert_empty(schema.fields)
+  end
+
+  def test_empty_union
+    schema = Avro::Schema.parse('[]')
+    assert_equal(schema.to_s, '[]')
+  end
+
+  def test_read
+    schema = Avro::Schema.parse('"string"')
+    writer_schema = Avro::Schema.parse('"int"')
+    assert_false(schema.read?(writer_schema))
+    assert_true(schema.read?(schema))
+  end
+
+  def test_be_read
+    schema = Avro::Schema.parse('"string"')
+    writer_schema = Avro::Schema.parse('"int"')
+    assert_false(schema.be_read?(writer_schema))
+    assert_true(schema.be_read?(schema))
+  end
+
+  def test_mutual_read
+    schema = Avro::Schema.parse('"string"')
+    writer_schema = Avro::Schema.parse('"int"')
+    default1 = Avro::Schema.parse('{"type":"record", "name":"Default", "fields":[{"name":"i", "type":"int", "default": 1}]}')
+    default2 = Avro::Schema.parse('{"type":"record", "name":"Default", "fields":[{"name:":"s", "type":"string", "default": ""}]}')
+    assert_false(schema.mutual_read?(writer_schema))
+    assert_true(schema.mutual_read?(schema))
+    assert_true(default1.mutual_read?(default2))
+  end
 end

data/test/test_schema_compatibility.rb

@@ -0,0 +1,463 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestSchemaCompatibility < Test::Unit::TestCase
+
+  def test_primitive_schema_compatibility
+    Avro::Schema::PRIMITIVE_TYPES.each do |schema_type|
+      assert_true(can_read?(send("#{schema_type}_schema"), send("#{schema_type}_schema")))
+    end
+  end
+
+  def test_compatible_reader_writer_pairs
+    [
+      long_schema, int_schema,
+      float_schema, int_schema,
+      float_schema, long_schema,
+      double_schema, long_schema,
+      double_schema, int_schema,
+      double_schema, float_schema,
+
+      int_array_schema, int_array_schema,
+      long_array_schema, int_array_schema,
+      int_map_schema, int_map_schema,
+      long_map_schema, int_map_schema,
+
+      enum1_ab_schema, enum1_ab_schema,
+      enum1_abc_schema, enum1_ab_schema,
+
+      string_schema, bytes_schema,
+      bytes_schema, string_schema,
+
+      empty_union_schema, empty_union_schema,
+      int_union_schema, int_union_schema,
+      int_string_union_schema, string_int_union_schema,
+      int_union_schema, empty_union_schema,
+      long_union_schema, int_union_schema,
+
+      int_union_schema, int_schema,
+      int_schema, int_union_schema,
+
+      empty_record1_schema, empty_record1_schema,
+      empty_record1_schema, a_int_record1_schema,
+
+      a_int_record1_schema, a_int_record1_schema,
+      a_dint_record1_schema, a_int_record1_schema,
+      a_dint_record1_schema, a_dint_record1_schema,
+      a_int_record1_schema, a_dint_record1_schema,
+
+      a_long_record1_schema, a_int_record1_schema,
+
+      a_int_record1_schema, a_int_b_int_record1_schema,
+      a_dint_record1_schema, a_int_b_int_record1_schema,
+
+      a_int_b_dint_record1_schema, a_int_record1_schema,
+      a_dint_b_dint_record1_schema, empty_record1_schema,
+      a_dint_b_dint_record1_schema, a_int_record1_schema,
+      a_int_b_int_record1_schema, a_dint_b_dint_record1_schema,
+
+      int_list_record_schema, int_list_record_schema,
+      long_list_record_schema, long_list_record_schema,
+      long_list_record_schema, int_list_record_schema,
+
+      null_schema, null_schema
+    ].each_slice(2) do |(reader, writer)|
+      assert_true(can_read?(writer, reader), "expecting #{reader} to read #{writer}")
+    end
+  end
+
+  def test_broken
+    assert_false(can_read?(int_string_union_schema, int_union_schema))
+  end
+
+  def test_incompatible_reader_writer_pairs
+    [
+      null_schema, int_schema,
+      null_schema, long_schema,
+
+      boolean_schema, int_schema,
+
+      int_schema, null_schema,
+      int_schema, boolean_schema,
+      int_schema, long_schema,
+      int_schema, float_schema,
+      int_schema, double_schema,
+
+      long_schema, float_schema,
+      long_schema, double_schema,
+
+      float_schema, double_schema,
+
+      string_schema, boolean_schema,
+      string_schema, int_schema,
+
+      bytes_schema, null_schema,
+      bytes_schema, int_schema,
+
+      int_array_schema, long_array_schema,
+      int_map_schema, int_array_schema,
+      int_array_schema, int_map_schema,
+      int_map_schema, long_map_schema,
+
+      enum1_ab_schema, enum1_abc_schema,
+      enum1_bc_schema, enum1_abc_schema,
+
+      enum1_ab_schema, enum2_ab_schema,
+      int_schema, enum2_ab_schema,
+      enum2_ab_schema, int_schema,
+
+      int_union_schema, int_string_union_schema,
+      string_union_schema, int_string_union_schema,
+
+      empty_record2_schema, empty_record1_schema,
+      a_int_record1_schema, empty_record1_schema,
+      a_int_b_dint_record1_schema, empty_record1_schema,
+
+      int_list_record_schema, long_list_record_schema,
+
+      null_schema, int_schema
+    ].each_slice(2) do |(reader, writer)|
+      assert_false(can_read?(writer, reader), "expecting #{reader} not to read #{writer}")
+    end
+  end
+
+  def writer_schema
+    Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"oldfield1", "type":"int"},
+        {"name":"oldfield2", "type":"string"}
+      ]}
+    SCHEMA
+  end
+
+  def test_missing_field
+    reader_schema = Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"oldfield1", "type":"int"}
+      ]}
+    SCHEMA
+    assert_true(can_read?(writer_schema, reader_schema))
+    assert_false(can_read?(reader_schema, writer_schema))
+  end
+
+  def test_missing_second_field
+    reader_schema = Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"oldfield2", "type":"string"}
+      ]}
+    SCHEMA
+    assert_true(can_read?(writer_schema, reader_schema))
+    assert_false(can_read?(reader_schema, writer_schema))
+  end
+
+  def test_all_fields
+    reader_schema = Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"oldfield1", "type":"int"},
+        {"name":"oldfield2", "type":"string"}
+      ]}
+    SCHEMA
+    assert_true(can_read?(writer_schema, reader_schema))
+    assert_true(can_read?(reader_schema, writer_schema))
+  end
+
+  def test_new_field_with_default
+    reader_schema = Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"oldfield1", "type":"int"},
+        {"name":"newfield1", "type":"int", "default":42}
+      ]}
+    SCHEMA
+    assert_true(can_read?(writer_schema, reader_schema))
+    assert_false(can_read?(reader_schema, writer_schema))
+  end
+
+  def test_new_field
+    reader_schema = Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"oldfield1", "type":"int"},
+        {"name":"newfield1", "type":"int"}
+      ]}
+    SCHEMA
+    assert_false(can_read?(writer_schema, reader_schema))
+    assert_false(can_read?(reader_schema, writer_schema))
+  end
+
+  def test_array_writer_schema
+    valid_reader = string_array_schema
+    invalid_reader = string_map_schema
+
+    assert_true(can_read?(string_array_schema, valid_reader))
+    assert_false(can_read?(string_array_schema, invalid_reader))
+  end
+
+  def test_primitive_writer_schema
+    valid_reader = string_schema
+    assert_true(can_read?(string_schema, valid_reader))
+    assert_false(can_read?(int_schema, string_schema))
+  end
+
+  def test_union_reader_writer_subset_incompatiblity
+    # reader union schema must contain all writer union branches
+    union_writer = union_schema(int_schema, string_schema)
+    union_reader = union_schema(string_schema)
+
+    assert_false(can_read?(union_writer, union_reader))
+    assert_true(can_read?(union_reader, union_writer))
+  end
+
+  def test_incompatible_record_field
+    string_schema = Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"MyRecord", "namespace":"ns", "fields": [
+        {"name":"field1", "type":"string"}
+      ]}
+    SCHEMA
+    int_schema = Avro::Schema.parse <<-SCHEMA2
+      {"type":"record", "name":"MyRecord", "namespace":"ns", "fields": [
+        {"name":"field1", "type":"int"}
+      ]}
+    SCHEMA2
+    assert_false(can_read?(string_schema, int_schema))
+  end
+
+  def test_enum_symbols
+    enum_schema1 = Avro::Schema.parse <<-SCHEMA
+      {"type":"enum", "name":"MyEnum", "symbols":["A","B"]}
+    SCHEMA
+    enum_schema2 = Avro::Schema.parse <<-SCHEMA
+      {"type":"enum", "name":"MyEnum", "symbols":["A","B","C"]}
+    SCHEMA
+    assert_false(can_read?(enum_schema2, enum_schema1))
+    assert_true(can_read?(enum_schema1, enum_schema2))
+  end
+
+  # Tests from lang/java/avro/src/test/java/org/apache/avro/io/parsing/TestResolvingGrammarGenerator2.java
+
+  def point_2d_schema
+    Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Point2D", "fields":[
+        {"name":"x", "type":"double"},
+        {"name":"y", "type":"double"}
+      ]}
+    SCHEMA
+  end
+
+  def point_2d_fullname_schema
+    Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Point", "namespace":"written", "fields":[
+        {"name":"x", "type":"double"},
+        {"name":"y", "type":"double"}
+      ]}
+    SCHEMA
+  end
+
+  def point_3d_no_default_schema
+    Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Point", "fields":[
+        {"name":"x", "type":"double"},
+        {"name":"y", "type":"double"},
+        {"name":"z", "type":"double"}
+      ]}
+    SCHEMA
+  end
+
+  def point_3d_schema
+    Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Point3D", "fields":[
+        {"name":"x", "type":"double"},
+        {"name":"y", "type":"double"},
+        {"name":"z", "type":"double", "default": 0.0}
+      ]}
+    SCHEMA
+  end
+
+  def point_3d_match_name_schema
+    Avro::Schema.parse <<-SCHEMA
+      {"type":"record", "name":"Point", "fields":[
+        {"name":"x", "type":"double"},
+        {"name":"y", "type":"double"},
+        {"name":"z", "type":"double", "default": 0.0}
+      ]}
+    SCHEMA
+  end
+
+  def test_union_resolution_no_structure_match
+    # short name match, but no structure match
+    read_schema = union_schema(null_schema, point_3d_no_default_schema)
+    assert_false(can_read?(point_2d_fullname_schema, read_schema))
+  end
+
+  def test_union_resolution_first_structure_match_2d
+    # multiple structure matches with no name matches
+    read_schema = union_schema(null_schema, point_3d_no_default_schema, point_2d_schema, point_3d_schema)
+    assert_false(can_read?(point_2d_fullname_schema, read_schema))
+  end
+
+  def test_union_resolution_first_structure_match_3d
+    # multiple structure matches with no name matches
+    read_schema = union_schema(null_schema, point_3d_no_default_schema, point_3d_schema, point_2d_schema)
+    assert_false(can_read?(point_2d_fullname_schema, read_schema))
+  end
+
+  def test_union_resolution_named_structure_match
+    # multiple structure matches with a short name match
+    read_schema = union_schema(null_schema, point_2d_schema, point_3d_match_name_schema, point_3d_schema)
+    assert_false(can_read?(point_2d_fullname_schema, read_schema))
+  end
+
+  def test_union_resolution_full_name_match
+    # there is a full name match that should be chosen
+    read_schema = union_schema(null_schema, point_2d_schema, point_3d_match_name_schema, point_3d_schema, point_2d_fullname_schema)
+    assert_true(can_read?(point_2d_fullname_schema, read_schema))
+  end
+
+  def can_read?(writer, reader)
+    Avro::SchemaCompatibility.can_read?(writer, reader)
+  end
+
+  def union_schema(*schemas)
+    schemas ||= []
+    Avro::Schema.parse("[#{schemas.map(&:to_s).join(',')}]")
+  end
+
+  Avro::Schema::PRIMITIVE_TYPES.each do |schema_type|
+    define_method("#{schema_type}_schema") do
+      Avro::Schema.parse("\"#{schema_type}\"")
+    end
+  end
+
+  def int_array_schema
+    Avro::Schema.parse('{"type":"array", "items":"int"}')
+  end
+
+  def long_array_schema
+    Avro::Schema.parse('{"type":"array", "items":"long"}')
+  end
+
+  def string_array_schema
+    Avro::Schema.parse('{"type":"array", "items":"string"}')
+  end
+
+  def int_map_schema
+    Avro::Schema.parse('{"type":"map", "values":"int"}')
+  end
+
+  def long_map_schema
+    Avro::Schema.parse('{"type":"map", "values":"long"}')
+  end
+
+  def string_map_schema
+    Avro::Schema.parse('{"type":"map", "values":"string"}')
+  end
+
+  def enum1_ab_schema
+    Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B"]}')
+  end
+
+  def enum1_abc_schema
+    Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}')
+  end
+
+  def enum1_bc_schema
+    Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["B","C"]}')
+  end
+
+  def enum2_ab_schema
+    Avro::Schema.parse('{"type":"enum", "name":"Enum2", "symbols":["A","B"]}')
+  end
+
+  def empty_record1_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record1"}')
+  end
+
+  def empty_record2_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record2"}')
+  end
+
+  def a_int_record1_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}]}')
+  end
+
+  def a_long_record1_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"long"}]}')
+  end
+
+  def a_int_b_int_record1_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int"}]}')
+  end
+
+  def a_dint_record1_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}]}')
+  end
+
+  def a_int_b_dint_record1_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int"}, {"name":"b", "type":"int", "default":0}]}')
+  end
+
+  def a_dint_b_dint_record1_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record1", "fields":[{"name":"a", "type":"int", "default":0}, {"name":"b", "type":"int", "default":0}]}')
+  end
+
+  def int_list_record_schema
+    Avro::Schema.parse <<-SCHEMA
+      {
+        "type":"record", "name":"List", "fields": [
+          {"name": "head", "type": "int"},
+          {"name": "tail", "type": "List"}
+      ]}
+    SCHEMA
+  end
+
+  def long_list_record_schema
+    Avro::Schema.parse <<-SCHEMA
+      {
+        "type":"record", "name":"List", "fields": [
+          {"name": "head", "type": "long"},
+          {"name": "tail", "type": "List"}
+      ]}
+    SCHEMA
+  end
+
+  def empty_union_schema
+    union_schema
+  end
+
+  def null_union_schema
+    union_schema(null_schema)
+  end
+
+  def int_union_schema
+    union_schema(int_schema)
+  end
+
+  def long_union_schema
+    union_schema(long_schema)
+  end
+
+  def string_union_schema
+    union_schema(string_schema)
+  end
+
+  def int_string_union_schema
+    union_schema(int_schema, string_schema)
+  end
+
+  def string_int_union_schema
+    union_schema(string_schema, int_schema)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro-salsify-fork
 version: !ruby/object:Gem::Version
-  version: 1.9.0.2
+  version: 1.9.0.3.pre1
 platform: ruby
 authors:
 - Apache Software Foundation / Salsify Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-06 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -72,6 +72,7 @@ files:
 - test/test_logical_types.rb
 - test/test_protocol.rb
 - test/test_schema.rb
+- test/test_schema_compatibility.rb
 - test/test_schema_normalization.rb
 - test/test_socket_transport.rb
 - test/tool.rb
@@ -110,5 +111,6 @@ test_files:
 - test/test_logical_types.rb
 - test/test_protocol.rb
 - test/test_schema.rb
+- test/test_schema_compatibility.rb
 - test/test_schema_normalization.rb
 - test/test_socket_transport.rb