avro 1.9.0 → 1.10.2

data/test/test_logical_types.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/test_protocol.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/test_schema.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -151,6 +151,20 @@ class TestSchema < Test::Unit::TestCase
     }
   end
 
+  def test_to_avro_includes_aliases
+    hash = {
+      'type' => 'record',
+      'name' => 'test_record',
+      'aliases' => %w(alt_record),
+      'fields' => [
+        { 'name' => 'f', 'type' => { 'type' => 'fixed', 'size' => 2, 'name' => 'test_fixed', 'aliases' => %w(alt_fixed) } },
+        { 'name' => 'e', 'type' => { 'type' => 'enum', 'symbols' => %w(A B), 'name' => 'test_enum', 'aliases' => %w(alt_enum) } }
+      ]
+    }
+    schema = hash_to_schema(hash)
+    assert_equal(schema.to_avro, hash)
+  end
+
   def test_unknown_named_type
     error = assert_raise Avro::UnknownSchemaError do
       Avro::Schema.parse <<-SCHEMA
@@ -163,6 +177,42 @@ class TestSchema < Test::Unit::TestCase
     assert_equal '"MissingType" is not a schema we know about.', error.message
   end
 
+  def test_invalid_name
+    error = assert_raise Avro::SchemaParseError do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my-invalid-name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+
+    assert_equal "Name my-invalid-name is invalid for type record!", error.message
+  end
+
+  def test_invalid_name_with_two_periods
+    error = assert_raise Avro::SchemaParseError do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my..invalid.name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+
+    assert_equal "Name my..invalid.name is invalid for type record!", error.message
+  end
+
+  def test_invalid_name_with_validation_disabled
+    Avro.disable_schema_name_validation = true
+    assert_nothing_raised do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my-invalid-name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+    Avro.disable_schema_name_validation = false
+  end
+
   def test_to_avro_handles_falsey_defaults
     schema = Avro::Schema.parse <<-SCHEMA
       {"type": "record", "name": "Record", "namespace": "my.name.space",
@@ -278,6 +328,40 @@ class TestSchema < Test::Unit::TestCase
     assert_equal enum_schema_hash, enum_schema_json.to_avro
   end
 
+  def test_enum_default_attribute
+    enum_schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "enum",
+        "name": "fruit",
+        "default": "apples",
+        "symbols": ["apples", "oranges"]
+      }
+    SCHEMA
+
+    enum_schema_hash = {
+      'type' => 'enum',
+      'name' => 'fruit',
+      'default' => 'apples',
+      'symbols' => %w(apples oranges)
+    }
+
+    assert_equal(enum_schema.default, "apples")
+    assert_equal(enum_schema_hash, enum_schema.to_avro)
+  end
+
+  def test_validate_enum_default
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'fruit',
+        default: 'bananas',
+        symbols: %w(apples oranges)
+      )
+    end
+    assert_equal("Default 'bananas' is not a valid symbol for enum fruit",
+                 exception.to_s)
+  end
+
   def test_empty_record
     schema = Avro::Schema.parse('{"type":"record", "name":"Empty"}')
     assert_empty(schema.fields)
@@ -455,5 +539,189 @@ class TestSchema < Test::Unit::TestCase
     end
     assert_equal('Error validating default for veggies: at . expected type null, got string with value "apple"',
                  exception.to_s)
+  end
+
+  def test_bytes_decimal_to_include_precision_scale
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 9,
+        "scale": 2
+      }
+    SCHEMA
+
+    schema_hash =
+      {
+        'type' => 'bytes',
+        'logicalType' => 'decimal',
+        'precision' => 9,
+        'scale' => 2
+      }
+
+    assert_equal schema_hash, schema.to_avro
+  end
+
+  def test_bytes_decimal_to_without_precision_scale
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes",
+        "logicalType": "decimal"
+      }
+    SCHEMA
+
+    schema_hash =
+      {
+        'type' => 'bytes',
+        'logicalType' => 'decimal'
+      }
+
+    assert_equal schema_hash, schema.to_avro
+  end
+
+  def test_bytes_schema
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes"
+      }
+    SCHEMA
+
+    schema_str = 'bytes'
+    assert_equal schema_str, schema.to_avro
+  end
+
+  def test_validate_duplicate_symbols
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'name',
+        symbols: ['erica', 'erica']
+      )
+    end
+    assert_equal(
+      'Duplicate symbol: ["erica", "erica"]',
+      exception.to_s
+    )
+  end
+
+  def test_validate_enum_symbols
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'things',
+        symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$']
+      )
+    end
+
+    assert_equal(
+      "Invalid symbols for things: 8ad_symbol, also-bad-symbol, >=, $ don't match #{Avro::Schema::EnumSchema::SYMBOL_REGEX.inspect}",
+      exception.to_s
+    )
+  end
+
+  def test_enum_symbol_validation_disabled_via_env
+    Avro.disable_enum_symbol_validation = nil
+    ENV['AVRO_DISABLE_ENUM_SYMBOL_VALIDATION'] = '1'
+
+    hash_to_schema(
+      type: 'enum',
+      name: 'things',
+      symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
+    )
+  ensure
+    ENV.delete('AVRO_DISABLE_ENUM_SYMBOL_VALIDATION')
+    Avro.disable_enum_symbol_validation = nil
+  end
+
+  def test_enum_symbol_validation_disabled_via_class_method
+    Avro.disable_enum_symbol_validation = true
+
+    hash_to_schema(
+      type: 'enum',
+      name: 'things',
+      symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
+    )
+  ensure
+    Avro.disable_enum_symbol_validation = nil
+  end
+  
+  def test_validate_field_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: 'banane' }
+        ]
+      )
+    end
+
+    assert_match(/Invalid aliases value "banane" for "string" banana/, exception.to_s)
+  end
+
+  def test_validate_same_alias_multiple_fields
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: %w(yellow) },
+          { name: 'lemo', type: 'string', aliases: %w(yellow) }
+        ]
+      )
+    end
+
+    assert_match('Alias ["yellow"] already in use', exception.to_s)
+  end
+
+  def test_validate_repeated_aliases
+    assert_nothing_raised do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: %w(yellow yellow) },
+        ]
+      )
+    end
+  end
+
+  def test_validate_record_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        aliases: ["foods", 2],
+        fields: []
+      )
+    end
+
+    assert_match(/Invalid aliases value \["foods", 2\] for record fruits/, exception.to_s)
+  end
+
+  def test_validate_enum_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'vowels',
+        aliases: [1, 2],
+        symbols: %w(A E I O U)
+      )
     end
+
+    assert_match(/Invalid aliases value \[1, 2\] for enum vowels/, exception.to_s)
+  end
+
+  def test_validate_fixed_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'fixed',
+        name: 'uuid',
+        size: 36,
+        aliases: "unique_id"
+      )
+    end
+
+    assert_match(/Invalid aliases value "unique_id" for fixed uuid/, exception.to_s)
+  end
 end

data/test/test_schema_compatibility.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -25,7 +25,9 @@ class TestSchemaCompatibility < Test::Unit::TestCase
   end
 
   def test_compatible_reader_writer_pairs
+    cached_schema = a_int_record1_schema
     [
+      cached_schema, cached_schema,
       long_schema, int_schema,
       float_schema, int_schema,
       float_schema, long_schema,
@@ -39,7 +41,12 @@ class TestSchemaCompatibility < Test::Unit::TestCase
       long_map_schema, int_map_schema,
 
       enum1_ab_schema, enum1_ab_schema,
+      enum1_ab_aliased_schema, enum1_ab_schema,
       enum1_abc_schema, enum1_ab_schema,
+      enum1_ab_default_schema, enum1_abc_schema,
+
+      fixed1_schema, fixed1_schema,
+      fixed1_aliased_schema, fixed1_schema,
 
       string_schema, bytes_schema,
       bytes_schema, string_schema,
@@ -55,6 +62,7 @@ class TestSchemaCompatibility < Test::Unit::TestCase
 
       empty_record1_schema, empty_record1_schema,
       empty_record1_schema, a_int_record1_schema,
+      empty_record1_aliased_schema, empty_record1_schema,
 
       a_int_record1_schema, a_int_record1_schema,
       a_dint_record1_schema, a_int_record1_schema,
@@ -117,16 +125,22 @@ class TestSchemaCompatibility < Test::Unit::TestCase
       int_map_schema, long_map_schema,
 
       enum1_ab_schema, enum1_abc_schema,
+      enum1_ab_schema, enum1_ab_aliased_schema,
       enum1_bc_schema, enum1_abc_schema,
 
       enum1_ab_schema, enum2_ab_schema,
       int_schema, enum2_ab_schema,
       enum2_ab_schema, int_schema,
 
+      fixed1_schema, fixed2_schema,
+      fixed1_schema, fixed1_size3_schema,
+      fixed1_schema, fixed1_aliased_schema,
+
       int_union_schema, int_string_union_schema,
       string_union_schema, int_string_union_schema,
 
       empty_record2_schema, empty_record1_schema,
+      empty_record1_schema, empty_record1_aliased_schema,
       a_int_record1_schema, empty_record1_schema,
       a_int_b_dint_record1_schema, empty_record1_schema,
 
@@ -169,6 +183,17 @@ class TestSchemaCompatibility < Test::Unit::TestCase
     assert_false(can_read?(reader_schema, writer_schema))
   end
 
+  def test_aliased_field
+    reader_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"newname1", "aliases":["oldfield1"], "type":"int"},
+        {"name":"oldfield2", "type":"string"}
+      ]}
+    SCHEMA
+    assert_true(can_read?(writer_schema, reader_schema))
+    assert_false(can_read?(reader_schema, writer_schema))
+  end
+
   def test_all_fields
     reader_schema = Avro::Schema.parse <<-SCHEMA
       {"type":"record", "name":"Record", "fields":[
@@ -250,6 +275,23 @@ class TestSchemaCompatibility < Test::Unit::TestCase
     assert_true(can_read?(enum_schema1, enum_schema2))
   end
 
+  def test_crossed_aliases
+    writer_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"field1", "type": "int"},
+        {"name":"field2", "type": "string"}
+      ]}
+    SCHEMA
+    reader_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Record", "fields":[
+        {"name":"field1", "aliases":["field2"], "type":"string"},
+        {"name":"field2", "aliases":["field1"], "type":"int"}
+      ]}
+    SCHEMA
+    # Not supported; alias is not used if there is a redirect match
+    assert_false(can_read?(writer_schema, reader_schema))
+  end
+
   # Tests from lang/java/avro/src/test/java/org/apache/avro/io/parsing/TestResolvingGrammarGenerator2.java
 
   def point_2d_schema
@@ -373,6 +415,14 @@ class TestSchemaCompatibility < Test::Unit::TestCase
     Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B"]}')
   end
 
+  def enum1_ab_default_schema
+    Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B"], "default":"A"}')
+  end
+
+  def enum1_ab_aliased_schema
+    Avro::Schema.parse('{"type":"enum", "name":"Enum2", "aliases":["Enum1"], "symbols":["A","B"]}')
+  end
+
   def enum1_abc_schema
     Avro::Schema.parse('{"type":"enum", "name":"Enum1", "symbols":["A","B","C"]}')
   end
@@ -385,10 +435,30 @@ class TestSchemaCompatibility < Test::Unit::TestCase
     Avro::Schema.parse('{"type":"enum", "name":"Enum2", "symbols":["A","B"]}')
   end
 
+  def fixed1_schema
+    Avro::Schema.parse('{"type":"fixed", "name":"Fixed1", "size": 2}')
+  end
+
+  def fixed1_aliased_schema
+    Avro::Schema.parse('{"type":"fixed", "name":"Fixed2", "aliases":["Fixed1"], "size": 2}')
+  end
+
+  def fixed2_schema
+    Avro::Schema.parse('{"type":"fixed", "name":"Fixed2", "size": 2}')
+  end
+
+  def fixed1_size3_schema
+    Avro::Schema.parse('{"type":"fixed", "name":"Fixed1", "size": 3}')
+  end
+
   def empty_record1_schema
     Avro::Schema.parse('{"type":"record", "name":"Record1"}')
   end
 
+  def empty_record1_aliased_schema
+    Avro::Schema.parse('{"type":"record", "name":"Record2", "aliases":["Record1"]}')
+  end
+
   def empty_record2_schema
     Avro::Schema.parse('{"type":"record", "name":"Record2"}')
   end