avro 1.8.2 → 1.10.1

data/test/test_protocol.rb

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
-# http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -139,7 +139,7 @@ EOS
 
 }
 EOS
-  ExampleProtocol.new(<<-EOS, true)
+  ExampleProtocol.new(<<-EOS, true),
 {"namespace": "org.apache.avro.test",
  "protocol": "BulkData",
 
@@ -160,6 +160,29 @@ EOS
  }
 
 }
+EOS
+  ExampleProtocol.new(<<-EOS, true),
+{
+  "namespace": "com.acme",
+  "protocol": "HelloWorld",
+  "doc": "protocol_documentation",
+
+  "types": [
+    {"name": "Greeting", "type": "record", "fields": [
+      {"name": "message", "type": "string"}]},
+    {"name": "Curse", "type": "error", "fields": [
+      {"name": "message", "type": "string"}]}
+  ],
+
+  "messages": {
+    "hello": {
+      "doc": "message_documentation",
+      "request": [{"name": "greeting", "type": "Greeting" }],
+      "response": "Greeting",
+      "errors": ["Curse"]
+    }
+  }
+}
 EOS
 ]
 
@@ -196,4 +219,14 @@ EOS
       assert_equal type.namespace, 'com.acme'
     end
   end
+
+  def test_protocol_doc_attribute
+    original = Protocol.parse(EXAMPLES.last.protocol_string)
+    assert_equal 'protocol_documentation', original.doc
+  end
+
+  def test_protocol_message_doc_attribute
+    original = Protocol.parse(EXAMPLES.last.protocol_string)
+    assert_equal 'message_documentation', original.messages['hello'].doc
+  end
 end

data/test/test_schema.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,6 +17,10 @@
 require 'test_help'
 
 class TestSchema < Test::Unit::TestCase
+  def hash_to_schema(hash)
+    Avro::Schema.parse(hash.to_json)
+  end
+
   def test_default_namespace
     schema = Avro::Schema.parse <<-SCHEMA
       {"type": "record", "name": "OuterRecord", "fields": [
@@ -27,13 +31,13 @@ class TestSchema < Test::Unit::TestCase
       ]}
     SCHEMA
 
-    assert_equal schema.name, 'OuterRecord'
-    assert_equal schema.fullname, 'OuterRecord'
+    assert_equal 'OuterRecord', schema.name
+    assert_equal 'OuterRecord', schema.fullname
     assert_nil schema.namespace
 
     schema.fields.each do |field|
-      assert_equal field.type.name, 'InnerRecord'
-      assert_equal field.type.fullname, 'InnerRecord'
+      assert_equal 'InnerRecord', field.type.name
+      assert_equal 'InnerRecord', field.type.fullname
       assert_nil field.type.namespace
     end
   end
@@ -50,13 +54,13 @@ class TestSchema < Test::Unit::TestCase
       ]}
     SCHEMA
 
-    assert_equal schema.name, 'OuterRecord'
-    assert_equal schema.fullname, 'my.name.space.OuterRecord'
-    assert_equal schema.namespace, 'my.name.space'
+    assert_equal 'OuterRecord', schema.name
+    assert_equal 'my.name.space.OuterRecord', schema.fullname
+    assert_equal 'my.name.space', schema.namespace
     schema.fields.each do |field|
-      assert_equal field.type.name, 'InnerRecord'
-      assert_equal field.type.fullname, 'my.name.space.InnerRecord'
-      assert_equal field.type.namespace, 'my.name.space'
+      assert_equal 'InnerRecord', field.type.name
+      assert_equal 'my.name.space.InnerRecord', field.type.fullname
+      assert_equal 'my.name.space', field.type.namespace
     end
   end
 
@@ -71,13 +75,13 @@ class TestSchema < Test::Unit::TestCase
       ]}
     SCHEMA
 
-    assert_equal schema.name, 'OuterRecord'
-    assert_equal schema.fullname, 'my.name.space.OuterRecord'
-    assert_equal schema.namespace, 'my.name.space'
+    assert_equal 'OuterRecord', schema.name
+    assert_equal 'my.name.space.OuterRecord', schema.fullname
+    assert_equal 'my.name.space', schema.namespace
     schema.fields.each do |field|
-      assert_equal field.type.name, 'InnerEnum'
-      assert_equal field.type.fullname, 'my.name.space.InnerEnum'
-      assert_equal field.type.namespace, 'my.name.space'
+      assert_equal 'InnerEnum', field.type.name
+      assert_equal 'my.name.space.InnerEnum', field.type.fullname
+      assert_equal 'my.name.space', field.type.namespace
     end
   end
 
@@ -96,18 +100,18 @@ class TestSchema < Test::Unit::TestCase
       ]}
     SCHEMA
 
-    assert_equal schema.name, 'OuterRecord'
-    assert_equal schema.fullname, 'outer.OuterRecord'
-    assert_equal schema.namespace, 'outer'
+    assert_equal 'OuterRecord', schema.name
+    assert_equal 'outer.OuterRecord', schema.fullname
+    assert_equal 'outer', schema.namespace
     middle = schema.fields.first.type
-    assert_equal middle.name, 'MiddleRecord'
-    assert_equal middle.fullname, 'middle.MiddleRecord'
-    assert_equal middle.namespace, 'middle'
+    assert_equal 'MiddleRecord', middle.name
+    assert_equal 'middle.MiddleRecord', middle.fullname
+    assert_equal 'middle', middle.namespace
     inner = middle.fields.first.type
-    assert_equal inner.name, 'InnerRecord'
-    assert_equal inner.fullname, 'middle.InnerRecord'
-    assert_equal inner.namespace, 'middle'
-    assert_equal inner.fields.first.type, middle
+    assert_equal 'InnerRecord', inner.name
+    assert_equal 'middle.InnerRecord', inner.fullname
+    assert_equal 'middle', inner.namespace
+    assert_equal middle, inner.fields.first.type
   end
 
   def test_to_avro_includes_namespaces
@@ -120,7 +124,7 @@ class TestSchema < Test::Unit::TestCase
       ]}
     SCHEMA
 
-    assert_equal schema.to_avro, {
+    assert_equal({
       'type' => 'record', 'name' => 'OuterRecord', 'namespace' => 'my.name.space',
       'fields' => [
         {'name' => 'definition', 'type' => {
@@ -129,7 +133,36 @@ class TestSchema < Test::Unit::TestCase
         }},
         {'name' => 'reference', 'type' => 'my.name.space.InnerFixed'}
       ]
+    }, schema.to_avro)
+  end
+
+  def test_to_avro_includes_logical_type
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "has_logical", "fields": [
+        {"name": "dt", "type": {"type": "int", "logicalType": "date"}}]
+      }
+    SCHEMA
+
+    assert_equal schema.to_avro, {
+      'type' => 'record', 'name' => 'has_logical',
+      'fields' => [
+        {'name' => 'dt', 'type' => {'type' => 'int', 'logicalType' => 'date'}}
+      ]
+    }
+  end
+
+  def test_to_avro_includes_aliases
+    hash = {
+      'type' => 'record',
+      'name' => 'test_record',
+      'aliases' => %w(alt_record),
+      'fields' => [
+        { 'name' => 'f', 'type' => { 'type' => 'fixed', 'size' => 2, 'name' => 'test_fixed', 'aliases' => %w(alt_fixed) } },
+        { 'name' => 'e', 'type' => { 'type' => 'enum', 'symbols' => %w(A B), 'name' => 'test_enum', 'aliases' => %w(alt_enum) } }
+      ]
     }
+    schema = hash_to_schema(hash)
+    assert_equal(schema.to_avro, hash)
   end
 
   def test_unknown_named_type
@@ -144,6 +177,42 @@ class TestSchema < Test::Unit::TestCase
     assert_equal '"MissingType" is not a schema we know about.', error.message
   end
 
+  def test_invalid_name
+    error = assert_raise Avro::SchemaParseError do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my-invalid-name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+
+    assert_equal "Name my-invalid-name is invalid for type record!", error.message
+  end
+
+  def test_invalid_name_with_two_periods
+    error = assert_raise Avro::SchemaParseError do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my..invalid.name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+
+    assert_equal "Name my..invalid.name is invalid for type record!", error.message
+  end
+
+  def test_invalid_name_with_validation_disabled
+    Avro.disable_schema_name_validation = true
+    assert_nothing_raised do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my-invalid-name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+    Avro.disable_schema_name_validation = false
+  end
+
   def test_to_avro_handles_falsey_defaults
     schema = Avro::Schema.parse <<-SCHEMA
       {"type": "record", "name": "Record", "namespace": "my.name.space",
@@ -160,4 +229,499 @@ class TestSchema < Test::Unit::TestCase
       ]
     }
   end
+
+  def test_record_field_doc_attribute
+    field_schema_json = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "record",
+        "name": "Record",
+        "namespace": "my.name.space",
+        "fields": [
+          {
+            "name": "name",
+            "type": "boolean",
+            "doc": "documentation"
+          }
+        ]
+      }
+    SCHEMA
+
+    field_schema_hash =
+      {
+        'type' => 'record',
+        'name' => 'Record',
+        'namespace' => 'my.name.space',
+        'fields' => [
+          {
+            'name' => 'name',
+            'type' => 'boolean',
+            'doc' => 'documentation'
+          }
+        ]
+      }
+
+    assert_equal field_schema_hash, field_schema_json.to_avro
+  end
+
+  def test_record_doc_attribute
+    record_schema_json = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "record",
+        "name": "Record",
+        "namespace": "my.name.space",
+        "doc": "documentation",
+        "fields": [
+          {
+            "name": "name",
+            "type": "boolean"
+          }
+        ]
+      }
+    SCHEMA
+
+    record_schema_hash =
+      {
+        'type' => 'record',
+        'name' => 'Record',
+        'namespace' => 'my.name.space',
+        'doc' => 'documentation',
+        'fields' => [
+          {
+            'name' => 'name',
+            'type' => 'boolean'
+          }
+        ]
+      }
+
+    assert_equal record_schema_hash, record_schema_json.to_avro
+  end
+
+  def test_enum_doc_attribute
+    enum_schema_json = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "enum",
+        "name": "Enum",
+        "namespace": "my.name.space",
+        "doc": "documentation",
+        "symbols" : [
+          "SPADES",
+          "HEARTS",
+          "DIAMONDS",
+          "CLUBS"
+        ]
+      }
+    SCHEMA
+
+    enum_schema_hash =
+      {
+        'type' => 'enum',
+        'name' => 'Enum',
+        'namespace' => 'my.name.space',
+        'doc' => 'documentation',
+        'symbols' => [
+          'SPADES',
+          'HEARTS',
+          'DIAMONDS',
+          'CLUBS'
+        ]
+      }
+    assert_equal enum_schema_hash, enum_schema_json.to_avro
+  end
+
+  def test_enum_default_attribute
+    enum_schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "enum",
+        "name": "fruit",
+        "default": "apples",
+        "symbols": ["apples", "oranges"]
+      }
+    SCHEMA
+
+    enum_schema_hash = {
+      'type' => 'enum',
+      'name' => 'fruit',
+      'default' => 'apples',
+      'symbols' => %w(apples oranges)
+    }
+
+    assert_equal(enum_schema.default, "apples")
+    assert_equal(enum_schema_hash, enum_schema.to_avro)
+  end
+
+  def test_validate_enum_default
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'fruit',
+        default: 'bananas',
+        symbols: %w(apples oranges)
+      )
+    end
+    assert_equal("Default 'bananas' is not a valid symbol for enum fruit",
+                 exception.to_s)
+  end
+
+  def test_empty_record
+    schema = Avro::Schema.parse('{"type":"record", "name":"Empty"}')
+    assert_empty(schema.fields)
+  end
+
+  def test_empty_union
+    schema = Avro::Schema.parse('[]')
+    assert_equal(schema.to_s, '[]')
+  end
+
+  def test_read
+    schema = Avro::Schema.parse('"string"')
+    writer_schema = Avro::Schema.parse('"int"')
+    assert_false(schema.read?(writer_schema))
+    assert_true(schema.read?(schema))
+  end
+
+  def test_be_read
+    schema = Avro::Schema.parse('"string"')
+    writer_schema = Avro::Schema.parse('"int"')
+    assert_false(schema.be_read?(writer_schema))
+    assert_true(schema.be_read?(schema))
+  end
+
+  def test_mutual_read
+    schema = Avro::Schema.parse('"string"')
+    writer_schema = Avro::Schema.parse('"int"')
+    default1 = Avro::Schema.parse('{"type":"record", "name":"Default", "fields":[{"name":"i", "type":"int", "default": 1}]}')
+    default2 = Avro::Schema.parse('{"type":"record", "name":"Default", "fields":[{"name:":"s", "type":"string", "default": ""}]}')
+    assert_false(schema.mutual_read?(writer_schema))
+    assert_true(schema.mutual_read?(schema))
+    assert_true(default1.mutual_read?(default2))
+  end
+
+  def test_validate_defaults
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          {
+            name: 'veggies',
+            type: 'string',
+            default: nil
+          }
+        ]
+      )
+    end
+    assert_equal('Error validating default for veggies: at . expected type string, got null',
+                 exception.to_s)
+  end
+
+  def test_field_default_validation_disabled
+    Avro.disable_field_default_validation = true
+    assert_nothing_raised do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          {
+            name: 'veggies',
+            type: 'string',
+            default: nil
+          }
+        ]
+      )
+    end
+  ensure
+    Avro.disable_field_default_validation = false
+  end
+
+  def test_field_default_validation_disabled_via_env
+    Avro.disable_field_default_validation = false
+    ENV['AVRO_DISABLE_FIELD_DEFAULT_VALIDATION'] = "1"
+
+    assert_nothing_raised do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          {
+            name: 'veggies',
+            type: 'string',
+            default: nil
+          }
+        ]
+      )
+    end
+  ensure
+    ENV.delete('AVRO_DISABLE_FIELD_DEFAULT_VALIDATION')
+    Avro.disable_field_default_validation = false
+  end
+
+  def test_validate_record_valid_default
+    assert_nothing_raised(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'with_subrecord',
+        fields: [
+          {
+            name: 'sub',
+            type: {
+              name: 'subrecord',
+              type: 'record',
+              fields: [
+                { type: 'string', name: 'x' }
+              ]
+            },
+            default: {
+              x: "y"
+            }
+          }
+        ]
+      )
+    end
+  end
+
+  def test_validate_record_invalid_default
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'with_subrecord',
+        fields: [
+          {
+            name: 'sub',
+            type: {
+              name: 'subrecord',
+              type: 'record',
+              fields: [
+                { type: 'string', name: 'x' }
+              ]
+            },
+            default: {
+              a: 1
+            }
+          }
+        ]
+      )
+    end
+    assert_equal('Error validating default for sub: at .x expected type string, got null',
+                 exception.to_s)
+  end
+
+  def test_validate_union_defaults
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          {
+            name: 'veggies',
+            type: %w(string null),
+            default: 5
+          }
+        ]
+      )
+    end
+    assert_equal('Error validating default for veggies: at . expected type string, got int with value 5',
+                 exception.to_s)
+  end
+
+  def test_validate_union_default_first_type
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          {
+            name: 'veggies',
+            type: %w(null string),
+            default: 'apple'
+          }
+        ]
+      )
+    end
+    assert_equal('Error validating default for veggies: at . expected type null, got string with value "apple"',
+                 exception.to_s)
+  end
+
+  def test_bytes_decimal_to_include_precision_scale
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 9,
+        "scale": 2
+      }
+    SCHEMA
+
+    schema_hash =
+      {
+        'type' => 'bytes',
+        'logicalType' => 'decimal',
+        'precision' => 9,
+        'scale' => 2
+      }
+
+    assert_equal schema_hash, schema.to_avro
+  end
+
+  def test_bytes_decimal_to_without_precision_scale
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes",
+        "logicalType": "decimal"
+      }
+    SCHEMA
+
+    schema_hash =
+      {
+        'type' => 'bytes',
+        'logicalType' => 'decimal'
+      }
+
+    assert_equal schema_hash, schema.to_avro
+  end
+
+  def test_bytes_schema
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes"
+      }
+    SCHEMA
+
+    schema_str = 'bytes'
+    assert_equal schema_str, schema.to_avro
+  end
+
+  def test_validate_duplicate_symbols
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'name',
+        symbols: ['erica', 'erica']
+      )
+    end
+    assert_equal(
+      'Duplicate symbol: ["erica", "erica"]',
+      exception.to_s
+    )
+  end
+
+  def test_validate_enum_symbols
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'things',
+        symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$']
+      )
+    end
+
+    assert_equal(
+      "Invalid symbols for things: 8ad_symbol, also-bad-symbol, >=, $ don't match #{Avro::Schema::EnumSchema::SYMBOL_REGEX.inspect}",
+      exception.to_s
+    )
+  end
+
+  def test_enum_symbol_validation_disabled_via_env
+    Avro.disable_enum_symbol_validation = nil
+    ENV['AVRO_DISABLE_ENUM_SYMBOL_VALIDATION'] = '1'
+
+    hash_to_schema(
+      type: 'enum',
+      name: 'things',
+      symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
+    )
+  ensure
+    ENV.delete('AVRO_DISABLE_ENUM_SYMBOL_VALIDATION')
+    Avro.disable_enum_symbol_validation = nil
+  end
+
+  def test_enum_symbol_validation_disabled_via_class_method
+    Avro.disable_enum_symbol_validation = true
+
+    hash_to_schema(
+      type: 'enum',
+      name: 'things',
+      symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
+    )
+  ensure
+    Avro.disable_enum_symbol_validation = nil
+  end
+  
+  def test_validate_field_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: 'banane' }
+        ]
+      )
+    end
+
+    assert_match(/Invalid aliases value "banane" for "string" banana/, exception.to_s)
+  end
+
+  def test_validate_same_alias_multiple_fields
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: %w(yellow) },
+          { name: 'lemo', type: 'string', aliases: %w(yellow) }
+        ]
+      )
+    end
+
+    assert_match('Alias ["yellow"] already in use', exception.to_s)
+  end
+
+  def test_validate_repeated_aliases
+    assert_nothing_raised do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: %w(yellow yellow) },
+        ]
+      )
+    end
+  end
+
+  def test_validate_record_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        aliases: ["foods", 2],
+        fields: []
+      )
+    end
+
+    assert_match(/Invalid aliases value \["foods", 2\] for record fruits/, exception.to_s)
+  end
+
+  def test_validate_enum_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'vowels',
+        aliases: [1, 2],
+        symbols: %w(A E I O U)
+      )
+    end
+
+    assert_match(/Invalid aliases value \[1, 2\] for enum vowels/, exception.to_s)
+  end
+
+  def test_validate_fixed_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'fixed',
+        name: 'uuid',
+        size: 36,
+        aliases: "unique_id"
+      )
+    end
+
+    assert_match(/Invalid aliases value "unique_id" for fixed uuid/, exception.to_s)
+  end
 end