avro 1.9.1 → 1.10.1

data/lib/avro/schema_compatibility.rb

@@ -28,10 +28,8 @@ module Avro
     end
 
     # Perform a basic check that a datum written with the writers_schema could
-    # be read using the readers_schema. This check only includes matching the types,
-    # including schema promotion, and matching the full name for named types.
-    # Aliases for named types are not supported here, and the ruby implementation
-    # of Avro in general does not include support for aliases.
+    # be read using the readers_schema. This check includes matching the types,
+    # including schema promotion, and matching the full name (including aliases) for named types.
     def self.match_schemas(writers_schema, readers_schema)
       w_type = writers_schema.type_sym
       r_type = readers_schema.type_sym
@@ -46,16 +44,16 @@ module Avro
 
         case r_type
         when :record
-          return writers_schema.fullname == readers_schema.fullname
+          return readers_schema.match_fullname?(writers_schema.fullname)
         when :error
-          return writers_schema.fullname == readers_schema.fullname
+          return readers_schema.match_fullname?(writers_schema.fullname)
         when :request
           return true
         when :fixed
-          return writers_schema.fullname == readers_schema.fullname &&
+          return readers_schema.match_fullname?(writers_schema.fullname) &&
             writers_schema.size == readers_schema.size
         when :enum
-          return writers_schema.fullname == readers_schema.fullname
+          return readers_schema.match_fullname?(writers_schema.fullname)
         when :map
           return match_schemas(writers_schema.values, readers_schema.values)
         when :array
@@ -118,8 +116,8 @@ module Avro
         when :union
           match_union_schemas(writers_schema, readers_schema)
         when :enum
-          # reader's symbols must contain all writer's symbols
-          (writers_schema.symbols - readers_schema.symbols).empty?
+          # reader's symbols must contain all writer's symbols or reader has default
+          (writers_schema.symbols - readers_schema.symbols).empty? || !readers_schema.default.nil?
         else
           if writers_schema.type_sym == :union && writers_schema.schemas.size == 1
             full_match_schemas(writers_schema.schemas.first, readers_schema)
@@ -148,7 +146,14 @@ module Avro
           if writer_fields_hash.key?(field.name)
             return false unless full_match_schemas(writer_fields_hash[field.name].type, field.type)
           else
-            return false unless field.default?
+            names = writer_fields_hash.keys & field.alias_names
+            if names.size > 1
+              return false
+            elsif names.size == 1
+              return false unless full_match_schemas(writer_fields_hash[names.first].type, field.type)
+            else
+              return false unless field.default?
+            end
           end
         end
 

data/test/case_finder.rb

@@ -44,13 +44,18 @@ class CaseFinder
   private
 
   def scan_case
-    if id = @scanner.scan(/\/\/ \d+\n/)
+    if (id = @scanner.scan(/\/\/ \d+\n/))
       while @scanner.skip(/\/\/ .*\n/); end
 
       input = scan_input
       canonical = scan_canonical
       fingerprint = scan_fingerprint
-
+      if not fingerprint and @cases
+        fingerprint = @cases[-1].fingerprint
+      end
+      if fingerprint
+        fingerprint = fingerprint.to_i & 0xFFFF_FFFF_FFFF_FFFF
+      end
       Case.new(id, input, canonical, fingerprint)
     else
       @scanner.skip(/.*\n/)
@@ -61,7 +66,7 @@ class CaseFinder
   def scan_item(name)
     if @scanner.scan(/<<#{name}\n/)
       lines = []
-      while line = @scanner.scan(/.+\n/)
+      while (line = @scanner.scan(/.+\n/))
         break if line.chomp == name
         lines << line
       end

data/test/random_data.rb

@@ -1,4 +1,3 @@
-#!/usr/bin/env ruby
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -85,8 +84,10 @@ class RandomData
     case schm.logical_type
     when 'date'
       Avro::LogicalTypes::IntDate.decode(rand_int)
-    when 'timestamp-millis', 'timestamp-micros'
+    when 'timestamp-micros'
       Avro::LogicalTypes::TimestampMicros.decode(rand_long)
+    when 'timestamp-millis'
+      Avro::LogicalTypes::TimestampMillis.decode(rand_long)
     end
   end
 

data/test/test_datafile.rb

@@ -180,6 +180,19 @@ JSON
     assert_equal records, ['a' * 10_000]
   end
 
+  def test_zstandard
+    Avro::DataFile.open('data.avr', 'w', '"string"', :zstandard) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size('data.avr') < 600)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
   def test_append_to_deflated_file
     schema = Avro::Schema.parse('"string"')
     writer = Avro::IO::DatumWriter.new(schema)

data/test/test_fingerprints.rb

@@ -34,4 +34,23 @@ class TestFingerprints < Test::Unit::TestCase
     assert_equal 28572620203319713300323544804233350633246234624932075150020181448463213378117,
       schema.sha256_fingerprint
   end
+
+  def test_crc_64_avro_fingerprint
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal 8247732601305521295, # hex: 0x7275d51a3f395c8f
+      schema.crc_64_avro_fingerprint
+  end
+
+  # This definitely belongs somewhere else
+  def test_single_object_encoding_header
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal ["c3", "01", "8f", "5c", "39", "3f", "1a", "D5", "75", "72"].map{|e| e.to_i(16) },
+      schema.single_object_encoding_header
+  end
 end

data/test/test_io.rb

@@ -90,7 +90,10 @@ EOS
        "name": "Test",
        "fields": [{"name": "ts",
                    "type": {"type": "long",
-                            "logicalType": "timestamp-micros"}}]}
+                            "logicalType": "timestamp-micros"}},
+                  {"name": "ts2",
+                   "type": {"type": "long",
+                            "logicalType": "timestamp-millis"}}]}
 EOS
     check(record_schema)
   end
@@ -112,6 +115,13 @@ EOS
     check_default(enum_schema, '"B"', "B")
   end
 
+  def test_enum_with_default
+    enum_schema = '{"type": "enum", "name": "Test", "symbols": ["A", "B"], "default": "A"}'
+    check(enum_schema)
+    # Field default is used for missing field.
+    check_default(enum_schema, '"B"', "B")
+  end
+
   def test_recursive
     recursive_schema = <<EOS
       {"type": "record",
@@ -158,6 +168,17 @@ EOS
     check_default(fixed_schema, '"a"', "a")
   end
 
+  def test_record_variable_key_types
+    datum = { sym: "foo", "str"=>"bar"}
+    ret_val = { "sym"=> "foo", "str"=>"bar"}
+    schema = Schema.parse('{"type":"record", "name":"rec", "fields":[{"name":"sym", "type":"string"}, {"name":"str", "type":"string"}]}')
+
+    writer, _encoder, _datum_writer = write_datum(datum, schema)
+
+    ret_datum = read_datum(writer, schema)
+    assert_equal ret_datum, ret_val
+  end
+
   def test_record_with_nil
     schema = Avro::Schema.parse('{"type":"record", "name":"rec", "fields":[{"type":"int", "name":"i"}]}')
     assert_raise(Avro::IO::AvroTypeError) do
@@ -390,6 +411,50 @@ EOS
     assert_equal(incorrect, 0)
   end
 
+  def test_unknown_enum_symbol
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["B", "C"]
+      }
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["A", "B"]
+      }
+    SCHEMA
+    datum_to_write = "C"
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    # Ruby implementation did not follow the spec and returns the writer's symbol here
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_unknown_enum_symbol_with_enum_default
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["B", "C"]
+      }
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["A", "B", "UNKNOWN"],
+        "default": "UNKNOWN"
+      }
+    SCHEMA
+    datum_to_write = "C"
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, "UNKNOWN")
+  end
+
   def test_array_schema_promotion
     writers_schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
     readers_schema = Avro::Schema.parse('{"type":"array", "items":"long"}')
@@ -408,6 +473,22 @@ EOS
     assert_equal(datum_read, datum_to_write)
   end
 
+  def test_aliased
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Rec1", "fields":[
+        {"name":"field1", "type":"int"}
+      ]}
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Rec2", "aliases":["Rec1"], "fields":[
+        {"name":"field2", "aliases":["field1"], "type":"int"}
+      ]}
+    SCHEMA
+    writer, * = write_datum({ 'field1' => 1 }, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, { 'field2' => 1 })
+  end
+
   def test_snappy_backward_compat
     # a snappy-compressed block payload without the checksum
     # this has no back-references, just one literal so the last 9

data/test/test_schema.rb

@@ -151,6 +151,20 @@ class TestSchema < Test::Unit::TestCase
     }
   end
 
+  def test_to_avro_includes_aliases
+    hash = {
+      'type' => 'record',
+      'name' => 'test_record',
+      'aliases' => %w(alt_record),
+      'fields' => [
+        { 'name' => 'f', 'type' => { 'type' => 'fixed', 'size' => 2, 'name' => 'test_fixed', 'aliases' => %w(alt_fixed) } },
+        { 'name' => 'e', 'type' => { 'type' => 'enum', 'symbols' => %w(A B), 'name' => 'test_enum', 'aliases' => %w(alt_enum) } }
+      ]
+    }
+    schema = hash_to_schema(hash)
+    assert_equal(schema.to_avro, hash)
+  end
+
   def test_unknown_named_type
     error = assert_raise Avro::UnknownSchemaError do
       Avro::Schema.parse <<-SCHEMA
@@ -163,6 +177,42 @@ class TestSchema < Test::Unit::TestCase
     assert_equal '"MissingType" is not a schema we know about.', error.message
   end
 
+  def test_invalid_name
+    error = assert_raise Avro::SchemaParseError do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my-invalid-name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+
+    assert_equal "Name my-invalid-name is invalid for type record!", error.message
+  end
+
+  def test_invalid_name_with_two_periods
+    error = assert_raise Avro::SchemaParseError do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my..invalid.name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+
+    assert_equal "Name my..invalid.name is invalid for type record!", error.message
+  end
+
+  def test_invalid_name_with_validation_disabled
+    Avro.disable_schema_name_validation = true
+    assert_nothing_raised do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my-invalid-name", "fields": [
+          {"name": "id", "type": "int"}
+        ]}
+      SCHEMA
+    end
+    Avro.disable_schema_name_validation = false
+  end
+
   def test_to_avro_handles_falsey_defaults
     schema = Avro::Schema.parse <<-SCHEMA
       {"type": "record", "name": "Record", "namespace": "my.name.space",
@@ -278,6 +328,40 @@ class TestSchema < Test::Unit::TestCase
     assert_equal enum_schema_hash, enum_schema_json.to_avro
   end
 
+  def test_enum_default_attribute
+    enum_schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "enum",
+        "name": "fruit",
+        "default": "apples",
+        "symbols": ["apples", "oranges"]
+      }
+    SCHEMA
+
+    enum_schema_hash = {
+      'type' => 'enum',
+      'name' => 'fruit',
+      'default' => 'apples',
+      'symbols' => %w(apples oranges)
+    }
+
+    assert_equal(enum_schema.default, "apples")
+    assert_equal(enum_schema_hash, enum_schema.to_avro)
+  end
+
+  def test_validate_enum_default
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'fruit',
+        default: 'bananas',
+        symbols: %w(apples oranges)
+      )
+    end
+    assert_equal("Default 'bananas' is not a valid symbol for enum fruit",
+                 exception.to_s)
+  end
+
   def test_empty_record
     schema = Avro::Schema.parse('{"type":"record", "name":"Empty"}')
     assert_empty(schema.fields)
@@ -455,5 +539,189 @@ class TestSchema < Test::Unit::TestCase
     end
     assert_equal('Error validating default for veggies: at . expected type null, got string with value "apple"',
                  exception.to_s)
+  end
+
+  def test_bytes_decimal_to_include_precision_scale
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 9,
+        "scale": 2
+      }
+    SCHEMA
+
+    schema_hash =
+      {
+        'type' => 'bytes',
+        'logicalType' => 'decimal',
+        'precision' => 9,
+        'scale' => 2
+      }
+
+    assert_equal schema_hash, schema.to_avro
+  end
+
+  def test_bytes_decimal_to_without_precision_scale
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes",
+        "logicalType": "decimal"
+      }
+    SCHEMA
+
+    schema_hash =
+      {
+        'type' => 'bytes',
+        'logicalType' => 'decimal'
+      }
+
+    assert_equal schema_hash, schema.to_avro
+  end
+
+  def test_bytes_schema
+    schema = Avro::Schema.parse <<-SCHEMA
+      {
+        "type": "bytes"
+      }
+    SCHEMA
+
+    schema_str = 'bytes'
+    assert_equal schema_str, schema.to_avro
+  end
+
+  def test_validate_duplicate_symbols
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'name',
+        symbols: ['erica', 'erica']
+      )
+    end
+    assert_equal(
+      'Duplicate symbol: ["erica", "erica"]',
+      exception.to_s
+    )
+  end
+
+  def test_validate_enum_symbols
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'things',
+        symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$']
+      )
+    end
+
+    assert_equal(
+      "Invalid symbols for things: 8ad_symbol, also-bad-symbol, >=, $ don't match #{Avro::Schema::EnumSchema::SYMBOL_REGEX.inspect}",
+      exception.to_s
+    )
+  end
+
+  def test_enum_symbol_validation_disabled_via_env
+    Avro.disable_enum_symbol_validation = nil
+    ENV['AVRO_DISABLE_ENUM_SYMBOL_VALIDATION'] = '1'
+
+    hash_to_schema(
+      type: 'enum',
+      name: 'things',
+      symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
+    )
+  ensure
+    ENV.delete('AVRO_DISABLE_ENUM_SYMBOL_VALIDATION')
+    Avro.disable_enum_symbol_validation = nil
+  end
+
+  def test_enum_symbol_validation_disabled_via_class_method
+    Avro.disable_enum_symbol_validation = true
+
+    hash_to_schema(
+      type: 'enum',
+      name: 'things',
+      symbols: ['good_symbol', '_GOOD_SYMBOL_2', '8ad_symbol', 'also-bad-symbol', '>=', '$'],
+    )
+  ensure
+    Avro.disable_enum_symbol_validation = nil
+  end
+  
+  def test_validate_field_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: 'banane' }
+        ]
+      )
+    end
+
+    assert_match(/Invalid aliases value "banane" for "string" banana/, exception.to_s)
+  end
+
+  def test_validate_same_alias_multiple_fields
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: %w(yellow) },
+          { name: 'lemo', type: 'string', aliases: %w(yellow) }
+        ]
+      )
+    end
+
+    assert_match('Alias ["yellow"] already in use', exception.to_s)
+  end
+
+  def test_validate_repeated_aliases
+    assert_nothing_raised do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        fields: [
+          { name: 'banana', type: 'string', aliases: %w(yellow yellow) },
+        ]
+      )
+    end
+  end
+
+  def test_validate_record_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'record',
+        name: 'fruits',
+        aliases: ["foods", 2],
+        fields: []
+      )
+    end
+
+    assert_match(/Invalid aliases value \["foods", 2\] for record fruits/, exception.to_s)
+  end
+
+  def test_validate_enum_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'enum',
+        name: 'vowels',
+        aliases: [1, 2],
+        symbols: %w(A E I O U)
+      )
     end
+
+    assert_match(/Invalid aliases value \[1, 2\] for enum vowels/, exception.to_s)
+  end
+
+  def test_validate_fixed_aliases
+    exception = assert_raise(Avro::SchemaParseError) do
+      hash_to_schema(
+        type: 'fixed',
+        name: 'uuid',
+        size: 36,
+        aliases: "unique_id"
+      )
+    end
+
+    assert_match(/Invalid aliases value "unique_id" for fixed uuid/, exception.to_s)
+  end
 end