avro 1.9.0 → 1.11.0

data/test/test_io.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -90,7 +91,10 @@ EOS
        "name": "Test",
        "fields": [{"name": "ts",
                    "type": {"type": "long",
-                            "logicalType": "timestamp-micros"}}]}
+                            "logicalType": "timestamp-micros"}},
+                  {"name": "ts2",
+                   "type": {"type": "long",
+                            "logicalType": "timestamp-millis"}}]}
 EOS
     check(record_schema)
   end
@@ -112,6 +116,13 @@ EOS
     check_default(enum_schema, '"B"', "B")
   end
 
+  def test_enum_with_default
+    enum_schema = '{"type": "enum", "name": "Test", "symbols": ["A", "B"], "default": "A"}'
+    check(enum_schema)
+    # Field default is used for missing field.
+    check_default(enum_schema, '"B"', "B")
+  end
+
   def test_recursive
     recursive_schema = <<EOS
       {"type": "record",
@@ -158,6 +169,17 @@ EOS
     check_default(fixed_schema, '"a"', "a")
   end
 
+  def test_record_variable_key_types
+    datum = { sym: "foo", "str"=>"bar"}
+    ret_val = { "sym"=> "foo", "str"=>"bar"}
+    schema = Schema.parse('{"type":"record", "name":"rec", "fields":[{"name":"sym", "type":"string"}, {"name":"str", "type":"string"}]}')
+
+    writer, _encoder, _datum_writer = write_datum(datum, schema)
+
+    ret_datum = read_datum(writer, schema)
+    assert_equal ret_datum, ret_val
+  end
+
   def test_record_with_nil
     schema = Avro::Schema.parse('{"type":"record", "name":"rec", "fields":[{"type":"int", "name":"i"}]}')
     assert_raise(Avro::IO::AvroTypeError) do
@@ -196,7 +218,7 @@ EOS
     [64, '80 01'],
     [8192, '80 80 01'],
     [-8193, '81 80 01'],
-  ]
+  ].freeze
 
   def avro_hexlify(reader)
     bytes = []
@@ -245,46 +267,46 @@ EOS
 
   def test_utf8_string_encoding
     [
-      "\xC3".force_encoding('ISO-8859-1'),
-      "\xC3\x83".force_encoding('UTF-8')
+      String.new("\xC3", encoding: 'ISO-8859-1'),
+      String.new("\xC3\x83", encoding: 'UTF-8')
     ].each do |value|
-      output = ''.force_encoding('BINARY')
+      output = String.new('', encoding: 'BINARY')
       encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
       datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"string"'))
       datum_writer.write(value, encoder)
 
-      assert_equal "\x04\xc3\x83".force_encoding('BINARY'), output
+      assert_equal String.new("\x04\xc3\x83", encoding: 'BINARY'), output
     end
   end
 
   def test_bytes_encoding
     [
-      "\xC3\x83".force_encoding('BINARY'),
-      "\xC3\x83".force_encoding('ISO-8859-1'),
-      "\xC3\x83".force_encoding('UTF-8')
+      String.new("\xC3\x83", encoding: 'BINARY'),
+      String.new("\xC3\x83", encoding: 'ISO-8859-1'),
+      String.new("\xC3\x83", encoding: 'UTF-8')
     ].each do |value|
-      output = ''.force_encoding('BINARY')
+      output = String.new('', encoding: 'BINARY')
       encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
       datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"bytes"'))
       datum_writer.write(value, encoder)
 
-      assert_equal "\x04\xc3\x83".force_encoding('BINARY'), output
+      assert_equal String.new("\x04\xc3\x83", encoding: 'BINARY'), output
     end
   end
 
   def test_fixed_encoding
     [
-      "\xC3\x83".force_encoding('BINARY'),
-      "\xC3\x83".force_encoding('ISO-8859-1'),
-      "\xC3\x83".force_encoding('UTF-8')
+      String.new("\xC3\x83", encoding: 'BINARY'),
+      String.new("\xC3\x83", encoding: 'ISO-8859-1'),
+      String.new("\xC3\x83", encoding: 'UTF-8')
     ].each do |value|
-      output = ''.force_encoding('BINARY')
+      output = String.new('', encoding: 'BINARY')
       encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
       schema = '{"type": "fixed", "name": "TwoBytes", "size": 2}'
       datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse(schema))
       datum_writer.write(value, encoder)
 
-      assert_equal "\xc3\x83".force_encoding('BINARY'), output
+      assert_equal String.new("\xC3\x83", encoding: 'BINARY'), output
     end
   end
 
@@ -390,6 +412,50 @@ EOS
     assert_equal(incorrect, 0)
   end
 
+  def test_unknown_enum_symbol
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["B", "C"]
+      }
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["A", "B"]
+      }
+    SCHEMA
+    datum_to_write = "C"
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    # Ruby implementation did not follow the spec and returns the writer's symbol here
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_unknown_enum_symbol_with_enum_default
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["B", "C"]
+      }
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["A", "B", "UNKNOWN"],
+        "default": "UNKNOWN"
+      }
+    SCHEMA
+    datum_to_write = "C"
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, "UNKNOWN")
+  end
+
   def test_array_schema_promotion
     writers_schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
     readers_schema = Avro::Schema.parse('{"type":"array", "items":"long"}')
@@ -408,6 +474,36 @@ EOS
     assert_equal(datum_read, datum_to_write)
   end
 
+  def test_aliased
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Rec1", "fields":[
+        {"name":"field1", "type":"int"}
+      ]}
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Rec2", "aliases":["Rec1"], "fields":[
+        {"name":"field2", "aliases":["field1"], "type":"int"}
+      ]}
+    SCHEMA
+    writer, * = write_datum({ 'field1' => 1 }, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, { 'field2' => 1 })
+  end
+
+  def test_big_decimal_datum_for_float
+    writers_schema = Avro::Schema.parse('"float"')
+    writer, * = write_datum(BigDecimal('1.2'), writers_schema)
+    datum_read = read_datum(writer, writers_schema)
+    assert_in_delta(1.2, datum_read)
+  end
+
+  def test_big_decimal_datum_for_double
+    writers_schema = Avro::Schema.parse('"double"')
+    writer, * = write_datum(BigDecimal("1.2"), writers_schema)
+    datum_read = read_datum(writer, writers_schema)
+    assert_in_delta(1.2, datum_read)
+  end
+
   def test_snappy_backward_compat
     # a snappy-compressed block payload without the checksum
     # this has no back-references, just one literal so the last 9
@@ -486,7 +582,7 @@ EOS
     datum = randomdata.next
     assert validate(schm, datum), 'datum is not valid for schema'
     w = Avro::IO::DatumWriter.new(schm)
-    writer = StringIO.new "", "w"
+    writer = StringIO.new(+"", "w")
     w.write(datum, Avro::IO::BinaryEncoder.new(writer))
     r = datum_reader(schm)
     reader = StringIO.new(writer.string)

data/test/test_logical_types.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -7,7 +8,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,6 +17,7 @@
 # limitations under the License.
 
 require 'test_help'
+require 'memory_profiler'
 
 class TestLogicalTypes < Test::Unit::TestCase
   def test_int_date
@@ -98,8 +100,143 @@ class TestLogicalTypes < Test::Unit::TestCase
     assert_equal 'duration', schema.logical_type
   end
 
+  def test_bytes_decimal
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 6 }
+    SCHEMA
+
+    assert_equal 'decimal', schema.logical_type
+    assert_equal 9, schema.precision
+    assert_equal 6, schema.scale
+
+    assert_encode_and_decode BigDecimal('-3.4562'), schema
+    assert_encode_and_decode BigDecimal('3.4562'), schema
+    assert_encode_and_decode 15.123, schema
+    assert_encode_and_decode 15, schema
+    assert_encode_and_decode BigDecimal('0.123456'), schema
+    assert_encode_and_decode BigDecimal('0'), schema
+    assert_encode_and_decode BigDecimal('1'), schema
+    assert_encode_and_decode BigDecimal('-1'), schema
+
+    assert_raise ArgumentError do
+      type = Avro::LogicalTypes::BytesDecimal.new(schema)
+      type.encode('1.23')
+    end
+  end
+
+  def test_bytes_decimal_range_errors
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
+    SCHEMA
+
+    type = Avro::LogicalTypes::BytesDecimal.new(schema)
+
+    assert_raises RangeError do
+      type.encode(BigDecimal('345'))
+    end
+
+    assert_raises RangeError do
+      type.encode(BigDecimal('1.5342'))
+    end
+
+    assert_raises RangeError do
+      type.encode(BigDecimal('-1.5342'))
+    end
+
+    assert_raises RangeError do
+      type.encode(BigDecimal('-100.2'))
+    end
+
+    assert_raises RangeError do
+      type.encode(BigDecimal('-99.991'))
+    end
+  end
+
+  def test_bytes_decimal_conversion
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
+    SCHEMA
+
+    type = Avro::LogicalTypes::BytesDecimal.new(schema)
+
+    enc = "\xcb\x43\x38".dup.force_encoding('BINARY')
+    assert_equal enc, type.encode(BigDecimal('-3.4562'))
+    assert_equal BigDecimal('-3.4562'), type.decode(enc)
+
+    assert_equal "\x34\xbc\xc8".dup.force_encoding('BINARY'), type.encode(BigDecimal('3.4562'))
+    assert_equal BigDecimal('3.4562'), type.decode("\x34\xbc\xc8".dup.force_encoding('BINARY'))
+
+    assert_equal "\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'), type.encode(BigDecimal('456123.123456'))
+    assert_equal BigDecimal('456123.123456'), type.decode("\x6a\x33\x0e\x87\x00".dup.force_encoding('BINARY'))
+  end
+
+  def test_logical_type_with_schema
+    exception = assert_raises(ArgumentError) do
+      Avro::LogicalTypes::LogicalTypeWithSchema.new(nil)
+    end
+    assert_equal exception.to_s, 'schema is required'
+
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "bytes", "logicalType": "decimal", "precision": 12, "scale": 6 }
+    SCHEMA
+
+    assert_nothing_raised do
+      Avro::LogicalTypes::LogicalTypeWithSchema.new(schema)
+    end
+
+    assert_raises NotImplementedError do
+      Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).encode(BigDecimal('2'))
+    end
+
+    assert_raises NotImplementedError do
+      Avro::LogicalTypes::LogicalTypeWithSchema.new(schema).decode('foo')
+    end
+  end
+
+  def test_bytes_decimal_object_allocations_encode
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
+    SCHEMA
+
+    type = Avro::LogicalTypes::BytesDecimal.new(schema)
+
+    positive_value = BigDecimal('5.2')
+    negative_value = BigDecimal('-5.2')
+
+    [positive_value, negative_value].each do |value|
+      report = MemoryProfiler.report do
+        type.encode(value)
+      end
+
+      assert_equal 5, report.total_allocated
+      # Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
+      assert_operator 1, :>=, report.total_retained
+    end
+  end
+
+  def test_bytes_decimal_object_allocations_decode
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2 }
+    SCHEMA
+
+    type = Avro::LogicalTypes::BytesDecimal.new(schema)
+
+    positive_enc = "\x02\b".dup.force_encoding('BINARY')
+    negative_enc = "\xFD\xF8".dup.force_encoding('BINARY')
+
+    [positive_enc, negative_enc].each do |encoded|
+      report = MemoryProfiler.report do
+        type.decode(encoded)
+      end
+
+      assert_equal 5, report.total_allocated
+      # Ruby 2.7 does not retain anything. Ruby 2.6 retains 1
+      assert_operator 1, :>=, report.total_retained
+    end
+  end
+
   def encode(datum, schema)
-    buffer = StringIO.new("")
+    buffer = StringIO.new
     encoder = Avro::IO::BinaryEncoder.new(buffer)
 
     datum_writer = Avro::IO::DatumWriter.new(schema)

data/test/test_protocol.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -184,7 +185,7 @@ EOS
   }
 }
 EOS
-]
+].freeze
 
   Protocol = Avro::Protocol
   def test_parse