avro 1.8.2 → 1.10.1

data/test/random_data.rb

@@ -1,4 +1,3 @@
-#!/usr/bin/env ruby
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -7,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -27,15 +26,17 @@ class RandomData
   end
 
   def nextdata(schm, d=0)
+    return logical_nextdata(schm, d=0) unless schm.type_adapter.eql?(Avro::LogicalTypes::Identity)
+
     case schm.type_sym
     when :boolean
       rand > 0.5
     when :string
       randstr()
     when :int
-      rand(Avro::Schema::INT_MAX_VALUE - Avro::Schema::INT_MIN_VALUE) + Avro::Schema::INT_MIN_VALUE
+      rand_int
     when :long
-      rand(Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) + Avro::Schema::LONG_MIN_VALUE
+      rand_long
     when :float
       (-1024 + 2048 * rand).round.to_f
     when :double
@@ -79,6 +80,17 @@ class RandomData
     end
   end
 
+  def logical_nextdata(schm, _d=0)
+    case schm.logical_type
+    when 'date'
+      Avro::LogicalTypes::IntDate.decode(rand_int)
+    when 'timestamp-micros'
+      Avro::LogicalTypes::TimestampMicros.decode(rand_long)
+    when 'timestamp-millis'
+      Avro::LogicalTypes::TimestampMillis.decode(rand_long)
+    end
+  end
+
   CHARPOOL = 'abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'
   BYTEPOOL = '12345abcd'
 
@@ -87,4 +99,12 @@ class RandomData
     rand(length+1).times { str << chars[rand(chars.size)] }
     str
   end
+
+  def rand_int
+    rand(Avro::Schema::INT_MAX_VALUE - Avro::Schema::INT_MIN_VALUE) + Avro::Schema::INT_MIN_VALUE
+  end
+
+  def rand_long
+    rand(Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) + Avro::Schema::LONG_MIN_VALUE
+  end
 end

data/test/sample_ipc_client.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/sample_ipc_http_client.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/sample_ipc_http_server.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/sample_ipc_server.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/test_datafile.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -20,13 +20,13 @@ require 'test_help'
 class TestDataFile < Test::Unit::TestCase
   HERE = File.expand_path File.dirname(__FILE__)
   def setup
-    if File.exists?(HERE + '/data.avr')
+    if File.exist?(HERE + '/data.avr')
       File.unlink(HERE + '/data.avr')
     end
   end
 
   def teardown
-    if File.exists?(HERE + '/data.avr')
+    if File.exist?(HERE + '/data.avr')
       File.unlink(HERE + '/data.avr')
     end
   end
@@ -38,7 +38,7 @@ class TestDataFile < Test::Unit::TestCase
   "fields" : [
     {"name": "username", "type": "string"},
     {"name": "age", "type": "int"},
-    {"name": "verified", "type": "boolean", "default": "false"}
+    {"name": "verified", "type": "boolean", "default": false}
   ]}
 JSON
 
@@ -180,6 +180,19 @@ JSON
     assert_equal records, ['a' * 10_000]
   end
 
+  def test_zstandard
+    Avro::DataFile.open('data.avr', 'w', '"string"', :zstandard) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size('data.avr') < 600)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
   def test_append_to_deflated_file
     schema = Avro::Schema.parse('"string"')
     writer = Avro::IO::DatumWriter.new(schema)

data/test/test_fingerprints.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -34,4 +34,23 @@ class TestFingerprints < Test::Unit::TestCase
     assert_equal 28572620203319713300323544804233350633246234624932075150020181448463213378117,
       schema.sha256_fingerprint
   end
+
+  def test_crc_64_avro_fingerprint
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal 8247732601305521295, # hex: 0x7275d51a3f395c8f
+      schema.crc_64_avro_fingerprint
+  end
+
+  # This definitely belongs somewhere else
+  def test_single_object_encoding_header
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal ["c3", "01", "8f", "5c", "39", "3f", "1a", "D5", "75", "72"].map{|e| e.to_i(16) },
+      schema.single_object_encoding_header
+  end
 end

data/test/test_help.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/test_io.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -84,6 +84,20 @@ EOS
     check_default(record_schema, '{"f": 11}', {"f" => 11})
   end
 
+  def test_record_with_logical_type
+    record_schema = <<EOS
+      {"type": "record",
+       "name": "Test",
+       "fields": [{"name": "ts",
+                   "type": {"type": "long",
+                            "logicalType": "timestamp-micros"}},
+                  {"name": "ts2",
+                   "type": {"type": "long",
+                            "logicalType": "timestamp-millis"}}]}
+EOS
+    check(record_schema)
+  end
+
   def test_error
     error_schema = <<EOS
       {"type": "error",
@@ -101,6 +115,13 @@ EOS
     check_default(enum_schema, '"B"', "B")
   end
 
+  def test_enum_with_default
+    enum_schema = '{"type": "enum", "name": "Test", "symbols": ["A", "B"], "default": "A"}'
+    check(enum_schema)
+    # Field default is used for missing field.
+    check_default(enum_schema, '"B"', "B")
+  end
+
   def test_recursive
     recursive_schema = <<EOS
       {"type": "record",
@@ -115,6 +136,7 @@ EOS
   def test_union
     union_schema = <<EOS
       ["string",
+       {"type": "int", "logicalType": "date"},
        "null",
        "long",
        {"type": "record",
@@ -146,10 +168,42 @@ EOS
     check_default(fixed_schema, '"a"', "a")
   end
 
+  def test_record_variable_key_types
+    datum = { sym: "foo", "str"=>"bar"}
+    ret_val = { "sym"=> "foo", "str"=>"bar"}
+    schema = Schema.parse('{"type":"record", "name":"rec", "fields":[{"name":"sym", "type":"string"}, {"name":"str", "type":"string"}]}')
+
+    writer, _encoder, _datum_writer = write_datum(datum, schema)
+
+    ret_datum = read_datum(writer, schema)
+    assert_equal ret_datum, ret_val
+  end
+
+  def test_record_with_nil
+    schema = Avro::Schema.parse('{"type":"record", "name":"rec", "fields":[{"type":"int", "name":"i"}]}')
+    assert_raise(Avro::IO::AvroTypeError) do
+      write_datum(nil, schema)
+    end
+  end
+
+  def test_array_with_nil
+    schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
+    assert_raise(Avro::IO::AvroTypeError) do
+      write_datum(nil, schema)
+    end
+  end
+
+  def test_map_with_nil
+    schema = Avro::Schema.parse('{"type":"map", "values":"long"}')
+    assert_raise(Avro::IO::AvroTypeError) do
+      write_datum(nil, schema)
+    end
+  end
+
   def test_enum_with_duplicate
     str = '{"type": "enum", "name": "Test","symbols" : ["AA", "AA"]}'
-    assert_raises(Avro::SchemaParseError) do
-      schema = Avro::Schema.parse str
+    assert_raises(Avro::SchemaParseError.new('Duplicate symbol: ["AA", "AA"]')) do
+      Avro::Schema.parse str
     end
   end
 
@@ -256,7 +310,7 @@ EOS
   end
 
   def test_skip_long
-    for value_to_skip, hex_encoding in BINARY_INT_ENCODINGS
+    for value_to_skip, _hex_encoding in BINARY_INT_ENCODINGS
       value_to_read = 6253
 
       # write some data in binary to string buffer
@@ -281,7 +335,7 @@ EOS
   end
 
   def test_skip_int
-    for value_to_skip, hex_encoding in BINARY_INT_ENCODINGS
+    for value_to_skip, _hex_encoding in BINARY_INT_ENCODINGS
       value_to_read = 6253
 
       writer = StringIO.new
@@ -331,7 +385,7 @@ EOS
       datum_to_write = 219
       for rs in promotable_schemas[(i + 1)..-1]
         readers_schema = Avro::Schema.parse(rs)
-        writer, enc, dw = write_datum(datum_to_write, writers_schema)
+        writer, _enc, _dw = write_datum(datum_to_write, writers_schema)
         datum_read = read_datum(writer, writers_schema, readers_schema)
         if datum_read != datum_to_write
           incorrect += 1
@@ -341,6 +395,100 @@ EOS
     end
   end
 
+  def test_interchangeable_schemas
+    interchangeable_schemas = ['"string"', '"bytes"']
+    incorrect = 0
+    interchangeable_schemas.each_with_index do |ws, i|
+      writers_schema = Avro::Schema.parse(ws)
+      datum_to_write = 'foo'
+      readers_schema = Avro::Schema.parse(interchangeable_schemas[i == 0 ? 1 : 0])
+      writer, * = write_datum(datum_to_write, writers_schema)
+      datum_read = read_datum(writer, writers_schema, readers_schema)
+      if datum_read != datum_to_write
+        incorrect += 1
+      end
+    end
+    assert_equal(incorrect, 0)
+  end
+
+  def test_unknown_enum_symbol
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["B", "C"]
+      }
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["A", "B"]
+      }
+    SCHEMA
+    datum_to_write = "C"
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    # Ruby implementation did not follow the spec and returns the writer's symbol here
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_unknown_enum_symbol_with_enum_default
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["B", "C"]
+      }
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {
+        "type": "enum",
+        "name": "test",
+        "symbols": ["A", "B", "UNKNOWN"],
+        "default": "UNKNOWN"
+      }
+    SCHEMA
+    datum_to_write = "C"
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, "UNKNOWN")
+  end
+
+  def test_array_schema_promotion
+    writers_schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
+    readers_schema = Avro::Schema.parse('{"type":"array", "items":"long"}')
+    datum_to_write = [1, 2]
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_map_schema_promotion
+    writers_schema = Avro::Schema.parse('{"type":"map", "values":"int"}')
+    readers_schema = Avro::Schema.parse('{"type":"map", "values":"long"}')
+    datum_to_write = { 'foo' => 1, 'bar' => 2 }
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_aliased
+    writers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Rec1", "fields":[
+        {"name":"field1", "type":"int"}
+      ]}
+    SCHEMA
+    readers_schema = Avro::Schema.parse(<<-SCHEMA)
+      {"type":"record", "name":"Rec2", "aliases":["Rec1"], "fields":[
+        {"name":"field2", "aliases":["field1"], "type":"int"}
+      ]}
+    SCHEMA
+    writer, * = write_datum({ 'field1' => 1 }, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, { 'field2' => 1 })
+  end
+
   def test_snappy_backward_compat
     # a snappy-compressed block payload without the checksum
     # this has no back-references, just one literal so the last 9
@@ -417,7 +565,7 @@ EOS
 
   def checkser(schm, randomdata)
     datum = randomdata.next
-    assert validate(schm, datum)
+    assert validate(schm, datum), 'datum is not valid for schema'
     w = Avro::IO::DatumWriter.new(schm)
     writer = StringIO.new "", "w"
     w.write(datum, Avro::IO::BinaryEncoder.new(writer))

data/test/test_logical_types.rb

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestLogicalTypes < Test::Unit::TestCase
+  def test_int_date
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int", "logicalType": "date" }
+    SCHEMA
+
+    assert_equal 'date', schema.logical_type
+    today = Date.today
+    assert_encode_and_decode today, schema
+    assert_preencoded Avro::LogicalTypes::IntDate.encode(today), schema, today
+  end
+
+  def test_int_date_conversion
+    type = Avro::LogicalTypes::IntDate
+
+    assert_equal 5, type.encode(Date.new(1970, 1, 6))
+    assert_equal 0, type.encode(Date.new(1970, 1, 1))
+    assert_equal(-5, type.encode(Date.new(1969, 12, 27)))
+
+    assert_equal Date.new(1970, 1, 6), type.decode(5)
+    assert_equal Date.new(1970, 1, 1), type.decode(0)
+    assert_equal Date.new(1969, 12, 27), type.decode(-5)
+  end
+
+  def test_timestamp_millis_long
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "long", "logicalType": "timestamp-millis" }
+    SCHEMA
+
+    # The Time.at format is (seconds, microseconds) since Epoch.
+    time = Time.at(628232400, 12000)
+
+    assert_equal 'timestamp-millis', schema.logical_type
+    assert_encode_and_decode time, schema
+    assert_preencoded Avro::LogicalTypes::TimestampMillis.encode(time), schema, time.utc
+  end
+
+  def test_timestamp_millis_long_conversion
+    type = Avro::LogicalTypes::TimestampMillis
+
+    now = Time.now.utc
+    now_millis = Time.utc(now.year, now.month, now.day, now.hour, now.min, now.sec, now.usec / 1000 * 1000)
+
+    assert_equal now_millis, type.decode(type.encode(now_millis))
+    assert_equal 1432849613221, type.encode(Time.utc(2015, 5, 28, 21, 46, 53, 221000))
+    assert_equal 1432849613221, type.encode(DateTime.new(2015, 5, 28, 21, 46, 53.221))
+    assert_equal Time.utc(2015, 5, 28, 21, 46, 53, 221000), type.decode(1432849613221)
+  end
+
+  def test_timestamp_micros_long
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "long", "logicalType": "timestamp-micros" }
+    SCHEMA
+
+    # The Time.at format is (seconds, microseconds) since Epoch.
+    time = Time.at(628232400, 12345)
+
+    assert_equal 'timestamp-micros', schema.logical_type
+    assert_encode_and_decode time, schema
+    assert_preencoded Avro::LogicalTypes::TimestampMicros.encode(time), schema, time.utc
+  end
+
+  def test_timestamp_micros_long_conversion
+    type = Avro::LogicalTypes::TimestampMicros
+
+    now = Time.now.utc
+
+    assert_equal Time.utc(now.year, now.month, now.day, now.hour, now.min, now.sec, now.usec), type.decode(type.encode(now))
+    assert_equal 1432849613221843, type.encode(Time.utc(2015, 5, 28, 21, 46, 53, 221843))
+    assert_equal 1432849613221843, type.encode(DateTime.new(2015, 5, 28, 21, 46, 53.221843))
+    assert_equal Time.utc(2015, 5, 28, 21, 46, 53, 221843), type.decode(1432849613221843)
+  end
+
+  def test_parse_fixed_duration
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "fixed", "size": 12, "name": "fixed_dur", "logicalType": "duration" }
+    SCHEMA
+
+    assert_equal 'duration', schema.logical_type
+  end
+
+  def encode(datum, schema)
+    buffer = StringIO.new("")
+    encoder = Avro::IO::BinaryEncoder.new(buffer)
+
+    datum_writer = Avro::IO::DatumWriter.new(schema)
+    datum_writer.write(datum, encoder)
+
+    buffer.string
+  end
+
+  def decode(encoded, schema)
+    buffer = StringIO.new(encoded)
+    decoder = Avro::IO::BinaryDecoder.new(buffer)
+
+    datum_reader = Avro::IO::DatumReader.new(schema, schema)
+    datum_reader.read(decoder)
+  end
+
+  def assert_encode_and_decode(datum, schema)
+    encoded = encode(datum, schema)
+    assert_equal datum, decode(encoded, schema)
+  end
+
+  def assert_preencoded(datum, schema, decoded)
+    encoded = encode(datum, schema)
+    assert_equal decoded, decode(encoded, schema)
+  end
+end