avro 1.8.2 → 1.9.0

data/lib/avro/schema_validator.rb

@@ -0,0 +1,242 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Avro
+  class SchemaValidator
+    ROOT_IDENTIFIER = '.'.freeze
+    PATH_SEPARATOR = '.'.freeze
+    INT_RANGE = Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE
+    LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
+    COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
+    BOOLEAN_VALUES = [true, false].freeze
+
+    class Result
+      attr_reader :errors
+
+      def initialize
+        @errors = []
+      end
+
+      def <<(error)
+        @errors << error
+      end
+
+      def add_error(path, message)
+        self << "at #{path} #{message}"
+      end
+
+      def failure?
+        @errors.any?
+      end
+
+      def to_s
+        errors.join("\n")
+      end
+    end
+
+    class ValidationError < StandardError
+      attr_reader :result
+
+      def initialize(result = Result.new)
+        @result = result
+        super
+      end
+
+      def to_s
+        result.to_s
+      end
+    end
+
+    TypeMismatchError = Class.new(ValidationError)
+
+    class << self
+      def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false, fail_on_extra_fields: false })
+        options ||= {}
+        options[:recursive] = true unless options.key?(:recursive)
+
+        result = Result.new
+        if options[:recursive]
+          validate_recursive(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
+        else
+          validate_simple(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
+        end
+        fail ValidationError, result if result.failure?
+        result
+      end
+
+      private
+
+      def validate_recursive(expected_schema, logical_datum, path, result, options = {})
+        datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
+
+        validate_simple(expected_schema, datum, path, result, encoded: true)
+
+        case expected_schema.type_sym
+        when :array
+          validate_array(expected_schema, datum, path, result, options)
+        when :map
+          validate_map(expected_schema, datum, path, result, options)
+        when :union
+          validate_union(expected_schema, datum, path, result, options)
+        when :record, :error, :request
+          fail TypeMismatchError unless datum.is_a?(Hash)
+          expected_schema.fields.each do |field|
+            deeper_path = deeper_path_for_hash(field.name, path)
+            validate_recursive(field.type, datum[field.name], deeper_path, result, options)
+          end
+          if options[:fail_on_extra_fields]
+            datum_fields = datum.keys.map(&:to_s)
+            schema_fields = expected_schema.fields.map(&:name)
+            (datum_fields - schema_fields).each do |extra_field|
+              result.add_error(path, "extra field '#{extra_field}' - not in schema")
+            end
+          end
+        end
+      rescue TypeMismatchError
+        result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
+      end
+
+      def validate_simple(expected_schema, logical_datum, path, result, options = {})
+        datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
+        validate_type(expected_schema)
+
+        case expected_schema.type_sym
+        when :null
+          fail TypeMismatchError unless datum.nil?
+        when :boolean
+          fail TypeMismatchError unless BOOLEAN_VALUES.include?(datum)
+        when :string, :bytes
+          fail TypeMismatchError unless datum.is_a?(String)
+        when :int
+          fail TypeMismatchError unless datum.is_a?(Integer)
+          result.add_error(path, "out of bound value #{datum}") unless INT_RANGE.cover?(datum)
+        when :long
+          fail TypeMismatchError unless datum.is_a?(Integer)
+          result.add_error(path, "out of bound value #{datum}") unless LONG_RANGE.cover?(datum)
+        when :float, :double
+          fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer)
+        when :fixed
+          if datum.is_a? String
+            result.add_error(path, fixed_string_message(expected_schema.size, datum)) unless datum.bytesize == expected_schema.size
+          else
+            result.add_error(path, "expected fixed with size #{expected_schema.size}, got #{actual_value_message(datum)}")
+          end
+        when :enum
+          result.add_error(path, enum_message(expected_schema.symbols, datum)) unless expected_schema.symbols.include?(datum)
+        end
+      rescue TypeMismatchError
+        result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
+      end
+
+      def resolve_datum(expected_schema, logical_datum, encoded)
+        if encoded
+          logical_datum
+        else
+          expected_schema.type_adapter.encode(logical_datum) rescue nil
+        end
+      end
+
+      def validate_type(expected_schema)
+        unless Avro::Schema::VALID_TYPES_SYM.include?(expected_schema.type_sym)
+          fail "Unexpected schema type #{expected_schema.type_sym} #{expected_schema.inspect}"
+        end
+      end
+
+      def fixed_string_message(size, datum)
+        "expected fixed with size #{size}, got \"#{datum}\" with size #{datum.bytesize}"
+      end
+
+      def enum_message(symbols, datum)
+        "expected enum with values #{symbols}, got #{actual_value_message(datum)}"
+      end
+
+      def validate_array(expected_schema, datum, path, result, options = {})
+        fail TypeMismatchError unless datum.is_a?(Array)
+        datum.each_with_index do |d, i|
+          validate_recursive(expected_schema.items, d, path + "[#{i}]", result, options)
+        end
+      end
+
+      def validate_map(expected_schema, datum, path, result, options = {})
+        fail TypeMismatchError unless datum.is_a?(Hash)
+        datum.keys.each do |k|
+          result.add_error(path, "unexpected key type '#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String)
+        end
+        datum.each do |k, v|
+          deeper_path = deeper_path_for_hash(k, path)
+          validate_recursive(expected_schema.values, v, deeper_path, result, options)
+        end
+      end
+
+      def validate_union(expected_schema, datum, path, result, options = {})
+        if expected_schema.schemas.size == 1
+          validate_recursive(expected_schema.schemas.first, datum, path, result, options)
+          return
+        end
+        failures = []
+        compatible_type = first_compatible_type(datum, expected_schema, path, failures, options)
+        return unless compatible_type.nil?
+
+        complex_type_failed = failures.detect { |r| COMPLEX_TYPES.include?(r[:type]) }
+        if complex_type_failed
+          complex_type_failed[:result].errors.each { |error| result << error }
+        else
+          types = expected_schema.schemas.map { |s| "'#{s.type_sym}'" }.join(', ')
+          result.add_error(path, "expected union of [#{types}], got #{actual_value_message(datum)}")
+        end
+      end
+
+      def first_compatible_type(datum, expected_schema, path, failures, options = {})
+        expected_schema.schemas.find do |schema|
+          result = Result.new
+          validate_recursive(schema, datum, path, result, options)
+          failures << { type: schema.type_sym, result: result } if result.failure?
+          !result.failure?
+        end
+      end
+
+      def deeper_path_for_hash(sub_key, path)
+        "#{path}#{PATH_SEPARATOR}#{sub_key}".squeeze(PATH_SEPARATOR)
+      end
+
+      def actual_value_message(value)
+        avro_type = if value.is_a?(Integer)
+                      ruby_integer_to_avro_type(value)
+                    else
+                      ruby_to_avro_type(value.class)
+                    end
+        if value.nil?
+          avro_type
+        else
+          "#{avro_type} with value #{value.inspect}"
+        end
+      end
+
+      def ruby_to_avro_type(ruby_class)
+        {
+          NilClass => 'null',
+          String => 'string',
+          Float => 'float',
+          Hash => 'record'
+        }.fetch(ruby_class, ruby_class)
+      end
+
+      def ruby_integer_to_avro_type(value)
+        INT_RANGE.cover?(value) ? 'int' : 'long'
+      end
+    end
+  end
+end

data/test/random_data.rb

@@ -27,15 +27,17 @@ class RandomData
   end
 
   def nextdata(schm, d=0)
+    return logical_nextdata(schm, d=0) unless schm.type_adapter.eql?(Avro::LogicalTypes::Identity)
+
     case schm.type_sym
     when :boolean
       rand > 0.5
     when :string
       randstr()
     when :int
-      rand(Avro::Schema::INT_MAX_VALUE - Avro::Schema::INT_MIN_VALUE) + Avro::Schema::INT_MIN_VALUE
+      rand_int
     when :long
-      rand(Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) + Avro::Schema::LONG_MIN_VALUE
+      rand_long
     when :float
       (-1024 + 2048 * rand).round.to_f
     when :double
@@ -79,6 +81,15 @@ class RandomData
     end
   end
 
+  def logical_nextdata(schm, _d=0)
+    case schm.logical_type
+    when 'date'
+      Avro::LogicalTypes::IntDate.decode(rand_int)
+    when 'timestamp-millis', 'timestamp-micros'
+      Avro::LogicalTypes::TimestampMicros.decode(rand_long)
+    end
+  end
+
   CHARPOOL = 'abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'
   BYTEPOOL = '12345abcd'
 
@@ -87,4 +98,12 @@ class RandomData
     rand(length+1).times { str << chars[rand(chars.size)] }
     str
   end
+
+  def rand_int
+    rand(Avro::Schema::INT_MAX_VALUE - Avro::Schema::INT_MIN_VALUE) + Avro::Schema::INT_MIN_VALUE
+  end
+
+  def rand_long
+    rand(Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) + Avro::Schema::LONG_MIN_VALUE
+  end
 end

data/test/test_datafile.rb

@@ -20,13 +20,13 @@ require 'test_help'
 class TestDataFile < Test::Unit::TestCase
   HERE = File.expand_path File.dirname(__FILE__)
   def setup
-    if File.exists?(HERE + '/data.avr')
+    if File.exist?(HERE + '/data.avr')
       File.unlink(HERE + '/data.avr')
     end
   end
 
   def teardown
-    if File.exists?(HERE + '/data.avr')
+    if File.exist?(HERE + '/data.avr')
       File.unlink(HERE + '/data.avr')
     end
   end
@@ -38,7 +38,7 @@ class TestDataFile < Test::Unit::TestCase
   "fields" : [
     {"name": "username", "type": "string"},
     {"name": "age", "type": "int"},
-    {"name": "verified", "type": "boolean", "default": "false"}
+    {"name": "verified", "type": "boolean", "default": false}
   ]}
 JSON
 

data/test/test_io.rb

@@ -84,6 +84,17 @@ EOS
     check_default(record_schema, '{"f": 11}', {"f" => 11})
   end
 
+  def test_record_with_logical_type
+    record_schema = <<EOS
+      {"type": "record",
+       "name": "Test",
+       "fields": [{"name": "ts",
+                   "type": {"type": "long",
+                            "logicalType": "timestamp-micros"}}]}
+EOS
+    check(record_schema)
+  end
+
   def test_error
     error_schema = <<EOS
       {"type": "error",
@@ -115,6 +126,7 @@ EOS
   def test_union
     union_schema = <<EOS
       ["string",
+       {"type": "int", "logicalType": "date"},
        "null",
        "long",
        {"type": "record",
@@ -146,10 +158,31 @@ EOS
     check_default(fixed_schema, '"a"', "a")
   end
 
+  def test_record_with_nil
+    schema = Avro::Schema.parse('{"type":"record", "name":"rec", "fields":[{"type":"int", "name":"i"}]}')
+    assert_raise(Avro::IO::AvroTypeError) do
+      write_datum(nil, schema)
+    end
+  end
+
+  def test_array_with_nil
+    schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
+    assert_raise(Avro::IO::AvroTypeError) do
+      write_datum(nil, schema)
+    end
+  end
+
+  def test_map_with_nil
+    schema = Avro::Schema.parse('{"type":"map", "values":"long"}')
+    assert_raise(Avro::IO::AvroTypeError) do
+      write_datum(nil, schema)
+    end
+  end
+
   def test_enum_with_duplicate
     str = '{"type": "enum", "name": "Test","symbols" : ["AA", "AA"]}'
-    assert_raises(Avro::SchemaParseError) do
-      schema = Avro::Schema.parse str
+    assert_raises(Avro::SchemaParseError.new('Duplicate symbol: ["AA", "AA"]')) do
+      Avro::Schema.parse str
     end
   end
 
@@ -256,7 +289,7 @@ EOS
   end
 
   def test_skip_long
-    for value_to_skip, hex_encoding in BINARY_INT_ENCODINGS
+    for value_to_skip, _hex_encoding in BINARY_INT_ENCODINGS
       value_to_read = 6253
 
       # write some data in binary to string buffer
@@ -281,7 +314,7 @@ EOS
   end
 
   def test_skip_int
-    for value_to_skip, hex_encoding in BINARY_INT_ENCODINGS
+    for value_to_skip, _hex_encoding in BINARY_INT_ENCODINGS
       value_to_read = 6253
 
       writer = StringIO.new
@@ -331,7 +364,7 @@ EOS
       datum_to_write = 219
       for rs in promotable_schemas[(i + 1)..-1]
         readers_schema = Avro::Schema.parse(rs)
-        writer, enc, dw = write_datum(datum_to_write, writers_schema)
+        writer, _enc, _dw = write_datum(datum_to_write, writers_schema)
         datum_read = read_datum(writer, writers_schema, readers_schema)
         if datum_read != datum_to_write
           incorrect += 1
@@ -341,6 +374,40 @@ EOS
     end
   end
 
+  def test_interchangeable_schemas
+    interchangeable_schemas = ['"string"', '"bytes"']
+    incorrect = 0
+    interchangeable_schemas.each_with_index do |ws, i|
+      writers_schema = Avro::Schema.parse(ws)
+      datum_to_write = 'foo'
+      readers_schema = Avro::Schema.parse(interchangeable_schemas[i == 0 ? 1 : 0])
+      writer, * = write_datum(datum_to_write, writers_schema)
+      datum_read = read_datum(writer, writers_schema, readers_schema)
+      if datum_read != datum_to_write
+        incorrect += 1
+      end
+    end
+    assert_equal(incorrect, 0)
+  end
+
+  def test_array_schema_promotion
+    writers_schema = Avro::Schema.parse('{"type":"array", "items":"int"}')
+    readers_schema = Avro::Schema.parse('{"type":"array", "items":"long"}')
+    datum_to_write = [1, 2]
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, datum_to_write)
+  end
+
+  def test_map_schema_promotion
+    writers_schema = Avro::Schema.parse('{"type":"map", "values":"int"}')
+    readers_schema = Avro::Schema.parse('{"type":"map", "values":"long"}')
+    datum_to_write = { 'foo' => 1, 'bar' => 2 }
+    writer, * = write_datum(datum_to_write, writers_schema)
+    datum_read = read_datum(writer, writers_schema, readers_schema)
+    assert_equal(datum_read, datum_to_write)
+  end
+
   def test_snappy_backward_compat
     # a snappy-compressed block payload without the checksum
     # this has no back-references, just one literal so the last 9
@@ -417,7 +484,7 @@ EOS
 
   def checkser(schm, randomdata)
     datum = randomdata.next
-    assert validate(schm, datum)
+    assert validate(schm, datum), 'datum is not valid for schema'
     w = Avro::IO::DatumWriter.new(schm)
     writer = StringIO.new "", "w"
     w.write(datum, Avro::IO::BinaryEncoder.new(writer))

data/test/test_logical_types.rb

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestLogicalTypes < Test::Unit::TestCase
+  def test_int_date
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int", "logicalType": "date" }
+    SCHEMA
+
+    assert_equal 'date', schema.logical_type
+    today = Date.today
+    assert_encode_and_decode today, schema
+    assert_preencoded Avro::LogicalTypes::IntDate.encode(today), schema, today
+  end
+
+  def test_int_date_conversion
+    type = Avro::LogicalTypes::IntDate
+
+    assert_equal 5, type.encode(Date.new(1970, 1, 6))
+    assert_equal 0, type.encode(Date.new(1970, 1, 1))
+    assert_equal(-5, type.encode(Date.new(1969, 12, 27)))
+
+    assert_equal Date.new(1970, 1, 6), type.decode(5)
+    assert_equal Date.new(1970, 1, 1), type.decode(0)
+    assert_equal Date.new(1969, 12, 27), type.decode(-5)
+  end
+
+  def test_timestamp_millis_long
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "long", "logicalType": "timestamp-millis" }
+    SCHEMA
+
+    # The Time.at format is (seconds, microseconds) since Epoch.
+    time = Time.at(628232400, 12000)
+
+    assert_equal 'timestamp-millis', schema.logical_type
+    assert_encode_and_decode time, schema
+    assert_preencoded Avro::LogicalTypes::TimestampMillis.encode(time), schema, time.utc
+  end
+
+  def test_timestamp_millis_long_conversion
+    type = Avro::LogicalTypes::TimestampMillis
+
+    now = Time.now.utc
+    now_millis = Time.utc(now.year, now.month, now.day, now.hour, now.min, now.sec, now.usec / 1000 * 1000)
+
+    assert_equal now_millis, type.decode(type.encode(now_millis))
+    assert_equal 1432849613221, type.encode(Time.utc(2015, 5, 28, 21, 46, 53, 221000))
+    assert_equal 1432849613221, type.encode(DateTime.new(2015, 5, 28, 21, 46, 53.221))
+    assert_equal Time.utc(2015, 5, 28, 21, 46, 53, 221000), type.decode(1432849613221)
+  end
+
+  def test_timestamp_micros_long
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "long", "logicalType": "timestamp-micros" }
+    SCHEMA
+
+    # The Time.at format is (seconds, microseconds) since Epoch.
+    time = Time.at(628232400, 12345)
+
+    assert_equal 'timestamp-micros', schema.logical_type
+    assert_encode_and_decode time, schema
+    assert_preencoded Avro::LogicalTypes::TimestampMicros.encode(time), schema, time.utc
+  end
+
+  def test_timestamp_micros_long_conversion
+    type = Avro::LogicalTypes::TimestampMicros
+
+    now = Time.now.utc
+
+    assert_equal Time.utc(now.year, now.month, now.day, now.hour, now.min, now.sec, now.usec), type.decode(type.encode(now))
+    assert_equal 1432849613221843, type.encode(Time.utc(2015, 5, 28, 21, 46, 53, 221843))
+    assert_equal 1432849613221843, type.encode(DateTime.new(2015, 5, 28, 21, 46, 53.221843))
+    assert_equal Time.utc(2015, 5, 28, 21, 46, 53, 221843), type.decode(1432849613221843)
+  end
+
+  def test_parse_fixed_duration
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "fixed", "size": 12, "name": "fixed_dur", "logicalType": "duration" }
+    SCHEMA
+
+    assert_equal 'duration', schema.logical_type
+  end
+
+  def encode(datum, schema)
+    buffer = StringIO.new("")
+    encoder = Avro::IO::BinaryEncoder.new(buffer)
+
+    datum_writer = Avro::IO::DatumWriter.new(schema)
+    datum_writer.write(datum, encoder)
+
+    buffer.string
+  end
+
+  def decode(encoded, schema)
+    buffer = StringIO.new(encoded)
+    decoder = Avro::IO::BinaryDecoder.new(buffer)
+
+    datum_reader = Avro::IO::DatumReader.new(schema, schema)
+    datum_reader.read(decoder)
+  end
+
+  def assert_encode_and_decode(datum, schema)
+    encoded = encode(datum, schema)
+    assert_equal datum, decode(encoded, schema)
+  end
+
+  def assert_preencoded(datum, schema, decoded)
+    encoded = encode(datum, schema)
+    assert_equal decoded, decode(encoded, schema)
+  end
+end