avro 1.8.2 → 1.9.1

data/lib/avro/schema_compatibility.rb

@@ -0,0 +1,170 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+module Avro
+  module SchemaCompatibility
+    # Perform a full, recursive check that a datum written using the writers_schema
+    # can be read using the readers_schema.
+    def self.can_read?(writers_schema, readers_schema)
+      Checker.new.can_read?(writers_schema, readers_schema)
+    end
+
+    # Perform a full, recursive check that a datum written using either the
+    # writers_schema or the readers_schema can be read using the other schema.
+    def self.mutual_read?(writers_schema, readers_schema)
+      Checker.new.mutual_read?(writers_schema, readers_schema)
+    end
+
+    # Perform a basic check that a datum written with the writers_schema could
+    # be read using the readers_schema. This check only includes matching the types,
+    # including schema promotion, and matching the full name for named types.
+    # Aliases for named types are not supported here, and the ruby implementation
+    # of Avro in general does not include support for aliases.
+    def self.match_schemas(writers_schema, readers_schema)
+      w_type = writers_schema.type_sym
+      r_type = readers_schema.type_sym
+
+      # This conditional is begging for some OO love.
+      if w_type == :union || r_type == :union
+        return true
+      end
+
+      if w_type == r_type
+        return true if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
+
+        case r_type
+        when :record
+          return writers_schema.fullname == readers_schema.fullname
+        when :error
+          return writers_schema.fullname == readers_schema.fullname
+        when :request
+          return true
+        when :fixed
+          return writers_schema.fullname == readers_schema.fullname &&
+            writers_schema.size == readers_schema.size
+        when :enum
+          return writers_schema.fullname == readers_schema.fullname
+        when :map
+          return match_schemas(writers_schema.values, readers_schema.values)
+        when :array
+          return match_schemas(writers_schema.items, readers_schema.items)
+        end
+      end
+
+      # Handle schema promotion
+      if w_type == :int && [:long, :float, :double].include?(r_type)
+        return true
+      elsif w_type == :long && [:float, :double].include?(r_type)
+        return true
+      elsif w_type == :float && r_type == :double
+        return true
+      elsif w_type == :string && r_type == :bytes
+        return true
+      elsif w_type == :bytes && r_type == :string
+        return true
+      end
+
+      return false
+    end
+
+    class Checker
+      SIMPLE_CHECKS = Schema::PRIMITIVE_TYPES_SYM.dup.add(:fixed).freeze
+
+      attr_reader :recursion_set
+      private :recursion_set
+
+      def initialize
+        @recursion_set = Set.new
+      end
+
+      def can_read?(writers_schema, readers_schema)
+        full_match_schemas(writers_schema, readers_schema)
+      end
+
+      def mutual_read?(writers_schema, readers_schema)
+        can_read?(writers_schema, readers_schema) && can_read?(readers_schema, writers_schema)
+      end
+
+      private
+
+      def full_match_schemas(writers_schema, readers_schema)
+        return true if recursion_in_progress?(writers_schema, readers_schema)
+
+        return false unless Avro::SchemaCompatibility.match_schemas(writers_schema, readers_schema)
+
+        if writers_schema.type_sym != :union && SIMPLE_CHECKS.include?(readers_schema.type_sym)
+          return true
+        end
+
+        case readers_schema.type_sym
+        when :record
+          match_record_schemas(writers_schema, readers_schema)
+        when :map
+          full_match_schemas(writers_schema.values, readers_schema.values)
+        when :array
+          full_match_schemas(writers_schema.items, readers_schema.items)
+        when :union
+          match_union_schemas(writers_schema, readers_schema)
+        when :enum
+          # reader's symbols must contain all writer's symbols
+          (writers_schema.symbols - readers_schema.symbols).empty?
+        else
+          if writers_schema.type_sym == :union && writers_schema.schemas.size == 1
+            full_match_schemas(writers_schema.schemas.first, readers_schema)
+          else
+            false
+          end
+        end
+      end
+
+      def match_union_schemas(writers_schema, readers_schema)
+        raise 'readers_schema must be a union' unless readers_schema.type_sym == :union
+
+        case writers_schema.type_sym
+        when :union
+          writers_schema.schemas.all? { |writer_type| full_match_schemas(writer_type, readers_schema) }
+        else
+          readers_schema.schemas.any? { |reader_type| full_match_schemas(writers_schema, reader_type) }
+        end
+      end
+
+      def match_record_schemas(writers_schema, readers_schema)
+        return false if writers_schema.type_sym == :union
+
+        writer_fields_hash = writers_schema.fields_hash
+        readers_schema.fields.each do |field|
+          if writer_fields_hash.key?(field.name)
+            return false unless full_match_schemas(writer_fields_hash[field.name].type, field.type)
+          else
+            return false unless field.default?
+          end
+        end
+
+        return true
+      end
+
+      def recursion_in_progress?(writers_schema, readers_schema)
+        key = [writers_schema.object_id, readers_schema.object_id]
+
+        if recursion_set.include?(key)
+          true
+        else
+          recursion_set.add(key)
+          false
+        end
+      end
+    end
+  end
+end

data/lib/avro/schema_normalization.rb

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/lib/avro/schema_validator.rb

@@ -0,0 +1,242 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Avro
+  class SchemaValidator
+    ROOT_IDENTIFIER = '.'.freeze
+    PATH_SEPARATOR = '.'.freeze
+    INT_RANGE = Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE
+    LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
+    COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
+    BOOLEAN_VALUES = [true, false].freeze
+
+    class Result
+      attr_reader :errors
+
+      def initialize
+        @errors = []
+      end
+
+      def <<(error)
+        @errors << error
+      end
+
+      def add_error(path, message)
+        self << "at #{path} #{message}"
+      end
+
+      def failure?
+        @errors.any?
+      end
+
+      def to_s
+        errors.join("\n")
+      end
+    end
+
+    class ValidationError < StandardError
+      attr_reader :result
+
+      def initialize(result = Result.new)
+        @result = result
+        super
+      end
+
+      def to_s
+        result.to_s
+      end
+    end
+
+    TypeMismatchError = Class.new(ValidationError)
+
+    class << self
+      def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false, fail_on_extra_fields: false })
+        options ||= {}
+        options[:recursive] = true unless options.key?(:recursive)
+
+        result = Result.new
+        if options[:recursive]
+          validate_recursive(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
+        else
+          validate_simple(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
+        end
+        fail ValidationError, result if result.failure?
+        result
+      end
+
+      private
+
+      def validate_recursive(expected_schema, logical_datum, path, result, options = {})
+        datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
+
+        validate_simple(expected_schema, datum, path, result, encoded: true)
+
+        case expected_schema.type_sym
+        when :array
+          validate_array(expected_schema, datum, path, result, options)
+        when :map
+          validate_map(expected_schema, datum, path, result, options)
+        when :union
+          validate_union(expected_schema, datum, path, result, options)
+        when :record, :error, :request
+          fail TypeMismatchError unless datum.is_a?(Hash)
+          expected_schema.fields.each do |field|
+            deeper_path = deeper_path_for_hash(field.name, path)
+            validate_recursive(field.type, datum[field.name], deeper_path, result, options)
+          end
+          if options[:fail_on_extra_fields]
+            datum_fields = datum.keys.map(&:to_s)
+            schema_fields = expected_schema.fields.map(&:name)
+            (datum_fields - schema_fields).each do |extra_field|
+              result.add_error(path, "extra field '#{extra_field}' - not in schema")
+            end
+          end
+        end
+      rescue TypeMismatchError
+        result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
+      end
+
+      def validate_simple(expected_schema, logical_datum, path, result, options = {})
+        datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
+        validate_type(expected_schema)
+
+        case expected_schema.type_sym
+        when :null
+          fail TypeMismatchError unless datum.nil?
+        when :boolean
+          fail TypeMismatchError unless BOOLEAN_VALUES.include?(datum)
+        when :string, :bytes
+          fail TypeMismatchError unless datum.is_a?(String)
+        when :int
+          fail TypeMismatchError unless datum.is_a?(Integer)
+          result.add_error(path, "out of bound value #{datum}") unless INT_RANGE.cover?(datum)
+        when :long
+          fail TypeMismatchError unless datum.is_a?(Integer)
+          result.add_error(path, "out of bound value #{datum}") unless LONG_RANGE.cover?(datum)
+        when :float, :double
+          fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer)
+        when :fixed
+          if datum.is_a? String
+            result.add_error(path, fixed_string_message(expected_schema.size, datum)) unless datum.bytesize == expected_schema.size
+          else
+            result.add_error(path, "expected fixed with size #{expected_schema.size}, got #{actual_value_message(datum)}")
+          end
+        when :enum
+          result.add_error(path, enum_message(expected_schema.symbols, datum)) unless expected_schema.symbols.include?(datum)
+        end
+      rescue TypeMismatchError
+        result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
+      end
+
+      def resolve_datum(expected_schema, logical_datum, encoded)
+        if encoded
+          logical_datum
+        else
+          expected_schema.type_adapter.encode(logical_datum) rescue nil
+        end
+      end
+
+      def validate_type(expected_schema)
+        unless Avro::Schema::VALID_TYPES_SYM.include?(expected_schema.type_sym)
+          fail "Unexpected schema type #{expected_schema.type_sym} #{expected_schema.inspect}"
+        end
+      end
+
+      def fixed_string_message(size, datum)
+        "expected fixed with size #{size}, got \"#{datum}\" with size #{datum.bytesize}"
+      end
+
+      def enum_message(symbols, datum)
+        "expected enum with values #{symbols}, got #{actual_value_message(datum)}"
+      end
+
+      def validate_array(expected_schema, datum, path, result, options = {})
+        fail TypeMismatchError unless datum.is_a?(Array)
+        datum.each_with_index do |d, i|
+          validate_recursive(expected_schema.items, d, path + "[#{i}]", result, options)
+        end
+      end
+
+      def validate_map(expected_schema, datum, path, result, options = {})
+        fail TypeMismatchError unless datum.is_a?(Hash)
+        datum.keys.each do |k|
+          result.add_error(path, "unexpected key type '#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String)
+        end
+        datum.each do |k, v|
+          deeper_path = deeper_path_for_hash(k, path)
+          validate_recursive(expected_schema.values, v, deeper_path, result, options)
+        end
+      end
+
+      def validate_union(expected_schema, datum, path, result, options = {})
+        if expected_schema.schemas.size == 1
+          validate_recursive(expected_schema.schemas.first, datum, path, result, options)
+          return
+        end
+        failures = []
+        compatible_type = first_compatible_type(datum, expected_schema, path, failures, options)
+        return unless compatible_type.nil?
+
+        complex_type_failed = failures.detect { |r| COMPLEX_TYPES.include?(r[:type]) }
+        if complex_type_failed
+          complex_type_failed[:result].errors.each { |error| result << error }
+        else
+          types = expected_schema.schemas.map { |s| "'#{s.type_sym}'" }.join(', ')
+          result.add_error(path, "expected union of [#{types}], got #{actual_value_message(datum)}")
+        end
+      end
+
+      def first_compatible_type(datum, expected_schema, path, failures, options = {})
+        expected_schema.schemas.find do |schema|
+          result = Result.new
+          validate_recursive(schema, datum, path, result, options)
+          failures << { type: schema.type_sym, result: result } if result.failure?
+          !result.failure?
+        end
+      end
+
+      def deeper_path_for_hash(sub_key, path)
+        "#{path}#{PATH_SEPARATOR}#{sub_key}".squeeze(PATH_SEPARATOR)
+      end
+
+      def actual_value_message(value)
+        avro_type = if value.is_a?(Integer)
+                      ruby_integer_to_avro_type(value)
+                    else
+                      ruby_to_avro_type(value.class)
+                    end
+        if value.nil?
+          avro_type
+        else
+          "#{avro_type} with value #{value.inspect}"
+        end
+      end
+
+      def ruby_to_avro_type(ruby_class)
+        {
+          NilClass => 'null',
+          String => 'string',
+          Float => 'float',
+          Hash => 'record'
+        }.fetch(ruby_class, ruby_class)
+      end
+
+      def ruby_integer_to_avro_type(value)
+        INT_RANGE.cover?(value) ? 'int' : 'long'
+      end
+    end
+  end
+end

data/test/case_finder.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/test/random_data.rb

@@ -7,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -27,15 +27,17 @@ class RandomData
   end
 
   def nextdata(schm, d=0)
+    return logical_nextdata(schm, d=0) unless schm.type_adapter.eql?(Avro::LogicalTypes::Identity)
+
     case schm.type_sym
     when :boolean
       rand > 0.5
     when :string
       randstr()
     when :int
-      rand(Avro::Schema::INT_MAX_VALUE - Avro::Schema::INT_MIN_VALUE) + Avro::Schema::INT_MIN_VALUE
+      rand_int
     when :long
-      rand(Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) + Avro::Schema::LONG_MIN_VALUE
+      rand_long
     when :float
       (-1024 + 2048 * rand).round.to_f
     when :double
@@ -79,6 +81,15 @@ class RandomData
     end
   end
 
+  def logical_nextdata(schm, _d=0)
+    case schm.logical_type
+    when 'date'
+      Avro::LogicalTypes::IntDate.decode(rand_int)
+    when 'timestamp-millis', 'timestamp-micros'
+      Avro::LogicalTypes::TimestampMicros.decode(rand_long)
+    end
+  end
+
   CHARPOOL = 'abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'
   BYTEPOOL = '12345abcd'
 
@@ -87,4 +98,12 @@ class RandomData
     rand(length+1).times { str << chars[rand(chars.size)] }
     str
   end
+
+  def rand_int
+    rand(Avro::Schema::INT_MAX_VALUE - Avro::Schema::INT_MIN_VALUE) + Avro::Schema::INT_MIN_VALUE
+  end
+
+  def rand_long
+    rand(Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) + Avro::Schema::LONG_MIN_VALUE
+  end
 end