avro 1.10.1 → 1.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 25662687b72649ae6bbc6a18a975ada73965b60d
-  data.tar.gz: 8ffb726920396bc1644c499440a1f7b844deb404
+  metadata.gz: 38a2f2d15367014464b4613e0fc80724b345d3b3
+  data.tar.gz: 823aa3ebb6ea719cbd4c94060757a545c7d626a9
 SHA512:
-  metadata.gz: 0542c4933a9cd95411c76b26cb88e456c1cd733749eccff6125058ecbf64fd184f64f7547f0b24376b80ba3b8349937926dfbd85a159d7b51e587ec439e59ef8
-  data.tar.gz: 76e8c8d2f1f30199a343fccfda5a4f6b972c6b33cef99b324a61a97554baebae59b62b8eec14682e6f10015d48c4740fb00f3c58fefdff39a313bdbbebbc9066
+  metadata.gz: da4d7d3ecf02bb18c30c044635188ba2b8a34ff13f9450330e850198610ee87f4e9116a79a95366bc23b5e3fb7f70906736d1ff7834e9293825c5c3eba072b9b
+  data.tar.gz: 3194b9379e1ae396acd0a120f583ade7127628d5adf78ac382fcf7215643c69c5341ee0a5f234db56eec3e62070683340b4a21bcff4ba0e2208a832c571f8dc9

data/avro.gemspec

@@ -1,14 +1,14 @@
 # -*- encoding: utf-8 -*-
-# stub: avro 1.10.1 ruby lib
+# stub: avro 1.10.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "avro".freeze
-  s.version = "1.10.1"
+  s.version = "1.10.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Apache Software Foundation".freeze]
-  s.date = "2020-11-18"
+  s.date = "2021-03-09"
   s.description = "Avro is a data serialization and RPC format".freeze
   s.email = "dev@avro.apache.org".freeze
   s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]

data/lib/avro/VERSION.txt

@@ -1 +1 @@
-1.10.1
+1.10.2

data/lib/avro/io.rb

@@ -510,6 +510,8 @@ module Avro
 
     # DatumWriter for generic ruby objects
     class DatumWriter
+      VALIDATION_OPTIONS = { recursive: false, encoded: true }.freeze
+
       attr_accessor :writers_schema
       def initialize(writers_schema=nil)
         @writers_schema = writers_schema
@@ -522,7 +524,7 @@ module Avro
       def write_data(writers_schema, logical_datum, encoder)
         datum = writers_schema.type_adapter.encode(logical_datum)
 
-        unless Schema.validate(writers_schema, datum, { recursive: false, encoded: true })
+        unless Schema.validate(writers_schema, datum, VALIDATION_OPTIONS)
           raise AvroTypeError.new(writers_schema, datum)
         end
 
@@ -580,12 +582,15 @@ module Avro
       end
 
       def write_union(writers_schema, datum, encoder)
-        index_of_schema = -1
-        found = writers_schema.schemas.
-          find{|e| index_of_schema += 1; found = Schema.validate(e, datum) }
-        unless found  # Because find_index doesn't exist in 1.8.6
+        index_of_schema = writers_schema.schemas.find_index do |schema|
+          # Optimize away expensive validation calls for the common null type
+          schema.type_sym == :null ? datum.nil? : Schema.validate(schema, datum)
+        end
+
+        unless index_of_schema
           raise AvroTypeError.new(writers_schema, datum)
         end
+
         encoder.write_long(index_of_schema)
         write_data(writers_schema.schemas[index_of_schema], datum, encoder)
       end

data/lib/avro/schema.rb

@@ -36,6 +36,8 @@ module Avro
     LONG_MIN_VALUE = -(1 << 63)
     LONG_MAX_VALUE = (1 << 63) - 1
 
+    DEFAULT_VALIDATE_OPTIONS = { recursive: true, encoded: false }.freeze
+
     def self.parse(json_string)
       real_parse(MultiJson.load(json_string), {})
     end
@@ -109,7 +111,7 @@ module Avro
     end
 
     # Determine if a ruby datum is an instance of a schema
-    def self.validate(expected_schema, logical_datum, options = { recursive: true, encoded: false })
+    def self.validate(expected_schema, logical_datum, options = DEFAULT_VALIDATE_OPTIONS)
       SchemaValidator.validate!(expected_schema, logical_datum, options)
       true
     rescue SchemaValidator::ValidationError

data/lib/avro/schema_compatibility.rb

@@ -15,6 +15,9 @@
 # limitations under the License.
 module Avro
   module SchemaCompatibility
+    INT_COERCIBLE_TYPES_SYM = [:long, :float, :double].freeze
+    LONG_COERCIBLE_TYPES_SYM = [:float, :double].freeze
+
     # Perform a full, recursive check that a datum written using the writers_schema
     # can be read using the readers_schema.
     def self.can_read?(writers_schema, readers_schema)
@@ -31,6 +34,9 @@ module Avro
     # be read using the readers_schema. This check includes matching the types,
     # including schema promotion, and matching the full name (including aliases) for named types.
     def self.match_schemas(writers_schema, readers_schema)
+      # Bypass deeper checks if the schemas are the same Ruby objects
+      return true if writers_schema.equal?(readers_schema)
+
       w_type = writers_schema.type_sym
       r_type = readers_schema.type_sym
 
@@ -62,9 +68,9 @@ module Avro
       end
 
       # Handle schema promotion
-      if w_type == :int && [:long, :float, :double].include?(r_type)
+      if w_type == :int && INT_COERCIBLE_TYPES_SYM.include?(r_type)
         return true
-      elsif w_type == :long && [:float, :double].include?(r_type)
+      elsif w_type == :long && LONG_COERCIBLE_TYPES_SYM.include?(r_type)
         return true
       elsif w_type == :float && r_type == :double
         return true

data/lib/avro/schema_validator.rb

@@ -22,16 +22,18 @@ module Avro
     LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
     COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
     BOOLEAN_VALUES = [true, false].freeze
+    DEFAULT_VALIDATION_OPTIONS = { recursive: true, encoded: false, fail_on_extra_fields: false }.freeze
+    RECURSIVE_SIMPLE_VALIDATION_OPTIONS = { encoded: true }.freeze
+    RUBY_CLASS_TO_AVRO_TYPE = {
+      NilClass => 'null'.freeze,
+      String => 'string'.freeze,
+      Float => 'float'.freeze,
+      Hash => 'record'.freeze
+    }.freeze
 
     class Result
-      attr_reader :errors
-
-      def initialize
-        @errors = []
-      end
-
       def <<(error)
-        @errors << error
+        errors << error
       end
 
       def add_error(path, message)
@@ -39,11 +41,16 @@ module Avro
       end
 
       def failure?
-        @errors.any?
+        defined?(@errors) && errors.any?
       end
 
       def to_s
-        errors.join("\n")
+        failure? ? errors.join("\n") : ''
+      end
+
+      def errors
+        # Use less memory for success results by lazily creating the errors array
+        @errors ||= []
       end
     end
 
@@ -63,12 +70,9 @@ module Avro
     TypeMismatchError = Class.new(ValidationError)
 
     class << self
-      def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false, fail_on_extra_fields: false })
-        options ||= {}
-        options[:recursive] = true unless options.key?(:recursive)
-
+      def validate!(expected_schema, logical_datum, options = DEFAULT_VALIDATION_OPTIONS)
         result = Result.new
-        if options[:recursive]
+        if options.fetch(:recursive, true)
           validate_recursive(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
         else
           validate_simple(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
@@ -79,10 +83,10 @@ module Avro
 
       private
 
-      def validate_recursive(expected_schema, logical_datum, path, result, options = {})
+      def validate_recursive(expected_schema, logical_datum, path, result, options)
         datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
 
-        validate_simple(expected_schema, datum, path, result, encoded: true)
+        validate_simple(expected_schema, datum, path, result, RECURSIVE_SIMPLE_VALIDATION_OPTIONS)
 
         case expected_schema.type_sym
         when :array
@@ -95,7 +99,8 @@ module Avro
           fail TypeMismatchError unless datum.is_a?(Hash)
           expected_schema.fields.each do |field|
             deeper_path = deeper_path_for_hash(field.name, path)
-            validate_recursive(field.type, datum[field.name], deeper_path, result, options)
+            nested_value = datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym]
+            validate_recursive(field.type, nested_value, deeper_path, result, options)
           end
           if options[:fail_on_extra_fields]
             datum_fields = datum.keys.map(&:to_s)
@@ -109,7 +114,7 @@ module Avro
         result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
       end
 
-      def validate_simple(expected_schema, logical_datum, path, result, options = {})
+      def validate_simple(expected_schema, logical_datum, path, result, options)
         datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
         validate_type(expected_schema)
 
@@ -163,14 +168,14 @@ module Avro
         "expected enum with values #{symbols}, got #{actual_value_message(datum)}"
       end
 
-      def validate_array(expected_schema, datum, path, result, options = {})
+      def validate_array(expected_schema, datum, path, result, options)
         fail TypeMismatchError unless datum.is_a?(Array)
         datum.each_with_index do |d, i|
-          validate_recursive(expected_schema.items, d, path + "[#{i}]", result, options)
+          validate_recursive(expected_schema.items, d, "#{path}[#{i}]", result, options)
         end
       end
 
-      def validate_map(expected_schema, datum, path, result, options = {})
+      def validate_map(expected_schema, datum, path, result, options)
         fail TypeMismatchError unless datum.is_a?(Hash)
         datum.keys.each do |k|
           result.add_error(path, "unexpected key type '#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String)
@@ -181,7 +186,7 @@ module Avro
         end
       end
 
-      def validate_union(expected_schema, datum, path, result, options = {})
+      def validate_union(expected_schema, datum, path, result, options)
         if expected_schema.schemas.size == 1
           validate_recursive(expected_schema.schemas.first, datum, path, result, options)
           return
@@ -201,6 +206,9 @@ module Avro
 
       def first_compatible_type(datum, expected_schema, path, failures, options = {})
         expected_schema.schemas.find do |schema|
+          # Avoid expensive validation if we're just validating a nil
+          next datum.nil? if schema.type_sym == :null
+
           result = Result.new
           validate_recursive(schema, datum, path, result, options)
           failures << { type: schema.type_sym, result: result } if result.failure?
@@ -209,7 +217,9 @@ module Avro
       end
 
       def deeper_path_for_hash(sub_key, path)
-        "#{path}#{PATH_SEPARATOR}#{sub_key}".squeeze(PATH_SEPARATOR)
+        deeper_path = "#{path}#{PATH_SEPARATOR}#{sub_key}"
+        deeper_path.squeeze!(PATH_SEPARATOR)
+        deeper_path
       end
 
       def actual_value_message(value)
@@ -226,16 +236,11 @@ module Avro
       end
 
       def ruby_to_avro_type(ruby_class)
-        {
-          NilClass => 'null',
-          String => 'string',
-          Float => 'float',
-          Hash => 'record'
-        }.fetch(ruby_class, ruby_class)
+        RUBY_CLASS_TO_AVRO_TYPE.fetch(ruby_class, ruby_class)
       end
 
       def ruby_integer_to_avro_type(value)
-        INT_RANGE.cover?(value) ? 'int' : 'long'
+        INT_RANGE.cover?(value) ? 'int'.freeze : 'long'.freeze
       end
     end
   end

data/test/test_schema_compatibility.rb

@@ -25,7 +25,9 @@ class TestSchemaCompatibility < Test::Unit::TestCase
   end
 
   def test_compatible_reader_writer_pairs
+    cached_schema = a_int_record1_schema
     [
+      cached_schema, cached_schema,
       long_schema, int_schema,
       float_schema, int_schema,
       float_schema, long_schema,

data/test/test_schema_validator.rb

@@ -17,7 +17,7 @@
 require 'test_help'
 
 class TestSchemaValidator < Test::Unit::TestCase
-  def validate!(schema, value, options=nil)
+  def validate!(schema, value, options = {})
     Avro::SchemaValidator.validate!(schema, value, options)
   end
 
@@ -196,6 +196,12 @@ class TestSchemaValidator < Test::Unit::TestCase
     assert_valid_schema(schema, [{ 'sub' => nil }], [{ 'sub' => 1 }])
   end
 
+  def test_validate_record_with_symbol_keys
+    schema = hash_to_schema(type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }])
+
+    assert_valid_schema(schema, [{ sub: 1 }], [{ sub: '1' }])
+  end
+
   def test_validate_shallow_record
     schema = hash_to_schema(
       type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro
 version: !ruby/object:Gem::Version
-  version: 1.10.1
+  version: 1.10.2
 platform: ruby
 authors:
 - Apache Software Foundation
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-18 00:00:00.000000000 Z
+date: 2021-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json