avro 1.9.0 → 1.11.0

data/lib/avro/schema_compatibility.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,6 +16,9 @@
 # limitations under the License.
 module Avro
   module SchemaCompatibility
+    INT_COERCIBLE_TYPES_SYM = [:long, :float, :double].freeze
+    LONG_COERCIBLE_TYPES_SYM = [:float, :double].freeze
+
     # Perform a full, recursive check that a datum written using the writers_schema
     # can be read using the readers_schema.
     def self.can_read?(writers_schema, readers_schema)
@@ -28,11 +32,12 @@ module Avro
     end
 
     # Perform a basic check that a datum written with the writers_schema could
-    # be read using the readers_schema. This check only includes matching the types,
-    # including schema promotion, and matching the full name for named types.
-    # Aliases for named types are not supported here, and the ruby implementation
-    # of Avro in general does not include support for aliases.
+    # be read using the readers_schema. This check includes matching the types,
+    # including schema promotion, and matching the full name (including aliases) for named types.
     def self.match_schemas(writers_schema, readers_schema)
+      # Bypass deeper checks if the schemas are the same Ruby objects
+      return true if writers_schema.equal?(readers_schema)
+
       w_type = writers_schema.type_sym
       r_type = readers_schema.type_sym
 
@@ -42,31 +47,25 @@ module Avro
       end
 
       if w_type == r_type
-        return true if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
+        return readers_schema.match_schema?(writers_schema) if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
 
         case r_type
-        when :record
-          return writers_schema.fullname == readers_schema.fullname
-        when :error
-          return writers_schema.fullname == readers_schema.fullname
         when :request
           return true
-        when :fixed
-          return writers_schema.fullname == readers_schema.fullname &&
-            writers_schema.size == readers_schema.size
-        when :enum
-          return writers_schema.fullname == readers_schema.fullname
         when :map
           return match_schemas(writers_schema.values, readers_schema.values)
         when :array
           return match_schemas(writers_schema.items, readers_schema.items)
+        else
+          return readers_schema.match_schema?(writers_schema)
         end
       end
 
       # Handle schema promotion
-      if w_type == :int && [:long, :float, :double].include?(r_type)
+      # rubocop:disable Lint/DuplicateBranch
+      if w_type == :int && INT_COERCIBLE_TYPES_SYM.include?(r_type)
         return true
-      elsif w_type == :long && [:float, :double].include?(r_type)
+      elsif w_type == :long && LONG_COERCIBLE_TYPES_SYM.include?(r_type)
         return true
       elsif w_type == :float && r_type == :double
         return true
@@ -75,8 +74,13 @@ module Avro
       elsif w_type == :bytes && r_type == :string
         return true
       end
+      # rubocop:enable Lint/DuplicateBranch
 
-      return false
+      if readers_schema.respond_to?(:match_schema?)
+        readers_schema.match_schema?(writers_schema)
+      else
+        false
+      end
     end
 
     class Checker
@@ -118,8 +122,8 @@ module Avro
         when :union
           match_union_schemas(writers_schema, readers_schema)
         when :enum
-          # reader's symbols must contain all writer's symbols
-          (writers_schema.symbols - readers_schema.symbols).empty?
+          # reader's symbols must contain all writer's symbols or reader has default
+          (writers_schema.symbols - readers_schema.symbols).empty? || !readers_schema.default.nil?
         else
           if writers_schema.type_sym == :union && writers_schema.schemas.size == 1
             full_match_schemas(writers_schema.schemas.first, readers_schema)
@@ -148,7 +152,14 @@ module Avro
           if writer_fields_hash.key?(field.name)
             return false unless full_match_schemas(writer_fields_hash[field.name].type, field.type)
           else
-            return false unless field.default?
+            names = writer_fields_hash.keys & field.alias_names
+            if names.size > 1
+              return false
+            elsif names.size == 1
+              return false unless full_match_schemas(writer_fields_hash[names.first].type, field.type)
+            else
+              return false unless field.default?
+            end
           end
         end
 

data/lib/avro/schema_normalization.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/lib/avro/schema_validator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,22 +17,24 @@
 
 module Avro
   class SchemaValidator
-    ROOT_IDENTIFIER = '.'.freeze
-    PATH_SEPARATOR = '.'.freeze
-    INT_RANGE = Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE
-    LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
+    ROOT_IDENTIFIER = '.'
+    PATH_SEPARATOR = '.'
+    INT_RANGE = (Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE).freeze
+    LONG_RANGE = (Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE).freeze
     COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
     BOOLEAN_VALUES = [true, false].freeze
+    DEFAULT_VALIDATION_OPTIONS = { recursive: true, encoded: false, fail_on_extra_fields: false }.freeze
+    RECURSIVE_SIMPLE_VALIDATION_OPTIONS = { encoded: true }.freeze
+    RUBY_CLASS_TO_AVRO_TYPE = {
+      NilClass => 'null',
+      String => 'string',
+      Float => 'float',
+      Hash => 'record'
+    }.freeze
 
     class Result
-      attr_reader :errors
-
-      def initialize
-        @errors = []
-      end
-
       def <<(error)
-        @errors << error
+        errors << error
       end
 
       def add_error(path, message)
@@ -39,11 +42,16 @@ module Avro
       end
 
       def failure?
-        @errors.any?
+        defined?(@errors) && errors.any?
       end
 
       def to_s
-        errors.join("\n")
+        failure? ? errors.join("\n") : ''
+      end
+
+      def errors
+        # Use less memory for success results by lazily creating the errors array
+        @errors ||= []
       end
     end
 
@@ -63,12 +71,9 @@ module Avro
     TypeMismatchError = Class.new(ValidationError)
 
     class << self
-      def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false, fail_on_extra_fields: false })
-        options ||= {}
-        options[:recursive] = true unless options.key?(:recursive)
-
+      def validate!(expected_schema, logical_datum, options = DEFAULT_VALIDATION_OPTIONS)
         result = Result.new
-        if options[:recursive]
+        if options.fetch(:recursive, true)
           validate_recursive(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
         else
           validate_simple(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
@@ -79,10 +84,10 @@ module Avro
 
       private
 
-      def validate_recursive(expected_schema, logical_datum, path, result, options = {})
+      def validate_recursive(expected_schema, logical_datum, path, result, options)
         datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
 
-        validate_simple(expected_schema, datum, path, result, encoded: true)
+        validate_simple(expected_schema, datum, path, result, RECURSIVE_SIMPLE_VALIDATION_OPTIONS)
 
         case expected_schema.type_sym
         when :array
@@ -95,7 +100,8 @@ module Avro
           fail TypeMismatchError unless datum.is_a?(Hash)
           expected_schema.fields.each do |field|
             deeper_path = deeper_path_for_hash(field.name, path)
-            validate_recursive(field.type, datum[field.name], deeper_path, result, options)
+            nested_value = datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym]
+            validate_recursive(field.type, nested_value, deeper_path, result, options)
           end
           if options[:fail_on_extra_fields]
             datum_fields = datum.keys.map(&:to_s)
@@ -109,7 +115,7 @@ module Avro
         result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
       end
 
-      def validate_simple(expected_schema, logical_datum, path, result, options = {})
+      def validate_simple(expected_schema, logical_datum, path, result, options)
         datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
         validate_type(expected_schema)
 
@@ -127,7 +133,7 @@ module Avro
           fail TypeMismatchError unless datum.is_a?(Integer)
           result.add_error(path, "out of bound value #{datum}") unless LONG_RANGE.cover?(datum)
         when :float, :double
-          fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer)
+          fail TypeMismatchError unless datum.is_a?(Float) || datum.is_a?(Integer) || datum.is_a?(BigDecimal)
         when :fixed
           if datum.is_a? String
             result.add_error(path, fixed_string_message(expected_schema.size, datum)) unless datum.bytesize == expected_schema.size
@@ -163,14 +169,14 @@ module Avro
         "expected enum with values #{symbols}, got #{actual_value_message(datum)}"
       end
 
-      def validate_array(expected_schema, datum, path, result, options = {})
+      def validate_array(expected_schema, datum, path, result, options)
         fail TypeMismatchError unless datum.is_a?(Array)
         datum.each_with_index do |d, i|
-          validate_recursive(expected_schema.items, d, path + "[#{i}]", result, options)
+          validate_recursive(expected_schema.items, d, "#{path}[#{i}]", result, options)
         end
       end
 
-      def validate_map(expected_schema, datum, path, result, options = {})
+      def validate_map(expected_schema, datum, path, result, options)
         fail TypeMismatchError unless datum.is_a?(Hash)
         datum.keys.each do |k|
           result.add_error(path, "unexpected key type '#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String)
@@ -181,7 +187,7 @@ module Avro
         end
       end
 
-      def validate_union(expected_schema, datum, path, result, options = {})
+      def validate_union(expected_schema, datum, path, result, options)
         if expected_schema.schemas.size == 1
           validate_recursive(expected_schema.schemas.first, datum, path, result, options)
           return
@@ -201,6 +207,9 @@ module Avro
 
       def first_compatible_type(datum, expected_schema, path, failures, options = {})
         expected_schema.schemas.find do |schema|
+          # Avoid expensive validation if we're just validating a nil
+          next datum.nil? if schema.type_sym == :null
+
           result = Result.new
           validate_recursive(schema, datum, path, result, options)
           failures << { type: schema.type_sym, result: result } if result.failure?
@@ -209,7 +218,9 @@ module Avro
       end
 
       def deeper_path_for_hash(sub_key, path)
-        "#{path}#{PATH_SEPARATOR}#{sub_key}".squeeze(PATH_SEPARATOR)
+        deeper_path = +"#{path}#{PATH_SEPARATOR}#{sub_key}"
+        deeper_path.squeeze!(PATH_SEPARATOR)
+        deeper_path.freeze
       end
 
       def actual_value_message(value)
@@ -226,12 +237,7 @@ module Avro
       end
 
       def ruby_to_avro_type(ruby_class)
-        {
-          NilClass => 'null',
-          String => 'string',
-          Float => 'float',
-          Hash => 'record'
-        }.fetch(ruby_class, ruby_class)
+        RUBY_CLASS_TO_AVRO_TYPE.fetch(ruby_class, ruby_class)
       end
 
       def ruby_integer_to_avro_type(value)

data/lib/avro.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,24 +23,36 @@ require 'stringio'
 require 'zlib'
 
 module Avro
-  VERSION = "FIXME"
+  VERSION = File.read("#{__dir__}/avro/VERSION.txt").freeze
 
   class AvroError < StandardError; end
 
   class AvroTypeError < Avro::AvroError
     def initialize(schm=nil, datum=nil, msg=nil)
-      msg ||= "Not a #{schm.to_s}: #{datum}"
+      msg ||= "Not a #{schm}: #{datum}"
       super(msg)
     end
   end
 
   class << self
+    attr_writer :disable_enum_symbol_validation
     attr_writer :disable_field_default_validation
+    attr_writer :disable_schema_name_validation
+
+    def disable_enum_symbol_validation
+      @disable_enum_symbol_validation ||=
+        ENV.fetch('AVRO_DISABLE_ENUM_SYMBOL_VALIDATION', '') != ''
+    end
 
     def disable_field_default_validation
       @disable_field_default_validation ||=
         ENV.fetch('AVRO_DISABLE_FIELD_DEFAULT_VALIDATION', '') != ''
     end
+
+    def disable_schema_name_validation
+      @disable_schema_name_validation ||=
+        ENV.fetch('AVRO_DISABLE_SCHEMA_NAME_VALIDATION', '') != ''
+    end
   end
 end
 

data/test/case_finder.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -7,7 +8,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -44,13 +45,18 @@ class CaseFinder
   private
 
   def scan_case
-    if id = @scanner.scan(/\/\/ \d+\n/)
+    if (id = @scanner.scan(/\/\/ \d+\n/))
       while @scanner.skip(/\/\/ .*\n/); end
 
       input = scan_input
       canonical = scan_canonical
       fingerprint = scan_fingerprint
-
+      if not fingerprint and @cases
+        fingerprint = @cases[-1].fingerprint
+      end
+      if fingerprint
+        fingerprint = fingerprint.to_i & 0xFFFF_FFFF_FFFF_FFFF
+      end
       Case.new(id, input, canonical, fingerprint)
     else
       @scanner.skip(/.*\n/)
@@ -61,7 +67,7 @@ class CaseFinder
   def scan_item(name)
     if @scanner.scan(/<<#{name}\n/)
       lines = []
-      while line = @scanner.scan(/.+\n/)
+      while (line = @scanner.scan(/.+\n/))
         break if line.chomp == name
         lines << line
       end

data/test/random_data.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/env ruby
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
-# http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -75,7 +75,7 @@ class RandomData
       return nil if len == 0
       symbols[rand(len)]
     when :fixed
-      f = ""
+      f = +""
       schm.size.times { f << BYTEPOOL[rand(BYTEPOOL.size), 1] }
       f
     end
@@ -85,8 +85,10 @@ class RandomData
     case schm.logical_type
     when 'date'
       Avro::LogicalTypes::IntDate.decode(rand_int)
-    when 'timestamp-millis', 'timestamp-micros'
+    when 'timestamp-micros'
       Avro::LogicalTypes::TimestampMicros.decode(rand_long)
+    when 'timestamp-millis'
+      Avro::LogicalTypes::TimestampMillis.decode(rand_long)
     end
   end
 
@@ -94,7 +96,7 @@ class RandomData
   BYTEPOOL = '12345abcd'
 
   def randstr(chars=CHARPOOL, length=20)
-    str = ''
+    str = +''
     rand(length+1).times { str << chars[rand(chars.size)] }
     str
   end

data/test/sample_ipc_client.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -7,7 +8,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/sample_ipc_http_client.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -7,7 +8,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/sample_ipc_http_server.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/sample_ipc_server.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -7,7 +8,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

data/test/test_datafile.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -7,7 +8,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -180,6 +181,19 @@ JSON
     assert_equal records, ['a' * 10_000]
   end
 
+  def test_zstandard
+    Avro::DataFile.open('data.avr', 'w', '"string"', :zstandard) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size('data.avr') < 600)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
   def test_append_to_deflated_file
     schema = Avro::Schema.parse('"string"')
     writer = Avro::IO::DatumWriter.new(schema)

data/test/test_fingerprints.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -34,4 +35,23 @@ class TestFingerprints < Test::Unit::TestCase
     assert_equal 28572620203319713300323544804233350633246234624932075150020181448463213378117,
       schema.sha256_fingerprint
   end
+
+  def test_crc_64_avro_fingerprint
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal 8247732601305521295, # hex: 0x7275d51a3f395c8f
+      schema.crc_64_avro_fingerprint
+  end
+
+  # This definitely belongs somewhere else
+  def test_single_object_encoding_header
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal ["c3", "01", "8f", "5c", "39", "3f", "1a", "D5", "75", "72"].map{|e| e.to_i(16) },
+      schema.single_object_encoding_header
+  end
 end

data/test/test_help.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -6,7 +7,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 # 
-# http://www.apache.org/licenses/LICENSE-2.0
+# https://www.apache.org/licenses/LICENSE-2.0
 # 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,