avro 1.9.2 → 1.11.0

data/lib/avro/logical_types.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,9 +17,188 @@
 # limitations under the License.
 
 require 'date'
+require 'bigdecimal'
+require 'bigdecimal/util'
 
 module Avro
   module LogicalTypes
+    ##
+    # Base class for logical types requiring a schema to be present
+    class LogicalTypeWithSchema
+      ##
+      # @return [Avro::Schema] The schema this logical type is dealing with
+      attr_reader :schema
+
+      ##
+      # Build a new instance of a logical type using the provided schema
+      #
+      # @param schema [Avro::Schema]
+      #     The schema to use with this instance
+      #
+      # @raise [ArgumentError]
+      #     If the provided schema is nil
+      def initialize(schema)
+        raise ArgumentError, 'schema is required' if schema.nil?
+
+        @schema = schema
+      end
+
+      ##
+      # Encode the provided datum
+      #
+      # @param datum [Object] The datum to encode
+      #
+      # @raise [NotImplementedError]
+      #     Subclass will need to override this method
+      def encode(datum)
+        raise NotImplementedError
+      end
+
+      ##
+      # Decode the provided datum
+      #
+      # @param datum [Object] The datum to decode
+      #
+      # @raise [NotImplementedError]
+      #     Subclass will need to override this method
+      def decode(datum)
+        raise NotImplementedError
+      end
+    end
+
+    ##
+    # Logical type to handle arbitrary-precision decimals using byte array.
+    #
+    # The byte array contains the two's-complement representation of the unscaled integer
+    # value in big-endian byte order.
+    class BytesDecimal < LogicalTypeWithSchema
+      # Messages for exceptions
+      ERROR_INSUFFICIENT_PRECISION = 'Precision is too small'
+      ERROR_ROUNDING_NECESSARY     = 'Rounding necessary'
+      ERROR_VALUE_MUST_BE_NUMERIC  = 'value must be numeric'
+
+      # The pattern used to pack up the byte array (8 bit unsigned integer/char)
+      PACK_UNSIGNED_CHARS = 'C*'
+
+      # The number 10 as BigDecimal
+      TEN = BigDecimal(10).freeze
+
+      ##
+      # @return [Integer] The number of total digits supported by the decimal
+      attr_reader :precision
+
+      ##
+      # @return [Integer] The number of fractional digits
+      attr_reader :scale
+
+      ##
+      # Build a new decimal logical type
+      #
+      # @param schema [Avro::Schema]
+      #     The schema defining precision and scale for the conversion
+      def initialize(schema)
+        super
+
+        @scale     = schema.scale.to_i
+        @precision = schema.precision.to_i
+        @factor    = TEN ** @scale
+      end
+
+      ##
+      # Encode the provided value into a byte array
+      #
+      # @param value [BigDecimal, Float, Integer]
+      #     The numeric value to encode
+      #
+      # @raise [ArgumentError]
+      #     If the provided value is not a numeric type
+      #
+      # @raise [RangeError]
+      #     If the provided value has a scale higher than the schema permits,
+      #     or does not fit into the schema's precision
+      def encode(value)
+        raise ArgumentError, ERROR_VALUE_MUST_BE_NUMERIC unless value.is_a?(Numeric)
+
+        to_byte_array(unscaled_value(value.to_d)).pack(PACK_UNSIGNED_CHARS).freeze
+      end
+
+      ##
+      # Decode a byte array (in form of a string) into a BigDecimal of the
+      # given precision and scale
+      #
+      # @param stream [String]
+      #     The byte array to decode
+      #
+      # @return [BigDecimal]
+      def decode(stream)
+        from_byte_array(stream) / @factor
+      end
+
+      private
+
+      ##
+      # Convert the provided stream of bytes into the unscaled value
+      #
+      # @param stream [String]
+      #     The stream of bytes to convert
+      #
+      # @return [Integer]
+      def from_byte_array(stream)
+        bytes    = stream.bytes
+        positive = bytes.first[7].zero?
+        total    = 0
+
+        bytes.each_with_index do |value, ix|
+          total += (positive ? value : (value ^ 0xff)) << (bytes.length - ix - 1) * 8
+        end
+
+        return total if positive
+
+        -(total + 1)
+      end
+
+      ##
+      # Convert the provided number into its two's complement representation
+      # in network order (big endian).
+      #
+      # @param number [Integer]
+      #     The number to convert
+      #
+      # @return [Array<Integer>]
+      #     The byte array in network order
+      def to_byte_array(number)
+        [].tap do |result|
+          loop do
+            result.unshift(number & 0xff)
+            number >>= 8
+
+            break if (number == 0 || number == -1) && (result.first[7] == number[7])
+          end
+        end
+      end
+
+      ##
+      # Get the unscaled value from a BigDecimal considering the schema's scale
+      #
+      # @param decimal [BigDecimal]
+      #     The decimal to get the unscaled value from
+      #
+      # @return [Integer]
+      def unscaled_value(decimal)
+        details = decimal.split
+        length  = details[1].length
+
+        fractional_part = length - details[3]
+        raise RangeError, ERROR_ROUNDING_NECESSARY if fractional_part > scale
+
+        if length > precision || (length - fractional_part) > (precision - scale)
+          raise RangeError, ERROR_INSUFFICIENT_PRECISION
+        end
+
+        (decimal * @factor).to_i
+      end
+    end
+
     module IntDate
       EPOCH_START = Date.new(1970, 1, 1)
 
@@ -72,6 +252,9 @@ module Avro
     end
 
     TYPES = {
+      "bytes" => {
+        "decimal" => BytesDecimal
+      },
       "int" => {
         "date" => IntDate
       },
@@ -81,10 +264,11 @@ module Avro
       },
     }.freeze
 
-    def self.type_adapter(type, logical_type)
+    def self.type_adapter(type, logical_type, schema = nil)
       return unless logical_type
 
-      TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
+      adapter = TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
+      adapter.is_a?(Class) ? adapter.new(schema) : adapter
     end
   end
 end

data/lib/avro/protocol.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

data/lib/avro/schema.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -29,11 +30,17 @@ module Avro
     NAMED_TYPES_SYM     = Set.new(NAMED_TYPES.map(&:to_sym))
     VALID_TYPES_SYM     = Set.new(VALID_TYPES.map(&:to_sym))
 
+    NAME_REGEX = /^([A-Za-z_][A-Za-z0-9_]*)(\.([A-Za-z_][A-Za-z0-9_]*))*$/.freeze
+
     INT_MIN_VALUE = -(1 << 31)
     INT_MAX_VALUE = (1 << 31) - 1
     LONG_MIN_VALUE = -(1 << 63)
     LONG_MAX_VALUE = (1 << 63) - 1
 
+    DEFAULT_VALIDATE_OPTIONS = { recursive: true, encoded: false }.freeze
+
+    DECIMAL_LOGICAL_TYPE = 'decimal'
+
     def self.parse(json_string)
       real_parse(MultiJson.load(json_string), {})
     end
@@ -53,23 +60,36 @@ module Avro
 
         type_sym = type.to_sym
         if PRIMITIVE_TYPES_SYM.include?(type_sym)
-          return PrimitiveSchema.new(type_sym, logical_type)
-
+          case type_sym
+          when :bytes
+            precision = json_obj['precision']
+            scale = json_obj['scale']
+            return BytesSchema.new(type_sym, logical_type, precision, scale)
+          else
+            return PrimitiveSchema.new(type_sym, logical_type)
+          end
         elsif NAMED_TYPES_SYM.include? type_sym
           name = json_obj['name']
+          if !Avro.disable_schema_name_validation && name !~ NAME_REGEX
+            raise SchemaParseError, "Name #{name} is invalid for type #{type}!"
+          end
           namespace = json_obj.include?('namespace') ? json_obj['namespace'] : default_namespace
+          aliases = json_obj['aliases']
           case type_sym
           when :fixed
             size = json_obj['size']
-            return FixedSchema.new(name, namespace, size, names, logical_type)
+            precision = json_obj['precision']
+            scale = json_obj['scale']
+            return FixedSchema.new(name, namespace, size, names, logical_type, aliases, precision, scale)
           when :enum
             symbols = json_obj['symbols']
             doc     = json_obj['doc']
-            return EnumSchema.new(name, namespace, symbols, names, doc)
+            default = json_obj['default']
+            return EnumSchema.new(name, namespace, symbols, names, doc, default, aliases)
           when :record, :error
             fields = json_obj['fields']
             doc    = json_obj['doc']
-            return RecordSchema.new(name, namespace, fields, names, type_sym, doc)
+            return RecordSchema.new(name, namespace, fields, names, type_sym, doc, aliases)
           else
             raise SchemaParseError.new("Unknown named type: #{type}")
           end
@@ -96,7 +116,7 @@ module Avro
     end
 
     # Determine if a ruby datum is an instance of a schema
-    def self.validate(expected_schema, logical_datum, options = { recursive: true, encoded: false })
+    def self.validate(expected_schema, logical_datum, options = DEFAULT_VALIDATE_OPTIONS)
       SchemaValidator.validate!(expected_schema, logical_datum, options)
       true
     rescue SchemaValidator::ValidationError
@@ -116,7 +136,7 @@ module Avro
     def type; @type_sym.to_s; end
 
     def type_adapter
-      @type_adapter ||= LogicalTypes.type_adapter(type, logical_type) || LogicalTypes::Identity
+      @type_adapter ||= LogicalTypes.type_adapter(type, logical_type, self) || LogicalTypes::Identity
     end
 
     # Returns the MD5 fingerprint of the schema as an Integer.
@@ -131,6 +151,49 @@ module Avro
       Digest::SHA256.hexdigest(parsing_form).to_i(16)
     end
 
+    CRC_EMPTY = 0xc15d213aa4d7a795
+
+    # The java library caches this value after initialized, so this pattern
+    # mimics that.
+    @@fp_table = nil
+    def initFPTable
+      @@fp_table = Array.new(256)
+      256.times do |i|
+        fp = i
+        8.times do
+          fp = (fp >> 1) ^ ( CRC_EMPTY & -( fp & 1 ) )
+        end
+        @@fp_table[i] = fp
+      end
+    end
+
+    def crc_64_avro_fingerprint
+      parsing_form = Avro::SchemaNormalization.to_parsing_form(self)
+      data_bytes = parsing_form.unpack("C*")
+
+      initFPTable unless @@fp_table
+
+      fp = CRC_EMPTY
+      data_bytes.each do |b|
+        fp = (fp >> 8) ^ @@fp_table[ (fp ^ b) & 0xff ]
+      end
+      fp
+    end
+
+    SINGLE_OBJECT_MAGIC_NUMBER = [0xC3, 0x01].freeze
+    def single_object_encoding_header
+      [SINGLE_OBJECT_MAGIC_NUMBER, single_object_schema_fingerprint].flatten
+    end
+    def single_object_schema_fingerprint
+      working = crc_64_avro_fingerprint
+      bytes = Array.new(8)
+      8.times do |i|
+        bytes[i] = (working & 0xff)
+        working = working >> 8
+      end
+      bytes
+    end
+
     def read?(writers_schema)
       SchemaCompatibility.can_read?(writers_schema, self)
     end
@@ -143,11 +206,11 @@ module Avro
       SchemaCompatibility.mutual_read?(other_schema, self)
     end
 
-    def ==(other, seen=nil)
+    def ==(other, _seen=nil)
       other.is_a?(Schema) && type_sym == other.type_sym
     end
 
-    def hash(seen=nil)
+    def hash(_seen=nil)
       type_sym.hash
     end
 
@@ -165,7 +228,7 @@ module Avro
       end
     end
 
-    def to_avro(names=nil)
+    def to_avro(_names=nil)
       props = {'type' => type}
       props['logicalType'] = logical_type if logical_type
       props
@@ -175,14 +238,26 @@ module Avro
       MultiJson.dump to_avro
     end
 
+    def validate_aliases!
+      unless aliases.nil? ||
+        (aliases.is_a?(Array) && aliases.all? { |a| a.is_a?(String) })
+
+        raise Avro::SchemaParseError,
+              "Invalid aliases value #{aliases.inspect} for #{type} #{name}. Must be an array of strings."
+      end
+    end
+    private :validate_aliases!
+
     class NamedSchema < Schema
-      attr_reader :name, :namespace
+      attr_reader :name, :namespace, :aliases
 
-      def initialize(type, name, namespace=nil, names=nil, doc=nil, logical_type=nil)
+      def initialize(type, name, namespace=nil, names=nil, doc=nil, logical_type=nil, aliases=nil)
         super(type, logical_type)
         @name, @namespace = Name.extract_namespace(name, namespace)
-        @doc  = doc
-        names = Name.add_name(names, self)
+        @doc = doc
+        @aliases = aliases
+        validate_aliases! if aliases
+        Name.add_name(names, self)
       end
 
       def to_avro(names=Set.new)
@@ -192,33 +267,57 @@ module Avro
         end
         props = {'name' => @name}
         props.merge!('namespace' => @namespace) if @namespace
-        props.merge!('doc' => @doc) if @doc
+        props['namespace'] = @namespace if @namespace
+        props['doc'] = @doc if @doc
+        props['aliases'] = aliases if aliases && aliases.any?
         super.merge props
       end
 
       def fullname
         @fullname ||= Name.make_fullname(@name, @namespace)
       end
+
+      def fullname_aliases
+        @fullname_aliases ||= if aliases
+                                aliases.map { |a| Name.make_fullname(a, namespace) }
+                              else
+                                []
+                              end
+      end
+
+      def match_fullname?(name)
+        name == fullname || fullname_aliases.include?(name)
+      end
+
+      def match_schema?(schema)
+        type_sym == schema.type_sym && match_fullname?(schema.fullname)
+      end
     end
 
     class RecordSchema < NamedSchema
       attr_reader :fields, :doc
 
       def self.make_field_objects(field_data, names, namespace=nil)
-        field_objects, field_names = [], Set.new
-        field_data.each_with_index do |field, i|
+        field_objects, field_names, alias_names = [], Set.new, Set.new
+        field_data.each do |field|
           if field.respond_to?(:[]) # TODO(jmhodges) wtffffff
             type = field['type']
             name = field['name']
             default = field.key?('default') ? field['default'] : :no_default
             order = field['order']
             doc = field['doc']
-            new_field = Field.new(type, name, default, order, names, namespace, doc)
+            aliases = field['aliases']
+            new_field = Field.new(type, name, default, order, names, namespace, doc, aliases)
             # make sure field name has not been used yet
             if field_names.include?(new_field.name)
               raise SchemaParseError, "Field name #{new_field.name.inspect} is already in use"
             end
             field_names << new_field.name
+            # make sure alias has not be been used yet
+            if new_field.aliases && alias_names.intersect?(new_field.aliases.to_set)
+              raise SchemaParseError, "Alias #{(alias_names & new_field.aliases).to_a} already in use"
+            end
+            alias_names.merge(new_field.aliases) if new_field.aliases
           else
             raise SchemaParseError, "Not a valid field: #{field}"
           end
@@ -227,14 +326,14 @@ module Avro
         field_objects
       end
 
-      def initialize(name, namespace, fields, names=nil, schema_type=:record, doc=nil)
+      def initialize(name, namespace, fields, names=nil, schema_type=:record, doc=nil, aliases=nil)
         if schema_type == :request || schema_type == 'request'
           @type_sym = schema_type.to_sym
           @namespace = namespace
           @name = nil
           @doc = nil
         else
-          super(schema_type, name, namespace, names, doc)
+          super(schema_type, name, namespace, names, doc, nil, aliases)
         end
         @fields = if fields
                     RecordSchema.make_field_objects(fields, names, self.namespace)
@@ -247,6 +346,16 @@ module Avro
         @fields_hash ||= fields.inject({}){|hsh, field| hsh[field.name] = field; hsh }
       end
 
+      def fields_by_alias
+        @fields_by_alias ||= fields.each_with_object({}) do |field, hash|
+          if field.aliases
+            field.aliases.each do |a|
+              hash[a] = field
+            end
+          end
+        end
+      end
+
       def to_avro(names=Set.new)
         hsh = super
         return hsh unless hsh.is_a?(Hash)
@@ -313,20 +422,41 @@ module Avro
     end
 
     class EnumSchema < NamedSchema
-      attr_reader :symbols, :doc
+      SYMBOL_REGEX = /^[A-Za-z_][A-Za-z0-9_]*$/.freeze
 
-      def initialize(name, space, symbols, names=nil, doc=nil)
+      attr_reader :symbols, :doc, :default
+
+      def initialize(name, space, symbols, names=nil, doc=nil, default=nil, aliases=nil)
         if symbols.uniq.length < symbols.length
           fail_msg = "Duplicate symbol: #{symbols}"
           raise Avro::SchemaParseError, fail_msg
         end
-        super(:enum, name, space, names, doc)
+
+        if !Avro.disable_enum_symbol_validation
+          invalid_symbols = symbols.select { |symbol| symbol !~ SYMBOL_REGEX }
+
+          if invalid_symbols.any?
+            raise SchemaParseError,
+              "Invalid symbols for #{name}: #{invalid_symbols.join(', ')} don't match #{SYMBOL_REGEX.inspect}"
+          end
+        end
+
+        if default && !symbols.include?(default)
+          raise Avro::SchemaParseError, "Default '#{default}' is not a valid symbol for enum #{name}"
+        end
+
+        super(:enum, name, space, names, doc, nil, aliases)
+        @default = default
         @symbols = symbols
       end
 
-      def to_avro(names=Set.new)
+      def to_avro(_names=Set.new)
         avro = super
-        avro.is_a?(Hash) ? avro.merge('symbols' => symbols) : avro
+        if avro.is_a?(Hash)
+          avro['symbols'] = symbols
+          avro['default'] = default if default
+        end
+        avro
       end
     end
 
@@ -346,34 +476,102 @@ module Avro
         hsh = super
         hsh.size == 1 ? type : hsh
       end
+
+      def match_schema?(schema)
+        return type_sym == schema.type_sym
+        # TODO: eventually this could handle schema promotion for primitive schemas too
+      end
+    end
+
+    class BytesSchema < PrimitiveSchema
+      ERROR_INVALID_SCALE         = 'Scale must be greater than or equal to 0'
+      ERROR_INVALID_PRECISION     = 'Precision must be positive'
+      ERROR_PRECISION_TOO_SMALL   = 'Precision must be greater than scale'
+
+      attr_reader :precision, :scale
+
+      def initialize(type, logical_type=nil, precision=nil, scale=nil)
+        super(type.to_sym, logical_type)
+
+        @precision = precision.to_i if precision
+        @scale = scale.to_i if scale
+
+        validate_decimal! if logical_type == DECIMAL_LOGICAL_TYPE
+      end
+
+      def to_avro(names=nil)
+        avro = super
+        return avro if avro.is_a?(String)
+
+        avro['precision'] = precision if precision
+        avro['scale'] = scale if scale
+        avro
+      end
+
+      def match_schema?(schema)
+        return true if super
+
+        if logical_type == DECIMAL_LOGICAL_TYPE && schema.logical_type == DECIMAL_LOGICAL_TYPE
+          return precision == schema.precision && (scale || 0) == (schema.scale || 0)
+        end
+
+        false
+      end
+
+      private
+
+      def validate_decimal!
+        raise Avro::SchemaParseError, ERROR_INVALID_PRECISION unless precision.to_i.positive?
+        raise Avro::SchemaParseError, ERROR_INVALID_SCALE if scale.to_i.negative?
+        raise Avro::SchemaParseError, ERROR_PRECISION_TOO_SMALL if precision < scale.to_i
+      end
     end
 
     class FixedSchema < NamedSchema
-      attr_reader :size
-      def initialize(name, space, size, names=nil, logical_type=nil)
+      attr_reader :size, :precision, :scale
+      def initialize(name, space, size, names=nil, logical_type=nil, aliases=nil, precision=nil, scale=nil)
         # Ensure valid cto args
         unless size.is_a?(Integer)
           raise AvroError, 'Fixed Schema requires a valid integer for size property.'
         end
-        super(:fixed, name, space, names, nil, logical_type)
+        super(:fixed, name, space, names, nil, logical_type, aliases)
         @size = size
+        @precision = precision
+        @scale = scale
       end
 
       def to_avro(names=Set.new)
         avro = super
-        avro.is_a?(Hash) ? avro.merge('size' => size) : avro
+        return avro if avro.is_a?(String)
+
+        avro['size'] = size
+        avro['precision'] = precision if precision
+        avro['scale'] = scale if scale
+        avro
+      end
+
+      def match_schema?(schema)
+        return true if super && size == schema.size
+
+        if logical_type == DECIMAL_LOGICAL_TYPE && schema.logical_type == DECIMAL_LOGICAL_TYPE
+          return precision == schema.precision && (scale || 0) == (schema.scale || 0)
+        end
+
+        false
       end
     end
 
     class Field < Schema
-      attr_reader :type, :name, :default, :order, :doc
+      attr_reader :type, :name, :default, :order, :doc, :aliases
 
-      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil, doc=nil)
+      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil, doc=nil, aliases=nil) # rubocop:disable Lint/MissingSuper
         @type = subparse(type, names, namespace)
         @name = name
         @default = default
         @order = order
         @doc = doc
+        @aliases = aliases
+        validate_aliases! if aliases
         validate_default! if default? && !Avro.disable_field_default_validation
       end
 
@@ -389,6 +587,10 @@ module Avro
         end
       end
 
+      def alias_names
+        @alias_names ||= Array(aliases)
+      end
+
       private
 
       def validate_default!