avromatic 0.13.0 → 0.14.0.rc0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e9a7c0252ba986f0babd9df45f33a044e4ff690
-  data.tar.gz: 4af262744ceb5baf95a9de4b227ab3c697ff1997
+  metadata.gz: dd0e7ff37e01961347eb5222e574d1e1942078ec
+  data.tar.gz: 3894c1ed617bc9525eac01c72969bed69702c015
 SHA512:
-  metadata.gz: b55e680ab5badd4ffde004b89030e1d8d9bf39c8f9f7d5725cdb5244106f551440780b9c5df0a85f86698a34394df115e621bbbb8ca0a31095d2ed100c90351f
-  data.tar.gz: b92bce0d36ecb7854fb642527fae21d7355cdcb0004a799d1739fa384d63ec61c18cc023cab0f775b746c5c37cdccda6847e32b039d98bb9b23905d6c82c53c1
+  metadata.gz: 87701549c8cb63b5636d2647f5b3025fef2f652ca886c876223b9d44457370827d8324536b7492fe786c5b9686f67991074130d69e29901f2739ca63b8098f43
+  data.tar.gz: 9da95256f8eb308390bcd64517525ac8e4410a812666efcc3eb92bca6865a7f0f481ca7a086062d8844db69fb4b09ce874e7b19f7d7608b33aaa74d3565e3fde

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # avromatic changelog
 
+## v0.14.0
+- Add `Avromatic::Messaging` and `Avromatic::IO::DatumReader` classes to
+  optimize the decoding of Avro unions to Avromatic models.
+
 ## v0.13.0
 - Add interfaces to deserialize as a hash of attributes instead of a model.
 

data/lib/avromatic/io/datum_reader.rb

@@ -0,0 +1,74 @@
+# rubocop:disable Style/WhenThen
+module Avromatic
+  module IO
+    # Subclass DatumReader to include additional information about the union
+    # index used.
+    class DatumReader < Avro::IO::DatumReader
+
+      UNION_MEMBER_INDEX = '__avromatic_member_index'.freeze
+
+      def read_data(writers_schema, readers_schema, decoder, initial_record = {})
+        # schema matching
+        unless self.class.match_schemas(writers_schema, readers_schema)
+          raise SchemaMatchException.new(writers_schema, readers_schema)
+        end
+
+        # schema resolution: reader's schema is a union, writer's schema is not
+        if writers_schema.type_sym != :union && readers_schema.type_sym == :union
+          rs_index = readers_schema.schemas.find_index do |s|
+            self.class.match_schemas(writers_schema, s)
+          end
+
+          union_info = { UNION_MEMBER_INDEX => rs_index }
+
+          return read_data(writers_schema, readers_schema.schemas[rs_index], decoder, union_info) if rs_index
+          raise SchemaMatchException.new(writers_schema, readers_schema)
+        end
+
+        # function dispatch for reading data based on type of writer's
+        # schema
+        datum = case writers_schema.type_sym
+                when :null;    decoder.read_null
+                when :boolean; decoder.read_boolean
+                when :string;  decoder.read_string
+                when :int;     decoder.read_int
+                when :long;    decoder.read_long
+                when :float;   decoder.read_float
+                when :double;  decoder.read_double
+                when :bytes;   decoder.read_bytes
+                when :fixed;   read_fixed(writers_schema, readers_schema, decoder)
+                when :enum;    read_enum(writers_schema, readers_schema, decoder)
+                when :array;   read_array(writers_schema, readers_schema, decoder)
+                when :map;     read_map(writers_schema, readers_schema, decoder)
+                when :union;   read_union(writers_schema, readers_schema, decoder)
+                when :record, :error, :request; read_record(writers_schema, readers_schema, decoder, initial_record)
+                else
+                  raise AvroError.new("Cannot read unknown schema type: #{writers_schema.type}")
+                end
+
+        if readers_schema.respond_to?(:logical_type)
+          readers_schema.type_adapter.decode(datum)
+        else
+          datum
+        end
+      end
+
+      # Override to specify initial record that may contain union index
+      def read_record(writers_schema, readers_schema, decoder, initial_record = {})
+        readers_fields_hash = readers_schema.fields_hash
+        read_record = Avromatic.use_custom_datum_reader ? initial_record : {}
+        writers_schema.fields.each do |field|
+          readers_field = readers_fields_hash[field.name]
+          if readers_field
+            field_val = read_data(field.type, readers_field.type, decoder)
+            read_record[field.name] = field_val
+          else
+            skip_data(field.type, decoder)
+          end
+        end
+
+        read_record
+      end
+    end
+  end
+end

data/lib/avromatic/messaging.rb

@@ -0,0 +1,32 @@
+require 'avro_turf/messaging'
+require 'avromatic/io/datum_reader'
+
+module Avromatic
+  # Subclass AvroTurf::Messaging to use a custom DatumReader
+  class Messaging < AvroTurf::Messaging
+    def decode(data, schema_name: nil, namespace: @namespace)
+      readers_schema = schema_name && @schema_store.find(schema_name, namespace)
+      stream = StringIO.new(data)
+      decoder = Avro::IO::BinaryDecoder.new(stream)
+
+      # The first byte is MAGIC!!!
+      magic_byte = decoder.read(1)
+
+      if magic_byte != MAGIC_BYTE
+        raise "Expected data to begin with a magic byte, got `#{magic_byte.inspect}`"
+      end
+
+      # The schema id is a 4-byte big-endian integer.
+      schema_id = decoder.read(4).unpack('N').first
+
+      writers_schema = @schemas_by_id.fetch(schema_id) do
+        schema_json = @registry.fetch(schema_id)
+        @schemas_by_id[schema_id] = Avro::Schema.parse(schema_json)
+      end
+
+      # The following line was changed to use a custom DatumReader
+      reader = Avromatic::IO::DatumReader.new(writers_schema, readers_schema)
+      reader.read(decoder)
+    end
+  end
+end

data/lib/avromatic/model/attribute/union.rb

@@ -1,4 +1,5 @@
 require 'avromatic/model/attribute_type/union'
+require 'avromatic/io/datum_reader'
 
 module Avromatic
   module Model
@@ -12,6 +13,8 @@ module Avromatic
       class Union < Virtus::Attribute
         primitive Avromatic::Model::AttributeType::Union
 
+        MEMBER_INDEX = ::Avromatic::IO::DatumReader::UNION_MEMBER_INDEX
+
         def initialize(*)
           super
 
@@ -24,12 +27,11 @@ module Avromatic
           return input if value_coerced?(input)
 
           result = nil
-          member_attributes.find do |union_attribute|
-            begin
-              coerced = union_attribute.coerce(input)
-              result = coerced unless coerced.is_a?(Avromatic::Model::Attributes) && coerced.invalid?
-            rescue
-              nil
+          if input.key?(MEMBER_INDEX)
+            result = safe_coerce(member_attributes[input.delete(MEMBER_INDEX)], input)
+          else
+            member_attributes.find do |union_attribute|
+              result = safe_coerce(union_attribute, input)
             end
           end
           result
@@ -43,6 +45,13 @@ module Avromatic
 
         private
 
+        def safe_coerce(member_attribute, input)
+          coerced = member_attribute.coerce(input)
+          coerced unless coerced.is_a?(Avromatic::Model::Attributes) && coerced.invalid?
+        rescue
+          nil
+        end
+
         def member_attributes
           @member_attributes ||= Array.new
         end

data/lib/avromatic/model/messaging_serialization.rb

@@ -1,5 +1,3 @@
-require 'avro_turf/messaging'
-
 module Avromatic
   module Model
 

data/lib/avromatic/model/raw_serialization.rb

@@ -88,11 +88,14 @@ module Avromatic
         end
 
         def custom_datum_reader(schema, key_or_value)
-          Avro::IO::DatumReader.new(schema, send("#{key_or_value}_avro_schema"))
+          datum_reader_class.new(schema, send("#{key_or_value}_avro_schema"))
         end
       end
 
       module ClassMethods
+        def datum_reader_class
+          Avromatic::IO::DatumReader
+        end
 
         # Store a hash of Procs by field name (as a symbol) to convert
         # the value before Avro serialization.
@@ -111,8 +114,8 @@ module Avromatic
 
         def datum_reader
           @datum_reader ||= begin
-            hash = { value: Avro::IO::DatumReader.new(value_avro_schema) }
-            hash[:key] = Avro::IO::DatumReader.new(key_avro_schema) if key_avro_schema
+            hash = { value: datum_reader_class.new(value_avro_schema) }
+            hash[:key] = datum_reader_class.new(key_avro_schema) if key_avro_schema
             hash
           end
         end

data/lib/avromatic/version.rb

@@ -1,3 +1,3 @@
 module Avromatic
-  VERSION = '0.13.0'.freeze
+  VERSION = '0.14.0.rc0'.freeze
 end

data/lib/avromatic.rb

@@ -1,14 +1,15 @@
 require 'avromatic/version'
+require 'avro_turf'
 require 'avromatic/model'
 require 'avromatic/model_registry'
-require 'avro_turf'
-require 'avro_turf/messaging'
+require 'avromatic/messaging'
 require 'active_support/core_ext/string/inflections'
 
 module Avromatic
   class << self
     attr_accessor :schema_registry, :registry_url, :schema_store, :logger,
-                  :messaging, :type_registry, :nested_models
+                  :messaging, :type_registry, :nested_models,
+                  :use_custom_datum_reader
 
     delegate :register_type, to: :type_registry
   end
@@ -16,6 +17,7 @@ module Avromatic
   self.nested_models = ModelRegistry.new
   self.logger = Logger.new($stdout)
   self.type_registry = Avromatic::Model::TypeRegistry.new
+  self.use_custom_datum_reader = true
 
   def self.configure
     yield self
@@ -31,7 +33,7 @@ module Avromatic
 
   def self.build_messaging
     raise 'Avromatic must be configured with a schema_store' unless schema_store
-    AvroTurf::Messaging.new(
+    Avromatic::Messaging.new(
       registry: schema_registry || build_schema_registry,
       schema_store: schema_store,
       logger: logger

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avromatic
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0.rc0
 platform: ruby
 authors:
 - Salsify Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2017-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -245,6 +245,8 @@ files:
 - gemfiles/rails4_1.gemfile
 - gemfiles/rails4_2.gemfile
 - lib/avromatic.rb
+- lib/avromatic/io/datum_reader.rb
+- lib/avromatic/messaging.rb
 - lib/avromatic/model.rb
 - lib/avromatic/model/attribute/abstract_timestamp.rb
 - lib/avromatic/model/attribute/record.rb
@@ -287,9 +289,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.8