avromatic 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0006753e54e4ba4a3b0a5f40fa71810417d1b37f
-  data.tar.gz: 00da3685e121175995d6908a76c23ff0997108fc
+  metadata.gz: c482b56c1e0e8d3f9271d7a7e61e763a9dffdd83
+  data.tar.gz: 0005f4ef6ff0abdc4a484b67873de53fab595244
 SHA512:
-  metadata.gz: fc9f465e19cb10d935fe191acdb21ad51eb4aa1e9dbe373355672048b32a81b71c2c61afde243c6ac22e6b14a6b33e3b52f6b16350504cf2026a3edebefc9f34
-  data.tar.gz: 6d3acedd8f81a97b065a27d684c95367fa841f3ec78f0e295dc6672e770ddef77cdc9189baa4ddfceb38977bf6d59c266c4fa4d9a65d2db302bc51a49374458a
+  metadata.gz: 3ad6910fa3bda579cc807aec5b2e374d7821f7fb8c47b43de99b58c738b822b4f95d884594ec89a6f083baac97b018621d6ff70e9795caf8c050b96f3c4cc974
+  data.tar.gz: d11cb1499b84dc7b4dfea172f0391eeee61e4f161d35a5e8e44c8181c775693b86902e2b0825e31fe4829058fbef51299a65da1d262c1aacfd689d74cacc9b43

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # avromatic changelog
 
+## v0.24.0
+- Add `Avromatic::IO::DatumWriter` to optimize the encoding of Avro unions
+  from Avromatic models.
+- Expose the `#key_attributes_for_avro` method on models.
+
 ## v0.23.0
 - Add mutable option when defining a model.
 

data/README.md

@@ -97,6 +97,14 @@ end
   is included in the hash returned by the `DatumReader` but can be omitted by
   setting this option to `false`.
 
+#### Encoding
+* **use_custom_datum_writer**: `Avromatic` includes a modified subclass of
+  `Avro::IO::DatumWriter`. This subclass uses additional information about
+  the index of union members to optimize the encoding of Avro messages.
+  By default this information is included in the hash passed to the encoder
+  but can be omitted by setting this option to `false`.
+
+
 ### Models
 
 Models are defined based on an Avro schema for a record.

data/lib/avromatic/io/datum_reader.rb

@@ -6,7 +6,7 @@ module Avromatic
     # branch 'salsify-master' with the tag 'v1.9.0.3'
     class DatumReader < Avro::IO::DatumReader
 
-      UNION_MEMBER_INDEX = '__avromatic_member_index'.freeze
+      UNION_MEMBER_INDEX = Avromatic::IO::UNION_MEMBER_INDEX
 
       def read_data(writers_schema, readers_schema, decoder, initial_record = {})
         # schema matching

data/lib/avromatic/io/datum_writer.rb

@@ -0,0 +1,25 @@
+module Avromatic
+  module IO
+    # Subclass DatumWriter to use additional information about union member
+    # index.
+    class DatumWriter < Avro::IO::DatumWriter
+      def write_union(writers_schema, datum, encoder)
+        optional = writers_schema.schemas.first.type_sym == :null
+        if datum.is_a?(Hash) && datum.key?(Avromatic::IO::UNION_MEMBER_INDEX)
+          index_of_schema = datum[Avromatic::IO::UNION_MEMBER_INDEX]
+          # Avromatic does not treat the null of an optional field as part of the union
+          index_of_schema += 1 if optional
+        else
+          index_of_schema = writers_schema.schemas.find_index do |schema|
+            Avro::Schema.validate(schema, datum)
+          end
+        end
+        unless index_of_schema
+          raise Avro::IO::AvroTypeError.new(writers_schema, datum)
+        end
+        encoder.write_long(index_of_schema)
+        write_data(writers_schema.schemas[index_of_schema], datum, encoder)
+      end
+    end
+  end
+end

data/lib/avromatic/io.rb

@@ -0,0 +1,8 @@
+module Avromatic
+  module IO
+    UNION_MEMBER_INDEX = '__avromatic_member_index'.freeze
+  end
+end
+
+require 'avromatic/io/datum_reader'
+require 'avromatic/io/datum_writer'

data/lib/avromatic/messaging.rb

@@ -1,8 +1,8 @@
 require 'avro_turf/messaging'
-require 'avromatic/io/datum_reader'
+require 'avromatic/io'
 
 module Avromatic
-  # Subclass AvroTurf::Messaging to use a custom DatumReader for decode.
+  # Subclass AvroTurf::Messaging to use a custom DatumReader and DatumWriter
   class Messaging < AvroTurf::Messaging
     attr_reader :registry
 
@@ -30,5 +30,30 @@ module Avromatic
       reader = Avromatic::IO::DatumReader.new(writers_schema, readers_schema)
       reader.read(decoder)
     end
+
+    def encode(message, schema_name: nil, namespace: @namespace, subject: nil)
+      schema = @schema_store.find(schema_name, namespace)
+
+      # Schemas are registered under the full name of the top level Avro record
+      # type, or `subject` if it's provided.
+      schema_id = @registry.register(subject || schema.fullname, schema)
+
+      stream = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(stream)
+
+      # Always start with the magic byte.
+      encoder.write(MAGIC_BYTE)
+
+      # The schema id is encoded as a 4-byte big-endian integer.
+      encoder.write([schema_id].pack('N'))
+
+      # The following line differs from the parent class to use a custom DatumWriter
+      writer = Avromatic::IO::DatumWriter.new(schema)
+
+      # The actual message comes last.
+      writer.write(message, encoder)
+
+      stream.string
+    end
   end
 end

data/lib/avromatic/model/attribute/union.rb

@@ -1,5 +1,5 @@
 require 'avromatic/model/attribute_type/union'
-require 'avromatic/io/datum_reader'
+require 'avromatic/io'
 
 module Avromatic
   module Model

data/lib/avromatic/model/raw_serialization.rb

@@ -15,18 +15,53 @@ module Avromatic
                  to: :class
         private :avro_serializer, :datum_writer, :datum_reader
 
+        EMPTY_ARRAY = [].freeze
+
+        included do
+          @attribute_member_types = {}
+        end
+
         module ClassMethods
-          def recursive_serialize(value, attribute_name = nil)
+          def recursive_serialize(value, name: nil, member_types: nil)
+            member_types = attribute_member_types(name) if name
+            member_types ||= EMPTY_ARRAY
+
             if value.is_a?(Avromatic::Model::Attributes)
-              value.value_attributes_for_avro
+              hash = value.value_attributes_for_avro
+              if Avromatic.use_custom_datum_writer
+                member_index = member_types.index(value.class) if member_types.any?
+                hash[Avromatic::IO::UNION_MEMBER_INDEX] = member_index if member_index
+              end
+              hash
             elsif value.is_a?(Array)
-              value.map { |v| recursive_serialize(v) }
+              value.map { |v| recursive_serialize(v, member_types: member_types) }
             elsif value.is_a?(Hash)
-              value.each_with_object({}) do |(k, v), hash|
-                hash[k] = recursive_serialize(v)
+              value.each_with_object({}) do |(k, v), map|
+                map[k] = recursive_serialize(v, member_types: member_types)
               end
             else
-              avro_serializer[attribute_name].call(value)
+              avro_serializer[name].call(value)
+            end
+          end
+
+          private
+
+          def attribute_member_types(name)
+            @attribute_member_types.fetch(name) do
+              member_types = nil
+              attribute = attribute_set[name] if name
+              if attribute
+                if attribute.primitive == Array &&
+                  attribute.member_type.is_a?(Avromatic::Model::Attribute::Union)
+                  member_types = attribute.member_type.primitive.types
+                elsif attribute.primitive == Hash &&
+                  attribute.value_type.is_a?(Avromatic::Model::Attribute::Union)
+                  member_types = attribute.value_type.primitive.types
+                elsif attribute.options[:primitive] == Avromatic::Model::AttributeType::Union
+                  member_types = attribute.primitive.types
+                end
+              end
+              @attribute_member_types[name] = member_types
             end
           end
         end
@@ -44,15 +79,15 @@ module Avromatic
           avro_hash(value_avro_field_names)
         end
 
-        private
-
         def key_attributes_for_avro
           avro_hash(key_avro_field_names)
         end
 
+        private
+
         def avro_hash(fields)
           attributes.slice(*fields).each_with_object(Hash.new) do |(key, value), result|
-            result[key.to_s] = self.class.recursive_serialize(value, key)
+            result[key.to_s] = self.class.recursive_serialize(value, name: key)
           end
         end
 
@@ -94,7 +129,11 @@ module Avromatic
 
       module ClassMethods
         def datum_reader_class
-          Avromatic::IO::DatumReader
+          Avromatic.use_custom_datum_reader ? Avromatic::IO::DatumReader : Avro::IO::DatumReader
+        end
+
+        def datum_writer_class
+          Avromatic.use_custom_datum_writer ? Avromatic::IO::DatumWriter : Avro::IO::DatumWriter
         end
 
         # Store a hash of Procs by field name (as a symbol) to convert
@@ -106,8 +145,8 @@ module Avromatic
 
         def datum_writer
           @datum_writer ||= begin
-                              hash = { value: Avro::IO::DatumWriter.new(value_avro_schema) }
-                              hash[:key] = Avro::IO::DatumWriter.new(key_avro_schema) if key_avro_schema
+                              hash = { value: datum_writer_class.new(value_avro_schema) }
+                              hash[:key] = datum_writer_class.new(key_avro_schema) if key_avro_schema
                               hash
                             end
         end

data/lib/avromatic/version.rb

@@ -1,3 +1,3 @@
 module Avromatic
-  VERSION = '0.23.0'.freeze
+  VERSION = '0.24.0'.freeze
 end

data/lib/avromatic.rb

@@ -9,7 +9,8 @@ module Avromatic
   class << self
     attr_accessor :schema_registry, :registry_url, :schema_store, :logger,
                   :messaging, :type_registry, :nested_models,
-                  :use_custom_datum_reader, :use_schema_fingerprint_lookup
+                  :use_custom_datum_reader, :use_custom_datum_writer,
+                  :use_schema_fingerprint_lookup
 
     delegate :register_type, to: :type_registry
   end
@@ -18,6 +19,7 @@ module Avromatic
   self.logger = Logger.new($stdout)
   self.type_registry = Avromatic::Model::TypeRegistry.new
   self.use_custom_datum_reader = true
+  self.use_custom_datum_writer = true
   self.use_schema_fingerprint_lookup = true
 
   def self.configure

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avromatic
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.0
 platform: ruby
 authors:
 - Salsify Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-02 00:00:00.000000000 Z
+date: 2017-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -273,7 +273,9 @@ files:
 - gemfiles/rails4_2.gemfile
 - gemfiles/rails5_0.gemfile
 - lib/avromatic.rb
+- lib/avromatic/io.rb
 - lib/avromatic/io/datum_reader.rb
+- lib/avromatic/io/datum_writer.rb
 - lib/avromatic/messaging.rb
 - lib/avromatic/model.rb
 - lib/avromatic/model/attribute/abstract_timestamp.rb