deimos-ruby 1.4.0.pre.beta7 → 1.5.0.pre.beta2

data/lib/deimos/base_consumer.rb

@@ -6,20 +6,20 @@ module Deimos
     include SharedConfig
 
     class << self
-      # @return [AvroDataEncoder]
+      # @return [Deimos::SchemaBackends::Base]
       def decoder
-        @decoder ||= AvroDataDecoder.new(schema: config[:schema],
-                                         namespace: config[:namespace])
+        @decoder ||= Deimos.schema_backend(schema: config[:schema],
+                                           namespace: config[:namespace])
       end
 
-      # @return [AvroDataEncoder]
+      # @return [Deimos::SchemaBackends::Base]
       def key_decoder
-        @key_decoder ||= AvroDataDecoder.new(schema: config[:key_schema],
-                                             namespace: config[:namespace])
+        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
+                                               namespace: config[:namespace])
       end
     end
 
-    # Helper method to decode an Avro-encoded key.
+    # Helper method to decode an encoded key.
     # @param key [String]
     # @return [Object] the decoded key.
     def decode_key(key)

data/lib/deimos/batch_consumer.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'deimos/avro_data_decoder'
 require 'deimos/base_consumer'
 require 'phobos/batch_handler'
 

data/lib/deimos/config/configuration.rb

@@ -251,6 +251,10 @@ module Deimos
     end
 
     setting :schema do
+
+      # Backend class to use when encoding/decoding messages.
+      setting :backend, :mock
+
       # URL of the Confluent schema registry.
       # @return [String]
       setting :registry_url, 'http://localhost:8081'

data/lib/deimos/consumer.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-require 'deimos/avro_data_decoder'
 require 'deimos/base_consumer'
 require 'deimos/shared_config'
 require 'phobos/handler'
 require 'active_support/all'
-require 'ddtrace'
 
 # Class to consume messages coming from the pipeline topic
 # Note: According to the docs, instances of your handler will be created

data/lib/deimos/kafka_source.rb

@@ -41,7 +41,7 @@ module Deimos
     def send_kafka_event_on_destroy
       return unless self.class.kafka_config[:delete]
 
-      self.class.kafka_producers.each { |p| p.send_event(self.deletion_payload) }
+      self.class.kafka_producers.each { |p| p.publish_list([self.deletion_payload]) }
     end
 
     # Payload to send after we are destroyed.

data/lib/deimos/kafka_topic_info.rb

@@ -14,7 +14,7 @@ module Deimos
         # Try to create it - it's fine if it already exists
         begin
           self.create(topic: topic)
-        rescue ActiveRecord::RecordNotUnique # rubocop:disable Lint/HandleExceptions
+        rescue ActiveRecord::RecordNotUnique # rubocop:disable Lint/SuppressedException
           # continue on
         end
 

data/lib/deimos/message.rb

@@ -18,23 +18,23 @@ module Deimos
 
     # Add message_id and timestamp default values if they are in the
     # schema and don't already have values.
-    # @param schema [Avro::Schema]
-    def add_fields(schema)
+    # @param fields [Array<String>] existing name fields in the schema.
+    def add_fields(fields)
       return if @payload.except(:payload_key, :partition_key).blank?
 
-      if schema.fields.any? { |f| f.name == 'message_id' }
+      if fields.include?('message_id')
         @payload['message_id'] ||= SecureRandom.uuid
       end
-      if schema.fields.any? { |f| f.name == 'timestamp' }
+      if fields.include?('timestamp')
         @payload['timestamp'] ||= Time.now.in_time_zone.to_s
       end
     end
 
-    # @param schema [Avro::Schema]
-    def coerce_fields(schema)
+    # @param encoder [Deimos::SchemaBackends::Base]
+    def coerce_fields(encoder)
       return if payload.nil?
 
-      @payload = SchemaCoercer.new(schema).coerce(@payload)
+      @payload = encoder.coerce(@payload)
     end
 
     # @return [Hash]

data/lib/deimos/producer.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require 'deimos/avro_data_encoder'
 require 'deimos/message'
 require 'deimos/shared_config'
-require 'deimos/schema_coercer'
 require 'phobos/producer'
 require 'active_support/notifications'
 
@@ -143,23 +141,23 @@ module Deimos
         backend.publish(producer_class: self, messages: batch)
       end
 
-      # @return [AvroDataEncoder]
+      # @return [Deimos::SchemaBackends::Base]
       def encoder
-        @encoder ||= AvroDataEncoder.new(schema: config[:schema],
-                                         namespace: config[:namespace])
+        @encoder ||= Deimos.schema_backend(schema: config[:schema],
+                                           namespace: config[:namespace])
       end
 
-      # @return [AvroDataEncoder]
+      # @return [Deimos::SchemaBackends::Base]
       def key_encoder
-        @key_encoder ||= AvroDataEncoder.new(schema: config[:key_schema],
-                                             namespace: config[:namespace])
+        @key_encoder ||= Deimos.schema_backend(schema: config[:key_schema],
+                                               namespace: config[:namespace])
       end
 
       # Override this in active record producers to add
       # non-schema fields to check for updates
       # @return [Array<String>] fields to check for updates
       def watched_attributes
-        self.encoder.avro_schema.fields.map(&:name)
+        self.encoder.schema_fields.map(&:name)
       end
 
     private
@@ -169,13 +167,13 @@ module Deimos
         # this violates the Law of Demeter but it has to happen in a very
         # specific order and requires a bunch of methods on the producer
         # to work correctly.
-        message.add_fields(encoder.avro_schema)
+        message.add_fields(encoder.schema_fields.map(&:name))
         message.partition_key = self.partition_key(message.payload)
         message.key = _retrieve_key(message.payload)
         # need to do this before _coerce_fields because that might result
         # in an empty payload which is an *error* whereas this is intended.
         message.payload = nil if message.payload.blank?
-        message.coerce_fields(encoder.avro_schema)
+        message.coerce_fields(encoder)
         message.encoded_key = _encode_key(message.key)
         message.topic = self.topic
         message.encoded_payload = if message.payload.nil?
@@ -200,7 +198,7 @@ module Deimos
         end
 
         if config[:key_field]
-          encoder.encode_key(config[:key_field], key, "#{config[:topic]}-key")
+          encoder.encode_key(config[:key_field], key, topic: "#{config[:topic]}-key")
         elsif config[:key_schema]
           key_encoder.encode(key, topic: "#{config[:topic]}-key")
         else

data/lib/deimos/schema_backends/avro_base.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'avro'
+require 'avro_turf'
+require 'avro_turf/mutable_schema_store'
+require_relative 'avro_schema_coercer'
+
+module Deimos
+  module SchemaBackends
+    # Encode / decode using Avro, either locally or via schema registry.
+    class AvroBase < Base
+      attr_accessor :schema_store
+
+      # @override
+      def initialize(schema:, namespace:)
+        super(schema: schema, namespace: namespace)
+        @schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
+      end
+
+      # @override
+      def encode_key(key_id, key, topic: nil)
+        @key_schema ||= _generate_key_schema(key_id)
+        field_name = _field_name_from_schema(@key_schema)
+        payload = { field_name => key }
+        encode(payload, schema: @key_schema['name'], topic: topic)
+      end
+
+      # @override
+      def decode_key(payload, key_id)
+        @key_schema ||= _generate_key_schema(key_id)
+        field_name = _field_name_from_schema(@key_schema)
+        decode(payload, schema: @key_schema['name'])[field_name]
+      end
+
+      # @override
+      def coerce_field(field, value)
+        AvroSchemaCoercer.new(avro_schema).coerce_type(field.type, value)
+      end
+
+      # @override
+      def schema_fields
+        avro_schema.fields.map { |field| SchemaField.new(field.name, field.type) }
+      end
+
+      # @override
+      def validate(payload, schema:)
+        Avro::SchemaValidator.validate!(avro_schema(schema), payload,
+                                        recursive: true,
+                                        fail_on_extra_fields: true)
+      end
+
+      # @override
+      def self.mock_backend
+        :avro_validation
+      end
+
+    private
+
+      # @param schema [String]
+      # @return [Avro::Schema]
+      def avro_schema(schema=nil)
+        schema ||= @schema
+        @schema_store.find(schema, @namespace)
+      end
+
+      # Generate a key schema from the given value schema and key ID. This
+      # is used when encoding or decoding keys from an existing value schema.
+      # @param key_id [Symbol]
+      # @return [Hash]
+      def _generate_key_schema(key_id)
+        key_field = avro_schema.fields.find { |f| f.name == key_id.to_s }
+        name = _key_schema_name(@schema)
+        key_schema = {
+          'type' => 'record',
+          'name' => name,
+          'namespace' => @namespace,
+          'doc' => "Key for #{@namespace}.#{@schema} - autogenerated by Deimos",
+          'fields' => [
+            {
+              'name' => key_id,
+              'type' => key_field.type.type_sym.to_s
+            }
+          ]
+        }
+        @schema_store.add_schema(key_schema)
+        key_schema
+      end
+
+      # @param value_schema [Hash]
+      # @return [String]
+      def _field_name_from_schema(value_schema)
+        raise "Schema #{@schema} not found!" if value_schema.nil?
+        if value_schema['fields'].nil? || value_schema['fields'].empty?
+          raise "Schema #{@schema} has no fields!"
+        end
+
+        value_schema['fields'][0]['name']
+      end
+
+      # @param schema [String]
+      # @return [String]
+      def _key_schema_name(schema)
+        "#{schema}_key"
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/avro_local.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'avro_base'
+
+module Deimos
+  module SchemaBackends
+    # Encode / decode using local Avro encoding.
+    class AvroLocal < AvroBase
+      # @override
+      def decode_payload(payload, schema:)
+        avro_turf.decode(payload, schema_name: schema, namespace: @namespace)
+      end
+
+      # @override
+      def encode_payload(payload, schema: nil, topic: nil)
+        avro_turf.encode(payload, schema_name: schema, namespace: @namespace)
+      end
+
+    private
+
+      # @return [AvroTurf]
+      def avro_turf
+        @avro_turf ||= AvroTurf.new(
+          schemas_path: Deimos.config.schema.path,
+          schema_store: @schema_store
+        )
+      end
+    end
+  end
+end

data/lib/deimos/{schema_coercer.rb → schema_backends/avro_schema_coercer.rb}

@@ -1,66 +1,20 @@
 # frozen_string_literal: true
 
+require 'active_support/time'
+
 module Deimos
   # Class to coerce values in a payload to match a schema.
-  class SchemaCoercer
+  class AvroSchemaCoercer
     # @param schema [Avro::Schema]
     def initialize(schema)
       @schema = schema
     end
 
-    # @param payload [Hash]
-    # @return [HashWithIndifferentAccess]
-    def coerce(payload)
-      result = {}
-      @schema.fields.each do |field|
-        name = field.name
-        next unless payload.key?(name)
-
-        val = payload[name]
-        result[name] = _coerce_type(field.type, val)
-      end
-      result.with_indifferent_access
-    end
-
-  private
-
-    # @param val [String]
-    # @return [Boolean]
-    def _is_integer_string?(val)
-      return false unless val.is_a?(String)
-
-      begin
-          true if Integer(val)
-      rescue StandardError
-        false
-        end
-    end
-
-    # @param val [String]
-    # @return [Boolean]
-    def _is_float_string?(val)
-      return false unless val.is_a?(String)
-
-      begin
-          true if Float(val)
-      rescue StandardError
-        false
-        end
-    end
-
-    # @param val [Object]
-    # @return [Boolean]
-    def _is_to_s_defined?(val)
-      return false if val.nil?
-
-      Object.instance_method(:to_s).bind(val).call != val.to_s
-    end
-
     # @param type [Symbol]
     # @param val [Object]
     # @return [Object]
-    def _coerce_type(type, val)
-      int_classes = [Time, DateTime, ActiveSupport::TimeWithZone]
+    def coerce_type(type, val)
+      int_classes = [Time, ActiveSupport::TimeWithZone]
       field_type = type.type.to_sym
       if field_type == :union
         union_types = type.schemas.map { |s| s.type.to_sym }
@@ -104,5 +58,39 @@ module Deimos
         val
       end
     end
+
+  private
+
+    # @param val [String]
+    # @return [Boolean]
+    def _is_integer_string?(val)
+      return false unless val.is_a?(String)
+
+      begin
+        true if Integer(val)
+      rescue StandardError
+        false
+      end
+    end
+
+    # @param val [String]
+    # @return [Boolean]
+    def _is_float_string?(val)
+      return false unless val.is_a?(String)
+
+      begin
+        true if Float(val)
+      rescue StandardError
+        false
+      end
+    end
+
+    # @param val [Object]
+    # @return [Boolean]
+    def _is_to_s_defined?(val)
+      return false if val.nil?
+
+      Object.instance_method(:to_s).bind(val).call != val.to_s
+    end
   end
 end

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'avro_base'
+require_relative 'avro_validation'
+require 'avro_turf/messaging'
+
+module Deimos
+  module SchemaBackends
+    # Encode / decode using the Avro schema registry.
+    class AvroSchemaRegistry < AvroBase
+      # @override
+      def decode_payload(payload, schema:)
+        avro_turf_messaging.decode(payload, schema_name: schema)
+      end
+
+      # @override
+      def encode_payload(payload, schema: nil, topic: nil)
+        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic)
+      end
+
+    private
+
+      # @return [AvroTurf::Messaging]
+      def avro_turf_messaging
+        @avro_turf_messaging ||= AvroTurf::Messaging.new(
+          schema_store: @schema_store,
+          registry_url: Deimos.config.schema.registry_url,
+          schemas_path: Deimos.config.schema.path,
+          namespace: @namespace
+        )
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/avro_validation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'avro_base'
+
+module Deimos
+  module SchemaBackends
+    # Leave Ruby hashes as is but validate them against the schema.
+    # Useful for unit tests.
+    class AvroValidation < AvroBase
+      # @override
+      def decode_payload(payload, schema: nil)
+        payload.with_indifferent_access
+      end
+
+      # @override
+      def encode_payload(payload, schema: nil, topic: nil)
+        payload.with_indifferent_access
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/base.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Deimos
+  # Represents a field in the schema.
+  class SchemaField
+    attr_accessor :name, :type
+
+    # @param name [String]
+    # @param type [Object]
+    def initialize(name, type)
+      @name = name
+      @type = type
+    end
+  end
+
+  module SchemaBackends
+    # Base class for encoding / decoding.
+    class Base
+      attr_accessor :schema, :namespace, :key_schema
+
+      # @param schema [String|Symbol]
+      # @param namespace [String]
+      def initialize(schema:, namespace: nil)
+        @schema = schema
+        @namespace = namespace
+      end
+
+      # Encode a payload with a schema. Public method.
+      # @param payload [Hash]
+      # @param schema [Symbol|String]
+      # @param topic [String]
+      # @return [String]
+      def encode(payload, schema: nil, topic: nil)
+        validate(payload, schema: schema || @schema)
+        encode_payload(payload, schema: schema || @schema, topic: topic)
+      end
+
+      # Decode a payload with a schema. Public method.
+      # @param payload [String]
+      # @param schema [Symbol|String]
+      # @return [Hash]
+      def decode(payload, schema: nil)
+        decode_payload(payload, schema: schema || @schema)
+      end
+
+      # Given a hash, coerce its types to our schema. To be defined by subclass.
+      # @param payload [Hash]
+      # @return [Hash]
+      def coerce(payload)
+        result = {}
+        self.schema_fields.each do |field|
+          name = field.name
+          next unless payload.key?(name)
+
+          val = payload[name]
+          result[name] = coerce_field(field, val)
+        end
+        result.with_indifferent_access
+      end
+
+      # Indicate a class which should act as a mocked version of this backend.
+      # This class should perform all validations but not actually do any
+      # encoding.
+      # Note that the "mock" version (e.g. avro_validation) should return
+      # its own symbol when this is called, since it may be called multiple
+      # times depending on the order of RSpec helpers.
+      # @return [Symbol]
+      def self.mock_backend
+        :mock
+      end
+
+      # Encode a payload. To be defined by subclass.
+      # @param payload [Hash]
+      # @param schema [Symbol|String]
+      # @param topic [String]
+      # @return [String]
+      def encode_payload(_payload, schema:, topic: nil)
+        raise NotImplementedError
+      end
+
+      # Decode a payload. To be defined by subclass.
+      # @param payload [String]
+      # @param schema [String|Symbol]
+      # @return [Hash]
+      def decode_payload(_payload, schema:)
+        raise NotImplementedError
+      end
+
+      # Validate that a payload matches the schema. To be defined by subclass.
+      # @param payload [Hash]
+      # @param schema [String|Symbol]
+      def validate(_payload, schema:)
+        raise NotImplementedError
+      end
+
+      # List of field names belonging to the schema. To be defined by subclass.
+      # @return [Array<SchemaField>]
+      def schema_fields
+        raise NotImplementedError
+      end
+
+      # Given a value and a field definition (as defined by whatever the
+      # underlying schema library is), coerce the given value to
+      # the given field type.
+      # @param field [SchemaField]
+      # @param value [Object]
+      # @return [Object]
+      def coerce_field(_field, _value)
+        raise NotImplementedError
+      end
+
+      # Encode a message key. To be defined by subclass.
+      # @param key [String|Hash] the value to use as the key.
+      # @param key_id [Symbol|String] the field name of the key.
+      # @param topic [String]
+      # @return [String]
+      def encode_key(_key, _key_id, topic: nil)
+        raise NotImplementedError
+      end
+
+      # Decode a message key. To be defined by subclass.
+      # @param payload [Hash] the message itself.
+      # @param key_id [Symbol|String] the field in the message to decode.
+      # @return [String]
+      def decode_key(_payload, _key_id)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/deimos/schema_backends/mock.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Deimos
+  module SchemaBackends
+    # Mock implementation of a schema backend that does no encoding or validation.
+    class Mock < Base
+      # @override
+      def decode_payload(payload, schema:)
+        payload.is_a?(String) ? 'payload-decoded' : payload.map { |k, v| [k, "decoded-#{v}"] }
+      end
+
+      # @override
+      def encode_payload(payload, schema:, topic: nil)
+        payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }
+      end
+
+      # @override
+      def validate(_payload, schema:)
+      end
+
+      # @override
+      def schema_fields
+        []
+      end
+
+      # @override
+      def coerce_field(_type, value)
+        value
+      end
+
+      # @override
+      def encode_key(key_id, key)
+        { key_id => key }
+      end
+
+      # @override
+      def decode_key(payload, key_id)
+        payload[key_id]
+      end
+    end
+  end
+end