deimos-ruby 1.4.0.pre.beta7 → 1.5.0.pre.beta2

data/lib/deimos/test_helpers.rb

@@ -2,7 +2,6 @@
 
 require 'active_support/concern'
 require 'active_support/core_ext'
-require 'avro_turf'
 require 'deimos/tracing/mock'
 require 'deimos/metrics/mock'
 
@@ -14,139 +13,55 @@ module Deimos
     extend ActiveSupport::Concern
 
     class << self
+      # for backwards compatibility
       # @return [Array<Hash>]
       def sent_messages
-        @sent_messages ||= []
+        Deimos::Backends::Test.sent_messages
       end
     end
 
     included do
-      # @param encoder_schema [String]
-      # @param namespace [String]
-      # @return [Deimos::AvroDataEncoder]
-      def create_encoder(encoder_schema, namespace)
-        encoder = Deimos::AvroDataEncoder.new(schema: encoder_schema,
-                                              namespace: namespace)
-
-        # we added and_wrap_original to RSpec 2 but the regular block
-        # syntax wasn't working for some reason - block wasn't being passed
-        # to the method
-        block = proc do |m, *args|
-          m.call(*args)
-          args[0]
-        end
-        allow(encoder).to receive(:encode_local).and_wrap_original(&block)
-        allow(encoder).to receive(:encode) do |payload, schema: nil, topic: nil|
-          encoder.encode_local(payload, schema: schema)
-        end
-
-        block = proc do |m, *args|
-          m.call(*args)&.values&.first
-        end
-        allow(encoder).to receive(:encode_key).and_wrap_original(&block)
-        encoder
-      end
-
-      # @param decoder_schema [String]
-      # @param namespace [String]
-      # @return [Deimos::AvroDataDecoder]
-      def create_decoder(decoder_schema, namespace)
-        decoder = Deimos::AvroDataDecoder.new(schema: decoder_schema,
-                                              namespace: namespace)
-        allow(decoder).to receive(:decode_local) { |payload| payload }
-        allow(decoder).to receive(:decode) do |payload, schema: nil|
-          schema ||= decoder.schema
-          if schema && decoder.namespace
-            # Validate against local schema.
-            encoder = Deimos::AvroDataEncoder.new(schema: schema,
-                                                  namespace: decoder.namespace)
-            encoder.schema_store = decoder.schema_store
-            payload = payload.respond_to?(:stringify_keys) ? payload.stringify_keys : payload
-            encoder.encode_local(payload)
-          end
-          payload
-        end
-        allow(decoder).to receive(:decode_key) do |payload, _key_id|
-          payload.values.first
-        end
-        decoder
-      end
 
       RSpec.configure do |config|
 
         config.before(:suite) do
-          Deimos.configure do |fr_config|
-            fr_config.logger = Logger.new(STDOUT)
-            fr_config.consumers.reraise_errors = true
-            fr_config.kafka.seed_brokers ||= ['test_broker']
+          Deimos.configure do |d_config|
+            d_config.logger = Logger.new(STDOUT)
+            d_config.consumers.reraise_errors = true
+            d_config.kafka.seed_brokers ||= ['test_broker']
+            d_config.schema.backend = Deimos.schema_backend_class.mock_backend
           end
         end
-
       end
+
       before(:each) do
         client = double('client').as_null_object
         allow(client).to receive(:time) do |*_args, &block|
           block.call
         end
+        Deimos.configure { |c| c.producers.backend = :test }
+        Deimos::Backends::Test.sent_messages.clear
       end
     end
 
-    # Stub all already-loaded producers and consumers for unit testing purposes.
+    # @deprecated
     def stub_producers_and_consumers!
-      Deimos::TestHelpers.sent_messages.clear
-
-      allow(Deimos::Producer).to receive(:produce_batch) do |_, batch|
-        Deimos::TestHelpers.sent_messages.concat(batch.map(&:to_h))
-      end
-
-      Deimos::Producer.descendants.each do |klass|
-        next if klass == Deimos::ActiveRecordProducer # "abstract" class
-
-        stub_producer(klass)
-      end
-
-      Deimos::Consumer.descendants.each do |klass|
-        # TODO: remove this when ActiveRecordConsumer uses batching
-        next if klass == Deimos::ActiveRecordConsumer # "abstract" class
-
-        stub_consumer(klass)
-      end
-
-      Deimos::BatchConsumer.descendants.each do |klass|
-        next if klass == Deimos::ActiveRecordConsumer # "abstract" class
-
-        stub_batch_consumer(klass)
-      end
+      warn('stub_producers_and_consumers! is no longer necessary and this method will be removed in 3.0')
     end
 
-    # Stub a given producer class.
-    # @param klass [Class < Deimos::Producer]
-    def stub_producer(klass)
-      allow(klass).to receive(:encoder) do
-        create_encoder(klass.config[:schema], klass.config[:namespace])
-      end
-      allow(klass).to receive(:key_encoder) do
-        create_encoder(klass.config[:key_schema], klass.config[:namespace])
-      end
+    # @deprecated
+    def stub_producer(_klass)
+      warn('Stubbing producers is no longer necessary and this method will be removed in 3.0')
     end
 
-    # Stub a given consumer class.
-    # @param klass [Class < Deimos::Consumer]
-    def stub_consumer(klass)
-      _stub_base_consumer(klass)
-      klass.class_eval do
-        alias_method(:old_consume, :consume) unless self.instance_methods.include?(:old_consume)
-      end
-      allow_any_instance_of(klass).to receive(:consume) do |instance, payload, metadata|
-        metadata[:key] = klass.new.decode_key(metadata[:key]) if klass.config[:key_configured]
-        instance.old_consume(payload, metadata)
-      end
+    # @deprecated
+    def stub_consumer(_klass)
+      warn('Stubbing consumers is no longer necessary and this method will be removed in 3.0')
     end
 
-    # Stub a given batch consumer class.
-    # @param klass [Class < Deimos::BatchConsumer]
-    def stub_batch_consumer(klass)
-      _stub_base_consumer(klass)
+    # @deprecated
+    def stub_batch_consumer(_klass)
+      warn('Stubbing batch consumers is no longer necessary and this method will be removed in 3.0')
     end
 
     # get the difference of 2 hashes.
@@ -166,7 +81,7 @@ module Deimos
 
     # :nodoc:
     def _frk_failure_message(topic, message, key=nil, partition_key=nil, was_negated=false)
-      messages = Deimos::TestHelpers.sent_messages.
+      messages = Deimos::Backends::Test.sent_messages.
         select { |m| m[:topic] == topic }.
         map { |m| m.except(:topic) }
       message_string = ''
@@ -200,7 +115,7 @@ module Deimos
                   msg
                 end
       match do |topic|
-        Deimos::TestHelpers.sent_messages.any? do |m|
+        Deimos::Backends::Test.sent_messages.any? do |m|
           hash_matcher = RSpec::Matchers::BuiltIn::Match.new(message)
           hash_matcher.send(:match,
                             message,
@@ -231,7 +146,7 @@ module Deimos
     # Clear all sent messages - e.g. if we want to check that
     # particular messages were sent or not sent after a point in time.
     def clear_kafka_messages!
-      Deimos::TestHelpers.sent_messages.clear
+      Deimos::Backends::Test.sent_messages.clear
     end
 
     # test that a message was sent on the given topic.
@@ -241,14 +156,14 @@ module Deimos
     # @param key [String|Integer]
     # @return [Boolean]
     def was_message_sent?(message, topic, key=nil)
-      Deimos::TestHelpers.sent_messages.any? do |m|
+      Deimos::Backends::Test.sent_messages.any? do |m|
         message == m[:payload] && m[:topic] == topic &&
           (key.present? ? m[:key] == key : true)
       end
     end
 
     # Test that a given handler will consume a given payload correctly, i.e.
-    # that the Avro schema is correct. If
+    # that the schema is correct. If
     # a block is given, that block will be executed when `consume` is called.
     # Otherwise it will just confirm that `consume` is called at all.
     # @param handler_class_or_topic [Class|String] Class which inherits from
@@ -300,33 +215,18 @@ module Deimos
       ).send(:process_message, payload)
     end
 
-    # Check to see that a given message will fail due to Avro errors.
+    # Check to see that a given message will fail due to validation errors.
     # @param handler_class [Class]
     # @param payload [Hash]
     def test_consume_invalid_message(handler_class, payload)
-      handler = handler_class.new
-      allow(handler_class).to receive(:new).and_return(handler)
-      listener = double('listener',
-                        handler_class: handler_class,
-                        encoding: nil)
-      message = double('message',
-                       key: _key_from_consumer(handler_class),
-                       partition_key: nil,
-                       partition: 1,
-                       offset: 1,
-                       value: payload)
-
       expect {
-        Phobos::Actions::ProcessMessage.new(
-          listener: listener,
-          message: message,
-          listener_metadata: { topic: 'my-topic' }
-        ).send(:process_message, payload)
-      }.to raise_error(Avro::SchemaValidator::ValidationError)
+        handler_class.decoder.validate(payload,
+                                       schema: handler_class.decoder.schema)
+      }.to raise_error
     end
 
     # Test that a given handler will consume a given batch payload correctly,
-    # i.e. that the Avro schema is correct. If
+    # i.e. that the schema is correct. If
     # a block is given, that block will be executed when `consume` is called.
     # Otherwise it will just confirm that `consume` is called at all.
     # @param handler_class_or_topic [Class|String] Class which inherits from
@@ -384,7 +284,7 @@ module Deimos
       action.send(:execute)
     end
 
-    # Check to see that a given message will fail due to Avro errors.
+    # Check to see that a given message will fail due to validation errors.
     # @param handler_class [Class]
     # @param payloads [Array<Hash>]
     def test_consume_batch_invalid_message(handler_class, payloads)
@@ -418,33 +318,21 @@ module Deimos
       allow(action).to receive(:handle_error) { |e| raise e }
 
       expect { action.send(:execute) }.
-        to raise_error(Avro::SchemaValidator::ValidationError)
-    end
-
-    # @param schema1 [String|Hash] a file path, JSON string, or
-    # hash representing a schema.
-    # @param schema2 [String|Hash] a file path, JSON string, or
-    # hash representing a schema.
-    # @return [Boolean] true if the schemas are compatible, false otherwise.
-    def self.schemas_compatible?(schema1, schema2)
-      json1, json2 = [schema1, schema2].map do |schema|
-        if schema.is_a?(String)
-          schema = File.read(schema) unless schema.strip.starts_with?('{') # file path
-          MultiJson.load(schema)
-        else
-          schema
-        end
-      end
-      avro_schema1 = Avro::Schema.real_parse(json1, {})
-      avro_schema2 = Avro::Schema.real_parse(json2, {})
-      Avro::SchemaCompatibility.mutual_read?(avro_schema1, avro_schema2)
+        to raise_error
     end
 
   private
 
     def _key_from_consumer(consumer)
-      if consumer.config[:key_field] || consumer.config[:key_schema]
-        { 'test' => 1 }
+      if consumer.config[:key_field]
+        { consumer.config[:key_field] => 1 }
+      elsif consumer.config[:key_schema]
+        backend = consumer.decoder
+        old_schema = backend.schema
+        backend.schema = consumer.config[:key_schema]
+        key = backend.schema_fields.map { |field| [field.name, 1] }.to_h
+        backend.schema = old_schema
+        key
       elsif consumer.config[:no_keys]
         nil
       else
@@ -461,19 +349,5 @@ module Deimos
 
       handler.handler.constantize
     end
-
-    # Stub shared methods between consumers/batch consumers
-    # @param [Class < Deimos::BaseConsumer] klass Consumer class to stub
-    def _stub_base_consumer(klass)
-      allow(klass).to receive(:decoder) do
-        create_decoder(klass.config[:schema], klass.config[:namespace])
-      end
-
-      if klass.config[:key_schema] # rubocop:disable Style/GuardClause
-        allow(klass).to receive(:key_decoder) do
-          create_decoder(klass.config[:key_schema], klass.config[:namespace])
-        end
-      end
-    end
   end
 end

data/lib/deimos/utils/db_producer.rb

@@ -93,6 +93,11 @@ module Deimos
           end
         end
         Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+        Deimos.config.metrics&.increment(
+          'db_producer.process',
+          tags: %W(topic:#{@current_topic}),
+          by: messages.size
+        )
         return false if batch_size < BATCH_SIZE
 
         KafkaTopicInfo.heartbeat(@current_topic, @id) # keep alive

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.4.0-beta7'
+  VERSION = '1.5.0-beta2'
 end

data/lib/deimos.rb

@@ -1,12 +1,8 @@
 # frozen_string_literal: true
 
-require 'avro-patches'
-require 'avro_turf'
 require 'phobos'
 require 'deimos/version'
 require 'deimos/config/configuration'
-require 'deimos/avro_data_encoder'
-require 'deimos/avro_data_decoder'
 require 'deimos/producer'
 require 'deimos/active_record_producer'
 require 'deimos/active_record_consumer'
@@ -15,16 +11,19 @@ require 'deimos/batch_consumer'
 require 'deimos/instrumentation'
 require 'deimos/utils/lag_reporter'
 
-require 'deimos/publish_backend'
+require 'deimos/backends/base'
 require 'deimos/backends/kafka'
 require 'deimos/backends/kafka_async'
+require 'deimos/backends/test'
+
+require 'deimos/schema_backends/base'
 
 require 'deimos/monkey_patches/ruby_kafka_heartbeat'
-require 'deimos/monkey_patches/schema_store'
 require 'deimos/monkey_patches/phobos_producer'
 require 'deimos/monkey_patches/phobos_cli'
 
 require 'deimos/railtie' if defined?(Rails)
+
 if defined?(ActiveRecord)
   require 'deimos/kafka_source'
   require 'deimos/kafka_topic_info'
@@ -33,6 +32,7 @@ if defined?(ActiveRecord)
   require 'deimos/utils/executor.rb'
   require 'deimos/utils/db_producer.rb'
 end
+
 require 'deimos/utils/inline_consumer'
 require 'yaml'
 require 'erb'
@@ -40,6 +40,22 @@ require 'erb'
 # Parent module.
 module Deimos
   class << self
+    # @return [Class < Deimos::SchemaBackends::Base]
+    def schema_backend_class
+      backend = Deimos.config.schema.backend.to_s
+
+      require "deimos/schema_backends/#{backend}"
+
+      "Deimos::SchemaBackends::#{backend.classify}".constantize
+    end
+
+    # @param schema [String|Symbol]
+    # @param namespace [String]
+    # @return [Deimos::SchemaBackends::Base]
+    def schema_backend(schema:, namespace:)
+      schema_backend_class.new(schema: schema, namespace: namespace)
+    end
+
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
     def start_db_backend!(thread_count: 1)

data/lib/tasks/deimos.rake

@@ -7,7 +7,7 @@ namespace :deimos do
   desc 'Starts Deimos in the rails environment'
   task start: :environment do
     Deimos.configure do |config|
-      config.producers.backend = :kafka_sync if config.producers.backend == :kafka_async
+      config.producers.backend = :kafka if config.producers.backend == :kafka_async
     end
     ENV['DEIMOS_RAKE_TASK'] = 'true'
     STDOUT.sync = true

data/spec/active_record_consumer_spec.rb

@@ -119,5 +119,12 @@ module ActiveRecordProducerTest
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
 
+    it 'should coerce int values to datetimes' do
+      column = Widget.columns.find { |c| c.name == 'some_datetime_int' }
+      expect(MyConsumer.new.send(:_coerce_field, column, 1_579_046_400)).to eq('2020-01-14 19:00:00 -0500')
+      expect(MyConsumer.new.send(:_coerce_field, column, '1579046400')).to eq('2020-01-14 19:00:00 -0500')
+      expect(MyConsumer.new.send(:_coerce_field, column, 'some-other-val')).to eq('some-other-val')
+    end
+
   end
 end

data/spec/{publish_backend_spec.rb → backends/base_spec.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-RSpec.describe Deimos::PublishBackend do
+RSpec.describe Deimos::Backends::Base do
   include_context 'with publish_backend'
   it 'should call execute' do
     expect(described_class).to receive(:execute).

data/spec/backends/db_spec.rb

@@ -4,6 +4,11 @@ each_db_config(Deimos::Backends::Db) do
   include_context 'with publish_backend'
 
   it 'should save to the database' do
+    expect(Deimos.config.metrics).to receive(:increment).with(
+      'db_producer.insert',
+      tags: %w(topic:my-topic),
+      by: 3
+    )
     described_class.publish(producer_class: MyProducer, messages: messages)
     records = Deimos::KafkaMessage.all
     expect(records.size).to eq(3)

data/spec/batch_consumer_spec.rb

@@ -80,11 +80,6 @@ module ConsumerTest
       end
 
       it 'should decode payloads for all messages in the batch' do
-        expect_any_instance_of(Deimos::AvroDataDecoder).
-          to receive(:decode).with(batch[0])
-        expect_any_instance_of(Deimos::AvroDataDecoder).
-          to receive(:decode).with(batch[1])
-
         test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
           # Mock decoder simply returns the payload
           expect(received).to eq(batch)
@@ -110,7 +105,6 @@ module ConsumerTest
           key_config plain: true
         end
         stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
-        stub_batch_consumer(consumer_class)
 
         test_consume_batch('my_batch_consume_topic', batch, keys: [1, 2]) do |_received, metadata|
           expect(metadata[:keys]).to eq([1, 2])
@@ -132,7 +126,6 @@ module ConsumerTest
           end
         end
         stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
-        stub_batch_consumer(consumer_class)
         allow(Deimos.config.metrics).to receive(:histogram)
       end
 
@@ -215,7 +208,6 @@ module ConsumerTest
           end
         end
         stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
-        stub_batch_consumer(consumer_class)
         allow(Deimos.config.metrics).to receive(:histogram)
       end
 

data/spec/config/configuration_spec.rb

@@ -5,7 +5,7 @@ end
 describe Deimos, 'configuration' do
   it 'should configure with deprecated fields' do
     logger = Logger.new(nil)
-    Deimos.configure do
+    described_class.configure do
       kafka_logger logger
       reraise_consumer_errors true
       schema_registry_url 'http://schema.registry'
@@ -22,25 +22,25 @@ describe Deimos, 'configuration' do
       report_lag true
     end
 
-    expect(Deimos.config.kafka.logger).to eq(logger)
-    expect(Deimos.config.consumers.reraise_errors).to eq(true)
-    expect(Deimos.config.schema.registry_url).to eq('http://schema.registry')
-    expect(Deimos.config.kafka.seed_brokers).to eq('whatever')
-    expect(Deimos.config.producers.schema_namespace).to eq('namespace')
-    expect(Deimos.config.producers.topic_prefix).to eq('prefix')
-    expect(Deimos.config.producers.disabled).to eq(true)
-    expect(Deimos.config.kafka.ssl.enabled).to eq(true)
-    expect(Deimos.config.kafka.ssl.ca_cert).to eq('cert')
-    expect(Deimos.config.kafka.ssl.client_cert).to eq('cert')
-    expect(Deimos.config.kafka.ssl.client_cert_key).to eq('key')
-    expect(Deimos.config.producers.backend).to eq('db')
-    expect(Deimos.config.consumers.report_lag).to eq(true)
+    expect(described_class.config.kafka.logger).to eq(logger)
+    expect(described_class.config.consumers.reraise_errors).to eq(true)
+    expect(described_class.config.schema.registry_url).to eq('http://schema.registry')
+    expect(described_class.config.kafka.seed_brokers).to eq('whatever')
+    expect(described_class.config.producers.schema_namespace).to eq('namespace')
+    expect(described_class.config.producers.topic_prefix).to eq('prefix')
+    expect(described_class.config.producers.disabled).to eq(true)
+    expect(described_class.config.kafka.ssl.enabled).to eq(true)
+    expect(described_class.config.kafka.ssl.ca_cert).to eq('cert')
+    expect(described_class.config.kafka.ssl.client_cert).to eq('cert')
+    expect(described_class.config.kafka.ssl.client_cert_key).to eq('key')
+    expect(described_class.config.producers.backend).to eq('db')
+    expect(described_class.config.consumers.report_lag).to eq(true)
   end
 
   it 'reads existing Phobos config YML files' do
-    Deimos.config.reset!
-    Deimos.configure { |c| c.phobos_config_file = File.join(File.dirname(__FILE__), '..', 'phobos.yml') }
-    expect(Deimos.config.phobos_config).to match(
+    described_class.config.reset!
+    described_class.configure { |c| c.phobos_config_file = File.join(File.dirname(__FILE__), '..', 'phobos.yml') }
+    expect(described_class.config.phobos_config).to match(
       logger: an_instance_of(Logger),
       backoff: { min_ms: 1000, max_ms: 60_000 },
       consumer: {
@@ -112,8 +112,8 @@ describe Deimos, 'configuration' do
   specify '#phobos_config' do
     logger1 = Logger.new(nil)
     logger2 = Logger.new(nil)
-    Deimos.config.reset!
-    Deimos.configure do
+    described_class.config.reset!
+    described_class.configure do
       phobos_logger logger1
       kafka do
         logger logger2
@@ -169,7 +169,7 @@ describe Deimos, 'configuration' do
       end
     end
 
-    expect(Deimos.config.phobos_config).
+    expect(described_class.config.phobos_config).
       to match(
         logger: an_instance_of(Logger),
         backoff: { min_ms: 5, max_ms: 10 },

data/spec/consumer_spec.rb

@@ -150,7 +150,6 @@ module ConsumerTest
           end
         end
         stub_const('ConsumerTest::MyConsumer', consumer_class)
-        stub_consumer(consumer_class)
       end
 
       it 'should consume a message' do

data/spec/deimos_spec.rb

@@ -67,10 +67,6 @@ describe Deimos do
   end
 
   describe '#start_db_backend!' do
-    before(:each) do
-      allow(described_class).to receive(:run_db_backend)
-    end
-
     it 'should start if backend is db and thread_count is > 0' do
       signal_handler = instance_double(Deimos::Utils::SignalHandler)
       allow(signal_handler).to receive(:run!)

data/spec/kafka_source_spec.rb

@@ -89,6 +89,14 @@ module KafkaSourceSpec
       expect('my-topic-the-second').to have_sent(nil, 1)
     end
 
+    it 'should not call generate_payload but still publish a nil payload for deletion' do
+      widget = Widget.create!(widget_id: '808', name: 'delete_me!')
+      expect(Deimos::ActiveRecordProducer).not_to receive(:generate_payload)
+      widget.destroy
+      expect('my-topic').to have_sent(nil, widget.id)
+      expect('my-topic-the-second').to have_sent(nil, widget.id)
+    end
+
     it 'should send events on import' do
       widgets = (1..3).map do |i|
         Widget.new(widget_id: i, name: "Widget #{i}")