deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

data/spec/consumer_spec.rb

@@ -4,33 +4,44 @@
 # rubocop:disable Metrics/ModuleLength
 module ConsumerTest
   describe Deimos::Consumer, 'Message Consumer' do
+    let(:use_schema_classes) { false }
+    let(:reraise_errors) { false }
     prepend_before(:each) do
       # :nodoc:
       consumer_class = Class.new(described_class) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config field: 'test_id'
 
         # :nodoc:
-        def fatal_error?(_exception, payload, _metadata)
-          payload.to_s == 'fatal'
+        def fatal_error?(_exception, messages)
+          messages.payloads.first&.dig(:test_id) == ['fatal']
         end
 
         # :nodoc:
-        def consume(_payload, _metadata)
-          raise 'This should not be called unless call_original is set'
+        def consume_message(message)
+          message.payload
         end
       end
       stub_const('ConsumerTest::MyConsumer', consumer_class)
+      route_usc = use_schema_classes
+      route_rre = reraise_errors
+      Karafka::App.routes.redraw do
+        topic 'my_consume_topic' do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config field: 'test_id'
+          consumer consumer_class
+          use_schema_classes route_usc
+          reraise_errors route_rre
+        end
+      end
     end
 
     describe 'consume' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
+        let(:use_schema_classes) { use_schema_classes }
         context "with Schema Class consumption #{setting}" do
 
           before(:each) do
             Deimos.configure do |config|
-              config.schema.use_schema_classes = use_schema_classes
               config.schema.use_full_namespace = true
             end
           end
@@ -45,26 +56,9 @@ module ConsumerTest
           end
 
           it 'should consume a nil message' do
-            test_consume_message(MyConsumer, nil) do |payload, _metadata|
-              expect(payload).to be_nil
-            end
-          end
-
-          it 'should consume a message idempotently' do
-            # testing for a crash and re-consuming the same message/metadata
-            key = { 'test_id' => 'foo' }
-            test_metadata = { key: key }
-            allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
-              k['test_id']
+            test_consume_message(MyConsumer, nil, key: 'foo') do
+              expect(messages).to be_empty
             end
-            MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                              expect(metadata[:key]).to eq('foo')
-                                            end
-            MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                              expect(metadata[:key]).to eq('foo')
-                                            end
           end
 
           it 'should consume a message on a topic' do
@@ -77,83 +71,82 @@ module ConsumerTest
           end
 
           it 'should fail on invalid message' do
-            test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
+            expect { test_consume_message(MyConsumer, { 'invalid' => 'key' }) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should fail if reraise is false but fatal_error is true' do
-            Deimos.configure { |config| config.consumers.reraise_errors = false }
-            test_consume_invalid_message(MyConsumer, 'fatal')
+            expect { test_consume_message(MyConsumer, {test_id: 'fatal'}) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should fail if fatal_error is true globally' do
-            Deimos.configure do |config|
-              config.consumers.fatal_error = proc { true }
-              config.consumers.reraise_errors = false
-            end
-            test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
+            set_karafka_config(:fatal_error, proc { true })
+            expect { test_consume_message(MyConsumer, { 'invalid' => 'key' }) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should fail on message with extra fields' do
-            test_consume_invalid_message(MyConsumer,
+            allow_any_instance_of(Deimos::SchemaBackends::AvroValidation).
+              to receive(:coerce) { |_, m| m.with_indifferent_access }
+            expect { test_consume_message(MyConsumer,
                                          { 'test_id' => 'foo',
                                          'some_int' => 123,
-                                         'extra_field' => 'field name' })
+                                         'extra_field' => 'field name' }) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should not fail when before_consume fails without reraising errors' do
-            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            set_karafka_config(:reraise_errors, false)
             expect {
               test_consume_message(
                 MyConsumer,
                 { 'test_id' => 'foo',
-                  'some_int' => 123 },
-                skip_expectation: true
-              ) { raise 'OH NOES' }
+                  'some_int' => 123 }) { raise 'OH NOES' }
             }.not_to raise_error
           end
 
           it 'should not fail when consume fails without reraising errors' do
-            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            set_karafka_config(:reraise_errors, false)
+            allow(Deimos::ProducerMiddleware).to receive(:call) { |m| m[:payload] = m[:payload].to_json; m }
             expect {
               test_consume_message(
                 MyConsumer,
-                { 'invalid' => 'key' },
-                skip_expectation: true
-              )
+                { 'invalid' => 'key' })
             }.not_to raise_error
           end
-
-          it 'should call original' do
-            expect {
-              test_consume_message(MyConsumer,
-                                   { 'test_id' => 'foo', 'some_int' => 123 },
-                                   call_original: true)
-            }.to raise_error('This should not be called unless call_original is set')
-          end
         end
       end
 
       context 'with overriden schema classes' do
 
         before(:each) do
+          set_karafka_config(:use_schema_classes, true)
           Deimos.configure do |config|
-            config.schema.use_schema_classes = true
             config.schema.use_full_namespace = true
           end
         end
 
         prepend_before(:each) do
           consumer_class = Class.new(described_class) do
-            schema 'MyUpdatedSchema'
-            namespace 'com.my-namespace'
-            key_config field: 'test_id'
-
             # :nodoc:
-            def consume(_payload, _metadata)
-              raise 'This should not be called unless call_original is set'
+            def consume_message(message)
+              message.payload
             end
           end
           stub_const('ConsumerTest::MyConsumer', consumer_class)
+          Deimos.config.schema.use_schema_classes = true
+          Karafka::App.routes.redraw do
+            topic 'my_consume_topic' do
+              schema 'MyUpdatedSchema'
+              namespace 'com.my-namespace'
+              key_config field: 'test_id'
+              consumer consumer_class
+            end
+          end
+        end
+        after(:each) do
+          Karafka::App.routes.clear
         end
 
         it 'should consume messages' do
@@ -169,93 +162,6 @@ module ConsumerTest
       end
     end
 
-    describe 'decode_key' do
-
-      it 'should use the key field in the value if set' do
-        # actual decoding is disabled in test
-        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
-        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
-      end
-
-      it 'should use the key schema if set' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config schema: 'MySchema_key'
-        end
-        stub_const('ConsumerTest::MySchemaConsumer', consumer_class)
-        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
-        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
-      end
-
-      it 'should not decode if plain is set' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
-        end
-        stub_const('ConsumerTest::MyNonEncodedConsumer', consumer_class)
-        expect(MyNonEncodedConsumer.new.decode_key('123')).to eq('123')
-      end
-
-      it 'should error with nothing set' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-        end
-        stub_const('ConsumerTest::MyErrorConsumer', consumer_class)
-        expect { MyErrorConsumer.new.decode_key('123') }.
-          to raise_error('No key config given - if you are not decoding keys, please use `key_config plain: true`')
-      end
-
-    end
-
-    describe 'timestamps' do
-      before(:each) do
-        # :nodoc:
-        consumer_class = Class.new(described_class) do
-          schema 'MySchemaWithDateTimes'
-          namespace 'com.my-namespace'
-          key_config plain: true
-
-          # :nodoc:
-          def consume(_payload, _metadata)
-            raise 'This should not be called unless call_original is set'
-          end
-        end
-        stub_const('ConsumerTest::MyConsumer', consumer_class)
-      end
-
-      it 'should consume a message' do
-        expect(Deimos.config.metrics).to receive(:histogram).twice
-        test_consume_message('my_consume_topic',
-                             { 'test_id' => 'foo',
-                             'some_int' => 123,
-                             'updated_at' => Time.now.to_i,
-                             'timestamp' => 2.minutes.ago.to_s }) do |payload, _metadata|
-                               expect(payload['test_id']).to eq('foo')
-                             end
-      end
-
-      it 'should fail nicely when timestamp wrong format' do
-        expect(Deimos.config.metrics).to receive(:histogram).twice
-        test_consume_message('my_consume_topic',
-                             { 'test_id' => 'foo',
-                             'some_int' => 123,
-                             'updated_at' => Time.now.to_i,
-                             'timestamp' => 'dffdf' }) do |payload, _metadata|
-                               expect(payload['test_id']).to eq('foo')
-                             end
-        test_consume_message('my_consume_topic',
-                             { 'test_id' => 'foo',
-                             'some_int' => 123,
-                             'updated_at' => Time.now.to_i,
-                             'timestamp' => '' }) do |payload, _metadata|
-                               expect(payload['test_id']).to eq('foo')
-                             end
-      end
-
-    end
   end
 end
 # rubocop:enable Metrics/ModuleLength

data/spec/deimos_spec.rb

@@ -2,72 +2,12 @@
 
 describe Deimos do
 
-  let(:phobos_configuration) do
-    { 'logger' =>
-        { 'file' => 'log/phobos.log',
-          'stdout_json' => false,
-          'level' => 'debug',
-          'ruby_kafka' =>
-              { 'level' => 'debug' } },
-      'kafka' =>
-        { 'client_id' => 'phobos',
-          'connect_timeout' => 15,
-          'socket_timeout' => 15,
-          'seed_brokers' => 'my_seed_broker.com',
-          'ssl_ca_cert' => 'my_ssl_ca_cert',
-          'ssl_client_cert' => 'my_ssl_client_cert',
-          'ssl_client_cert_key' => 'my_ssl_client_cert_key' },
-      'producer' =>
-        { 'ack_timeout' => 5,
-          'required_acks' => :all,
-          'max_retries' => 2,
-          'retry_backoff' => 1,
-          'max_buffer_size' => 10_000,
-          'max_buffer_bytesize' => 10_000_000,
-          'compression_codec' => nil,
-          'compression_threshold' => 1,
-          'max_queue_size' => 10_000,
-          'delivery_threshold' => 0,
-          'delivery_interval' => 0 },
-      'consumer' =>
-        { 'session_timeout' => 300,
-          'offset_commit_interval' => 10,
-          'offset_commit_threshold' => 0,
-          'heartbeat_interval' => 10 },
-      'backoff' =>
-        { 'min_ms' => 1000,
-          'max_ms' => 60_000 },
-      'listeners' => [
-        { 'handler' => 'ConsumerTest::MyConsumer',
-          'topic' => 'my_consume_topic',
-          'group_id' => 'my_group_id',
-          'max_bytes_per_partition' => 524_288 },
-        { 'handler' => 'ConsumerTest::MyBatchConsumer',
-          'topic' => 'my_batch_consume_topic',
-          'group_id' => 'my_batch_group_id',
-          'delivery' => 'inline_batch' }
-      ],
-      'custom_logger' => nil,
-      'custom_kafka_logger' => nil }
-  end
-
-  let(:config_path) { File.join(File.dirname(__FILE__), 'phobos.yml') }
-
   it 'should have a version number' do
     expect(Deimos::VERSION).not_to be_nil
   end
 
-  it 'should error if required_acks is not all' do
-    expect {
-      described_class.configure do |config|
-        config.producers.backend = :db
-        config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.bad_db.yml')
-      end
-    }.to raise_error('Cannot set producers.backend to :db unless producers.required_acks is set to ":all"!')
-  end
-
-  describe '#start_db_backend!' do
-    it 'should start if backend is db and thread_count is > 0' do
+  describe '#start_outbox_backend!' do
+    it 'should start if backend is outbox and thread_count is > 0' do
       signal_handler = instance_double(Sigurd::SignalHandler)
       allow(signal_handler).to receive(:run!)
       expect(Sigurd::Executor).to receive(:new).
@@ -77,9 +17,9 @@ describe Deimos do
         signal_handler
       end
       described_class.configure do |config|
-        config.producers.backend = :db
+        config.producers.backend = :outbox
       end
-      described_class.start_db_backend!(thread_count: 2)
+      described_class.start_outbox_backend!(thread_count: 2)
     end
 
     it 'should not start if backend is not db' do
@@ -87,83 +27,27 @@ describe Deimos do
       described_class.configure do |config|
         config.producers.backend = :kafka
       end
-      expect { described_class.start_db_backend!(thread_count: 2) }.
-        to raise_error('Publish backend is not set to :db, exiting')
+      expect { described_class.start_outbox_backend!(thread_count: 2) }.
+        to raise_error('Publish backend is not set to :outbox, exiting')
     end
 
     it 'should not start if thread_count is nil' do
       expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
-        config.producers.backend = :db
+        config.producers.backend = :outbox
       end
-      expect { described_class.start_db_backend!(thread_count: nil) }.
+      expect { described_class.start_outbox_backend!(thread_count: nil) }.
         to raise_error('Thread count is not given or set to zero, exiting')
     end
 
     it 'should not start if thread_count is 0' do
       expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
-        config.producers.backend = :db
+        config.producers.backend = :outbox
       end
-      expect { described_class.start_db_backend!(thread_count: 0) }.
+      expect { described_class.start_outbox_backend!(thread_count: 0) }.
         to raise_error('Thread count is not given or set to zero, exiting')
     end
   end
 
-  describe 'delivery configuration' do
-    before(:each) do
-      allow(YAML).to receive(:load).and_return(phobos_configuration)
-    end
-
-    it 'should not raise an error with properly configured handlers' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-            delivery :message
-          end
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-            delivery :batch
-          end
-          consumer do
-            class_name 'ConsumerTest::MyBatchConsumer'
-            delivery :inline_batch
-          end
-        end
-      }.not_to raise_error
-    end
-
-    it 'should raise an error if inline_batch listeners do not implement consume_batch' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-            delivery :inline_batch
-          end
-        end
-      }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
-    end
-
-    it 'should raise an error if Consumers do not have message or batch delivery' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyBatchConsumer'
-            delivery :message
-          end
-        end
-      }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
-    end
-
-    it 'should treat nil as `batch`' do
-      expect {
-        described_class.configure do
-          consumer do
-            class_name 'ConsumerTest::MyConsumer'
-          end
-        end
-      }.not_to raise_error
-    end
-  end
 end

data/spec/kafka_source_spec.rb

@@ -17,18 +17,10 @@ module KafkaSourceSpec
 
       # Dummy producer which mimicks the behavior of a real producer
       class WidgetProducer < Deimos::ActiveRecordProducer
-        topic 'my-topic'
-        namespace 'com.my-namespace'
-        schema 'Widget'
-        key_config field: :id
       end
 
       # Dummy producer which mimicks the behavior of a real producer
       class WidgetProducerTheSecond < Deimos::ActiveRecordProducer
-        topic 'my-topic-the-second'
-        namespace 'com.my-namespace'
-        schema 'WidgetTheSecond'
-        key_config field: :id
       end
 
       # Dummy class we can include the mixin in. Has a backing table created
@@ -51,6 +43,22 @@ module KafkaSourceSpec
 
     before(:each) do
       Widget.delete_all
+      Karafka::App.routes.redraw do
+        topic 'my-topic' do
+          namespace 'com.my-namespace'
+          schema 'Widget'
+          key_config field: :id
+          producer_class WidgetProducer
+        end
+
+        topic 'my-topic-the-second' do
+          namespace 'com.my-namespace'
+          schema 'WidgetTheSecond'
+          key_config field: :id
+          producer_class WidgetProducerTheSecond
+        end
+
+      end
     end
 
     it 'should send events on creation, update, and deletion' do
@@ -206,10 +214,9 @@ module KafkaSourceSpec
     context 'with DB backend' do
       before(:each) do
         Deimos.configure do |config|
-          config.producers.backend = :db
+          config.producers.backend = :outbox
         end
         setup_db(DB_OPTIONS.last) # sqlite
-        allow(Deimos::Producer).to receive(:produce_batch).and_call_original
       end
 
       it 'should save to the DB' do
@@ -309,46 +316,6 @@ module KafkaSourceSpec
       end
     end
 
-    context 'with AR models that implement the kafka_producer interface' do
-      before(:each) do
-        # Dummy class we can include the mixin in. Has a backing table created
-        # earlier and has the import hook disabled
-        deprecated_class = Class.new(ActiveRecord::Base) do
-          include Deimos::KafkaSource
-          self.table_name = 'widgets'
-
-          # :nodoc:
-          def self.kafka_config
-            {
-              update: true,
-              delete: true,
-              import: false,
-              create: true
-            }
-          end
-
-          # :nodoc:
-          def self.kafka_producer
-            WidgetProducer
-          end
-        end
-        stub_const('WidgetDeprecated', deprecated_class)
-        WidgetDeprecated.reset_column_information
-      end
-
-      it 'logs a warning and sends the message as usual' do
-        expect(Deimos.config.logger).to receive(:warn).with({ message: WidgetDeprecated::DEPRECATION_WARNING })
-        widget = WidgetDeprecated.create(widget_id: 1, name: 'Widget 1')
-        expect('my-topic').to have_sent({
-                                          widget_id: 1,
-                                          name: 'Widget 1',
-                                          id: widget.id,
-                                          created_at: anything,
-                                          updated_at: anything
-                                        }, widget.id)
-      end
-    end
-
     context 'with AR models that do not implement any producer interface' do
       before(:each) do
         # Dummy class we can include the mixin in. Has a backing table created
@@ -371,10 +338,10 @@ module KafkaSourceSpec
         WidgetBuggy.reset_column_information
       end
 
-      it 'raises a NotImplementedError exception' do
+      it 'raises a MissingImplementationError exception' do
         expect {
           WidgetBuggy.create(widget_id: 1, name: 'Widget 1')
-        }.to raise_error(NotImplementedError)
+        }.to raise_error(Deimos::MissingImplementationError)
       end
     end
   end

data/spec/karafka/karafka.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+class KarafkaApp < Karafka::App
+  setup do |config|
+    config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
+    config.client_id = 'example_app'
+    # Recreate consumers with each batch. This will allow Rails code reload to work in the
+    # development mode. Otherwise Karafka process would not be aware of code changes
+    config.consumer_persistence = !Rails.env.development?
+  end
+
+  # Comment out this part if you are not using instrumentation and/or you are not
+  # interested in logging events for certain environments. Since instrumentation
+  # notifications add extra boilerplate, if you want to achieve max performance,
+  # listen to only what you really need for given environment.
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
+  # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
+
+  # This logger prints the producer development info using the Karafka logger.
+  # It is similar to the consumer logger listener but producer oriented.
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(
+      # Log producer operations using the Karafka logger
+      Karafka.logger,
+      # If you set this to true, logs will contain each message details
+      # Please note, that this can be extensive
+      log_messages: false
+    )
+  )
+
+  # You can subscribe to all consumer related errors and record/track then that way
+  #
+  # Karafka.monitor.subscribe 'error.occurred' do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  # You can subscribe to all producer related errors and record/track then that way
+  # Please note, that producer and consumer have their own notifications pipeline so you need to
+  # setup error tracking independently for each of them
+  #
+  # Karafka.producer.monitor.subscribe('error.occurred') do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  routes.draw do
+    # Uncomment this if you use Karafka with ActiveJob
+    # You need to define the topic per each queue name you use
+    # active_job_topic :default
+    #    topic :example do
+      # Uncomment this if you want Karafka to manage your topics configuration
+      # Managing topics configuration via routing will allow you to ensure config consistency
+      # across multiple environments
+      #
+      # config(partitions: 2, 'cleanup.policy': 'compact')
+    # consumer ExampleConsumer
+    # end
+  end
+end
+
+# Karafka now features a Web UI!
+# Visit the setup documentation to get started and enhance your experience.
+#
+# https://karafka.io/docs/Web-UI-Getting-Started
+Deimos.setup_karafka