deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

data/spec/active_record_consume/batch_slicer_spec.rb

@@ -4,12 +4,12 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
   describe '#slice' do
     let(:batch) do
       [
-        Deimos::Message.new({ v: 1 }, nil, key: 'C'),
-        Deimos::Message.new({ v: 123 }, nil, key: 'A'),
-        Deimos::Message.new({ v: 999 }, nil, key: 'B'),
-        Deimos::Message.new({ v: 456 }, nil, key: 'A'),
-        Deimos::Message.new({ v: 2 }, nil, key: 'C'),
-        Deimos::Message.new({ v: 3 }, nil, key: 'C')
+        Deimos::Message.new({ v: 1 }, key: 'C'),
+        Deimos::Message.new({ v: 123 }, key: 'A'),
+        Deimos::Message.new({ v: 999 }, key: 'B'),
+        Deimos::Message.new({ v: 456 }, key: 'A'),
+        Deimos::Message.new({ v: 2 }, key: 'C'),
+        Deimos::Message.new({ v: 3 }, key: 'C')
       ]
     end
 
@@ -19,16 +19,16 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
       expect(slices).
         to match([
                    match_array([
-                                 Deimos::Message.new({ v: 1 }, nil, key: 'C'),
-                                 Deimos::Message.new({ v: 123 }, nil, key: 'A'),
-                                 Deimos::Message.new({ v: 999 }, nil, key: 'B')
+                                 Deimos::Message.new({ v: 1 }, key: 'C'),
+                                 Deimos::Message.new({ v: 123 }, key: 'A'),
+                                 Deimos::Message.new({ v: 999 }, key: 'B')
                                ]),
                    match_array([
-                                 Deimos::Message.new({ v: 456 }, nil, key: 'A'),
-                                 Deimos::Message.new({ v: 2 }, nil, key: 'C')
+                                 Deimos::Message.new({ v: 456 }, key: 'A'),
+                                 Deimos::Message.new({ v: 2 }, key: 'C')
                                ]),
                    match_array([
-                                 Deimos::Message.new({ v: 3 }, nil, key: 'C')
+                                 Deimos::Message.new({ v: 3 }, key: 'C')
                                ])
                  ])
     end

data/spec/active_record_consumer_spec.rb

@@ -33,25 +33,16 @@ module ActiveRecordConsumerTest
     prepend_before(:each) do
 
       consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
-        schema 'MySchemaWithDateTimes'
-        namespace 'com.my-namespace'
-        key_config plain: true
         record_class Widget
       end
       stub_const('MyConsumer', consumer_class)
 
       consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
-        schema 'MySchemaWithDateTimes'
-        namespace 'com.my-namespace'
-        key_config schema: 'MySchemaId_key'
         record_class Widget
       end
       stub_const('MyConsumerWithKey', consumer_class)
 
       consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config none: true
         record_class Widget
 
         # :nodoc:
@@ -135,14 +126,39 @@ module ActiveRecordConsumerTest
         end
       end
       stub_const('Schemas::MySchemaWithDateTimes', schema_datetime_class)
+
+      Karafka::App.routes.redraw do
+        topic "my-topic" do
+          consumer MyConsumer
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config plain: true
+        end
+        topic "my-topic2" do
+          consumer MyConsumerWithKey
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchemaId_key'
+        end
+        topic "my-topic3" do
+          consumer MyCustomFetchConsumer
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config none: true
+        end
+      end
     end
 
     describe 'consume' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
         context "with Schema Class consumption #{setting}" do
           before(:each) do
+            Karafka::App.routes.draw do
+              defaults do
+                use_schema_classes use_schema_classes
+              end
+            end
             Deimos.configure do |config|
-              config.schema.use_schema_classes = use_schema_classes
               config.schema.use_full_namespace = true
             end
           end
@@ -200,7 +216,7 @@ module ActiveRecordConsumerTest
             test_consume_message(MyCustomFetchConsumer, {
                                    test_id: 'id1',
                                    some_int: 3
-                                 }, call_original: true)
+                                 })
             expect(widget1.reload.updated_at.in_time_zone).
               to eq(Time.local(2020, 5, 6, 5, 5, 5))
             travel_back
@@ -225,13 +241,13 @@ module ActiveRecordConsumerTest
             test_consume_message(MyCustomFetchConsumer, {
                                    test_id: 'id1',
                                    some_int: 3
-                                 }, call_original: true)
+                                 })
             expect(widget1.reload.some_int).to eq(3)
             expect(Widget.count).to eq(1)
             test_consume_message(MyCustomFetchConsumer, {
                                    test_id: 'id2',
                                    some_int: 4
-                                 }, call_original: true)
+                                 })
             expect(Widget.count).to eq(2)
             expect(Widget.find_by_test_id('id1').some_int).to eq(3)
             expect(Widget.find_by_test_id('id2').some_int).to eq(4)

data/spec/active_record_producer_spec.rb

@@ -5,28 +5,13 @@ describe Deimos::ActiveRecordProducer do
   include_context 'with widgets'
 
   prepend_before(:each) do
-
-    producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchema'
-      namespace 'com.my-namespace'
-      topic 'my-topic'
-      key_config none: true
-    end
+    producer_class = Class.new(Deimos::ActiveRecordProducer)
     stub_const('MyProducer', producer_class)
 
-    producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithBooleans'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-boolean'
-      key_config none: true
-    end
+    producer_class = Class.new(Deimos::ActiveRecordProducer)
     stub_const('MyBooleanProducer', producer_class)
 
     producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithId'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-id'
-      key_config none: true
       record_class Widget
 
       # :nodoc:
@@ -38,19 +23,11 @@ describe Deimos::ActiveRecordProducer do
     stub_const('MyProducerWithID', producer_class)
 
     producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithUniqueId'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-unique-id'
-      key_config field: :id
       record_class Widget
     end
     stub_const('MyProducerWithUniqueID', producer_class)
 
     producer_class = Class.new(Deimos::ActiveRecordProducer) do
-      schema 'MySchemaWithUniqueId'
-      namespace 'com.my-namespace'
-      topic 'my-topic-with-unique-id'
-      key_config field: :id
       record_class Widget
 
       # :nodoc:
@@ -63,6 +40,39 @@ describe Deimos::ActiveRecordProducer do
     end
 
     stub_const('MyProducerWithPostProcess', producer_class)
+    Karafka::App.routes.redraw do
+      topic 'my-topic' do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducer
+      end
+      topic 'my-topic-with-boolean' do
+        producer_class MyBooleanProducer
+        schema 'MySchemaWithBooleans'
+        namespace 'com.my-namespace'
+        key_config none: true
+      end
+      topic 'my-topic-with-id' do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducerWithID
+      end
+      topic 'my-topic-with-unique-id' do
+        schema 'MySchemaWithUniqueId'
+        namespace 'com.my-namespace'
+        key_config field: :id
+        producer_class MyProducerWithUniqueID
+      end
+      topic 'my-topic-with-post-process' do
+        schema 'MySchemaWithUniqueId'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class MyProducerWithPostProcess
+      end
+    end
+
   end
 
   describe 'produce' do
@@ -117,7 +127,7 @@ describe Deimos::ActiveRecordProducer do
   end
 
   specify '#watched_attributes' do
-    expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
+    expect(MyProducer.watched_attributes(nil)).to eq(%w(test_id some_int))
   end
 
 end

data/spec/backends/base_spec.rb

@@ -8,27 +8,4 @@ RSpec.describe Deimos::Backends::Base do
     described_class.publish(producer_class: MyProducer, messages: messages)
   end
 
-  describe 'payload_log method' do
-    it 'should return whole payload (default behavior)' do
-      log_message = described_class.send(:log_message, messages)
-      expect(log_message[:payloads].count).to eq(3)
-      expect(log_message[:payloads].first[:payload]).to eq({ 'foo' => 1 })
-      expect(log_message[:payloads].first[:key]).to eq('foo1')
-    end
-
-    it 'should return only keys of messages' do
-      Deimos.config.payload_log = :keys
-      log_message = described_class.send(:log_message, messages)
-      expect(log_message[:payload_keys].count).to eq(3)
-      expect(log_message[:payload_keys]).to be_a(Array)
-      expect(log_message[:payload_keys].first).to eq('foo1')
-    end
-
-    it 'should return only messages count' do
-      Deimos.config.payload_log = :count
-      log_message = described_class.send(:log_message, messages)
-      expect(log_message[:payloads_count]).to be_a(Integer)
-      expect(log_message[:payloads_count]).to eq(3)
-    end
-  end
 end

data/spec/backends/kafka_async_spec.rb

@@ -3,9 +3,7 @@
 RSpec.describe Deimos::Backends::KafkaAsync do
   include_context 'with publish_backend'
   it 'should publish to Kafka asynchronously' do
-    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
-    expect(producer).to receive(:async_publish_list).with(messages.map(&:encoded_hash))
-    expect(described_class).to receive(:producer).and_return(producer)
+    expect(Karafka.producer).to receive(:produce_many_async).with(messages)
     described_class.publish(producer_class: MyProducer, messages: messages)
   end
 end

data/spec/backends/kafka_spec.rb

@@ -3,9 +3,7 @@
 RSpec.describe Deimos::Backends::Kafka do
   include_context 'with publish_backend'
   it 'should publish to Kafka synchronously' do
-    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
-    expect(producer).to receive(:publish_list).with(messages.map(&:encoded_hash))
-    expect(described_class).to receive(:producer).and_return(producer)
+    expect(Karafka.producer).to receive(:produce_many_sync).with(messages)
     described_class.publish(producer_class: MyProducer, messages: messages)
   end
 end

data/spec/backends/{db_spec.rb → outbox_spec.rb}

@@ -1,31 +1,31 @@
 # frozen_string_literal: true
 
-each_db_config(Deimos::Backends::Db) do
+each_db_config(Deimos::Backends::Outbox) do
   include_context 'with publish_backend'
 
   it 'should save to the database' do
-    expect(Deimos.config.metrics).to receive(:increment).with(
-      'db_producer.insert',
-      tags: %w(topic:my-topic),
-      by: 3
-    )
+    # expect(Deimos.config.metrics).to receive(:increment).with(
+    #   'outbox.insert',
+    #   tags: %w(topic:my-topic),
+    #   by: 3
+    # )
     described_class.publish(producer_class: MyProducer, messages: messages)
     records = Deimos::KafkaMessage.all
     expect(records.size).to eq(3)
     expect(records[0].attributes.to_h).to include(
-      'message' => '{"foo"=>1}',
+      'message' => '{"test_id":"foo1","some_int":1}',
       'topic' => 'my-topic',
-      'key' => 'foo1'
+      'key' => '{"test_id":"foo1"}'
     )
     expect(records[1].attributes.to_h).to include(
-      'message' => '{"foo"=>2}',
+      'message' => '{"test_id":"foo2","some_int":2}',
       'topic' => 'my-topic',
-      'key' => 'foo2'
+      'key' => '{"test_id":"foo2"}'
     )
     expect(records[2].attributes.to_h).to include(
-      'message' => '{"foo"=>3}',
+      'message' => '{"test_id":"foo3","some_int":3}',
       'topic' => 'my-topic',
-      'key' => 'foo3'
+      'key' => '{"test_id":"foo3"}'
     )
   end
 
@@ -37,18 +37,12 @@ each_db_config(Deimos::Backends::Db) do
   end
 
   it 'should add to non-keyed messages' do
+    orig_messages = messages.deep_dup
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: messages)
     expect(Deimos::KafkaMessage.count).to eq(3)
     described_class.publish(producer_class: MyNoKeyProducer,
-                            messages: [messages.first])
+                            messages: [orig_messages.first])
     expect(Deimos::KafkaMessage.count).to eq(4)
   end
-
-  it 'should add messages with Hash keys with JSON encoding' do
-    described_class.publish(producer_class: MyProducer,
-                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
-    expect(Deimos::KafkaMessage.count).to eq(1)
-    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
-  end
 end

data/spec/batch_consumer_spec.rb

@@ -3,19 +3,34 @@
 # :nodoc:
 module ConsumerTest
   describe Deimos::Consumer, 'Batch Consumer' do
-    prepend_before(:each) do
-      # :nodoc:
-      consumer_class = Class.new(described_class) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config field: 'test_id'
-
+    let(:schema) { 'MySchema' }
+    let(:use_schema_classes) { false }
+    let(:reraise_errors) { false }
+    let(:key_config) { { field: 'test_id' } }
+    let(:consumer_class) do
+      Class.new(described_class) do
         # :nodoc:
-        def consume_batch(_payloads, _metadata)
-          raise 'This should not be called unless call_original is set'
+        def consume_batch
         end
       end
+    end
+    before(:each) do
+      # :nodoc:
+      stub_const('MyBatchConsumer', consumer_class)
       stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+      klass = consumer_class
+      route_schema = schema
+      route_key = key_config
+      route_use_classes = use_schema_classes
+      Karafka::App.routes.redraw do
+        topic 'my-topic' do
+          consumer klass
+          schema route_schema
+          namespace 'com.my-namespace'
+          key_config route_key
+          use_schema_classes route_use_classes
+        end
+      end
     end
 
     let(:batch) do
@@ -38,41 +53,23 @@ module ConsumerTest
               Deimos::Utils::SchemaClass.instance(p, 'MySchema', 'com.my-namespace')
             end
           end
+          let(:use_schema_classes) { true }
 
           before(:each) do
             Deimos.configure do |config|
-              config.schema.use_schema_classes = use_schema_classes
               config.schema.use_full_namespace = true
             end
           end
 
-          it 'should provide backwards compatibility for BatchConsumer class' do
-            consumer_class = Class.new(Deimos::BatchConsumer) do
-              schema 'MySchema'
-              namespace 'com.my-namespace'
-              key_config field: 'test_id'
-
-              # :nodoc:
-              def consume_batch(_payloads, _metadata)
-                raise 'This should not be called unless call_original is set'
-              end
-            end
-            stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
-
-            test_consume_batch(MyOldBatchConsumer, schema_class_batch) do |received, _metadata|
-              expect(received).to eq(schema_class_batch)
-            end
-          end
-
           it 'should consume a batch of messages' do
-            test_consume_batch(MyBatchConsumer, schema_class_batch) do |received, _metadata|
-              expect(received).to eq(schema_class_batch)
+            test_consume_batch(MyBatchConsumer, schema_class_batch) do |received|
+              expect(received.payloads).to eq(schema_class_batch)
             end
           end
 
           it 'should consume a message on a topic' do
-            test_consume_batch('my_batch_consume_topic', schema_class_batch) do |received, _metadata|
-              expect(received).to eq(schema_class_batch)
+            test_consume_batch('my-topic', schema_class_batch) do |received|
+              expect(received.payloads).to eq(schema_class_batch)
             end
           end
         end
@@ -80,29 +77,19 @@ module ConsumerTest
     end
 
     describe 'when reraising errors is disabled' do
-      before(:each) do
-        Deimos.configure { |config| config.consumers.reraise_errors = false }
-      end
+      let(:reraise_errors) { false }
 
       it 'should not fail when before_consume_batch fails' do
         expect {
           test_consume_batch(
             MyBatchConsumer,
-            batch,
-            skip_expectation: true
-          ) { raise 'OH NOES' }
+            batch
+          ) do
+            raise 'OH NOES'
+          end
         }.not_to raise_error
       end
 
-      it 'should not fail when consume_batch fails' do
-        expect {
-          test_consume_batch(
-            MyBatchConsumer,
-            invalid_payloads,
-            skip_expectation: true
-          )
-        }.not_to raise_error
-      end
     end
 
     describe 'decoding' do
@@ -111,53 +98,32 @@ module ConsumerTest
       end
 
       it 'should decode payloads for all messages in the batch' do
-        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+        test_consume_batch('my-topic', batch) do
           # Mock decoder simply returns the payload
-          expect(received).to eq(batch)
+          expect(messages.payloads).to eq(batch)
         end
       end
 
       it 'should decode keys for all messages in the batch' do
-        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
-          to receive(:decode_key).with(keys[0]).and_call_original
-        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
-          to receive(:decode_key).with(keys[1]).and_call_original
-
-        test_consume_batch('my_batch_consume_topic', batch, keys: keys) do |_received, metadata|
-          # Mock decode_key extracts the value of the first field as the key
-          expect(metadata[:keys]).to eq(%w(foo bar))
-          expect(metadata[:first_offset]).to eq(1)
+        test_consume_batch('my-topic', batch, keys: keys) do
+          expect(messages.map(&:key)).to eq([{'test_id' => 'foo'}, {'test_id' => 'bar'}])
         end
       end
 
-      it 'should decode plain keys for all messages in the batch' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
-        end
-        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
-
-        test_consume_batch('my_batch_consume_topic', batch, keys: [1, 2]) do |_received, metadata|
-          expect(metadata[:keys]).to eq([1, 2])
+      context 'with plain keys' do
+        let(:key_config) { { plain: true } }
+        it 'should decode plain keys for all messages in the batch' do
+          test_consume_batch('my-topic', batch, keys: [1, 2]) do |_received, metadata|
+            expect(metadata[:keys]).to eq([1, 2])
+          end
         end
       end
     end
 
     describe 'timestamps' do
+      let(:schema) { 'MySchemaWithDateTimes' }
+      let(:key_config) { { none: true } }
       before(:each) do
-        # :nodoc:
-        consumer_class = Class.new(described_class) do
-          schema 'MySchemaWithDateTimes'
-          namespace 'com.my-namespace'
-          key_config plain: true
-
-          # :nodoc:
-          def consume_batch(_payloads, _metadata)
-            raise 'This should not be called unless call_original is set'
-          end
-        end
-        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
         allow(Deimos.config.metrics).to receive(:histogram)
       end
 
@@ -202,45 +168,36 @@ module ConsumerTest
       end
 
       it 'should consume a batch' do
-        expect(Deimos.config.metrics).
-          to receive(:histogram).with('handler',
-                                      a_kind_of(Numeric),
-                                      tags: %w(time:time_delayed topic:my-topic)).twice
+        # expect(Deimos.config.metrics).
+        #   to receive(:histogram).with('handler',
+        #                               a_kind_of(Numeric),
+        #                               tags: %w(time:time_delayed topic:my-topic)).twice
 
-        test_consume_batch('my_batch_consume_topic', batch_with_time) do |received, _metadata|
-          expect(received).to eq(batch_with_time)
+        test_consume_batch('my-topic', batch_with_time) do
+          expect(messages.payloads).to eq(batch_with_time)
         end
       end
 
       it 'should fail nicely and ignore timestamps with the wrong format' do
         batch = invalid_times.concat(batch_with_time)
 
-        expect(Deimos.config.metrics).
-          to receive(:histogram).with('handler',
-                                      a_kind_of(Numeric),
-                                      tags: %w(time:time_delayed topic:my-topic)).twice
+        # expect(Deimos.config.metrics).
+        #   to receive(:histogram).with('handler',
+        #                               a_kind_of(Numeric),
+        #                               tags: %w(time:time_delayed topic:my-topic)).twice
 
-        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
-          expect(received).to eq(batch)
+        test_consume_batch('my-topic', batch) do
+          expect(messages.payloads).to eq(batch)
         end
       end
     end
 
     describe 'logging' do
+      let(:schema) { 'MySchemaWithUniqueId' }
+      let(:key_config) { { plain: true } }
       before(:each) do
-        # :nodoc:
-        consumer_class = Class.new(described_class) do
-          schema 'MySchemaWithUniqueId'
-          namespace 'com.my-namespace'
-          key_config plain: true
-
-          # :nodoc:
-          def consume_batch(_payloads, _metadata)
-            raise 'This should not be called unless call_original is set'
-          end
-        end
-        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
         allow(Deimos.config.metrics).to receive(:histogram)
+        set_karafka_config(:payload_log, :keys)
       end
 
       it 'should log message identifiers' do
@@ -251,20 +208,13 @@ module ConsumerTest
             'timestamp' => 2.minutes.ago.to_s, 'message_id' => 'two' }
         ]
 
-        allow(Deimos.config.logger).
-          to receive(:info)
+        allow(Deimos::Logging).to receive(:log_info)
 
-        expect(Deimos.config.logger).
-          to receive(:info).
-          with(hash_including(
-                 message_ids: [
-                   { key: 1, message_id: 'one' },
-                   { key: 2, message_id: 'two' }
-                 ]
-               )).
-          twice
+        expect(Deimos::Logging).
+          to receive(:log_info).
+          with(hash_including(payload_keys: ["1", "2"]))
 
-        test_consume_batch('my_batch_consume_topic', batch_with_message_id, keys: [1, 2])
+        test_consume_batch('my-topic', batch_with_message_id, keys: [1, 2])
       end
     end
   end