deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

data/spec/batch_consumer_spec.rb

@@ -3,19 +3,34 @@
 # :nodoc:
 module ConsumerTest
   describe Deimos::Consumer, 'Batch Consumer' do
-    prepend_before(:each) do
-      # :nodoc:
-      consumer_class = Class.new(described_class) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config field: 'test_id'
-
+    let(:schema) { 'MySchema' }
+    let(:use_schema_classes) { false }
+    let(:reraise_errors) { false }
+    let(:key_config) { { field: 'test_id' } }
+    let(:consumer_class) do
+      Class.new(described_class) do
         # :nodoc:
-        def consume_batch(_payloads, _metadata)
-          raise 'This should not be called unless call_original is set'
+        def consume_batch
         end
       end
+    end
+    before(:each) do
+      # :nodoc:
+      stub_const('MyBatchConsumer', consumer_class)
       stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+      klass = consumer_class
+      route_schema = schema
+      route_key = key_config
+      route_use_classes = use_schema_classes
+      Karafka::App.routes.redraw do
+        topic 'my-topic' do
+          consumer klass
+          schema route_schema
+          namespace 'com.my-namespace'
+          key_config route_key
+          use_schema_classes route_use_classes
+        end
+      end
     end
 
     let(:batch) do
@@ -38,41 +53,23 @@ module ConsumerTest
               Deimos::Utils::SchemaClass.instance(p, 'MySchema', 'com.my-namespace')
             end
           end
+          let(:use_schema_classes) { true }
 
           before(:each) do
             Deimos.configure do |config|
-              config.schema.use_schema_classes = use_schema_classes
               config.schema.use_full_namespace = true
             end
           end
 
-          it 'should provide backwards compatibility for BatchConsumer class' do
-            consumer_class = Class.new(Deimos::BatchConsumer) do
-              schema 'MySchema'
-              namespace 'com.my-namespace'
-              key_config field: 'test_id'
-
-              # :nodoc:
-              def consume_batch(_payloads, _metadata)
-                raise 'This should not be called unless call_original is set'
-              end
-            end
-            stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
-
-            test_consume_batch(MyOldBatchConsumer, schema_class_batch) do |received, _metadata|
-              expect(received).to eq(schema_class_batch)
-            end
-          end
-
           it 'should consume a batch of messages' do
-            test_consume_batch(MyBatchConsumer, schema_class_batch) do |received, _metadata|
-              expect(received).to eq(schema_class_batch)
+            test_consume_batch(MyBatchConsumer, schema_class_batch) do |received|
+              expect(received.payloads).to eq(schema_class_batch)
             end
           end
 
           it 'should consume a message on a topic' do
-            test_consume_batch('my_batch_consume_topic', schema_class_batch) do |received, _metadata|
-              expect(received).to eq(schema_class_batch)
+            test_consume_batch('my-topic', schema_class_batch) do |received|
+              expect(received.payloads).to eq(schema_class_batch)
             end
           end
         end
@@ -80,29 +77,19 @@ module ConsumerTest
     end
 
     describe 'when reraising errors is disabled' do
-      before(:each) do
-        Deimos.configure { |config| config.consumers.reraise_errors = false }
-      end
+      let(:reraise_errors) { false }
 
       it 'should not fail when before_consume_batch fails' do
         expect {
           test_consume_batch(
             MyBatchConsumer,
-            batch,
-            skip_expectation: true
-          ) { raise 'OH NOES' }
+            batch
+          ) do
+            raise 'OH NOES'
+          end
         }.not_to raise_error
       end
 
-      it 'should not fail when consume_batch fails' do
-        expect {
-          test_consume_batch(
-            MyBatchConsumer,
-            invalid_payloads,
-            skip_expectation: true
-          )
-        }.not_to raise_error
-      end
     end
 
     describe 'decoding' do
@@ -111,53 +98,32 @@ module ConsumerTest
       end
 
       it 'should decode payloads for all messages in the batch' do
-        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+        test_consume_batch('my-topic', batch) do
           # Mock decoder simply returns the payload
-          expect(received).to eq(batch)
+          expect(messages.payloads).to eq(batch)
         end
       end
 
       it 'should decode keys for all messages in the batch' do
-        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
-          to receive(:decode_key).with(keys[0]).and_call_original
-        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
-          to receive(:decode_key).with(keys[1]).and_call_original
-
-        test_consume_batch('my_batch_consume_topic', batch, keys: keys) do |_received, metadata|
-          # Mock decode_key extracts the value of the first field as the key
-          expect(metadata[:keys]).to eq(%w(foo bar))
-          expect(metadata[:first_offset]).to eq(1)
+        test_consume_batch('my-topic', batch, keys: keys) do
+          expect(messages.map(&:key)).to eq([{'test_id' => 'foo'}, {'test_id' => 'bar'}])
         end
       end
 
-      it 'should decode plain keys for all messages in the batch' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
-        end
-        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
-
-        test_consume_batch('my_batch_consume_topic', batch, keys: [1, 2]) do |_received, metadata|
-          expect(metadata[:keys]).to eq([1, 2])
+      context 'with plain keys' do
+        let(:key_config) { { plain: true } }
+        it 'should decode plain keys for all messages in the batch' do
+          test_consume_batch('my-topic', batch, keys: [1, 2]) do |_received, metadata|
+            expect(metadata[:keys]).to eq([1, 2])
+          end
         end
       end
     end
 
     describe 'timestamps' do
+      let(:schema) { 'MySchemaWithDateTimes' }
+      let(:key_config) { { none: true } }
       before(:each) do
-        # :nodoc:
-        consumer_class = Class.new(described_class) do
-          schema 'MySchemaWithDateTimes'
-          namespace 'com.my-namespace'
-          key_config plain: true
-
-          # :nodoc:
-          def consume_batch(_payloads, _metadata)
-            raise 'This should not be called unless call_original is set'
-          end
-        end
-        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
         allow(Deimos.config.metrics).to receive(:histogram)
       end
 
@@ -202,45 +168,36 @@ module ConsumerTest
       end
 
       it 'should consume a batch' do
-        expect(Deimos.config.metrics).
-          to receive(:histogram).with('handler',
-                                      a_kind_of(Numeric),
-                                      tags: %w(time:time_delayed topic:my-topic)).twice
+        # expect(Deimos.config.metrics).
+        #   to receive(:histogram).with('handler',
+        #                               a_kind_of(Numeric),
+        #                               tags: %w(time:time_delayed topic:my-topic)).twice
 
-        test_consume_batch('my_batch_consume_topic', batch_with_time) do |received, _metadata|
-          expect(received).to eq(batch_with_time)
+        test_consume_batch('my-topic', batch_with_time) do
+          expect(messages.payloads).to eq(batch_with_time)
         end
       end
 
       it 'should fail nicely and ignore timestamps with the wrong format' do
         batch = invalid_times.concat(batch_with_time)
 
-        expect(Deimos.config.metrics).
-          to receive(:histogram).with('handler',
-                                      a_kind_of(Numeric),
-                                      tags: %w(time:time_delayed topic:my-topic)).twice
+        # expect(Deimos.config.metrics).
+        #   to receive(:histogram).with('handler',
+        #                               a_kind_of(Numeric),
+        #                               tags: %w(time:time_delayed topic:my-topic)).twice
 
-        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
-          expect(received).to eq(batch)
+        test_consume_batch('my-topic', batch) do
+          expect(messages.payloads).to eq(batch)
         end
       end
     end
 
     describe 'logging' do
+      let(:schema) { 'MySchemaWithUniqueId' }
+      let(:key_config) { { plain: true } }
       before(:each) do
-        # :nodoc:
-        consumer_class = Class.new(described_class) do
-          schema 'MySchemaWithUniqueId'
-          namespace 'com.my-namespace'
-          key_config plain: true
-
-          # :nodoc:
-          def consume_batch(_payloads, _metadata)
-            raise 'This should not be called unless call_original is set'
-          end
-        end
-        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
         allow(Deimos.config.metrics).to receive(:histogram)
+        set_karafka_config(:payload_log, :keys)
       end
 
       it 'should log message identifiers' do
@@ -251,20 +208,13 @@ module ConsumerTest
             'timestamp' => 2.minutes.ago.to_s, 'message_id' => 'two' }
         ]
 
-        allow(Deimos.config.logger).
-          to receive(:info)
+        allow(Deimos::Logging).to receive(:log_info)
 
-        expect(Deimos.config.logger).
-          to receive(:info).
-          with(hash_including(
-                 message_ids: [
-                   { key: 1, message_id: 'one' },
-                   { key: 2, message_id: 'two' }
-                 ]
-               )).
-          twice
+        expect(Deimos::Logging).
+          to receive(:log_info).
+          with(hash_including(payload_keys: ["1", "2"]))
 
-        test_consume_batch('my_batch_consume_topic', batch_with_message_id, keys: [1, 2])
+        test_consume_batch('my-topic', batch_with_message_id, keys: [1, 2])
       end
     end
   end

data/spec/consumer_spec.rb

@@ -4,33 +4,44 @@
 # rubocop:disable Metrics/ModuleLength
 module ConsumerTest
   describe Deimos::Consumer, 'Message Consumer' do
+    let(:use_schema_classes) { false }
+    let(:reraise_errors) { false }
     prepend_before(:each) do
       # :nodoc:
       consumer_class = Class.new(described_class) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config field: 'test_id'
 
         # :nodoc:
-        def fatal_error?(_exception, payload, _metadata)
-          payload.to_s == 'fatal'
+        def fatal_error?(_exception, messages)
+          messages.payloads.first&.dig(:test_id) == ['fatal']
         end
 
         # :nodoc:
-        def consume(_payload, _metadata)
-          raise 'This should not be called unless call_original is set'
+        def consume_message(message)
+          message.payload
         end
       end
       stub_const('ConsumerTest::MyConsumer', consumer_class)
+      route_usc = use_schema_classes
+      route_rre = reraise_errors
+      Karafka::App.routes.redraw do
+        topic 'my_consume_topic' do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config field: 'test_id'
+          consumer consumer_class
+          use_schema_classes route_usc
+          reraise_errors route_rre
+        end
+      end
     end
 
     describe 'consume' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
+        let(:use_schema_classes) { use_schema_classes }
         context "with Schema Class consumption #{setting}" do
 
           before(:each) do
             Deimos.configure do |config|
-              config.schema.use_schema_classes = use_schema_classes
               config.schema.use_full_namespace = true
             end
           end
@@ -45,26 +56,9 @@ module ConsumerTest
           end
 
           it 'should consume a nil message' do
-            test_consume_message(MyConsumer, nil) do |payload, _metadata|
-              expect(payload).to be_nil
-            end
-          end
-
-          it 'should consume a message idempotently' do
-            # testing for a crash and re-consuming the same message/metadata
-            key = { 'test_id' => 'foo' }
-            test_metadata = { key: key }
-            allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
-              k['test_id']
+            test_consume_message(MyConsumer, nil, key: 'foo') do
+              expect(messages).to be_empty
             end
-            MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                              expect(metadata[:key]).to eq('foo')
-                                            end
-            MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                              expect(metadata[:key]).to eq('foo')
-                                            end
           end
 
           it 'should consume a message on a topic' do
@@ -77,83 +71,82 @@ module ConsumerTest
           end
 
           it 'should fail on invalid message' do
-            test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
+            expect { test_consume_message(MyConsumer, { 'invalid' => 'key' }) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should fail if reraise is false but fatal_error is true' do
-            Deimos.configure { |config| config.consumers.reraise_errors = false }
-            test_consume_invalid_message(MyConsumer, 'fatal')
+            expect { test_consume_message(MyConsumer, {test_id: 'fatal'}) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should fail if fatal_error is true globally' do
-            Deimos.configure do |config|
-              config.consumers.fatal_error = proc { true }
-              config.consumers.reraise_errors = false
-            end
-            test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
+            set_karafka_config(:fatal_error, proc { true })
+            expect { test_consume_message(MyConsumer, { 'invalid' => 'key' }) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should fail on message with extra fields' do
-            test_consume_invalid_message(MyConsumer,
+            allow_any_instance_of(Deimos::SchemaBackends::AvroValidation).
+              to receive(:coerce) { |_, m| m.with_indifferent_access }
+            expect { test_consume_message(MyConsumer,
                                          { 'test_id' => 'foo',
                                          'some_int' => 123,
-                                         'extra_field' => 'field name' })
+                                         'extra_field' => 'field name' }) }.
+              to raise_error(Avro::SchemaValidator::ValidationError)
           end
 
           it 'should not fail when before_consume fails without reraising errors' do
-            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            set_karafka_config(:reraise_errors, false)
             expect {
               test_consume_message(
                 MyConsumer,
                 { 'test_id' => 'foo',
-                  'some_int' => 123 },
-                skip_expectation: true
-              ) { raise 'OH NOES' }
+                  'some_int' => 123 }) { raise 'OH NOES' }
             }.not_to raise_error
           end
 
           it 'should not fail when consume fails without reraising errors' do
-            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            set_karafka_config(:reraise_errors, false)
+            allow(Deimos::ProducerMiddleware).to receive(:call) { |m| m[:payload] = m[:payload].to_json; m }
             expect {
               test_consume_message(
                 MyConsumer,
-                { 'invalid' => 'key' },
-                skip_expectation: true
-              )
+                { 'invalid' => 'key' })
             }.not_to raise_error
           end
-
-          it 'should call original' do
-            expect {
-              test_consume_message(MyConsumer,
-                                   { 'test_id' => 'foo', 'some_int' => 123 },
-                                   call_original: true)
-            }.to raise_error('This should not be called unless call_original is set')
-          end
         end
       end
 
       context 'with overriden schema classes' do
 
         before(:each) do
+          set_karafka_config(:use_schema_classes, true)
           Deimos.configure do |config|
-            config.schema.use_schema_classes = true
             config.schema.use_full_namespace = true
           end
         end
 
         prepend_before(:each) do
           consumer_class = Class.new(described_class) do
-            schema 'MyUpdatedSchema'
-            namespace 'com.my-namespace'
-            key_config field: 'test_id'
-
             # :nodoc:
-            def consume(_payload, _metadata)
-              raise 'This should not be called unless call_original is set'
+            def consume_message(message)
+              message.payload
             end
           end
           stub_const('ConsumerTest::MyConsumer', consumer_class)
+          Deimos.config.schema.use_schema_classes = true
+          Karafka::App.routes.redraw do
+            topic 'my_consume_topic' do
+              schema 'MyUpdatedSchema'
+              namespace 'com.my-namespace'
+              key_config field: 'test_id'
+              consumer consumer_class
+            end
+          end
+        end
+        after(:each) do
+          Karafka::App.routes.clear
         end
 
         it 'should consume messages' do
@@ -169,93 +162,6 @@ module ConsumerTest
       end
     end
 
-    describe 'decode_key' do
-
-      it 'should use the key field in the value if set' do
-        # actual decoding is disabled in test
-        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
-        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
-      end
-
-      it 'should use the key schema if set' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config schema: 'MySchema_key'
-        end
-        stub_const('ConsumerTest::MySchemaConsumer', consumer_class)
-        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
-        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
-      end
-
-      it 'should not decode if plain is set' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
-        end
-        stub_const('ConsumerTest::MyNonEncodedConsumer', consumer_class)
-        expect(MyNonEncodedConsumer.new.decode_key('123')).to eq('123')
-      end
-
-      it 'should error with nothing set' do
-        consumer_class = Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-        end
-        stub_const('ConsumerTest::MyErrorConsumer', consumer_class)
-        expect { MyErrorConsumer.new.decode_key('123') }.
-          to raise_error('No key config given - if you are not decoding keys, please use `key_config plain: true`')
-      end
-
-    end
-
-    describe 'timestamps' do
-      before(:each) do
-        # :nodoc:
-        consumer_class = Class.new(described_class) do
-          schema 'MySchemaWithDateTimes'
-          namespace 'com.my-namespace'
-          key_config plain: true
-
-          # :nodoc:
-          def consume(_payload, _metadata)
-            raise 'This should not be called unless call_original is set'
-          end
-        end
-        stub_const('ConsumerTest::MyConsumer', consumer_class)
-      end
-
-      it 'should consume a message' do
-        expect(Deimos.config.metrics).to receive(:histogram).twice
-        test_consume_message('my_consume_topic',
-                             { 'test_id' => 'foo',
-                             'some_int' => 123,
-                             'updated_at' => Time.now.to_i,
-                             'timestamp' => 2.minutes.ago.to_s }) do |payload, _metadata|
-                               expect(payload['test_id']).to eq('foo')
-                             end
-      end
-
-      it 'should fail nicely when timestamp wrong format' do
-        expect(Deimos.config.metrics).to receive(:histogram).twice
-        test_consume_message('my_consume_topic',
-                             { 'test_id' => 'foo',
-                             'some_int' => 123,
-                             'updated_at' => Time.now.to_i,
-                             'timestamp' => 'dffdf' }) do |payload, _metadata|
-                               expect(payload['test_id']).to eq('foo')
-                             end
-        test_consume_message('my_consume_topic',
-                             { 'test_id' => 'foo',
-                             'some_int' => 123,
-                             'updated_at' => Time.now.to_i,
-                             'timestamp' => '' }) do |payload, _metadata|
-                               expect(payload['test_id']).to eq('foo')
-                             end
-      end
-
-    end
   end
 end
 # rubocop:enable Metrics/ModuleLength