deimos-ruby 1.22.4 → 1.23.0

data/spec/config/configuration_spec.rb

@@ -91,7 +91,8 @@ describe Deimos, 'configuration' do
           heartbeat_interval: 10,
           handler: 'ConsumerTest::MyConsumer',
           use_schema_classes: nil,
-          max_db_batch_size: nil
+          max_db_batch_size: nil,
+          bulk_import_id_generator: nil
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
@@ -109,7 +110,8 @@ describe Deimos, 'configuration' do
           heartbeat_interval: 10,
           handler: 'ConsumerTest::MyBatchConsumer',
           use_schema_classes: nil,
-          max_db_batch_size: nil
+          max_db_batch_size: nil,
+          bulk_import_id_generator: nil
         }
       ],
       producer: {
@@ -261,7 +263,8 @@ describe Deimos, 'configuration' do
             heartbeat_interval: 13,
             handler: 'MyConfigConsumer',
             use_schema_classes: false,
-            max_db_batch_size: nil
+            max_db_batch_size: nil,
+            bulk_import_id_generator: nil
           }
         ],
         producer: {
@@ -279,4 +282,40 @@ describe Deimos, 'configuration' do
         }
       )
   end
+
+  it 'should override global configurations' do
+    described_class.configure do
+      consumers.bulk_import_id_generator(-> { 'global' })
+      consumers.replace_associations true
+
+      consumer do
+        class_name 'MyConfigConsumer'
+        schema 'blah'
+        topic 'blah'
+        group_id 'myconsumerid'
+        bulk_import_id_generator(-> { 'consumer' })
+        replace_associations false
+      end
+
+      consumer do
+        class_name 'MyConfigConsumer2'
+        schema 'blah'
+        topic 'blah'
+        group_id 'myconsumerid'
+      end
+    end
+
+    consumers = described_class.config.consumers
+    expect(consumers.replace_associations).to eq(true)
+    expect(consumers.bulk_import_id_generator.call).to eq('global')
+
+    custom = MyConfigConsumer.config
+    expect(custom[:replace_associations]).to eq(false)
+    expect(custom[:bulk_import_id_generator].call).to eq('consumer')
+
+    default = MyConfigConsumer2.config
+    expect(default[:replace_associations]).to eq(true)
+    expect(default[:bulk_import_id_generator].call).to eq('global')
+
+  end
 end

data/spec/consumer_spec.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 # :nodoc:
+# rubocop:disable Metrics/ModuleLength
 module ConsumerTest
   describe Deimos::Consumer, 'Message Consumer' do
     prepend_before(:each) do
@@ -26,8 +27,12 @@ module ConsumerTest
     describe 'consume' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
         context "with Schema Class consumption #{setting}" do
+
           before(:each) do
-            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+            Deimos.configure do |config|
+              config.schema.use_schema_classes = use_schema_classes
+              config.schema.generate_namespace_folders = true
+            end
           end
 
           it 'should consume a message' do
@@ -127,6 +132,41 @@ module ConsumerTest
           end
         end
       end
+
+      context 'with overriden schema classes' do
+
+        before(:each) do
+          Deimos.configure do |config|
+            config.schema.use_schema_classes = true
+            config.schema.generate_namespace_folders = true
+          end
+        end
+
+        prepend_before(:each) do
+          consumer_class = Class.new(described_class) do
+            schema 'MyUpdatedSchema'
+            namespace 'com.my-namespace'
+            key_config field: 'test_id'
+
+            # :nodoc:
+            def consume(_payload, _metadata)
+              raise 'This should not be called unless call_original is set'
+            end
+          end
+          stub_const('ConsumerTest::MyConsumer', consumer_class)
+        end
+
+        it 'should consume messages' do
+          test_consume_message('my_consume_topic',
+                               { 'test_id' => 'foo',
+                                 'some_int' => 1 }) do |payload, _metadata|
+            expect(payload['test_id']).to eq('foo')
+            expect(payload['some_int']).to eq(1)
+            expect(payload['super_int']).to eq(9000)
+                                 end
+        end
+
+      end
     end
 
     describe 'decode_key' do
@@ -218,3 +258,4 @@ module ConsumerTest
     end
   end
 end
+# rubocop:enable Metrics/ModuleLength

data/spec/schemas/my_namespace/my_updated_schema.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas; module MyNamespace
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.MySchema
+  class MyUpdatedSchema < Schemas::MyNamespace::MySchema
+
+    attr_accessor :super_int
+
+    def initialize(test_id: nil,
+                   some_int: nil)
+      super
+      self.super_int = some_int.nil? ? 10 : some_int * 9000
+    end
+  end
+end
+end

data/spec/utils/db_producer_spec.rb

@@ -2,12 +2,13 @@
 
 each_db_config(Deimos::Utils::DbProducer) do
   let(:producer) do
-    producer = described_class.new
+    producer = described_class.new(logger)
     allow(producer).to receive(:sleep)
     allow(producer).to receive(:producer).and_return(phobos_producer)
     producer
   end
 
+  let(:logger) { nil }
   let(:phobos_producer) do
     pp = instance_double(Phobos::Producer::PublicAPI)
     allow(pp).to receive(:publish_list)
@@ -308,35 +309,6 @@ each_db_config(Deimos::Utils::DbProducer) do
       Deimos.unsubscribe(subscriber)
     end
 
-    it 'should delete messages on buffer overflow' do
-      messages = (1..4).map do |i|
-        Deimos::KafkaMessage.create!(
-          id: i,
-          topic: 'my-topic',
-          message: "mess#{i}",
-          partition_key: "key#{i}"
-        )
-      end
-      (5..8).each do |i|
-        Deimos::KafkaMessage.create!(
-          id: i,
-          topic: 'my-topic2',
-          message: "mess#{i}",
-          partition_key: "key#{i}"
-        )
-      end
-
-      expect(Deimos::KafkaTopicInfo).to receive(:lock).
-        with('my-topic', 'abc').and_return(true)
-      expect(producer).to receive(:produce_messages).and_raise(Kafka::BufferOverflow)
-      expect(producer).to receive(:retrieve_messages).and_return(messages)
-      expect(Deimos::KafkaTopicInfo).to receive(:register_error)
-
-      expect(Deimos::KafkaMessage.count).to eq(8)
-      producer.process_topic('my-topic')
-      expect(Deimos::KafkaMessage.count).to eq(4)
-    end
-
     it 'should retry deletes and not re-publish' do
       messages = (1..4).map do |i|
         Deimos::KafkaMessage.create!(
@@ -388,6 +360,102 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
     end
 
+    context 'with buffer overflow exception' do
+      let(:messages) do
+        (1..4).map do |i|
+          Deimos::KafkaMessage.create!(
+            id: i,
+            key: i,
+            topic: 'my-topic',
+            message: { message: "mess#{i}" },
+            partition_key: "key#{i}"
+          )
+        end
+      end
+      let(:logger) do
+        logger = instance_double(Logger)
+        allow(logger).to receive(:error)
+        logger
+      end
+      let(:message_producer) do
+        Deimos.config.schema.backend = :mock
+        Deimos::ActiveRecordProducer.topic('my-topic')
+        Deimos::ActiveRecordProducer.key_config
+        Deimos::ActiveRecordProducer
+      end
+
+      around(:each) do |example|
+        config = Deimos::ActiveRecordProducer.config.clone
+        backend = Deimos.config.schema.backend
+
+        example.run
+      ensure
+        Deimos::ActiveRecordProducer.instance_variable_set(:@config, config)
+        Deimos.config.schema.backend = backend
+      end
+
+      before(:each) do
+        message_producer
+        (5..8).each do |i|
+          Deimos::KafkaMessage.create!(
+            id: i,
+            topic: 'my-topic2',
+            message: "mess#{i}",
+            partition_key: "key#{i}"
+          )
+        end
+        allow(Deimos::KafkaTopicInfo).to receive(:lock).
+          with('my-topic', 'abc').and_return(true)
+        allow(producer).to receive(:produce_messages).and_raise(Kafka::BufferOverflow)
+        allow(producer).to receive(:retrieve_messages).and_return(messages)
+        allow(Deimos::KafkaTopicInfo).to receive(:register_error)
+      end
+
+      it 'should delete messages on buffer overflow' do
+        expect(Deimos::KafkaMessage.count).to eq(8)
+        producer.process_topic('my-topic')
+        expect(Deimos::KafkaMessage.count).to eq(4)
+      end
+
+      it 'should notify on buffer overflow' do
+        subscriber = Deimos.subscribe('db_producer.produce') do |event|
+          expect(event.payload[:exception_object].message).to eq('Kafka::BufferOverflow')
+          expect(event.payload[:messages]).to eq(messages)
+        end
+        producer.process_topic('my-topic')
+        Deimos.unsubscribe(subscriber)
+        expect(logger).to have_received(:error).with('Message batch too large, deleting...')
+        expect(logger).to have_received(:error).with(
+          [
+            { key: '1', payload: 'payload-decoded' },
+            { key: '2', payload: 'payload-decoded' },
+            { key: '3', payload: 'payload-decoded' },
+            { key: '4', payload: 'payload-decoded' }
+          ]
+        )
+      end
+
+      context 'with exception on error logging attempt' do
+        let(:message_producer) do
+          Deimos::ActiveRecordProducer.topic('my-topic')
+          Deimos::ActiveRecordProducer
+        end
+
+        it 'should notify on buffer overflow disregarding decoding exception' do
+          subscriber = Deimos.subscribe('db_producer.produce') do |event|
+            expect(event.payload[:exception_object].message).to eq('Kafka::BufferOverflow')
+            expect(event.payload[:messages]).to eq(messages)
+          end
+          producer.process_topic('my-topic')
+          Deimos.unsubscribe(subscriber)
+          expect(logger).to have_received(:error).with('Message batch too large, deleting...')
+          expect(logger).to have_received(:error).with(
+            'Large message details logging failure: '\
+            'No key config given - if you are not decoding keys, please use `key_config plain: true`'
+          )
+        end
+      end
+    end
   end
 
   describe '#send_pending_metrics' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.22.4
+  version: 1.23.0
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-05 00:00:00.000000000 Z
+date: 2024-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -579,6 +579,7 @@ files:
 - spec/schemas/my_namespace/my_schema_with_date_time.rb
 - spec/schemas/my_namespace/my_schema_with_id.rb
 - spec/schemas/my_namespace/my_schema_with_unique_id.rb
+- spec/schemas/my_namespace/my_updated_schema.rb
 - spec/schemas/my_namespace/wibble.rb
 - spec/schemas/my_namespace/widget.rb
 - spec/schemas/my_namespace/widget_the_second.rb
@@ -632,7 +633,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.20
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
@@ -706,6 +707,7 @@ test_files:
 - spec/schemas/my_namespace/my_schema_with_date_time.rb
 - spec/schemas/my_namespace/my_schema_with_id.rb
 - spec/schemas/my_namespace/my_schema_with_unique_id.rb
+- spec/schemas/my_namespace/my_updated_schema.rb
 - spec/schemas/my_namespace/wibble.rb
 - spec/schemas/my_namespace/widget.rb
 - spec/schemas/my_namespace/widget_the_second.rb