deimos-ruby 2.6.0.pre.beta1 → 2.6.0

data/spec/active_record_batch_consumer_spec.rb

@@ -86,7 +86,7 @@ module ActiveRecordBatchConsumerTest
           before(:each) do
             Deimos.configure do |config|
               config.schema.use_schema_classes = use_schema_classes
-              config.avrogen.use_full_namespace = true
+              config.schema.use_full_namespace = true
             end
           end
 
@@ -460,6 +460,291 @@ module ActiveRecordBatchConsumerTest
       end
     end
 
+    describe 'individual fallback when a batch operation fails' do
+      # `test_id: ''` passes Avro validation (it's a string) but fails the model's
+      # `validates :test_id, presence: true`, so `MassUpdater` raises before it issues any SQL.
+      # That's the poison-message case: one record that can never be persisted.
+      let(:poison_payload) { { test_id: '', some_int: 3 } }
+
+      let(:consumer_class) do
+        Class.new(described_class) do
+          record_class Widget
+          compacted false
+        end
+      end
+
+      before(:each) do
+        register_consumer(consumer_class, 'MySchema',
+                          key_config: { plain: true },
+                          configs: { reraise_errors: true })
+      end
+
+      it 'should not raise when the batch succeeds' do
+        publish_batch(
+          [
+            { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+            { key: 2, payload: { test_id: 'def', some_int: 2 } }
+          ]
+        )
+
+        expect(all_widgets).
+          to contain_exactly(have_attributes(id: 1, test_id: 'abc'),
+                             have_attributes(id: 2, test_id: 'def'))
+      end
+
+      context 'when one record in the batch cannot be persisted' do
+
+        it 'should save every record except the bad one and report the failed key' do
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+                { key: 2, payload: poison_payload },
+                { key: 3, payload: { test_id: 'ghi', some_int: 3 } }
+              ]
+            )
+          }.to raise_error(Deimos::BatchFallbackError, /Failed keys: "2"/)
+
+          expect(all_widgets).
+            to contain_exactly(have_attributes(id: 1, test_id: 'abc', some_int: 1),
+                               have_attributes(id: 3, test_id: 'ghi', some_int: 3))
+        end
+
+        it 'should collect the failures of every bad record in the batch' do
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: poison_payload },
+                { key: 2, payload: { test_id: 'def', some_int: 2 } },
+                { key: 3, payload: poison_payload }
+              ]
+            )
+          }.to raise_error(Deimos::BatchFallbackError) { |error|
+            expect(error.failures.map { |message, _| message.key }).to contain_exactly('1', '3')
+            expect(error.failures.map(&:last)).to all(be_a(ActiveRecord::RecordInvalid))
+          }
+
+          expect(all_widgets).to contain_exactly(have_attributes(id: 2, test_id: 'def'))
+        end
+
+        it 'should raise the original error when no record could be saved on its own' do
+          # Isolating salvaged nothing, so the failure was never about one bad message. The
+          # original error is more useful than a BatchFallbackError listing every key.
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: poison_payload },
+                { key: 2, payload: poison_payload }
+              ]
+            )
+          }.to raise_error(ActiveRecord::RecordInvalid)
+
+          expect(all_widgets).to be_empty
+        end
+
+        it 'should emit an instrumentation event naming the topic and operation' do
+          events = []
+          Karafka.monitor.subscribe('deimos.batch_consumption.initial_failure') do |event|
+            events << event.payload
+          end
+
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+                { key: 2, payload: poison_payload }
+              ]
+            )
+          }.to raise_error(Deimos::BatchFallbackError)
+
+          expect(events.size).to eq(1)
+          expect(events.first).to include(consumer: consumer_class,
+                                          topic: 'my-topic',
+                                          operation: :upsert_records,
+                                          count: 2)
+        end
+
+        it 'should only re-run the database write, not the message-level processing' do
+          # Retrying by breaking the batch into writes of one must not re-run anything that
+          # already ran for the batch as a whole. pre_process is called once for the group, and
+          # valid_records is announced once with every record that made it, rather than once per
+          # record.
+          pre_processed = []
+          allow_any_instance_of(consumer_class).to receive(:pre_process) do |_, messages|
+            pre_processed << messages.map(&:key)
+          end
+
+          valid_records = []
+          Karafka.monitor.subscribe('deimos.batch_consumption.valid_records') do |event|
+            valid_records << event.payload[:records]
+          end
+
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+                { key: 2, payload: poison_payload },
+                { key: 3, payload: { test_id: 'ghi', some_int: 3 } }
+              ]
+            )
+          }.to raise_error(Deimos::BatchFallbackError)
+
+          expect(pre_processed).to eq([%w(1 2 3)])
+          expect(valid_records.size).to eq(1)
+          expect(valid_records.first.map(&:test_id)).to contain_exactly('abc', 'ghi')
+        end
+
+        it 'should not retry individually when the batch failed on a deadlock' do
+          allow(Deimos::Utils::DeadlockRetry).to receive(:sleep)
+          allow(Widget).to receive(:import!).
+            and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded'))
+
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+                { key: 2, payload: { test_id: 'def', some_int: 2 } }
+              ]
+            )
+          }.to raise_error(ActiveRecord::Deadlocked)
+
+          # Only DeadlockRetry's own 3 attempts - no per-message retries on top.
+          expect(Widget).to have_received(:import!).exactly(3).times
+        end
+      end
+
+      context 'with a single-message batch' do
+        it 'should raise the original error rather than wrapping it' do
+          expect {
+            publish_batch([{ key: 1, payload: poison_payload }])
+          }.to raise_error(ActiveRecord::RecordInvalid)
+
+          expect(all_widgets).to be_empty
+        end
+      end
+
+      context 'when batch_message_fallback is turned off' do
+        before(:each) do
+          register_consumer(consumer_class, 'MySchema',
+                            key_config: { plain: true },
+                            configs: { reraise_errors: true,
+                                       batch_message_fallback: false })
+        end
+
+        it 'should lose the whole batch to a single bad record' do
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+                { key: 2, payload: poison_payload },
+                { key: 3, payload: { test_id: 'ghi', some_int: 3 } }
+              ]
+            )
+          }.to raise_error(ActiveRecord::RecordInvalid)
+
+          expect(all_widgets).to be_empty
+        end
+
+        it 'should not attempt any individual writes' do
+          allow(Widget).to receive(:import!).and_call_original
+
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+                { key: 2, payload: poison_payload }
+              ]
+            )
+          }.to raise_error(ActiveRecord::RecordInvalid)
+
+          # The batch write raises during validation, before any import is issued.
+          expect(Widget).not_to have_received(:import!)
+        end
+      end
+
+      context 'with keys repeated across BatchSlicer slices' do
+        it 'should still process the later slices after an earlier one fails' do
+          # Key 1 appears twice, so BatchSlicer splits the batch into two slices:
+          # [key 1 (poison), key 2] and [key 1 (valid)]. The valid update for key 1 sits in the
+          # second slice, so it only lands if a failure in the first slice doesn't abort the rest.
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: poison_payload },
+                { key: 1, payload: { test_id: 'later', some_int: 2 } },
+                { key: 2, payload: { test_id: 'ok', some_int: 3 } }
+              ]
+            )
+          }.to raise_error(Deimos::BatchFallbackError, /Failed keys: "1"/)
+
+          expect(all_widgets).
+            to contain_exactly(have_attributes(id: 1, test_id: 'later', some_int: 2),
+                               have_attributes(id: 2, test_id: 'ok', some_int: 3))
+        end
+      end
+
+      context 'with max_db_batch_size' do
+        let(:consumer_class) do
+          Class.new(described_class) do
+            record_class Widget
+            compacted false
+            max_db_batch_size 2
+          end
+        end
+
+        it 'should still process the later groups after an earlier one fails' do
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: { test_id: 'abc', some_int: 1 } },
+                { key: 2, payload: poison_payload },
+                { key: 3, payload: { test_id: 'ghi', some_int: 3 } },
+                { key: 4, payload: { test_id: 'jkl', some_int: 4 } }
+              ]
+            )
+          }.to raise_error(Deimos::BatchFallbackError, /Failed keys: "2"/)
+
+          expect(all_widgets).
+            to contain_exactly(have_attributes(id: 1, test_id: 'abc'),
+                               have_attributes(id: 3, test_id: 'ghi'),
+                               have_attributes(id: 4, test_id: 'jkl'))
+        end
+      end
+
+      context 'when removing records fails' do
+        let(:consumer_class) do
+          Class.new(described_class) do
+            record_class Widget
+            compacted false
+
+            def remove_records(messages)
+              raise 'cannot delete widget 2' if messages.any? { |m| m.key.to_s == '2' }
+
+              super
+            end
+          end
+        end
+
+        it 'should delete the records it can and report the one it cannot' do
+          Widget.create!(id: 1, test_id: 'abc', some_int: 1)
+          Widget.create!(id: 2, test_id: 'def', some_int: 2)
+          Widget.create!(id: 3, test_id: 'ghi', some_int: 3)
+
+          expect {
+            publish_batch(
+              [
+                { key: 1, payload: nil },
+                { key: 2, payload: nil },
+                { key: 3, payload: nil }
+              ]
+            )
+          }.to raise_error(Deimos::BatchFallbackError, /Failed keys: "2"/)
+
+          expect(all_widgets).to contain_exactly(have_attributes(id: 2, test_id: 'def'))
+        end
+      end
+    end
+
     describe 'skipping records' do
       before(:each) do
         register_consumer(consumer_class,

data/spec/active_record_consume/batch_consumption_spec.rb

@@ -119,7 +119,7 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchConsumption do
     end
 
     it 'for schema classes' do
-      klass = Class.new(AvroGen::SchemaClass::Record) do
+      klass = Class.new(Deimos::SchemaClass::Record) do
         attr_accessor :some_name
 
         # @override

data/spec/active_record_consumer_spec.rb

@@ -59,7 +59,7 @@ module ActiveRecordConsumerTest
 
       Time.zone = 'Eastern Time (US & Canada)'
 
-      schema_class = Class.new(AvroGen::SchemaClass::Record) do
+      schema_class = Class.new(Deimos::SchemaClass::Record) do
         def schema
           'MySchema'
         end
@@ -88,7 +88,7 @@ module ActiveRecordConsumerTest
       end
       stub_const('Schemas::MySchema', schema_class)
 
-      schema_datetime_class = Class.new(AvroGen::SchemaClass::Record) do
+      schema_datetime_class = Class.new(Deimos::SchemaClass::Record) do
         def schema
           'MySchemaWithDateTimes'
         end
@@ -164,7 +164,7 @@ module ActiveRecordConsumerTest
               end
             end
             Deimos.configure do |config|
-              config.avrogen.use_full_namespace = true
+              config.schema.use_full_namespace = true
             end
           end
 

data/spec/active_record_producer_spec.rb

@@ -95,7 +95,7 @@ describe Deimos::ActiveRecordProducer do
         before(:each) do
           Deimos.configure do |config|
             config.schema.use_schema_classes = use_schema_classes
-            config.avrogen.use_full_namespace = true
+            config.schema.use_full_namespace = true
           end
         end
 

data/spec/batch_consumer_spec.rb

@@ -51,14 +51,14 @@ module ConsumerTest
 
           let(:schema_class_batch) do
             batch.map do |p|
-              AvroGen::SchemaClass.instance(p, 'MySchema', 'com.my-namespace')
+              Deimos::Utils::SchemaClass.instance(p, 'MySchema', 'com.my-namespace')
             end
           end
           let(:use_schema_classes) { true }
 
           before(:each) do
             Deimos.configure do |config|
-              config.avrogen.use_full_namespace = true
+              config.schema.use_full_namespace = true
             end
           end
 

data/spec/consumer_spec.rb

@@ -48,8 +48,8 @@ module ConsumerTest
 
           before(:each) do
             Deimos.configure do |config|
-              config.avrogen.use_full_namespace = true
-              config.avrogen.schema_namespace_map = {
+              config.schema.use_full_namespace = true
+              config.schema.schema_namespace_map = {
                 'com' => 'Schemas',
                 'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
               }
@@ -160,7 +160,7 @@ module ConsumerTest
         before(:each) do
           set_karafka_config(:use_schema_classes, true)
           Deimos.configure do |config|
-            config.avrogen.use_full_namespace = true
+            config.schema.use_full_namespace = true
           end
         end
 
@@ -205,8 +205,8 @@ module ConsumerTest
         before(:each) do
           set_karafka_config(:use_schema_classes, true)
           Deimos.configure do |config|
-            config.avrogen.use_full_namespace = true
-            config.avrogen.schema_namespace_map = {
+            config.schema.use_full_namespace = true
+            config.schema.schema_namespace_map = {
               'com' => 'Schemas',
               'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
             }

data/spec/generators/schema_class/my_schema_spec.rb

@@ -0,0 +1,16 @@
+RSpec.describe Schemas::MyNamespace::MySchema do
+  let(:key) { Schemas::MyNamespace::MySchemaKey.new(test_id: 123) }
+
+  it 'should produce a tombstone with a hash' do
+    result = described_class.tombstone({ test_id: 123 })
+    expect(result.payload_key).to eq(key)
+    expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
+  end
+
+  it 'should work with a record' do
+    key = Schemas::MyNamespace::MySchemaKey.new(test_id: 123)
+    result = described_class.tombstone(key)
+    expect(result.payload_key).to eq(key)
+    expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
+  end
+end

data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# For testing the generated class.
+RSpec.describe Schemas::MyNamespace::MySchemaWithCircularReference do
+  let(:payload_hash) do
+    {
+      properties: {
+        a_boolean: {
+          'property' => true
+        },
+        an_integer: {
+          'property' => 1
+        },
+        a_float: {
+          'property' => 4.5
+        },
+        a_string: {
+          'property' => 'string'
+        },
+        an_array: {
+          'property' => [1, 2, 3]
+        },
+        an_hash: {
+          'property' => {
+            'a_key' => 'a_value'
+          }
+        }
+      }
+    }
+  end
+
+  describe 'class initialization' do
+    it 'should initialize the class from keyword arguments' do
+      klass = described_class.new(properties: payload_hash[:properties])
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class from a hash with symbols as keys' do
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class when missing attributes' do
+      payload_hash.delete(:properties)
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+  end
+
+  describe 'base class methods' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    it 'should return the name of the schema and namespace' do
+      expect(klass.schema).to eq('MySchemaWithCircularReference')
+      expect(klass.namespace).to eq('com.my-namespace')
+      expect(klass.full_schema).to eq('com.my-namespace.MySchemaWithCircularReference')
+    end
+
+    it 'should return a json version of the payload' do
+      described_class.new(**payload_hash)
+      payload_h = {
+        'properties' => {
+          a_boolean: {
+            'property' =>true
+          },
+          an_integer: {
+            'property' =>1
+          },
+          a_float: {
+            'property' =>4.5
+          },
+          a_string: {
+            'property' =>'string'
+          },
+          an_array: {
+            'property' =>[1, 2, 3]
+          },
+          an_hash: {
+            'property' =>{
+              'a_key' => 'a_value'
+            }
+          }
+        }
+      }
+
+      expect(klass.as_json).to eq(payload_h)
+    end
+
+    it 'should return a JSON string of the payload' do
+      s = '{"properties":{"a_boolean":{"property":true},"an_integer":{"property":1},"a_float":{"property":4.5},"a_str' \
+          'ing":{"property":"string"},"an_array":{"property":[1,2,3]},"an_hash":{"property":{"a_key":"a_value"}}}}'
+      expect(klass.to_json).to eq(s)
+    end
+  end
+end