deimos-ruby 1.22.5 → 1.23.0

data/spec/active_record_batch_consumer_spec.rb

@@ -11,6 +11,7 @@ module ActiveRecordBatchConsumerTest
         t.string(:part_two)
         t.integer(:some_int)
         t.boolean(:deleted, default: false)
+        t.string(:bulk_import_id)
         t.timestamps
 
         t.index(%i(part_one part_two), unique: true)
@@ -73,8 +74,12 @@ module ActiveRecordBatchConsumerTest
     describe 'consume_batch' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
         context "with Schema Class consumption #{setting}" do
+
           before(:each) do
-            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+            Deimos.configure do |config|
+              config.schema.use_schema_classes = use_schema_classes
+              config.schema.generate_namespace_folders = true
+            end
           end
 
           it 'should handle an empty batch' do
@@ -205,6 +210,48 @@ module ActiveRecordBatchConsumerTest
                 ]
               )
           end
+
+          it 'should handle deletes with deadlock retries' do
+            allow(Deimos::Utils::DeadlockRetry).to receive(:sleep)
+            allow(instance_double(ActiveRecord::Relation)).to receive(:delete_all).and_raise(
+              ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
+            ).twice.ordered
+
+            Widget.create!(id: 1, test_id: 'abc', some_int: 2)
+
+            publish_batch(
+              [
+                { key: 1,
+                  payload: nil },
+                { key: 1,
+                  payload: nil }
+              ]
+            )
+
+            expect(all_widgets).to be_empty
+          end
+
+          it 'should not delete after multiple deadlock retries' do
+            allow(Deimos::Utils::DeadlockRetry).to receive(:sleep)
+            allow(instance_double(ActiveRecord::Relation)).to receive(:delete_all).and_raise(
+              ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
+            ).exactly(3).times
+
+            Widget.create!(id: 1, test_id: 'abc', some_int: 2)
+
+            publish_batch(
+              [
+                { key: 1,
+                  payload: nil },
+                { key: 1,
+                  payload: nil }
+              ]
+            )
+
+            expect(Widget.count).to eq(0)
+
+          end
+
         end
       end
     end
@@ -254,64 +301,6 @@ module ActiveRecordBatchConsumerTest
       end
     end
 
-    describe 'batch atomicity' do
-      it 'should roll back if there was an exception while deleting' do
-        Widget.create!(id: 1, test_id: 'abc', some_int: 2)
-
-        travel 1.day
-
-        expect(Widget.connection).to receive(:delete).and_raise('Some error')
-
-        expect {
-          publish_batch(
-            [
-              { key: 1,
-                payload: { test_id: 'def', some_int: 3 } },
-              { key: 1,
-                payload: nil }
-            ]
-          )
-        }.to raise_error('Some error')
-
-        expect(all_widgets).
-          to match_array(
-            [
-              have_attributes(id: 1, test_id: 'abc', some_int: 2, updated_at: start, created_at: start)
-            ]
-          )
-      end
-
-      it 'should roll back if there was an invalid instance while upserting' do
-        Widget.create!(id: 1, test_id: 'abc', some_int: 2) # Updated but rolled back
-        Widget.create!(id: 3, test_id: 'ghi', some_int: 3) # Removed but rolled back
-
-        travel 1.day
-
-        expect {
-          publish_batch(
-            [
-              { key: 1,
-                payload: { test_id: 'def', some_int: 3 } },
-              { key: 2,
-                payload: nil },
-              { key: 2,
-                payload: { test_id: '', some_int: 4 } }, # Empty string is not valid for test_id
-              { key: 3,
-                payload: nil }
-            ]
-          )
-        }.to raise_error(ActiveRecord::RecordInvalid)
-
-        expect(all_widgets).
-          to match_array(
-            [
-              have_attributes(id: 1, test_id: 'abc', some_int: 2, updated_at: start, created_at: start),
-              have_attributes(id: 3, test_id: 'ghi', some_int: 3, updated_at: start, created_at: start)
-            ]
-          )
-      end
-    end
-
     describe 'compound keys' do
       let(:consumer_class) do
         Class.new(described_class) do
@@ -493,5 +482,333 @@ module ActiveRecordBatchConsumerTest
       end
     end
 
+    describe 'pre processing' do
+      context 'with uncompacted messages' do
+        let(:consumer_class) do
+          Class.new(described_class) do
+            schema 'MySchema'
+            namespace 'com.my-namespace'
+            key_config plain: true
+            record_class Widget
+            compacted false
+
+            def pre_process(messages)
+              messages.each do |message|
+                message.payload[:some_int] = -message.payload[:some_int]
+              end
+            end
+
+          end
+        end
+
+        it 'should pre-process records' do
+          Widget.create!(id: 1, test_id: 'abc', some_int: 1)
+          Widget.create!(id: 2, test_id: 'def', some_int: 2)
+
+          publish_batch(
+            [
+              { key: 1,
+                payload: { test_id: 'abc', some_int: 11 } },
+              { key: 2,
+                payload: { test_id: 'def', some_int: 20 } }
+            ]
+          )
+
+          widget_one, widget_two = Widget.all.to_a
+
+          expect(widget_one.some_int).to eq(-11)
+          expect(widget_two.some_int).to eq(-20)
+        end
+      end
+    end
+
+    describe 'global configurations' do
+
+      context 'with a global bulk_import_id_generator' do
+
+        before(:each) do
+          Deimos.configure do
+            consumers.bulk_import_id_generator(proc { 'global' })
+          end
+        end
+
+        it 'should call the default bulk_import_id_generator proc' do
+          publish_batch(
+            [
+              { key: 1,
+                payload: { test_id: 'abc', some_int: 3 } }
+            ]
+          )
+
+          expect(all_widgets).
+            to match_array(
+              [
+                have_attributes(id: 1,
+                                test_id: 'abc',
+                                some_int: 3,
+                                updated_at: start,
+                                created_at: start,
+                                bulk_import_id: 'global')
+              ]
+            )
+
+        end
+
+      end
+
+      context 'with a class defined bulk_import_id_generator' do
+
+        before(:each) do
+          Deimos.configure do
+            consumers.bulk_import_id_generator(proc { 'global' })
+          end
+          consumer_class.config[:bulk_import_id_generator] = proc { 'custom' }
+        end
+
+        it 'should call the default bulk_import_id_generator proc' do
+
+          publish_batch(
+            [
+              { key: 1,
+                payload: { test_id: 'abc', some_int: 3 } }
+            ]
+          )
+
+          expect(all_widgets).
+            to match_array(
+              [
+                have_attributes(id: 1,
+                                test_id: 'abc',
+                                some_int: 3,
+                                updated_at: start,
+                                created_at: start,
+                                bulk_import_id: 'custom')
+              ]
+            )
+
+        end
+      end
+
+    end
+
+    describe 'should_consume?' do
+
+      let(:consumer_class) do
+        Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+          record_class Widget
+          compacted false
+
+          def should_consume?(record)
+            record.test_id != 'def'
+          end
+
+          def self.process_invalid_records(_)
+            nil
+          end
+
+          ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
+            payload = ActiveSupport::Notifications::Event.new(*args).payload
+            payload[:consumer].process_invalid_records(payload[:records])
+          end
+
+        end
+      end
+
+      it "should skip records that shouldn't be consumed" do
+        Widget.create!(id: 1, test_id: 'abc', some_int: 1)
+        Widget.create!(id: 2, test_id: 'def', some_int: 2)
+        publish_batch(
+          [
+            { key: 1,
+              payload: { test_id: 'abc', some_int: 11 } },
+            { key: 2,
+              payload: { test_id: 'def', some_int: 20 } }
+          ]
+        )
+
+        expect(Widget.count).to eq(2)
+        expect(Widget.all.to_a).to match_array([
+                                                 have_attributes(id: 1,
+                                                                 test_id: 'abc',
+                                                                 some_int: 11,
+                                                                 updated_at: start,
+                                                                 created_at: start),
+                                                 have_attributes(id: 2,
+                                                                 test_id: 'def',
+                                                                 some_int: 2,
+                                                                 updated_at: start,
+                                                                 created_at: start)
+                                               ])
+      end
+
+    end
+
+    describe 'post processing' do
+
+      context 'with uncompacted messages' do
+        let(:consumer_class) do
+          Class.new(described_class) do
+            schema 'MySchema'
+            namespace 'com.my-namespace'
+            key_config plain: true
+            record_class Widget
+            compacted false
+
+            def should_consume?(record)
+              record.some_int.even?
+            end
+
+            def self.process_valid_records(valid)
+              # Success
+              attrs = valid.first.attributes
+              Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: 2000)
+            end
+
+            def self.process_invalid_records(invalid)
+              # Invalid
+              attrs = invalid.first.record.attributes
+              Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: attrs['some_int'])
+            end
+
+            ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
+              payload = ActiveSupport::Notifications::Event.new(*args).payload
+              payload[:consumer].process_invalid_records(payload[:records])
+            end
+
+            ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
+              payload = ActiveSupport::Notifications::Event.new(*args).payload
+              payload[:consumer].process_valid_records(payload[:records])
+            end
+
+          end
+        end
+
+        it 'should process successful and failed records' do
+          Widget.create!(id: 1, test_id: 'abc', some_int: 1)
+          Widget.create!(id: 2, test_id: 'def', some_int: 2)
+
+          publish_batch(
+            [
+              { key: 1,
+                payload: { test_id: 'abc', some_int: 11 } },
+              { key: 2,
+                payload: { test_id: 'def', some_int: 20 } }
+            ]
+          )
+
+          widget_one, widget_two = Widget.all.to_a
+
+          expect(widget_one.some_int).to eq(11)
+          expect(widget_two.some_int).to eq(2000)
+        end
+      end
+
+      context 'with compacted messages' do
+        let(:consumer_class) do
+          Class.new(described_class) do
+            schema 'MySchema'
+            namespace 'com.my-namespace'
+            key_config plain: true
+            record_class Widget
+            compacted true
+
+            def should_consume?(record)
+              record.some_int.even?
+            end
+
+            def self.process_valid_records(valid)
+              # Success
+              attrs = valid.first.attributes
+              Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: 2000)
+            end
+
+            def self.process_invalid_records(invalid)
+              # Invalid
+              attrs = invalid.first.record.attributes
+              Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: attrs['some_int'])
+            end
+
+            ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
+              payload = ActiveSupport::Notifications::Event.new(*args).payload
+              payload[:consumer].process_invalid_records(payload[:records])
+            end
+
+            ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
+              payload = ActiveSupport::Notifications::Event.new(*args).payload
+              payload[:consumer].process_valid_records(payload[:records])
+            end
+
+          end
+        end
+
+        it 'should process successful and failed records' do
+          Widget.create!(id: 1, test_id: 'abc', some_int: 1)
+          Widget.create!(id: 2, test_id: 'def', some_int: 2)
+
+          publish_batch(
+            [
+              { key: 1,
+                payload: { test_id: 'abc', some_int: 11 } },
+              { key: 2,
+                payload: { test_id: 'def', some_int: 20 } }
+            ]
+          )
+
+          widget_one, widget_two = Widget.all.to_a
+
+          expect(widget_one.some_int).to eq(11)
+          expect(widget_two.some_int).to eq(2000)
+        end
+      end
+
+      context 'with post processing errors' do
+        let(:consumer_class) do
+          Class.new(described_class) do
+            schema 'MySchema'
+            namespace 'com.my-namespace'
+            key_config plain: true
+            record_class Widget
+            compacted false
+
+            def self.process_valid_records(_)
+              raise StandardError, 'Something went wrong'
+            end
+
+            ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
+              payload = ActiveSupport::Notifications::Event.new(*args).payload
+              payload[:consumer].process_valid_records(payload[:records])
+            end
+
+          end
+        end
+
+        it 'should save records if an exception occurs in post processing' do
+          Widget.create!(id: 1, test_id: 'abc', some_int: 1)
+          Widget.create!(id: 2, test_id: 'def', some_int: 2)
+
+          expect {
+            publish_batch(
+              [
+                { key: 1,
+                  payload: { test_id: 'abc', some_int: 11 } },
+                { key: 2,
+                  payload: { test_id: 'def', some_int: 20 } }
+              ]
+            )
+          }.to raise_error(StandardError, 'Something went wrong')
+
+          widget_one, widget_two = Widget.all.to_a
+
+          expect(widget_one.some_int).to eq(11)
+          expect(widget_two.some_int).to eq(20)
+
+        end
+      end
+
+    end
+
   end
 end

data/spec/active_record_consume/mass_updater_spec.rb

@@ -39,6 +39,8 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
       end
     end
 
+    let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
     before(:each) do
       stub_const('Widget', widget_class)
       stub_const('Detail', detail_class)
@@ -52,24 +54,65 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
             Deimos::ActiveRecordConsume::BatchRecord.new(
               klass: Widget,
               attributes: { test_id: 'id1', some_int: 5, detail: { title: 'Title 1' } },
-              bulk_import_column: 'bulk_import_id'
+              bulk_import_column: 'bulk_import_id',
+              bulk_import_id_generator: bulk_id_generator
             ),
             Deimos::ActiveRecordConsume::BatchRecord.new(
               klass: Widget,
               attributes: { test_id: 'id2', some_int: 10, detail: { title: 'Title 2' } },
-              bulk_import_column: 'bulk_import_id'
+              bulk_import_column: 'bulk_import_id',
+              bulk_import_id_generator: bulk_id_generator
             )
           ]
         )
       end
 
       it 'should mass update the batch' do
-        described_class.new(Widget).mass_update(batch)
+        allow(SecureRandom).to receive(:uuid).and_return('1', '2')
+        results = described_class.new(Widget, bulk_import_id_generator: bulk_id_generator).mass_update(batch)
+        expect(results.count).to eq(2)
+        expect(results.map(&:test_id)).to match(%w(id1 id2))
         expect(Widget.count).to eq(2)
+        expect(Widget.all.to_a.map(&:bulk_import_id)).to match(%w(1 2))
         expect(Detail.count).to eq(2)
         expect(Widget.first.detail).not_to be_nil
         expect(Widget.last.detail).not_to be_nil
       end
 
+      context 'with deadlock retries' do
+        before(:each) do
+          allow(Deimos::Utils::DeadlockRetry).to receive(:sleep)
+        end
+
+        it 'should upsert rows after deadlocks' do
+          allow(Widget).to receive(:import!).and_raise(
+            ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
+          ).twice.ordered
+          allow(Widget).to receive(:import!).and_raise(
+            ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
+          ).once.and_call_original
+
+          results = described_class.new(Widget, bulk_import_id_generator: bulk_id_generator).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(results.map(&:test_id)).to match(%w(id1 id2))
+          expect(Widget.count).to eq(2)
+          expect(Detail.count).to eq(2)
+          expect(Widget.first.detail).not_to be_nil
+          expect(Widget.last.detail).not_to be_nil
+        end
+
+        it 'should not upsert after encountering multiple deadlocks' do
+          allow(Widget).to receive(:import!).and_raise(
+            ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
+          ).exactly(3).times
+          expect {
+            described_class.new(Widget, bulk_import_id_generator: bulk_id_generator).mass_update(batch)
+          }.to raise_error(ActiveRecord::Deadlocked)
+          expect(Widget.count).to eq(0)
+          expect(Detail.count).to eq(0)
+        end
+
+      end
+
     end
 end

data/spec/active_record_consumer_spec.rb

@@ -3,6 +3,7 @@
 require 'date'
 
 # Wrapped in a module to prevent class leakage
+# rubocop:disable Metrics/ModuleLength
 module ActiveRecordConsumerTest
   describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
     before(:all) do
@@ -66,13 +67,84 @@ module ActiveRecordConsumerTest
       stub_const('MyCustomFetchConsumer', consumer_class)
 
       Time.zone = 'Eastern Time (US & Canada)'
+
+      schema_class = Class.new(Deimos::SchemaClass::Record) do
+        def schema
+          'MySchema'
+        end
+
+        def namespace
+          'com.my-namespace'
+        end
+
+        attr_accessor :test_id
+        attr_accessor :some_int
+
+        def initialize(test_id: nil,
+                       some_int: nil)
+          self.test_id = test_id
+          self.some_int = some_int
+        end
+
+        def as_json(_opts={})
+          {
+            'test_id' => @test_id,
+            'some_int' => @some_int,
+            'payload_key' => @payload_key&.as_json
+          }
+        end
+      end
+      stub_const('Schemas::MySchema', schema_class)
+
+      schema_datetime_class = Class.new(Deimos::SchemaClass::Record) do
+        def schema
+          'MySchemaWithDateTimes'
+        end
+
+        def namespace
+          'com.my-namespace'
+        end
+
+        attr_accessor :test_id
+        attr_accessor :some_int
+        attr_accessor :updated_at
+        attr_accessor :some_datetime_int
+        attr_accessor :timestamp
+
+        def initialize(test_id: nil,
+                       some_int: nil,
+                       updated_at: nil,
+                       some_datetime_int: nil,
+                       timestamp: nil)
+          self.test_id = test_id
+          self.some_int = some_int
+          self.updated_at = updated_at
+          self.some_datetime_int = some_datetime_int
+          self.timestamp = timestamp
+        end
+
+        def as_json(_opts={})
+          {
+            'test_id' => @test_id,
+            'some_int' => @some_int,
+            'updated_at' => @updated_at,
+            'some_datetime_int' => @some_datetime_int,
+            'timestamp' => @timestamp,
+            'payload_key' => @payload_key&.as_json
+          }
+        end
+      end
+      stub_const('Schemas::MySchemaWithDateTimes', schema_datetime_class)
     end
 
     describe 'consume' do
       SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
         context "with Schema Class consumption #{setting}" do
           before(:each) do
-            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+            Deimos.configure do |config|
+              config.schema.use_schema_classes = use_schema_classes
+              config.schema.generate_namespace_folders = true
+            end
           end
 
           it 'should receive events correctly' do
@@ -182,3 +254,4 @@ module ActiveRecordConsumerTest
     end
   end
 end
+# rubocop:enable Metrics/ModuleLength

data/spec/active_record_producer_spec.rb

@@ -69,7 +69,10 @@ describe Deimos::ActiveRecordProducer do
     SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
       context "with Schema Class consumption #{setting}" do
         before(:each) do
-          Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+          Deimos.configure do |config|
+            config.schema.use_schema_classes = use_schema_classes
+            config.schema.generate_namespace_folders = true
+          end
         end
 
         it 'should send events correctly' do

data/spec/batch_consumer_spec.rb

@@ -40,7 +40,10 @@ module ConsumerTest
           end
 
           before(:each) do
-            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+            Deimos.configure do |config|
+              config.schema.use_schema_classes = use_schema_classes
+              config.schema.generate_namespace_folders = true
+            end
           end
 
           it 'should provide backwards compatibility for BatchConsumer class' do