deimos-ruby 1.24.2 → 1.24.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2be475993ca3cec87d78682f7724b973f5aadb885612f98d5978169da2932bf
-  data.tar.gz: 5ab0f052676184f800965bf68aebecac80583d1da2f38561a305e5f5b57a68c9
+  metadata.gz: de8ca3bf7233f799c3bf9169fa647dabebd8e08cd2b05ec00511c132a5f3920d
+  data.tar.gz: 03177d9ac87184fe3ce6ef916bce0ad39e3b81a6576f60ba71c33b298f82dfd8
 SHA512:
-  metadata.gz: 918a363d9994537a221ccc6d00131c75a97f36c8cf89bdda47e0a6e6770b36fd1c3b07e5177d0ef284494020d90dd9081bc9d40ec41f4dc8bbe45e9c4165ea1f
-  data.tar.gz: 60d316ef05ce427234e366e32f44addd7188adb45345b025bfb9fd21f339d0c579f317f1e55db20185b9f5f825af3226c27cb53d5e9336ec805200d2a9ce5bd9
+  metadata.gz: 40ecdddd9c0b8f5f9675f8058f4a8c8adf1eb69a293e04870a46b9d8ae1972cdc70088611c898f0ab91e072f6627c4760bca5ca5399b698316edda8f3212f5aa
+  data.tar.gz: 3f2fa463d5c777ce7e4dfc4cf610a7dd4ed3f6dba5b6dd459f1ca8fad037f0baa47278436c65c91dc90679f88e9d42a792b84ccbdc2cde4b58d1239d03b2fd4a

data/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.24.3 - 2024-05-13
+
+- Feature: Enable `producers.persistent_connections` phobos setting
+- Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
+
 # 1.24.2 - 2024-05-01
 - Fix: Deprecation notice with Rails 7.
 

data/docs/CONFIGURATION.md

@@ -102,6 +102,7 @@ heartbeat_interval|10|Interval between heartbeats; must be less than the session
 backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
 replace_associations|nil| Whether to delete existing associations for records during bulk consumption for this consumer. If no value is specified the provided/default value from the `consumers` configuration will be used.
 bulk_import_id_generator|nil| Block to determine the `bulk_import_id` generated during bulk consumption. If no block is specified the provided/default block from the `consumers` configuration will be used.
+save_associations_first|false|Whether to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
 
 ## Defining Database Pollers
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -83,7 +83,7 @@ module Deimos
       def deleted_query(records)
         keys = records.
           map { |m| record_key(m.key)[@klass.primary_key] }.
-          reject(&:nil?)
+          compact
 
         @klass.unscoped.where(@klass.primary_key => keys)
       end
@@ -168,7 +168,9 @@ module Deimos
                                   key_col_proc: key_col_proc,
                                   col_proc: col_proc,
                                   replace_associations: self.class.replace_associations,
-                                  bulk_import_id_generator: self.class.bulk_import_id_generator)
+                                  bulk_import_id_generator: self.class.bulk_import_id_generator,
+                                  save_associations_first: self.class.save_associations_first,
+                                  bulk_import_id_column: self.class.bulk_import_id_column)
         ActiveSupport::Notifications.instrument('batch_consumption.valid_records', {
                                                   records: updater.mass_update(record_list),
                                                   consumer: self.class

data/lib/deimos/active_record_consume/mass_updater.rb

@@ -20,10 +20,13 @@ module Deimos
       # @param col_proc [Proc<Class < ActiveRecord::Base>]
       # @param replace_associations [Boolean]
       def initialize(klass, key_col_proc: nil, col_proc: nil,
-                     replace_associations: true, bulk_import_id_generator: nil)
+                     replace_associations: true, bulk_import_id_generator: nil, save_associations_first: false,
+                     bulk_import_id_column: nil)
         @klass = klass
         @replace_associations = replace_associations
         @bulk_import_id_generator = bulk_import_id_generator
+        @save_associations_first = save_associations_first
+        @bulk_import_id_column = bulk_import_id_column&.to_s
 
         @key_cols = {}
         @key_col_proc = key_col_proc
@@ -84,15 +87,67 @@ module Deimos
         end
       end
 
+      # Assign associated records to corresponding primary records
+      # @param record_list [BatchRecordList] RecordList of primary records for this consumer
+      # @return [Hash]
+      def assign_associations(record_list)
+        associations_info = {}
+        record_list.associations.each do |assoc|
+          col = @bulk_import_id_column if assoc.klass.column_names.include?(@bulk_import_id_column)
+          associations_info[[assoc, col]] = []
+        end
+        record_list.batch_records.each do |primary_batch_record|
+          associations_info.each_key do |assoc, col|
+            batch_record = BatchRecord.new(klass: assoc.klass,
+                                           attributes: primary_batch_record.associations[assoc.name],
+                                           bulk_import_column: col,
+                                           bulk_import_id_generator: @bulk_import_id_generator)
+            # Associate this associated batch record's record with the primary record to
+            # retrieve foreign_key after associated records have been saved and primary
+            # keys have been filled
+            primary_batch_record.record.assign_attributes({ assoc.name => batch_record.record })
+            associations_info[[assoc, col]] << batch_record
+          end
+        end
+        associations_info
+      end
+
+      # Save associated records and fill foreign keys on RecordList records
+      # @param record_list [BatchRecordList] RecordList of primary records for this consumer
+      # @param associations_info [Hash] Contains association info
+      def save_associations_first(record_list, associations_info)
+        associations_info.each_value do |records|
+          assoc_record_list = BatchRecordList.new(records)
+          Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+            save_records_to_database(assoc_record_list)
+          end
+          import_associations(assoc_record_list)
+        end
+        record_list.records.each do |record|
+          associations_info.each_key do |assoc, _|
+            record.assign_attributes({ assoc.foreign_key => record.send(assoc.name).id })
+          end
+        end
+      end
+
       # @param record_list [BatchRecordList]
       # @return [Array<ActiveRecord::Base>]
       def mass_update(record_list)
         # The entire batch should be treated as one transaction so that if
         # any message fails, the whole thing is rolled back or retried
         # if there is deadlock
-        Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
-          save_records_to_database(record_list)
-          import_associations(record_list) if record_list.associations.any?
+
+        if @save_associations_first
+          associations_info = assign_associations(record_list)
+          save_associations_first(record_list, associations_info)
+          Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+            save_records_to_database(record_list)
+          end
+        else
+          Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+            save_records_to_database(record_list)
+            import_associations(record_list) if record_list.associations.any?
+          end
         end
         record_list.records
       end

data/lib/deimos/active_record_consumer.rb

@@ -45,6 +45,11 @@ module Deimos
         config[:replace_associations]
       end
 
+      # @return [Boolean]
+      def save_associations_first
+        config[:save_associations_first]
+      end
+
       # @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
       # only the last message for each unique key in a batch is processed.
       # @return [void]

data/lib/deimos/config/configuration.rb

@@ -97,7 +97,8 @@ module Deimos # rubocop:disable Metrics/ModuleLength
                                   kafka_config.replace_associations
                                 end,
           bulk_import_id_generator: kafka_config.bulk_import_id_generator ||
-            Deimos.config.consumers.bulk_import_id_generator
+            Deimos.config.consumers.bulk_import_id_generator,
+          save_associations_first: kafka_config.save_associations_first
         )
       end
     end
@@ -476,6 +477,11 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # @return [Block]
       setting :bulk_import_id_generator, nil
 
+      # If enabled save associated records prior to saving the main record class
+      # This will also set foreign keys for associated records
+      # @return [Boolean]
+      setting :save_associations_first, false
+
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.
       setting :group_id

data/lib/deimos/config/phobos_config.rb

@@ -52,7 +52,8 @@ module Deimos
           compression_threshold: self.producers.compression_threshold,
           max_queue_size: self.producers.max_queue_size,
           delivery_threshold: self.producers.delivery_threshold,
-          delivery_interval: self.producers.delivery_interval
+          delivery_interval: self.producers.delivery_interval,
+          persistent_connections: self.producers.persistent_connections
         },
         consumer: {
           session_timeout: self.consumers.session_timeout,

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.24.2'
+  VERSION = '1.24.3'
 end

data/spec/active_record_consume/mass_updater_spec.rb

@@ -114,5 +114,142 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
 
       end
 
+      context 'with save_associations_first' do
+        before(:all) do
+          ActiveRecord::Base.connection.create_table(:fidgets, force: true) do |t|
+            t.string(:test_id)
+            t.integer(:some_int)
+            t.string(:bulk_import_id)
+            t.timestamps
+          end
+
+          ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
+            t.string(:title)
+            t.string(:bulk_import_id)
+            t.belongs_to(:fidget)
+
+            t.index(%i(title), unique: true)
+          end
+
+          ActiveRecord::Base.connection.create_table(:widget_fidgets, force: true, id: false) do |t|
+            t.belongs_to(:fidget)
+            t.belongs_to(:widget)
+            t.string(:bulk_import_id)
+            t.string(:note)
+            t.index(%i(widget_id fidget_id), unique: true)
+          end
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.drop_table(:fidgets)
+          ActiveRecord::Base.connection.drop_table(:fidget_details)
+          ActiveRecord::Base.connection.drop_table(:widget_fidgets)
+        end
+
+        let(:fidget_detail_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidget_details'
+            belongs_to :fidget
+          end
+        end
+
+        let(:fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidgets'
+            has_one :fidget_detail
+          end
+        end
+
+        let(:widget_fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'widget_fidgets'
+            belongs_to :fidget
+            belongs_to :widget
+          end
+        end
+
+        let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
+        let(:key_proc) do
+          lambda do |klass|
+            case klass.to_s
+            when 'Widget', 'Fidget'
+              %w(id)
+            when 'WidgetFidget'
+              %w(widget_id fidget_id)
+            when 'FidgetDetail', 'Detail'
+              %w(title)
+            else
+              raise "Key Columns for #{klass} not defined"
+            end
+
+          end
+        end
+
+        before(:each) do
+          stub_const('Fidget', fidget_class)
+          stub_const('FidgetDetail', fidget_detail_class)
+          stub_const('WidgetFidget', widget_fidget_class)
+          Widget.reset_column_information
+          Fidget.reset_column_information
+          WidgetFidget.reset_column_information
+        end
+
+        # rubocop:disable RSpec/MultipleExpectations, RSpec/ExampleLength
+        it 'should backfill the associations when upserting primary records' do
+          batch = Deimos::ActiveRecordConsume::BatchRecordList.new(
+            [
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: WidgetFidget,
+                attributes: {
+                  widget: { test_id: 'id1', some_int: 10, detail: { title: 'Widget Title 1' } },
+                  fidget: { test_id: 'id1', some_int: 10, fidget_detail: { title: 'Fidget Title 1' } },
+                  note: 'Stuff 1'
+                },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              ),
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: WidgetFidget,
+                attributes: {
+                  widget: { test_id: 'id2', some_int: 20, detail: { title: 'Widget Title 2' } },
+                  fidget: { test_id: 'id2', some_int: 20, fidget_detail: { title: 'Fidget Title 2' } },
+                  note: 'Stuff 2'
+                },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              )
+            ]
+          )
+
+          results = described_class.new(WidgetFidget,
+                                        bulk_import_id_generator: bulk_id_generator,
+                                        bulk_import_id_column: 'bulk_import_id',
+                                        key_col_proc: key_proc,
+                                        save_associations_first: true).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(Widget.count).to eq(2)
+          expect(Detail.count).to eq(2)
+          expect(Fidget.count).to eq(2)
+          expect(FidgetDetail.count).to eq(2)
+
+          WidgetFidget.all.each_with_index do |widget_fidget, ind|
+            widget = Widget.find_by(id: widget_fidget.widget_id)
+            expect(widget.test_id).to eq("id#{ind + 1}")
+            expect(widget.some_int).to eq((ind + 1) * 10)
+            detail = Detail.find_by(widget_id: widget_fidget.widget_id)
+            expect(detail.title).to eq("Widget Title #{ind + 1}")
+            fidget = Fidget.find_by(id: widget_fidget.fidget_id)
+            expect(fidget.test_id).to eq("id#{ind + 1}")
+            expect(fidget.some_int).to eq((ind + 1) * 10)
+            fidget_detail = FidgetDetail.find_by(fidget_id: widget_fidget.fidget_id)
+            expect(fidget_detail.title).to eq("Fidget Title #{ind + 1}")
+            expect(widget_fidget.note).to eq("Stuff #{ind + 1}")
+          end
+        end
+        # rubocop:enable RSpec/MultipleExpectations, RSpec/ExampleLength
+
+      end
+
     end
 end

data/spec/config/configuration_spec.rb

@@ -92,7 +92,8 @@ describe Deimos, 'configuration' do
           handler: 'ConsumerTest::MyConsumer',
           use_schema_classes: nil,
           max_db_batch_size: nil,
-          bulk_import_id_generator: nil
+          bulk_import_id_generator: nil,
+          save_associations_first: false
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
@@ -111,7 +112,8 @@ describe Deimos, 'configuration' do
           handler: 'ConsumerTest::MyBatchConsumer',
           use_schema_classes: nil,
           max_db_batch_size: nil,
-          bulk_import_id_generator: nil
+          bulk_import_id_generator: nil,
+          save_associations_first: false
         }
       ],
       producer: {
@@ -125,7 +127,8 @@ describe Deimos, 'configuration' do
         compression_threshold: 1,
         max_queue_size: 10_000,
         delivery_threshold: 0,
-        delivery_interval: 0
+        delivery_interval: 0,
+        persistent_connections: false
       }
     )
   end
@@ -264,7 +267,8 @@ describe Deimos, 'configuration' do
             handler: 'MyConfigConsumer',
             use_schema_classes: false,
             max_db_batch_size: nil,
-            bulk_import_id_generator: nil
+            bulk_import_id_generator: nil,
+            save_associations_first: false
           }
         ],
         producer: {
@@ -278,7 +282,8 @@ describe Deimos, 'configuration' do
           compression_threshold: 2,
           max_queue_size: 10,
           delivery_threshold: 1,
-          delivery_interval: 1
+          delivery_interval: 1,
+          persistent_connections: true
         }
       )
   end
@@ -295,6 +300,7 @@ describe Deimos, 'configuration' do
         group_id 'myconsumerid'
         bulk_import_id_generator(-> { 'consumer' })
         replace_associations false
+        save_associations_first true
       end
 
       consumer do
@@ -312,10 +318,12 @@ describe Deimos, 'configuration' do
     custom = MyConfigConsumer.config
     expect(custom[:replace_associations]).to eq(false)
     expect(custom[:bulk_import_id_generator].call).to eq('consumer')
+    expect(custom[:save_associations_first]).to eq(true)
 
     default = MyConfigConsumer2.config
     expect(default[:replace_associations]).to eq(true)
     expect(default[:bulk_import_id_generator].call).to eq('global')
+    expect(default[:save_associations_first]).to eq(false)
 
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.24.2
+  version: 1.24.3
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-01 00:00:00.000000000 Z
+date: 2024-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf