deimos-ruby 1.24.1 → 1.24.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9f5a1b17f598db05ccd6c5086c3f7f34607c3272d15ba425a9a9af3d61fa6d2
-  data.tar.gz: 1324ecd619446398b84ce822f47ea73ffd7a7dfaed8d2b5a226194cda45ed86b
+  metadata.gz: de8ca3bf7233f799c3bf9169fa647dabebd8e08cd2b05ec00511c132a5f3920d
+  data.tar.gz: 03177d9ac87184fe3ce6ef916bce0ad39e3b81a6576f60ba71c33b298f82dfd8
 SHA512:
-  metadata.gz: 2a0e0fb2113acce5735ca3812a7dc3e5e5898880de2f81f406a5a68252c13a6ac9ca91b5906a0392ba5609db936c4c29c13458b5e2fefd18e0671738dafb5f0d
-  data.tar.gz: aa8104d7be9d74cf778decee10f73b4284ebb042772406efd6c24ec484331a1119495902d39e6e5b088a01503920658d01d24d0cdb537fbeef06cd645f02b43d
+  metadata.gz: 40ecdddd9c0b8f5f9675f8058f4a8c8adf1eb69a293e04870a46b9d8ae1972cdc70088611c898f0ab91e072f6627c4760bca5ca5399b698316edda8f3212f5aa
+  data.tar.gz: 3f2fa463d5c777ce7e4dfc4cf610a7dd4ed3f6dba5b6dd459f1ca8fad037f0baa47278436c65c91dc90679f88e9d42a792b84ccbdc2cde4b58d1239d03b2fd4a

data/.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Ruby 2.7
+      - name: Set up Ruby 3.3
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: 3.3
           bundler-cache: true
 
       - name: Bundle install
@@ -36,7 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0', '3.1' ]
+        ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3' ]
 
     steps:
       - uses: actions/checkout@v3

data/.ruby-version

@@ -1,2 +1 @@
-2.7.0
-
+3.2.2

data/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.24.3 - 2024-05-13
+
+- Feature: Enable `producers.persistent_connections` phobos setting
+- Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
+
+# 1.24.2 - 2024-05-01
+- Fix: Deprecation notice with Rails 7.
+
 # 1.24.1 - 2024-03-26
 - Fix: ActiveRecordConsumer batch consumption was not working with the trilogy adapter.
 

data/docs/CONFIGURATION.md

@@ -102,6 +102,7 @@ heartbeat_interval|10|Interval between heartbeats; must be less than the session
 backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
 replace_associations|nil| Whether to delete existing associations for records during bulk consumption for this consumer. If no value is specified the provided/default value from the `consumers` configuration will be used.
 bulk_import_id_generator|nil| Block to determine the `bulk_import_id` generated during bulk consumption. If no block is specified the provided/default block from the `consumers` configuration will be used.
+save_associations_first|false|Whether to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
 
 ## Defining Database Pollers
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -83,7 +83,7 @@ module Deimos
       def deleted_query(records)
         keys = records.
           map { |m| record_key(m.key)[@klass.primary_key] }.
-          reject(&:nil?)
+          compact
 
         @klass.unscoped.where(@klass.primary_key => keys)
       end
@@ -168,7 +168,9 @@ module Deimos
                                   key_col_proc: key_col_proc,
                                   col_proc: col_proc,
                                   replace_associations: self.class.replace_associations,
-                                  bulk_import_id_generator: self.class.bulk_import_id_generator)
+                                  bulk_import_id_generator: self.class.bulk_import_id_generator,
+                                  save_associations_first: self.class.save_associations_first,
+                                  bulk_import_id_column: self.class.bulk_import_id_column)
         ActiveSupport::Notifications.instrument('batch_consumption.valid_records', {
                                                   records: updater.mass_update(record_list),
                                                   consumer: self.class

data/lib/deimos/active_record_consume/mass_updater.rb

@@ -20,10 +20,13 @@ module Deimos
       # @param col_proc [Proc<Class < ActiveRecord::Base>]
       # @param replace_associations [Boolean]
       def initialize(klass, key_col_proc: nil, col_proc: nil,
-                     replace_associations: true, bulk_import_id_generator: nil)
+                     replace_associations: true, bulk_import_id_generator: nil, save_associations_first: false,
+                     bulk_import_id_column: nil)
         @klass = klass
         @replace_associations = replace_associations
         @bulk_import_id_generator = bulk_import_id_generator
+        @save_associations_first = save_associations_first
+        @bulk_import_id_column = bulk_import_id_column&.to_s
 
         @key_cols = {}
         @key_col_proc = key_col_proc
@@ -84,15 +87,67 @@ module Deimos
         end
       end
 
+      # Assign associated records to corresponding primary records
+      # @param record_list [BatchRecordList] RecordList of primary records for this consumer
+      # @return [Hash]
+      def assign_associations(record_list)
+        associations_info = {}
+        record_list.associations.each do |assoc|
+          col = @bulk_import_id_column if assoc.klass.column_names.include?(@bulk_import_id_column)
+          associations_info[[assoc, col]] = []
+        end
+        record_list.batch_records.each do |primary_batch_record|
+          associations_info.each_key do |assoc, col|
+            batch_record = BatchRecord.new(klass: assoc.klass,
+                                           attributes: primary_batch_record.associations[assoc.name],
+                                           bulk_import_column: col,
+                                           bulk_import_id_generator: @bulk_import_id_generator)
+            # Associate this associated batch record's record with the primary record to
+            # retrieve foreign_key after associated records have been saved and primary
+            # keys have been filled
+            primary_batch_record.record.assign_attributes({ assoc.name => batch_record.record })
+            associations_info[[assoc, col]] << batch_record
+          end
+        end
+        associations_info
+      end
+
+      # Save associated records and fill foreign keys on RecordList records
+      # @param record_list [BatchRecordList] RecordList of primary records for this consumer
+      # @param associations_info [Hash] Contains association info
+      def save_associations_first(record_list, associations_info)
+        associations_info.each_value do |records|
+          assoc_record_list = BatchRecordList.new(records)
+          Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+            save_records_to_database(assoc_record_list)
+          end
+          import_associations(assoc_record_list)
+        end
+        record_list.records.each do |record|
+          associations_info.each_key do |assoc, _|
+            record.assign_attributes({ assoc.foreign_key => record.send(assoc.name).id })
+          end
+        end
+      end
+
       # @param record_list [BatchRecordList]
       # @return [Array<ActiveRecord::Base>]
       def mass_update(record_list)
         # The entire batch should be treated as one transaction so that if
         # any message fails, the whole thing is rolled back or retried
         # if there is deadlock
-        Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
-          save_records_to_database(record_list)
-          import_associations(record_list) if record_list.associations.any?
+
+        if @save_associations_first
+          associations_info = assign_associations(record_list)
+          save_associations_first(record_list, associations_info)
+          Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+            save_records_to_database(record_list)
+          end
+        else
+          Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+            save_records_to_database(record_list)
+            import_associations(record_list) if record_list.associations.any?
+          end
         end
         record_list.records
       end

data/lib/deimos/active_record_consumer.rb

@@ -45,6 +45,11 @@ module Deimos
         config[:replace_associations]
       end
 
+      # @return [Boolean]
+      def save_associations_first
+        config[:save_associations_first]
+      end
+
       # @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
       # only the last message for each unique key in a batch is processed.
       # @return [void]

data/lib/deimos/config/configuration.rb

@@ -97,7 +97,8 @@ module Deimos # rubocop:disable Metrics/ModuleLength
                                   kafka_config.replace_associations
                                 end,
           bulk_import_id_generator: kafka_config.bulk_import_id_generator ||
-            Deimos.config.consumers.bulk_import_id_generator
+            Deimos.config.consumers.bulk_import_id_generator,
+          save_associations_first: kafka_config.save_associations_first
         )
       end
     end
@@ -476,6 +477,11 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # @return [Block]
       setting :bulk_import_id_generator, nil
 
+      # If enabled save associated records prior to saving the main record class
+      # This will also set foreign keys for associated records
+      # @return [Boolean]
+      setting :save_associations_first, false
+
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.
       setting :group_id

data/lib/deimos/config/phobos_config.rb

@@ -52,7 +52,8 @@ module Deimos
           compression_threshold: self.producers.compression_threshold,
           max_queue_size: self.producers.max_queue_size,
           delivery_threshold: self.producers.delivery_threshold,
-          delivery_interval: self.producers.delivery_interval
+          delivery_interval: self.producers.delivery_interval,
+          persistent_connections: self.producers.persistent_connections
         },
         consumer: {
           session_timeout: self.consumers.session_timeout,

data/lib/deimos/kafka_topic_info.rb

@@ -6,6 +6,11 @@ module Deimos
     self.table_name = 'kafka_topic_info'
 
     class << self
+
+      def quote_time(time)
+        time.respond_to?(:to_fs) ? time.to_fs(:db) : time.to_s(:db)
+      end
+
       # Lock a topic for the given ID. Returns whether the lock was successful.
       # @param topic [String]
       # @param lock_id [String]
@@ -22,9 +27,9 @@ module Deimos
         qtopic = self.connection.quote(topic)
         qlock_id = self.connection.quote(lock_id)
         qtable = self.connection.quote_table_name('kafka_topic_info')
-        qnow = self.connection.quote(Time.zone.now.to_s(:db))
+        qnow = self.connection.quote(quote_time(Time.zone.now))
         qfalse = self.connection.quoted_false
-        qtime = self.connection.quote(1.minute.ago.to_s(:db))
+        qtime = self.connection.quote(quote_time(1.minute.ago))
 
         # If a record is marked as error and less than 1 minute old,
         # we don't want to pick it up even if not currently locked because

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.24.1'
+  VERSION = '1.24.3'
 end

data/spec/active_record_consume/mass_updater_spec.rb

@@ -114,5 +114,142 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
 
       end
 
+      context 'with save_associations_first' do
+        before(:all) do
+          ActiveRecord::Base.connection.create_table(:fidgets, force: true) do |t|
+            t.string(:test_id)
+            t.integer(:some_int)
+            t.string(:bulk_import_id)
+            t.timestamps
+          end
+
+          ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
+            t.string(:title)
+            t.string(:bulk_import_id)
+            t.belongs_to(:fidget)
+
+            t.index(%i(title), unique: true)
+          end
+
+          ActiveRecord::Base.connection.create_table(:widget_fidgets, force: true, id: false) do |t|
+            t.belongs_to(:fidget)
+            t.belongs_to(:widget)
+            t.string(:bulk_import_id)
+            t.string(:note)
+            t.index(%i(widget_id fidget_id), unique: true)
+          end
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.drop_table(:fidgets)
+          ActiveRecord::Base.connection.drop_table(:fidget_details)
+          ActiveRecord::Base.connection.drop_table(:widget_fidgets)
+        end
+
+        let(:fidget_detail_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidget_details'
+            belongs_to :fidget
+          end
+        end
+
+        let(:fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidgets'
+            has_one :fidget_detail
+          end
+        end
+
+        let(:widget_fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'widget_fidgets'
+            belongs_to :fidget
+            belongs_to :widget
+          end
+        end
+
+        let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
+        let(:key_proc) do
+          lambda do |klass|
+            case klass.to_s
+            when 'Widget', 'Fidget'
+              %w(id)
+            when 'WidgetFidget'
+              %w(widget_id fidget_id)
+            when 'FidgetDetail', 'Detail'
+              %w(title)
+            else
+              raise "Key Columns for #{klass} not defined"
+            end
+
+          end
+        end
+
+        before(:each) do
+          stub_const('Fidget', fidget_class)
+          stub_const('FidgetDetail', fidget_detail_class)
+          stub_const('WidgetFidget', widget_fidget_class)
+          Widget.reset_column_information
+          Fidget.reset_column_information
+          WidgetFidget.reset_column_information
+        end
+
+        # rubocop:disable RSpec/MultipleExpectations, RSpec/ExampleLength
+        it 'should backfill the associations when upserting primary records' do
+          batch = Deimos::ActiveRecordConsume::BatchRecordList.new(
+            [
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: WidgetFidget,
+                attributes: {
+                  widget: { test_id: 'id1', some_int: 10, detail: { title: 'Widget Title 1' } },
+                  fidget: { test_id: 'id1', some_int: 10, fidget_detail: { title: 'Fidget Title 1' } },
+                  note: 'Stuff 1'
+                },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              ),
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: WidgetFidget,
+                attributes: {
+                  widget: { test_id: 'id2', some_int: 20, detail: { title: 'Widget Title 2' } },
+                  fidget: { test_id: 'id2', some_int: 20, fidget_detail: { title: 'Fidget Title 2' } },
+                  note: 'Stuff 2'
+                },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              )
+            ]
+          )
+
+          results = described_class.new(WidgetFidget,
+                                        bulk_import_id_generator: bulk_id_generator,
+                                        bulk_import_id_column: 'bulk_import_id',
+                                        key_col_proc: key_proc,
+                                        save_associations_first: true).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(Widget.count).to eq(2)
+          expect(Detail.count).to eq(2)
+          expect(Fidget.count).to eq(2)
+          expect(FidgetDetail.count).to eq(2)
+
+          WidgetFidget.all.each_with_index do |widget_fidget, ind|
+            widget = Widget.find_by(id: widget_fidget.widget_id)
+            expect(widget.test_id).to eq("id#{ind + 1}")
+            expect(widget.some_int).to eq((ind + 1) * 10)
+            detail = Detail.find_by(widget_id: widget_fidget.widget_id)
+            expect(detail.title).to eq("Widget Title #{ind + 1}")
+            fidget = Fidget.find_by(id: widget_fidget.fidget_id)
+            expect(fidget.test_id).to eq("id#{ind + 1}")
+            expect(fidget.some_int).to eq((ind + 1) * 10)
+            fidget_detail = FidgetDetail.find_by(fidget_id: widget_fidget.fidget_id)
+            expect(fidget_detail.title).to eq("Fidget Title #{ind + 1}")
+            expect(widget_fidget.note).to eq("Stuff #{ind + 1}")
+          end
+        end
+        # rubocop:enable RSpec/MultipleExpectations, RSpec/ExampleLength
+
+      end
+
     end
 end

data/spec/config/configuration_spec.rb

@@ -92,7 +92,8 @@ describe Deimos, 'configuration' do
           handler: 'ConsumerTest::MyConsumer',
           use_schema_classes: nil,
           max_db_batch_size: nil,
-          bulk_import_id_generator: nil
+          bulk_import_id_generator: nil,
+          save_associations_first: false
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
@@ -111,7 +112,8 @@ describe Deimos, 'configuration' do
           handler: 'ConsumerTest::MyBatchConsumer',
           use_schema_classes: nil,
           max_db_batch_size: nil,
-          bulk_import_id_generator: nil
+          bulk_import_id_generator: nil,
+          save_associations_first: false
         }
       ],
       producer: {
@@ -125,7 +127,8 @@ describe Deimos, 'configuration' do
         compression_threshold: 1,
         max_queue_size: 10_000,
         delivery_threshold: 0,
-        delivery_interval: 0
+        delivery_interval: 0,
+        persistent_connections: false
       }
     )
   end
@@ -264,7 +267,8 @@ describe Deimos, 'configuration' do
             handler: 'MyConfigConsumer',
             use_schema_classes: false,
             max_db_batch_size: nil,
-            bulk_import_id_generator: nil
+            bulk_import_id_generator: nil,
+            save_associations_first: false
           }
         ],
         producer: {
@@ -278,7 +282,8 @@ describe Deimos, 'configuration' do
           compression_threshold: 2,
           max_queue_size: 10,
           delivery_threshold: 1,
-          delivery_interval: 1
+          delivery_interval: 1,
+          persistent_connections: true
         }
       )
   end
@@ -295,6 +300,7 @@ describe Deimos, 'configuration' do
         group_id 'myconsumerid'
         bulk_import_id_generator(-> { 'consumer' })
         replace_associations false
+        save_associations_first true
       end
 
       consumer do
@@ -312,10 +318,12 @@ describe Deimos, 'configuration' do
     custom = MyConfigConsumer.config
     expect(custom[:replace_associations]).to eq(false)
     expect(custom[:bulk_import_id_generator].call).to eq('consumer')
+    expect(custom[:save_associations_first]).to eq(true)
 
     default = MyConfigConsumer2.config
     expect(default[:replace_associations]).to eq(true)
     expect(default[:bulk_import_id_generator].call).to eq('global')
+    expect(default[:save_associations_first]).to eq(false)
 
   end
 end

data/spec/spec_helper.rb

@@ -14,6 +14,8 @@ require 'handlers/my_batch_consumer'
 require 'handlers/my_consumer'
 require 'rspec/rails'
 require 'rspec/snapshot'
+require "trilogy_adapter/connection"
+ActiveRecord::Base.public_send :extend, TrilogyAdapter::Connection
 Dir['./spec/schemas/**/*.rb'].sort.each { |f| require f }
 
 # Constants used for consumer specs

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.24.1
+  version: 1.24.3
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-26 00:00:00.000000000 Z
+date: 2024-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -415,7 +415,6 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".gitmodules"

data/.circleci/config.yml

@@ -1,83 +0,0 @@
-defaults: &defaults
-    parallelism: 1
-    working_directory: ~/workspace
-    docker:
-      - image: ruby:2.6
-        environment:
-          RAILS_ENV: test
-          DB_HOST_IP: 127.0.0.1
-version: 2.1
-jobs:
-  build:
-    <<: *defaults
-    steps:
-      - checkout
-
-      # Restore bundle cache & npm cache
-      - restore_cache:
-          key: 'rails-{{ checksum "Gemfile.lock" }}'
-
-      # Bundle install dependencies in /tmp/
-      # so Dockerfile does not copy them since
-      # its base image is different than CircleCI
-      - run:
-          name: Install bundler
-          command: gem install bundler:2.1.4
-      - run:
-          name: Bundle install
-          command: bundle install --path vendor/bundle --jobs=4 --retry=3
-
-      # Store bundle cache
-      - save_cache:
-          key: 'rails-{{ checksum "Gemfile.lock" }}'
-          paths:
-            - ~/workspace/vendor/bundle
-
-      - persist_to_workspace:
-          root: ~/workspace
-          paths:
-            - .
-
-  lint:
-    <<: *defaults
-    steps:
-      - attach_workspace:
-          at: ~/workspace
-      - run:
-          name: Install bundler
-          command: gem install bundler:2.1.4
-      - run:
-           name: Point bundle to vendor/bundle
-           command: bundle --path vendor/bundle
-      - run: bundle exec rubocop --display-only-fail-level-offenses --fail-level C
-
-  test-rspec:
-    <<: *defaults
-    steps:
-      - attach_workspace:
-          at: ~/workspace
-      - run:
-          name: Install bundler
-          command: gem install bundler:2.1.4
-      - run:
-           name: Point bundle to vendor/bundle
-           command: bundle --path vendor/bundle
-      - run: mkdir result
-      - run:
-          name: Running rspec
-          command: bundle exec rspec --format progress --format RspecJunitFormatter -o result/rspec.xml
-          when: always
-      - store_test_results:
-          path: ~/workspace/result
-
-workflows:
-  version: 2
-  build-and-test:
-    jobs:
-      - build
-      - test-rspec:
-          requires:
-            - build
-      - lint:
-          requires:
-            - build