deimos-ruby 2.1.6 → 2.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83e7562bf4350bb2a223a8aaa6c4ad710c2c20741b39a8349132f4bac131100c
-  data.tar.gz: 5238a10a42512e0588c3295c2aea72888a203012906967f1c9aec71059123db9
+  metadata.gz: 3890d253c7d9b137c55d02e1968697b9a2921a8cd57dec3be49b6eb3ea50d670
+  data.tar.gz: 4538ffbef398ea60d5fe1f3cd20e30a1fe4e45c00e2369717ea160491554d687
 SHA512:
-  metadata.gz: 73bd43e24c3d2ff030ee3e09c572e68f18fb41da135df627d13e8298de2a888c3a69e7a4e7f51d15f6a687c01c26a5858724118dc4ea156033c5ec4fba01be8a
-  data.tar.gz: cd177bc3f323848882a1a31c9d1bdb526c8e75f21ce10f78361927cd6d9bce3eeb0bc3b30646f3e9f6112bd3dc256d279c91d6519f756c088a2d46e081140fbc
+  metadata.gz: 221bf0e05c603dda8f6451d4205ed4ea86f209fe8bdc4d6c43c5ad1a19aff57c9df80f9a82a3c134a31e78b6072eab076d6f3e8594fb8a01f4892857068f9aa0
+  data.tar.gz: 41cec49b3a6913617398bf194fd1d8fb65304231b34aca051fad5c37307b2e617b571c220ff95aefec57b2200564a9827bddd37ae80553a8091d784561f0c6d5

data/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 2.1.7 - 2025-07-23
+
+- Feature: Skip batch_record_list fill_primary_keys! if the primary key has already been filled during consumption
+
 # 2.1.6 - 2025-07-23
 
 - Fix: Allows rails to handle querying bulk_import_id when replacing associations for non-string bulk_import_ids

data/docs/CONFIGURATION.md

@@ -120,16 +120,16 @@ The following are additional settings that can be added to the `topic` block in
 
 ### Consumer Settings
 
-| Config name              | Default           | Description                                                                                                                                                                                                                                        |
-|--------------------------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| each_message             | false             | If true, use `consume_message` for each message rather than `consume_batch` for the full batch.                                                                                                                                                    |
-| reraise_errors           | false             | Default behavior is to swallow uncaught exceptions and log to the metrics provider. Set this to true to instead raise all errors. Note that raising an error will ensure that the message cannot be processed - if there is a bad message which will always raise that error, your consumer will not be able to proceed past it and will be stuck forever until you fix your code. See also the fatal_error configuration.                                                                                                                       |
-| fatal_error              | `proc { false }`  | Block taking an exception, payload and metadata and returning true if this should be considered a fatal error and false otherwise. E.g. you can use this to always fail if the database is available. Not needed if reraise_errors is set to true. |
-| max_db_batch_size        | nil               | Maximum limit for batching database calls to reduce the load on the db.                                                                                                                                                                            |
-| bulk_import_id_column    | `:bulk_import_id` | Name of the column to use for multi-table imports.                                                                                                                                                                                                 |
-| replace_associations     | true              | If false, append to associations in multi-table imports rather than replacing them.                                                                                                                                                                |
-| bulk_import_id_generator | nil               | Block to determine the bulk_import_id generated during bulk consumption. If no block is specified the provided/default block from the consumers configuration will be used.                                                                        |
-| save_associations_first  |false|Whether to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
+| Config name              | Default           | Description                                                                                                                                                                                                                                                                                                                                                                                                                |
+|--------------------------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| each_message             | false             | If true, use `consume_message` for each message rather than `consume_batch` for the full batch.                                                                                                                                                                                                                                                                                                                            |
+| reraise_errors           | false             | Default behavior is to swallow uncaught exceptions and log to the metrics provider. Set this to true to instead raise all errors. Note that raising an error will ensure that the message cannot be processed - if there is a bad message which will always raise that error, your consumer will not be able to proceed past it and will be stuck forever until you fix your code. See also the fatal_error configuration. |
+| fatal_error              | `proc { false }`  | Block taking an exception, payload and metadata and returning true if this should be considered a fatal error and false otherwise. E.g. you can use this to always fail if the database is available. Not needed if reraise_errors is set to true.                                                                                                                                                                         |
+| max_db_batch_size        | nil               | Maximum limit for batching database calls to reduce the load on the db.                                                                                                                                                                                                                                                                                                                                                    |
+| bulk_import_id_column    | `:bulk_import_id` | Name of the column to use for multi-table imports.                                                                                                                                                                                                                                                                                                                                                                         |
+| replace_associations     | true              | If false, append to associations in multi-table imports rather than replacing them.                                                                                                                                                                                                                                                                                                                                        |
+| bulk_import_id_generator | nil               | Block to determine the bulk_import_id generated during bulk consumption. If no block is specified the provided/default block from the consumers configuration will be used.                                                                                                                                                                                                                                                |
+| save_associations_first  | false             | Whether to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class                                                                                                                                                                                                                                |
 
 ### Defining Consumers
 

data/lib/deimos/active_record_consume/batch_record_list.rb

@@ -52,6 +52,9 @@ module Deimos
       # records.
       def fill_primary_keys!
         primary_col = self.klass.primary_key
+
+        return if self.batch_records.empty? || self.batch_records.first.send(primary_col).present?
+
         bulk_import_map = self.klass.
           where(self.bulk_import_column => self.batch_records.map(&:bulk_import_id)).
           select(primary_col, self.bulk_import_column).

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.1.6'
+  VERSION = '2.1.7'
 end

data/spec/active_record_consume/mass_updater_spec.rb

@@ -251,5 +251,192 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
 
       end
 
+      context 'with recorded primary_keys' do
+        before(:all) do
+          ActiveRecord::Base.connection.create_table(:fidgets, force: true, id: false) do |t|
+            t.string :test_id, primary_key: true
+            t.integer(:some_int)
+            t.string(:bulk_import_id)
+            t.timestamps
+          end
+
+          ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
+            t.string(:title)
+            t.string(:bulk_import_id)
+            t.belongs_to(:fidget)
+
+            t.index(%i(title), unique: true)
+          end
+
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.drop_table(:fidgets)
+          ActiveRecord::Base.connection.drop_table(:fidget_details)
+        end
+
+        let(:fidget_detail_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidget_details'
+            belongs_to :fidget
+          end
+        end
+
+        let(:fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidgets'
+            has_one :fidget_detail
+          end
+        end
+
+        let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
+        let(:key_proc) do
+          lambda do |klass|
+            case klass.to_s
+            when 'Fidget'
+              %w(test_id)
+            when 'FidgetDetail'
+              %w(title)
+            else
+              raise "Key Columns for #{klass} not defined"
+            end
+
+          end
+        end
+
+        before(:each) do
+          stub_const('Fidget', fidget_class)
+          stub_const('FidgetDetail', fidget_detail_class)
+          Fidget.reset_column_information
+        end
+
+
+        let(:batch) do
+          Deimos::ActiveRecordConsume::BatchRecordList.new(
+          [
+            Deimos::ActiveRecordConsume::BatchRecord.new(
+              klass: Fidget,
+              attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
+              bulk_import_column: 'bulk_import_id',
+              bulk_import_id_generator: bulk_id_generator
+            ),
+            Deimos::ActiveRecordConsume::BatchRecord.new(
+              klass: Fidget,
+              attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
+              bulk_import_column: 'bulk_import_id',
+              bulk_import_id_generator: bulk_id_generator
+            )
+          ]
+        )
+        end
+
+        it 'should not backfill the primary key when the primary_key exists' do
+          allow(Fidget).to receive(:where).and_call_original
+          results = described_class.new(Widget,
+                                        bulk_import_id_generator: bulk_id_generator,
+                                        bulk_import_id_column: 'bulk_import_id',
+                                        key_col_proc: key_proc).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(Fidget.count).to eq(2)
+          expect(Fidget).not_to have_received(:where).with(:bulk_import_id => [instance_of(String), instance_of(String)])
+        end
+
+      end
+
+      context 'without recorded primary_keys' do
+        before(:all) do
+          ActiveRecord::Base.connection.create_table(:fidgets, force: true) do |t|
+            t.string(:test_id)
+            t.integer(:some_int)
+            t.string(:bulk_import_id)
+            t.timestamps
+          end
+
+          ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
+            t.string(:title)
+            t.string(:bulk_import_id)
+            t.belongs_to(:fidget)
+
+            t.index(%i(title), unique: true)
+          end
+
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.drop_table(:fidgets)
+          ActiveRecord::Base.connection.drop_table(:fidget_details)
+        end
+
+        let(:fidget_detail_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidget_details'
+            belongs_to :fidget
+          end
+        end
+
+        let(:fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidgets'
+            has_one :fidget_detail
+          end
+        end
+
+        let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
+        let(:key_proc) do
+          lambda do |klass|
+            case klass.to_s
+            when 'Fidget'
+              %w(id)
+            when 'FidgetDetail'
+              %w(title)
+            else
+              raise "Key Columns for #{klass} not defined"
+            end
+
+          end
+        end
+
+        before(:each) do
+          stub_const('Fidget', fidget_class)
+          stub_const('FidgetDetail', fidget_detail_class)
+          Fidget.reset_column_information
+        end
+
+
+        let(:batch) do
+          Deimos::ActiveRecordConsume::BatchRecordList.new(
+            [
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: Fidget,
+                attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              ),
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: Fidget,
+                attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              )
+            ]
+          )
+        end
+
+        it 'should not backfill the primary key when the primary_key exists' do
+          allow(Fidget).to receive(:where).and_call_original
+          results = described_class.new(Widget,
+                                        bulk_import_id_generator: bulk_id_generator,
+                                        bulk_import_id_column: 'bulk_import_id',
+                                        key_col_proc: key_proc).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(Fidget.count).to eq(2)
+          expect(Fidget).to have_received(:where).with(:bulk_import_id => [instance_of(String), instance_of(String)])
+          expect(batch.records.map(&:id)).to eq([1,2])
+        end
+
+      end
+
     end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.1.7
 platform: ruby
 authors:
 - Daniel Orner