deimos-ruby 2.1.6 → 2.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83e7562bf4350bb2a223a8aaa6c4ad710c2c20741b39a8349132f4bac131100c
-  data.tar.gz: 5238a10a42512e0588c3295c2aea72888a203012906967f1c9aec71059123db9
+  metadata.gz: db2890edc1a3d0148f1d2d17d488956140403691807b782b14fb59ab85e5eb70
+  data.tar.gz: 559be5246d5793112335575066f02ec07ed4310baf86bdb630a076d448770714
 SHA512:
-  metadata.gz: 73bd43e24c3d2ff030ee3e09c572e68f18fb41da135df627d13e8298de2a888c3a69e7a4e7f51d15f6a687c01c26a5858724118dc4ea156033c5ec4fba01be8a
-  data.tar.gz: cd177bc3f323848882a1a31c9d1bdb526c8e75f21ce10f78361927cd6d9bce3eeb0bc3b30646f3e9f6112bd3dc256d279c91d6519f756c088a2d46e081140fbc
+  metadata.gz: 75f32bba6112ba1cabb233c9333bb0f6e56029d30a85d24ccca5057302f103b76482e364fb183f3f8fa545f3dbb75f4f03404f4b12392ec4e8c4bea199a8ccf7
+  data.tar.gz: a8a745e864fd29d442fd4a4f474f58428f3ffe7fee1ea41e50ebebff8121da0aba8c91c5605ac6c991b91fe7ccb6ff03fbabf203dc296beb43e60e13f3b9c991

data/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 2.1.8 - 2025-08-11
+
+- Feature: Producers can now customize the deletion payload, so different producers using the same model can send different delete messages.
+
+# 2.1.7 - 2025-07-23
+
+- Feature: Skip batch_record_list fill_primary_keys! if the primary key has already been filled during consumption
+
 # 2.1.6 - 2025-07-23
 
 - Fix: Allows rails to handle querying bulk_import_id when replacing associations for non-string bulk_import_ids
@@ -316,7 +324,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 # 1.12.2 - 2021-12-10
 
 ### Features :star:
-
 - Added `Deimos.encode` and `Deimos.decode` for non-topic-related encoding and decoding.
 
 # 1.12.1 - 2021-11-02
@@ -327,7 +334,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 # 1.12.0 - 2021-11-01
 
 ### Features :star:
-
 - Generate Schema classes from Avro Schemas
 - Use Schema Classes in your consumer and producer 
 
@@ -361,7 +367,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added a `save_record` method to ActiveRecordConsumer in case calling code wants to work with the record before saving.
 
 - ### Fixes :wrench:
-
 - Fixed a regression where the default values for consumer / Phobos listener configs were not correct (they were all nil). This is technically a breaking change, but it puts the configs back the way they were at version 1.4 and matches the documentation.
 
 ## 1.9.2 - 2021-01-29
@@ -378,7 +383,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Bumped the version of ruby-kafka to latest
 
 - ### Fixes :wrench:
-
 - Prevents DB Poller from reconnecting to DB if there is an open transaction
 - Replaces `before` by `prepend_before` for more consistent test setups.
 - Adds validation in the `kafka_producers` method (fixes [#90](https://github.com/flipp-oss/deimos/issues/90))

data/docs/CONFIGURATION.md

@@ -120,16 +120,16 @@ The following are additional settings that can be added to the `topic` block in
 
 ### Consumer Settings
 
-| Config name              | Default           | Description                                                                                                                                                                                                                                        |
-|--------------------------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| each_message             | false             | If true, use `consume_message` for each message rather than `consume_batch` for the full batch.                                                                                                                                                    |
-| reraise_errors           | false             | Default behavior is to swallow uncaught exceptions and log to the metrics provider. Set this to true to instead raise all errors. Note that raising an error will ensure that the message cannot be processed - if there is a bad message which will always raise that error, your consumer will not be able to proceed past it and will be stuck forever until you fix your code. See also the fatal_error configuration.                                                                                                                       |
-| fatal_error              | `proc { false }`  | Block taking an exception, payload and metadata and returning true if this should be considered a fatal error and false otherwise. E.g. you can use this to always fail if the database is available. Not needed if reraise_errors is set to true. |
-| max_db_batch_size        | nil               | Maximum limit for batching database calls to reduce the load on the db.                                                                                                                                                                            |
-| bulk_import_id_column    | `:bulk_import_id` | Name of the column to use for multi-table imports.                                                                                                                                                                                                 |
-| replace_associations     | true              | If false, append to associations in multi-table imports rather than replacing them.                                                                                                                                                                |
-| bulk_import_id_generator | nil               | Block to determine the bulk_import_id generated during bulk consumption. If no block is specified the provided/default block from the consumers configuration will be used.                                                                        |
-| save_associations_first  |false|Whether to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
+| Config name              | Default           | Description                                                                                                                                                                                                                                                                                                                                                                                                                |
+|--------------------------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| each_message             | false             | If true, use `consume_message` for each message rather than `consume_batch` for the full batch.                                                                                                                                                                                                                                                                                                                            |
+| reraise_errors           | false             | Default behavior is to swallow uncaught exceptions and log to the metrics provider. Set this to true to instead raise all errors. Note that raising an error will ensure that the message cannot be processed - if there is a bad message which will always raise that error, your consumer will not be able to proceed past it and will be stuck forever until you fix your code. See also the fatal_error configuration. |
+| fatal_error              | `proc { false }`  | Block taking an exception, payload and metadata and returning true if this should be considered a fatal error and false otherwise. E.g. you can use this to always fail if the database is available. Not needed if reraise_errors is set to true.                                                                                                                                                                         |
+| max_db_batch_size        | nil               | Maximum limit for batching database calls to reduce the load on the db.                                                                                                                                                                                                                                                                                                                                                    |
+| bulk_import_id_column    | `:bulk_import_id` | Name of the column to use for multi-table imports.                                                                                                                                                                                                                                                                                                                                                                         |
+| replace_associations     | true              | If false, append to associations in multi-table imports rather than replacing them.                                                                                                                                                                                                                                                                                                                                        |
+| bulk_import_id_generator | nil               | Block to determine the bulk_import_id generated during bulk consumption. If no block is specified the provided/default block from the consumers configuration will be used.                                                                                                                                                                                                                                                |
+| save_associations_first  | false             | Whether to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class                                                                                                                                                                                                                                |
 
 ### Defining Consumers
 

data/lib/deimos/active_record_consume/batch_record_list.rb

@@ -52,6 +52,9 @@ module Deimos
       # records.
       def fill_primary_keys!
         primary_col = self.klass.primary_key
+
+        return if self.batch_records.empty? || self.batch_records.first.send(primary_col).present?
+
         bulk_import_map = self.klass.
           where(self.bulk_import_column => self.batch_records.map(&:bulk_import_id)).
           select(primary_col, self.bulk_import_column).

data/lib/deimos/active_record_producer.rb

@@ -76,6 +76,15 @@ module Deimos
         Utils::SchemaClass.instance(payload, encoder.schema, encoder.namespace)
       end
 
+      # Deletion payload for a record by default, delegate to the
+      # model's deletion_payload. Producers may override this method to
+      # customize the deletion key/payload per producer.
+      # @param record [ActiveRecord::Base]
+      # @return [Hash]
+      def generate_deletion_payload(record)
+        record.deletion_payload
+      end
+
       # Query to use when polling the database with the DbPoller. Add
       # includes, joins, or wheres as necessary, or replace entirely.
       # @param time_from [Time] the time to start the query from.
@@ -100,7 +109,7 @@ module Deimos
       end
 
       # Post process records after publishing
-      # @param records [Array<ActiveRecord::Base>]
+      # @param _records [Array<ActiveRecord::Base>]
       def post_process(_records)
       end
 

data/lib/deimos/kafka_source.rb

@@ -46,7 +46,9 @@ module Deimos
     def send_kafka_event_on_destroy
       return unless self.class.kafka_config[:delete]
 
-      self.class.kafka_producers.each { |p| p.publish_list([self.deletion_payload]) }
+      self.class.kafka_producers.each do |p|
+        p.publish_list([p.generate_deletion_payload(self)])
+      end
     end
 
     # Payload to send after we are destroyed.

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.1.6'
+  VERSION = '2.1.8'
 end

data/spec/active_record_consume/mass_updater_spec.rb

@@ -251,5 +251,192 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
 
       end
 
+      context 'with recorded primary_keys' do
+        before(:all) do
+          ActiveRecord::Base.connection.create_table(:fidgets, force: true, id: false) do |t|
+            t.string :test_id, primary_key: true
+            t.integer(:some_int)
+            t.string(:bulk_import_id)
+            t.timestamps
+          end
+
+          ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
+            t.string(:title)
+            t.string(:bulk_import_id)
+            t.belongs_to(:fidget)
+
+            t.index(%i(title), unique: true)
+          end
+
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.drop_table(:fidgets)
+          ActiveRecord::Base.connection.drop_table(:fidget_details)
+        end
+
+        let(:fidget_detail_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidget_details'
+            belongs_to :fidget
+          end
+        end
+
+        let(:fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidgets'
+            has_one :fidget_detail
+          end
+        end
+
+        let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
+        let(:key_proc) do
+          lambda do |klass|
+            case klass.to_s
+            when 'Fidget'
+              %w(test_id)
+            when 'FidgetDetail'
+              %w(title)
+            else
+              raise "Key Columns for #{klass} not defined"
+            end
+
+          end
+        end
+
+        before(:each) do
+          stub_const('Fidget', fidget_class)
+          stub_const('FidgetDetail', fidget_detail_class)
+          Fidget.reset_column_information
+        end
+
+
+        let(:batch) do
+          Deimos::ActiveRecordConsume::BatchRecordList.new(
+          [
+            Deimos::ActiveRecordConsume::BatchRecord.new(
+              klass: Fidget,
+              attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
+              bulk_import_column: 'bulk_import_id',
+              bulk_import_id_generator: bulk_id_generator
+            ),
+            Deimos::ActiveRecordConsume::BatchRecord.new(
+              klass: Fidget,
+              attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
+              bulk_import_column: 'bulk_import_id',
+              bulk_import_id_generator: bulk_id_generator
+            )
+          ]
+        )
+        end
+
+        it 'should not backfill the primary key when the primary_key exists' do
+          allow(Fidget).to receive(:where).and_call_original
+          results = described_class.new(Widget,
+                                        bulk_import_id_generator: bulk_id_generator,
+                                        bulk_import_id_column: 'bulk_import_id',
+                                        key_col_proc: key_proc).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(Fidget.count).to eq(2)
+          expect(Fidget).not_to have_received(:where).with(:bulk_import_id => [instance_of(String), instance_of(String)])
+        end
+
+      end
+
+      context 'without recorded primary_keys' do
+        before(:all) do
+          ActiveRecord::Base.connection.create_table(:fidgets, force: true) do |t|
+            t.string(:test_id)
+            t.integer(:some_int)
+            t.string(:bulk_import_id)
+            t.timestamps
+          end
+
+          ActiveRecord::Base.connection.create_table(:fidget_details, force: true) do |t|
+            t.string(:title)
+            t.string(:bulk_import_id)
+            t.belongs_to(:fidget)
+
+            t.index(%i(title), unique: true)
+          end
+
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.drop_table(:fidgets)
+          ActiveRecord::Base.connection.drop_table(:fidget_details)
+        end
+
+        let(:fidget_detail_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidget_details'
+            belongs_to :fidget
+          end
+        end
+
+        let(:fidget_class) do
+          Class.new(ActiveRecord::Base) do
+            self.table_name = 'fidgets'
+            has_one :fidget_detail
+          end
+        end
+
+        let(:bulk_id_generator) { proc { SecureRandom.uuid } }
+
+        let(:key_proc) do
+          lambda do |klass|
+            case klass.to_s
+            when 'Fidget'
+              %w(id)
+            when 'FidgetDetail'
+              %w(title)
+            else
+              raise "Key Columns for #{klass} not defined"
+            end
+
+          end
+        end
+
+        before(:each) do
+          stub_const('Fidget', fidget_class)
+          stub_const('FidgetDetail', fidget_detail_class)
+          Fidget.reset_column_information
+        end
+
+
+        let(:batch) do
+          Deimos::ActiveRecordConsume::BatchRecordList.new(
+            [
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: Fidget,
+                attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              ),
+              Deimos::ActiveRecordConsume::BatchRecord.new(
+                klass: Fidget,
+                attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
+                bulk_import_column: 'bulk_import_id',
+                bulk_import_id_generator: bulk_id_generator
+              )
+            ]
+          )
+        end
+
+        it 'should not backfill the primary key when the primary_key exists' do
+          allow(Fidget).to receive(:where).and_call_original
+          results = described_class.new(Widget,
+                                        bulk_import_id_generator: bulk_id_generator,
+                                        bulk_import_id_column: 'bulk_import_id',
+                                        key_col_proc: key_proc).mass_update(batch)
+          expect(results.count).to eq(2)
+          expect(Fidget.count).to eq(2)
+          expect(Fidget).to have_received(:where).with(:bulk_import_id => [instance_of(String), instance_of(String)])
+          expect(batch.records.map(&:id)).to eq([1,2])
+        end
+
+      end
+
     end
 end

data/spec/kafka_source_spec.rb

@@ -105,23 +105,96 @@ module KafkaSourceSpec
       expect('my-topic-the-second').to have_sent(nil, widget.id)
     end
 
+    context 'multi-producer model using two different key fields' do
+      before(:each) do
+        class MultiKeyWidget < ActiveRecord::Base
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          def self.kafka_producers
+            [WidgetProducer, WidgetStringKeyProducer]
+          end
+        end
+
+        class WidgetStringKeyProducer < Deimos::ActiveRecordProducer
+          class << self
+
+            def generate_payload(attributes, record)
+              payload = super(attributes, record)
+              payload.merge('id' => record.model_id)
+
+            end
+
+            def generate_deletion_payload(record)
+              { payload_key: record.model_id }
+            end
+          end
+        end
+
+        Karafka::App.routes.redraw do
+          topic 'my-topic' do
+            namespace 'com.my-namespace'
+            schema 'Widget'
+            key_config field: :id
+            producer_class WidgetProducer
+          end
+
+          topic 'my-topic-the-Third' do
+            namespace 'com.my-namespace'
+            schema 'WidgetTheThird'
+            key_config field: :model_id
+            producer_class WidgetStringKeyProducer
+          end
+
+        end
+      end
+
+      it 'publishes create/delete with correct keys per producer (primary: id, secondary: model_id)' do
+        multi_key_widget = MultiKeyWidget.create!(widget_id: 7, model_id: 'model-id-123', name: 'name-123')
+
+        expect('my-topic').to have_sent({
+                                          widget_id: 7,
+                                          name: 'name-123',
+                                          id: multi_key_widget.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, multi_key_widget.id)
+
+        expect('my-topic-the-Third').to have_sent({
+                                                    widget_id: 7,
+                                                    model_id: multi_key_widget.model_id,
+                                                    id: multi_key_widget.model_id,
+                                                    created_at: anything,
+                                                    updated_at: anything
+                                                  }, multi_key_widget.model_id)
+
+        expect(WidgetProducer).to receive(:generate_deletion_payload).and_call_original
+        expect(WidgetStringKeyProducer).to receive(:generate_deletion_payload).and_call_original
+
+        multi_key_widget.destroy
+
+        expect('my-topic').to have_sent(nil, multi_key_widget.id)
+        expect('my-topic-the-Third').to have_sent(nil, multi_key_widget.model_id)
+      end
+    end
+
     context 'with truncation off' do
       before(:each) do
         Deimos.config.producers.truncate_columns = false
       end
       it 'should not truncate values' do
-        widget = Widget.create!(widget_id: 1, name: 'a'*500)
+        widget = Widget.create!(widget_id: 1, name: 'a' * 500)
         expect('my-topic').to have_sent({
                                           widget_id: 1,
-                                          name: 'a'*500,
+                                          name: 'a' * 500,
                                           id: widget.id,
                                           created_at: anything,
                                           updated_at: anything
                                         }, 1)
-        widget.update_attribute(:name, 'b'*500)
+        widget.update_attribute(:name, 'b' * 500)
         expect('my-topic').to have_sent({
                                           widget_id: 1,
-                                          name: 'b'*500,
+                                          name: 'b' * 500,
                                           id: widget.id,
                                           created_at: anything,
                                           updated_at: anything
@@ -134,18 +207,18 @@ module KafkaSourceSpec
         Deimos.config.producers.truncate_columns = true
       end
       it 'should truncate values' do
-        widget = Widget.create!(widget_id: 1, name: 'a'*500)
+        widget = Widget.create!(widget_id: 1, name: 'a' * 500)
         expect('my-topic').to have_sent({
                                           widget_id: 1,
-                                          name: 'a'*100,
+                                          name: 'a' * 100,
                                           id: widget.id,
                                           created_at: anything,
                                           updated_at: anything
                                         }, 1)
-        widget.update_attribute(:name, 'b'*500)
+        widget.update_attribute(:name, 'b' * 500)
         expect('my-topic').to have_sent({
                                           widget_id: 1,
-                                          name: 'b'*100,
+                                          name: 'b' * 100,
                                           id: widget.id,
                                           created_at: anything,
                                           updated_at: anything
@@ -304,6 +377,7 @@ module KafkaSourceSpec
             [WidgetProducer]
           end
         end
+
         WidgetNoImportHook.reset_column_information
       end
 

data/spec/schemas/com/my-namespace/WidgetTheThird.avsc

@@ -0,0 +1,27 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "WidgetTheThird",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "string"
+    },
+    {
+      "name": "widget_id",
+      "type": "long"
+    },
+    {
+      "name": "model_id",
+      "type": "string"
+    },
+    {
+      "name": "updated_at",
+      "type": "long"
+    },
+    {
+      "name": "created_at",
+      "type": "long"
+    }
+  ]
+}

data/spec/schemas/my_namespace/widget_the_third.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas; module MyNamespace
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.WidgetTheThird
+  class WidgetTheThird < Deimos::SchemaClass::Record
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :id
+    # @return [Integer]
+    attr_accessor :widget_id
+    # @return [String]
+    attr_accessor :model_id
+    # @return [Integer]
+    attr_accessor :updated_at
+    # @return [Integer]
+    attr_accessor :created_at
+
+    # @override
+    def initialize(_from_message: false, id: nil,
+                   widget_id: nil,
+                   model_id: nil,
+                   updated_at: nil,
+                   created_at: nil)
+      @_from_message = _from_message
+      super
+      self.id = id
+      self.widget_id = widget_id
+      self.model_id = model_id
+      self.updated_at = updated_at
+      self.created_at = created_at
+    end
+
+    # @override
+    def schema
+      'WidgetTheThird'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace'
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'id' => @id,
+        'widget_id' => @widget_id,
+        'model_id' => @model_id,
+        'updated_at' => @updated_at,
+        'created_at' => @created_at
+      }
+    end
+  end
+end; end

data/spec/snapshots/consumers-no-nest.snap

@@ -1648,6 +1648,66 @@ module Schemas
 end
 
 
+spec/app/lib/schema_classes/widget_the_third.rb:
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.WidgetTheThird
+  class WidgetTheThird < Deimos::SchemaClass::Record
+
+    ### Attribute Accessors ###
+    # @return [String]
+    attr_accessor :id
+    # @return [Integer]
+    attr_accessor :widget_id
+    # @return [String]
+    attr_accessor :model_id
+    # @return [Integer]
+    attr_accessor :updated_at
+    # @return [Integer]
+    attr_accessor :created_at
+
+    # @override
+    def initialize(_from_message: false, id: nil,
+                   widget_id: nil,
+                   model_id: nil,
+                   updated_at: nil,
+                   created_at: nil)
+      @_from_message = _from_message
+      super
+      self.id = id
+      self.widget_id = widget_id
+      self.model_id = model_id
+      self.updated_at = updated_at
+      self.created_at = created_at
+    end
+
+    # @override
+    def schema
+      'WidgetTheThird'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace'
+    end
+
+    # @override
+    def as_json(_opts={})
+      {
+        'id' => @id,
+        'widget_id' => @widget_id,
+        'model_id' => @model_id,
+        'updated_at' => @updated_at,
+        'created_at' => @created_at
+      }
+    end
+  end
+end
+
+
 spec/app/lib/schema_classes/yet_another_enum.rb:
 # frozen_string_literal: true