deimos-ruby 1.20.1 → 1.22.1

data/lib/deimos/config/configuration.rb

@@ -7,7 +7,7 @@ require_relative '../tracing/mock'
 require 'active_support/core_ext/numeric'
 
 # :nodoc:
-module Deimos
+module Deimos # rubocop:disable Metrics/ModuleLength
   include FigTree
 
   # :nodoc:
@@ -87,6 +87,12 @@ module Deimos
       namespace(kafka_config.namespace) if kafka_config.namespace.present?
       key_config(**kafka_config.key_config) if kafka_config.key_config.present?
       schema_class_config(kafka_config.use_schema_classes) if kafka_config.use_schema_classes.present?
+      if kafka_config.respond_to?(:bulk_import_id_column) # consumer
+        klass.config.merge!(
+          bulk_import_id_column: kafka_config.bulk_import_id_column,
+          replace_associations: kafka_config.replace_associations
+        )
+      end
     end
   end
 
@@ -402,6 +408,10 @@ module Deimos
       # Configure the usage of generated schema classes for this producer
       # @return [Boolean]
       setting :use_schema_classes
+      # If true, and using the multi-table feature of ActiveRecordConsumers, replace associations
+      # instead of appending to them.
+      # @return [Boolean]
+      setting :replace_associations
     end
 
     setting_object :consumer do
@@ -430,6 +440,12 @@ module Deimos
       # Optional maximum limit for batching database calls to reduce the load on the db.
       # @return [Integer]
       setting :max_db_batch_size
+      # Column to use for bulk imports, for multi-table feature.
+      # @return [String]
+      setting :bulk_import_id_column, :bulk_import_id
+      # If true, multi-table consumers will blow away associations rather than appending to them.
+      # @return [Boolean]
+      setting :replace_associations, true
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/config/phobos_config.rb

@@ -67,7 +67,8 @@ module Deimos
         next nil if consumer.disabled
 
         hash = consumer.to_h.reject do |k, _|
-          %i(class_name schema namespace key_config backoff disabled).include?(k)
+          %i(class_name schema namespace key_config backoff disabled replace_associations
+             bulk_import_id_column).include?(k)
         end
         hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
         hash[:handler] = consumer.class_name

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.20.1'
+  VERSION = '1.22.1'
 end

data/spec/active_record_batch_consumer_association_spec.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
+  describe Deimos::ActiveRecordConsumer,
+           'Batch Consumer with MySQL handling associations',
+           :integration,
+           db_config: DbConfigs::DB_OPTIONS.second do
+    include_context('with DB')
+
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+        t.string(:test_id)
+        t.string(:part_one)
+        t.string(:part_two)
+        t.integer(:some_int)
+        t.string(:bulk_import_id)
+        t.boolean(:deleted, default: false)
+        t.timestamps
+
+        t.index(%i(part_one part_two), unique: true)
+      end
+
+      # create one-to-one association -- Details
+      ActiveRecord::Base.connection.create_table(:details, force: true) do |t|
+        t.string(:title)
+        t.string(:bulk_import_id)
+        t.belongs_to(:widget)
+
+        t.index(%i(title), unique: true)
+      end
+
+      # Create one-to-many association Locales
+      ActiveRecord::Base.connection.create_table(:locales, force: true) do |t|
+        t.string(:title)
+        t.string(:language)
+        t.string(:bulk_import_id)
+        t.belongs_to(:widget)
+
+        t.index(%i(widget_id title language), unique: true)
+      end
+
+      class Detail < ActiveRecord::Base
+        validates :title, presence: true
+      end
+
+      class Locale < ActiveRecord::Base
+        validates :title, presence: true
+        validates :language, presence: true
+      end
+
+      # Sample model
+      class Widget < ActiveRecord::Base
+        has_one :detail
+        has_many :locales, dependent: :destroy
+        validates :test_id, presence: true
+
+        default_scope -> { where(deleted: false) }
+      end
+
+      Widget.reset_column_information
+      Detail.reset_column_information
+      Locale.reset_column_information
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+      ActiveRecord::Base.connection.drop_table(:details)
+      ActiveRecord::Base.connection.drop_table(:locales)
+    end
+
+    before(:each) do
+      ActiveRecord::Base.connection.truncate_tables(%i(widgets details locales))
+      Widget.create!(test_id: 'bad_id', some_int: 100) # should not show up
+    end
+
+    prepend_before(:each) do
+      consumer_class.config[:bulk_import_id_column] = :bulk_import_id
+      stub_const('MyBatchConsumer', consumer_class)
+      stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+    end
+
+    # Helper to publish a list of messages and call the consumer
+    def publish_batch(messages)
+      keys = messages.map { |m| m[:key] }
+      payloads = messages.map { |m| m[:payload] }
+
+      test_consume_batch(MyBatchConsumer, payloads, keys: keys, call_original: true)
+    end
+
+    let(:consumer_class) do
+      klass = Class.new(described_class) do
+        cattr_accessor :record_attributes_proc
+        cattr_accessor :should_consume_proc
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config plain: true
+        record_class Widget
+
+        def should_consume?(record)
+          if self.should_consume_proc
+            return self.should_consume_proc.call(record)
+          end
+
+          true
+        end
+
+        def record_attributes(payload, _key)
+          if self.record_attributes_proc
+            return self.record_attributes_proc.call(payload)
+          end
+
+          {
+            test_id: payload['test_id'],
+            some_int: payload['some_int'],
+            detail: {
+              title: payload['title']
+            }
+          }
+        end
+
+        def key_columns(klass)
+          case klass.to_s
+          when Widget.to_s
+            nil
+          when Detail.to_s
+            %w(title widget_id)
+          when Locale.to_s
+            %w(widget_id title language)
+          else
+            []
+          end
+        end
+
+        def columns(record_class)
+          all_cols = record_class.columns.map(&:name)
+
+          case record_class.to_s
+          when Widget.to_s
+            nil
+          when Detail.to_s, Locale.to_s
+            all_cols - ['id']
+          else
+            []
+          end
+        end
+      end
+      klass
+    end
+
+    context 'when association configured in consumer without model changes' do
+      before(:each) do
+        consumer_class.config[:bulk_import_id_column] = :bulk_import_id
+        ActiveRecord::Base.connection.remove_column(:widgets, :bulk_import_id)
+        Widget.reset_column_information
+      end
+
+      after(:each) do
+        ActiveRecord::Base.connection.add_column(:widgets, :bulk_import_id, :string)
+      end
+
+      it 'should raise error when bulk_import_id is not found' do
+        expect {
+          publish_batch([{ key: 2,
+                           payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        }.to raise_error('Create bulk_import_id on the widgets table. Run rails g deimos:bulk_import_id {table}'\
+        ' to create the migration.')
+      end
+    end
+
+    context 'with one-to-one relation in association and custom bulk_import_id' do
+      before(:each) do
+        consumer_class.config[:bulk_import_id_column] = :custom_id
+        consumer_class.config[:replace_associations] = false
+      end
+
+      before(:all) do
+        ActiveRecord::Base.connection.add_column(:widgets, :custom_id, :string, if_not_exists: true)
+        Widget.reset_column_information
+      end
+
+      it 'should save item to widget and associated detail' do
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        expect(Widget.count).to eq(2)
+        expect(Detail.count).to eq(1)
+        expect(Widget.last.id).to eq(Detail.first.widget_id)
+      end
+    end
+
+    context 'with one-to-many relationship in association and default bulk_import_id' do
+      before(:each) do
+        consumer_class.config[:replace_associations] = false
+        consumer_class.record_attributes_proc = proc do |payload|
+          {
+            test_id: payload['test_id'],
+            some_int: payload['some_int'],
+            locales: [
+              {
+                title: payload['title'],
+                language: 'en'
+              },
+              {
+                title: payload['title'],
+                language: 'fr'
+              }
+            ]
+          }
+        end
+      end
+
+      it 'should save item to widget and associated details' do
+        consumer_class.config[:replace_associations] = false
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        expect(Widget.count).to eq(2)
+        expect(Locale.count).to eq(2)
+        expect(Widget.last.id).to eq(Locale.first.widget_id)
+        expect(Widget.last.id).to eq(Locale.second.widget_id)
+
+        # publish again - should add locales to the widget
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 7, title: 'Widget Title 2' } }])
+        expect(Widget.count).to eq(2)
+        expect(Widget.last.some_int).to eq(7)
+        expect(Locale.count).to eq(4)
+        expect(Locale.all.map(&:widget_id).uniq).to eq([Widget.last.id])
+      end
+    end
+
+    context 'with replace_associations on' do
+      before(:each) do
+        consumer_class.config[:replace_associations] = true
+        consumer_class.record_attributes_proc = proc do |payload|
+          {
+            test_id: payload['test_id'],
+            some_int: payload['some_int'],
+            locales: [
+              {
+                title: payload['title'],
+                language: 'en'
+              },
+              {
+                title: payload['title'],
+                language: 'fr'
+              }
+            ]
+          }
+        end
+      end
+
+      it 'should save item to widget and replace associated details' do
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        expect(Widget.count).to eq(2)
+        expect(Locale.count).to eq(2)
+        expect(Widget.last.id).to eq(Locale.first.widget_id)
+        expect(Widget.last.id).to eq(Locale.second.widget_id)
+
+        # publish again - should replace locales
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 7, title: 'Widget Title 2' } }])
+        expect(Widget.count).to eq(2)
+        expect(Widget.last.some_int).to eq(7)
+        expect(Locale.count).to eq(2)
+        expect(Locale.all.map(&:title).uniq).to contain_exactly('Widget Title 2')
+        expect(Locale.all.map(&:widget_id).uniq).to contain_exactly(Widget.last.id)
+      end
+    end
+
+    context 'with invalid models' do
+      before(:each) do
+        consumer_class.should_consume_proc = proc { |val| val.some_int <= 10 }
+      end
+
+      it 'should only save valid models' do
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } },
+                       { key: 3,
+                         payload: { test_id: 'abc', some_int: 15, title: 'Widget Title 2' } }])
+        expect(Widget.count).to eq(2)
+      end
+    end
+           end
+end

data/spec/active_record_batch_consumer_spec.rb

@@ -32,6 +32,8 @@ module ActiveRecordBatchConsumerTest
 
     prepend_before(:each) do
       stub_const('MyBatchConsumer', consumer_class)
+      stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+      consumer_class.config[:bulk_import_id_column] = :bulk_import_id # default
     end
 
     around(:each) do |ex|
@@ -337,6 +339,9 @@ module ActiveRecordBatchConsumerTest
         Widget.create!(test_id: 'xxx', some_int: 2, part_one: 'ghi', part_two: 'jkl')
         Widget.create!(test_id: 'yyy', some_int: 7, part_one: 'mno', part_two: 'pqr')
 
+        allow_any_instance_of(MyBatchConsumer).to receive(:key_columns).
+          and_return(%w(part_one part_two))
+
         publish_batch(
           [
             { key: { part_one: 'abc', part_two: 'def' }, # To be created
@@ -488,24 +493,5 @@ module ActiveRecordBatchConsumerTest
       end
     end
 
-    describe 'association_list feature for SQLite database' do
-      let(:consumer_class) do
-        Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
-          record_class Widget
-          association_list :locales
-        end
-      end
-
-      it 'should throw NotImplemented error' do
-        stub_const('MyBatchConsumer', consumer_class)
-        expect {
-          publish_batch([{ key: 2, payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
-        }.to raise_error(Deimos::MissingImplementationError)
-      end
-    end
-
   end
 end

data/spec/active_record_consume/mass_updater_spec.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
+
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+        t.string(:test_id)
+        t.integer(:some_int)
+        t.string(:bulk_import_id)
+        t.timestamps
+      end
+
+      # create one-to-one association -- Details
+      ActiveRecord::Base.connection.create_table(:details, force: true) do |t|
+        t.string(:title)
+        t.string(:bulk_import_id)
+        t.belongs_to(:widget)
+
+        t.index(%i(title), unique: true)
+      end
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+      ActiveRecord::Base.connection.drop_table(:details)
+    end
+
+    let(:detail_class) do
+      Class.new(ActiveRecord::Base) do
+        self.table_name = 'details'
+        belongs_to :widget
+      end
+    end
+
+    let(:widget_class) do
+      Class.new(ActiveRecord::Base) do
+        self.table_name = 'widgets'
+        has_one :detail
+      end
+    end
+
+    before(:each) do
+      stub_const('Widget', widget_class)
+      stub_const('Detail', detail_class)
+      Widget.reset_column_information
+    end
+
+    describe '#mass_update' do
+      let(:batch) do
+        Deimos::ActiveRecordConsume::BatchRecordList.new(
+          [
+            Deimos::ActiveRecordConsume::BatchRecord.new(
+              klass: Widget,
+              attributes: { test_id: 'id1', some_int: 5, detail: { title: 'Title 1' } },
+              bulk_import_column: 'bulk_import_id'
+            ),
+            Deimos::ActiveRecordConsume::BatchRecord.new(
+              klass: Widget,
+              attributes: { test_id: 'id2', some_int: 10, detail: { title: 'Title 2' } },
+              bulk_import_column: 'bulk_import_id'
+            )
+          ]
+        )
+      end
+
+      it 'should mass update the batch' do
+        described_class.new(Widget).mass_update(batch)
+        expect(Widget.count).to eq(2)
+        expect(Detail.count).to eq(2)
+        expect(Widget.first.detail).not_to be_nil
+        expect(Widget.last.detail).not_to be_nil
+      end
+
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.20.1
+  version: 1.22.1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-18 00:00:00.000000000 Z
+date: 2023-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -410,7 +410,6 @@ files:
 - ".rubocop_todo.yml"
 - ".ruby-version"
 - CHANGELOG.md
-- CHANGELOG.md.orig
 - CODE_OF_CONDUCT.md
 - Dockerfile
 - Gemfile
@@ -431,7 +430,10 @@ files:
 - docs/UPGRADING.md
 - lib/deimos.rb
 - lib/deimos/active_record_consume/batch_consumption.rb
+- lib/deimos/active_record_consume/batch_record.rb
+- lib/deimos/active_record_consume/batch_record_list.rb
 - lib/deimos/active_record_consume/batch_slicer.rb
+- lib/deimos/active_record_consume/mass_updater.rb
 - lib/deimos/active_record_consume/message_consumption.rb
 - lib/deimos/active_record_consume/schema_model_converter.rb
 - lib/deimos/active_record_consumer.rb
@@ -509,10 +511,11 @@ files:
 - sig/avro.rbs
 - sig/defs.rbs
 - sig/fig_tree.rbs
-- spec/active_record_batch_consumer_mysql_spec.rb
+- spec/active_record_batch_consumer_association_spec.rb
 - spec/active_record_batch_consumer_spec.rb
 - spec/active_record_consume/batch_consumption_spec.rb
 - spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/mass_updater_spec.rb
 - spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
@@ -635,10 +638,11 @@ signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
 test_files:
-- spec/active_record_batch_consumer_mysql_spec.rb
+- spec/active_record_batch_consumer_association_spec.rb
 - spec/active_record_batch_consumer_spec.rb
 - spec/active_record_consume/batch_consumption_spec.rb
 - spec/active_record_consume/batch_slicer_spec.rb
+- spec/active_record_consume/mass_updater_spec.rb
 - spec/active_record_consume/schema_model_converter_spec.rb
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb