deimos-ruby 1.18.2 → 1.19.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3f5f0c1fdba2cf641706b6c18733f433c04f42e3e06f5871ab0050a0215cc61
-  data.tar.gz: 9c758f1d7e62dd9cd91e1631b3211cc9388adb0bb7b3dfea7b29099b46224bc7
+  metadata.gz: e73c9ad8e132353f906ffd5ffe91820e8ad9d5225dbf091c7889752c8d699703
+  data.tar.gz: ac5d117e07aa5881bfd484ce74bd1715cb1f483004b2a0609b2b9e45a1a42937
 SHA512:
-  metadata.gz: 30b074497e6efe25e87d6e4736ecba0725b5bb4b20dcfba9bc82a655c9ee246986b9a655ceeec05b6bfa57b0633fecafb3cf7e1335322bd40a4254b3d973944f
-  data.tar.gz: 798513006a509d7439e4d09c18a6a547bac683bcbb4a56013447fad59ba941702072f828d6ec8966da6fec8096659a686de7f796d858923e17fdcf7f21235318
+  metadata.gz: 1762491acddff361c0290837911eaf862697c9c1a1f2cb054636fdbc097290efb5babe43c9b6e85217a4c43892cf34f899011e5c22485b568bed1a690fd2fcc3
+  data.tar.gz: 56ce518a0b9d8bb62ebb8e799fa51f95ccf5f275e4e3231d7b01084d8a35c73c1d08998035694dde72ff530e6b72c001b6c99ab44ebde82357aee0d456ba6575

data/CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+- Feature: When consuming data in batch mode, this adds ability to save data to multiple tables with `association_list`.
+
 # 1.18.2 - 2022-11-14
 
 -  Fixes bug related to wait between runs when no records are fetched by updating poll_info

data/README.md

@@ -353,6 +353,54 @@ class MyBatchConsumer < Deimos::Consumer
   end
 end
 ```
+#### Saving data to Multiple Database tables
+
+> This feature is implemented and tested with MySQL database ONLY.
+
+Sometimes, the Kafka message needs to be saved to multiple database tables. For example, if a `User` topic provides you metadata and profile image for users, we might want to save it to multiple tables: `User` and `Image`.
+
+- The `association_list` configuration allows you to achieve this use case.
+- The optional `bulk_import_id_column` config allows you to specify column_name on `record_class` which can be used to retrieve IDs after save. Defaults to `bulk_import_id`
+
+You must override the `build_records` and `bulk_import_columns` methods on your ActiveRecord class for this feature to work.
+- `build_records` - This method is required to set the value of the `bulk_import_id` column and map Kafka messages to ActiveRecord model objects.
+- `columns(klass)` - Should return an array of column names that should be used by ActiveRecord klass during SQL insert operation.
+- `key_columns(messages, klass)` -  Should return an array of column name(s) that makes a row unique.
+```ruby
+class MyBatchConsumer < Deimos::ActiveRecordConsumer
+
+  record_class User
+  association_list :images
+
+  def build_records(messages)
+    # Initialise bulk_import_id and build ActiveRecord objects out of Kafka message attributes
+    messages.each do |m|
+      u = User.new(first_name: m.first_name, bulk_import_id: SecureRandom.uuid)
+      i = Image.new(attr1: m.image_url)
+      u.images << i
+      u
+    end
+  end
+  
+  def key_columns(_records, klass)
+    case klass
+    when User
+      super
+    when Image
+      ["image_url", "image_name"]
+    end
+  end
+
+  def columns(klass)
+    case klass
+    when User
+      super
+    when Image
+      klass.columns.map(&:name) - [:created_at, :updated_at, :id]
+    end
+  end
+end
+```
 
 # Rails Integration
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -3,6 +3,7 @@
 require 'deimos/active_record_consume/batch_slicer'
 require 'deimos/utils/deadlock_retry'
 require 'deimos/message'
+require 'deimos/exceptions'
 
 module Deimos
   module ActiveRecordConsume
@@ -86,38 +87,63 @@ module Deimos
       # records to either be updated or inserted.
       # @return [void]
       def upsert_records(messages)
-        key_cols = key_columns(messages)
-
-        # Create payloads with payload + key attributes
-        upserts = messages.map do |m|
-          attrs = if self.method(:record_attributes).parameters.size == 2
-                    record_attributes(m.payload, m.key)
-                  else
-                    record_attributes(m.payload)
-                  end
-
-          attrs&.merge(record_key(m.key))
-        end
+        key_cols = key_columns(messages, nil)
 
+        # Create ActiveRecord Models with payload + key attributes
+        upserts = build_records(messages)
         # If overridden record_attributes indicated no record, skip
         upserts.compact!
+        # apply ActiveRecord validations and fetch valid Records
+        valid_upserts = filter_records(upserts)
+
+        return if valid_upserts.empty?
+
+        save_records_to_database(@klass, key_cols, valid_upserts)
+        import_associations(valid_upserts) unless @association_list.blank?
+      end
+
+      def save_records_to_database(record_class, key_cols, records)
+        columns = columns(record_class)
 
         options = if key_cols.empty?
                     {} # Can't upsert with no key, just do regular insert
-                  elsif ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
+                  elsif mysql_adapter?
                     {
-                      on_duplicate_key_update: :all
+                      on_duplicate_key_update: columns
                     }
                   else
                     {
                       on_duplicate_key_update: {
                         conflict_target: key_cols,
-                        columns: :all
+                        columns: columns
                       }
                     }
                   end
+        record_class.import!(columns, records, options)
+      end
+
+      # Imports associated objects and import them to database table
+      # The base table is expected to contain bulk_import_id column for indexing associated objects with id
+      # @association_list configured on the consumer helps identify the ones required to be saved.
+      def import_associations(entities)
+        _validate_associations(entities)
+        _fill_primary_key_on_entities(entities)
+
+        # Select associations from config parameter association_list and
+        # fill id to associated_objects foreign_key column
+        @klass.reflect_on_all_associations.select { |assoc| @association_list.include?(assoc.name) }.
+          each do |assoc|
+            sub_records = entities.map { |entity|
+              # Get associated `has_one` or `has_many` records for each entity
+              sub_records = Array(entity.send(assoc.name))
+              # Set IDS from master to each of the records in `has_one` or `has_many` relation
+              sub_records.each { |d| d.send("#{assoc.send(:foreign_key)}=", entity.id) }
+              sub_records
+            }.flatten
 
-        @klass.import!(upserts, options)
+            columns = key_columns(nil, assoc.klass)
+            save_records_to_database(assoc.klass, columns, sub_records) if sub_records.any?
+          end
       end
 
       # Delete any records with a tombstone.
@@ -144,16 +170,31 @@ module Deimos
 
       # Get the set of attribute names that uniquely identify messages in the
       # batch. Requires at least one record.
+      # The parameters are mutually exclusive. records is used by default implementation.
       # @param records [Array<Message>] Non-empty list of messages.
+      # @param klass [ActiveRecord::Class] Class Name can be used to fetch columns
       # @return [Array<String>] List of attribute names.
       # @raise If records is empty.
-      def key_columns(records)
+      def key_columns(records, klass)
+        raise 'Must implement key_columns method for associations!' unless klass.nil?
+
         raise 'Cannot determine key from empty batch' if records.empty?
 
         first_key = records.first.key
         record_key(first_key).keys
       end
 
+      # Get the list of database table column names that should be saved to the database
+      # @param record_class [Class] ActiveRecord class associated to the Entity Object
+      # @return Array[String] list of table columns
+      def columns(record_class)
+        # In-memory records contain created_at and updated_at as nil
+        # which messes up ActiveRecord-Import bulk_import.
+        # It is necessary to ignore timestamp columns when using ActiveRecord objects
+        ignored_columns = %w(created_at updated_at)
+        record_class.columns.map(&:name) - ignored_columns
+      end
+
       # Compact a batch of messages, taking only the last message for each
       # unique key.
       # @param batch [Array<Message>] Batch of messages.
@@ -163,6 +204,61 @@ module Deimos
 
         batch.reverse.uniq(&:key).reverse!
       end
+
+      # Turns Kafka payload into ActiveRecord Objects by mapping relevant fields
+      # Override this method to build object and associations with message payload
+      # @param messages [Array<Deimos::Message>] the array of deimos messages in batch mode
+      # @return [Array<ActiveRecord>] Array of ActiveRecord objects
+      def build_records(messages)
+        messages.map do |m|
+          attrs = if self.method(:record_attributes).parameters.size == 2
+                    record_attributes(m.payload, m.key)
+                  else
+                    record_attributes(m.payload)
+                  end
+
+          attrs = attrs&.merge(record_key(m.key))
+          @klass.new(attrs) unless attrs.nil?
+        end
+      end
+
+      # Filters list of Active Records by applying active record validations.
+      # Tip: Add validates_associated in ActiveRecord model to validate associated models
+      # Optionally inherit this method and apply more filters in the application code
+      # The default implementation throws ActiveRecord::RecordInvalid by default
+      # @param records Array<ActiveRecord> - List of active records which will be subjected to model validations
+      # @return valid Array<ActiveRecord> - Subset of records that passed the model validations
+      def filter_records(records)
+        records.each(&:validate!)
+      end
+
+      # Returns true if MySQL Adapter is currently used
+      def mysql_adapter?
+        ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
+      end
+
+      # Checks whether the entities has necessary columns for `association_list` to work
+      # @return void
+      def _validate_associations(entities)
+        raise Deimos::MissingImplementationError unless mysql_adapter?
+
+        return if entities.first.respond_to?(@bulk_import_id_column)
+
+        raise "Create bulk_import_id on #{entities.first.class} and set it in `build_records` for associations." \
+              ' Run rails g deimos:bulk_import_id:setup to create the migration.'
+      end
+
+      # Fills Primary Key ID on in-memory objects.
+      # Uses @bulk_import_id_column on in-memory records to fetch saved records in database.
+      # @return void
+      def _fill_primary_key_on_entities(entities)
+        table_by_bulk_import_id = @klass.
+          where(@bulk_import_id_column => entities.map { |e| e[@bulk_import_id_column] }).
+          select(:id, @bulk_import_id_column).
+          index_by { |e| e[@bulk_import_id_column] }
+        # update IDs in upsert entity
+        entities.each { |entity| entity.id = table_by_bulk_import_id[entity[@bulk_import_id_column]].id }
+      end
     end
   end
 end

data/lib/deimos/active_record_consumer.rb

@@ -30,6 +30,18 @@ module Deimos
         config[:record_class] = klass
       end
 
+      # @param associations [List<String>] Optional list of associations that the consumer
+      # should save in addition to @klass
+      # @return [void]
+      def association_list(associations)
+        config[:association_list] = Array(associations)
+      end
+
+      # @param
+      def bulk_import_id_column(name)
+        config[:bulk_import_id_column] = name
+      end
+
       # @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
       # only the last message for each unique key in a batch is processed.
       # @return [void]
@@ -41,6 +53,8 @@ module Deimos
     # Setup
     def initialize
       @klass = self.class.config[:record_class]
+      @association_list = self.class.config[:association_list]
+      @bulk_import_id_column = self.class.config[:bulk_import_id_column] || :bulk_import_id
       @converter = ActiveRecordConsume::SchemaModelConverter.new(self.class.decoder, @klass)
 
       if self.class.config[:key_schema]

data/lib/deimos/exceptions.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Deimos
+  class MissingImplementationError < StandardError; end
+end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.18.2'
+  VERSION = '1.19.beta1'
 end

data/lib/generators/deimos/bulk_import_id/templates/migration.rb.tt

@@ -0,0 +1,7 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+
+  def change
+    add_column :<%= table_name %>, :<%= column_name %>, :string, index: true
+  end
+
+end

data/lib/generators/deimos/bulk_import_id_generator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record/migration'
+require 'rails/version'
+
+# Generates a migration for bulk import ID in consumer.
+module Deimos
+  module Generators
+    # Generator for ActiveRecord model and migration.
+    class BulkImportIdGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      include ActiveRecord::Generators::Migration
+
+      namespace 'deimos:bulk_import_id:setup'
+
+      argument :table_name, desc: 'The table to add bulk import column.', required: true
+      argument :column_name, desc: 'The bulk import ID column name.', default: 'bulk_import_id'
+
+      source_root File.expand_path('bulk_import_id/templates', __dir__)
+      desc 'Add column migration to the given table and name'
+
+      no_commands do
+        # @return [String]
+        def db_migrate_path
+          if defined?(Rails.application) && Rails.application
+            paths = Rails.application.config.paths['db/migrate']
+            paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
+          else
+            'db/migrate'
+          end
+        end
+
+        # @return [String]
+        def migration_version
+          "[#{ActiveRecord::Migration.current_version}]"
+        rescue StandardError
+          ''
+        end
+      end
+
+      # For a given table_name and column_name, create a migration to add the column
+      # column_name defaults to bulk_import_id
+      def generate
+        Rails.logger.info("Arguments: #{table_name},#{column_name}")
+        migration_template('migration.rb',
+                           "#{db_migrate_path}/add_#{column_name}_column_to_#{table_name}.rb")
+      end
+    end
+  end
+end

data/spec/active_record_batch_consumer_mysql_spec.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+module ActiveRecordBatchConsumerTest
+  describe Deimos::ActiveRecordConsumer,
+           'Batch Consumer with MySQL handling associations',
+           :integration,
+           db_config: DbConfigs::DB_OPTIONS.second do
+    include_context('with DB')
+
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+        t.string(:test_id)
+        t.string(:part_one)
+        t.string(:part_two)
+        t.integer(:some_int)
+        t.boolean(:deleted, default: false)
+        t.timestamps
+
+        t.index(%i(part_one part_two), unique: true)
+      end
+
+      # create one-to-one association -- Details
+      ActiveRecord::Base.connection.create_table(:details, force: true) do |t|
+        t.string(:title)
+        t.belongs_to(:widget)
+
+        t.index(%i(title), unique: true)
+      end
+
+      # Create one-to-many association Locales
+      ActiveRecord::Base.connection.create_table(:locales, force: true) do |t|
+        t.string(:title)
+        t.string(:language)
+        t.belongs_to(:widget)
+
+        t.index(%i(title language), unique: true)
+      end
+
+      class Detail < ActiveRecord::Base
+        validates :title, presence: true
+      end
+
+      class Locale < ActiveRecord::Base
+        validates :title, presence: true
+        validates :language, presence: true
+      end
+
+      # Sample model
+      class Widget < ActiveRecord::Base
+        has_one :detail
+        has_many :locales
+        validates :test_id, presence: true
+
+        default_scope -> { where(deleted: false) }
+      end
+
+      Widget.reset_column_information
+      Detail.reset_column_information
+      Locale.reset_column_information
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+      ActiveRecord::Base.connection.drop_table(:details)
+      ActiveRecord::Base.connection.drop_table(:locales)
+    end
+
+    before(:each) do
+      ActiveRecord::Base.connection.truncate_tables(%i(widgets details locales))
+    end
+
+    prepend_before(:each) do
+      stub_const('MyBatchConsumer', consumer_class)
+    end
+
+    # Helper to publish a list of messages and call the consumer
+    def publish_batch(messages)
+      keys = messages.map { |m| m[:key] }
+      payloads = messages.map { |m| m[:payload] }
+
+      test_consume_batch(MyBatchConsumer, payloads, keys: keys, call_original: true)
+    end
+
+    context 'when association_list configured in consumer without model changes' do
+      let(:consumer_class) do
+        Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+          record_class Widget
+          association_list :detail
+
+          def build_records(messages)
+            messages.map do |m|
+              payload = m.payload
+              w = Widget.new(test_id: payload['test_id'], some_int: payload['some_int'])
+              d = Detail.new(title: payload['title'])
+              w.detail = d
+              w
+            end
+          end
+        end
+      end
+
+      it 'should raise error when bulk_import_id is not found' do
+        stub_const('MyBatchConsumer', consumer_class)
+        expect {
+          publish_batch([{ key: 2,
+                           payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        }.to raise_error('Create bulk_import_id on ActiveRecordBatchConsumerTest::Widget'\
+        ' and set it in `build_records` for associations. Run rails g deimos:bulk_import_id:setup'\
+        ' to create the migration.')
+      end
+    end
+
+    context 'with one-to-one relation in association_list and custom bulk_import_id' do
+      let(:consumer_class) do
+        Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+          record_class Widget
+          association_list :detail
+          bulk_import_id_column :custom_id
+
+          def build_records(messages)
+            messages.map do |m|
+              payload = m.payload
+              w = Widget.new(test_id: payload['test_id'],
+                             some_int: payload['some_int'],
+                             custom_id: SecureRandom.uuid)
+              d = Detail.new(title: payload['title'])
+              w.detail = d
+              w
+            end
+          end
+
+          def key_columns(messages, klass)
+            case klass.to_s
+            when Widget.to_s
+              super
+            when Detail.to_s
+              %w(title widget_id)
+            else
+              []
+            end
+          end
+
+          def columns(record_class)
+            all_cols = record_class.columns.map(&:name)
+
+            case record_class.to_s
+            when Widget.to_s
+              super
+            when Detail.to_s
+              all_cols - ['id']
+            else
+              []
+            end
+          end
+        end
+      end
+
+      before(:all) do
+        ActiveRecord::Base.connection.add_column(:widgets, :custom_id, :string, if_not_exists: true)
+        Widget.reset_column_information
+      end
+
+      it 'should save item to widget and associated detail' do
+        stub_const('MyBatchConsumer', consumer_class)
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        expect(Widget.count).to eq(1)
+        expect(Detail.count).to eq(1)
+        expect(Widget.first.id).to eq(Detail.first.widget_id)
+      end
+    end
+
+    context 'with one-to-many relationship in association_list and default bulk_import_id' do
+      let(:consumer_class) do
+        Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+          record_class Widget
+          association_list :locales
+
+          def build_records(messages)
+            messages.map do |m|
+              payload = m.payload
+              w = Widget.new(test_id: payload['test_id'],
+                             some_int: payload['some_int'],
+                             bulk_import_id: SecureRandom.uuid)
+              w.locales << Locale.new(title: payload['title'], language: 'en')
+              w.locales << Locale.new(title: payload['title'], language: 'fr')
+              w
+            end
+          end
+
+          def key_columns(messages, klass)
+            case klass.to_s
+            when Widget.to_s
+              super
+            when Detail.to_s
+              %w(title widget_id)
+            when Locale.to_s
+              %w(title language)
+            else
+              []
+            end
+          end
+
+          def columns(record_class)
+            all_cols = record_class.columns.map(&:name)
+
+            case record_class.to_s
+            when Widget.to_s
+              super
+            when Detail.to_s, Locale.to_s
+              all_cols - ['id']
+            else
+              []
+            end
+          end
+        end
+      end
+
+      before(:all) do
+        ActiveRecord::Base.connection.add_column(:widgets, :bulk_import_id, :string, if_not_exists: true)
+        Widget.reset_column_information
+      end
+
+      it 'should save item to widget and associated details' do
+        stub_const('MyBatchConsumer', consumer_class)
+        publish_batch([{ key: 2,
+                         payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        expect(Widget.count).to eq(1)
+        expect(Locale.count).to eq(2)
+        expect(Widget.first.id).to eq(Locale.first.widget_id)
+        expect(Widget.first.id).to eq(Locale.second.widget_id)
+      end
+    end
+           end
+end

data/spec/active_record_batch_consumer_spec.rb

@@ -487,5 +487,25 @@ module ActiveRecordBatchConsumerTest
           to match_array([have_attributes(id: 2, test_id: 'abc123')])
       end
     end
+
+    describe 'association_list feature for SQLite database' do
+      let(:consumer_class) do
+        Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+          record_class Widget
+          association_list :locales
+        end
+      end
+
+      it 'should throw NotImplemented error' do
+        stub_const('MyBatchConsumer', consumer_class)
+        expect {
+          publish_batch([{ key: 2, payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
+        }.to raise_error(Deimos::MissingImplementationError)
+      end
+    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.18.2
+  version: 1.19.beta1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-14 00:00:00.000000000 Z
+date: 2023-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -446,6 +446,7 @@ files:
 - lib/deimos/consume/batch_consumption.rb
 - lib/deimos/consume/message_consumption.rb
 - lib/deimos/consumer.rb
+- lib/deimos/exceptions.rb
 - lib/deimos/instrumentation.rb
 - lib/deimos/kafka_message.rb
 - lib/deimos/kafka_source.rb
@@ -488,6 +489,8 @@ files:
 - lib/generators/deimos/active_record/templates/migration.rb.tt
 - lib/generators/deimos/active_record/templates/model.rb.tt
 - lib/generators/deimos/active_record_generator.rb
+- lib/generators/deimos/bulk_import_id/templates/migration.rb.tt
+- lib/generators/deimos/bulk_import_id_generator.rb
 - lib/generators/deimos/db_backend/templates/migration
 - lib/generators/deimos/db_backend/templates/rails3_migration
 - lib/generators/deimos/db_backend_generator.rb
@@ -505,6 +508,7 @@ files:
 - sig/avro.rbs
 - sig/defs.rbs
 - sig/fig_tree.rbs
+- spec/active_record_batch_consumer_mysql_spec.rb
 - spec/active_record_batch_consumer_spec.rb
 - spec/active_record_consume/batch_slicer_spec.rb
 - spec/active_record_consume/schema_model_converter_spec.rb
@@ -620,15 +624,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.3.20
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
 test_files:
+- spec/active_record_batch_consumer_mysql_spec.rb
 - spec/active_record_batch_consumer_spec.rb
 - spec/active_record_consume/batch_slicer_spec.rb
 - spec/active_record_consume/schema_model_converter_spec.rb