deimos-ruby 1.20.1 → 1.22.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 645fc0e8316bb64277119e8e89d6eceace9df56462c67675afca4e26235a8209
-  data.tar.gz: 2706c96862d317ebace1ec1a93e9b257d71256081417b2b576533d6eb9e46b06
+  metadata.gz: b1819694e6ec5bc9cad69f0a3d7edaad53d4daae80ee0af95312044d503e40d1
+  data.tar.gz: 1577ab483b8fc8a16dc614c12156e2161a3ab2d372b45ee7825f30837fdc36e4
 SHA512:
-  metadata.gz: 7623b491e584f8ad6fae59750fec0fbf7331f436147ace80b38d559bb2f697037d83a13f9ce70884575d1e9a22bf21e9096d1bddf4cb56c9edb875d2ffa13dec
-  data.tar.gz: 740f964ed1ad6d4f9ab675522454d58fe7c52fc26758eb906b1be4b51daf4742466394bc43f780541328a4e30a74a253d53dd17b97c8cecdf59e9b945b48ff4a
+  metadata.gz: 1fc162b425d00a308679662d4a48753dbe1fac0ab252d329ad10622f894301de2e35dcc2ca3c0d9366b516ebbce7e8d008fd37ee1c2526d9f5a5be3188d6380f
+  data.tar.gz: b09c3c0a0c7980b9bd2431754001c8b3f18d6a21981dbb7dd5b1336f60a3411e26c5c938b3d55e48039c2306ffac54bc2f95c4beac4700b994fdcf4168900ddc

data/CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.22.1 - 2023-05-01
+
+- Fix: Bug introduced with 1.22 when associations are not used
+
+# 1.22 - 2023-05-01
+
+- Feature: Added `replace_associations` and changed default behavior for multi-table consuming. No longer relies on Rails direct associations and wonky magic for new vs. existing records.
+- Fix: `bulk_import_id` is now handled by Deimos and does not need to be set by application code.
+- ***BREAKING CHANGE*** Replaced `filter_records` with `should_consume?` on ActiveRecordConsumer.
+- ***BREAKING CHANGE*** Replaced the behavior of `build_records` on ActiveRecordConsumer with a more powerful `record_attributes`.
+- ***BREAKING CHANGE*** Removed the `association_list` config as it can now be inferred from the data.
+
 # 1.21.1 - 2023-04-18
 
 - Fix: Datadog tracing now works with Datadog 1.x

data/README.md

@@ -359,33 +359,40 @@ end
 
 Sometimes, the Kafka message needs to be saved to multiple database tables. For example, if a `User` topic provides you metadata and profile image for users, we might want to save it to multiple tables: `User` and `Image`.
 
-- The `association_list` configuration allows you to achieve this use case.
+- Return associations as keys in `record_attributes` to enable this feature.
 - The `bulk_import_id_column` config allows you to specify column_name on `record_class` which can be used to retrieve IDs after save. Defaults to `bulk_import_id`. This config is *required* if you have associations but optional if you do not.
 
-You must override the `build_records` and `bulk_import_columns` methods on your ActiveRecord class for this feature to work.
-- `build_records` - This method is required to set the value of the `bulk_import_id` column and map Kafka messages to ActiveRecord model objects.
+You must override the `record_attributes` (and optionally `column` and `key_columns`) methods on your consumer class for this feature to work.
+- `record_attributes` - This method is required to map Kafka messages to ActiveRecord model objects.
 - `columns(klass)` - Should return an array of column names that should be used by ActiveRecord klass during SQL insert operation.
 - `key_columns(messages, klass)` -  Should return an array of column name(s) that makes a row unique.
 ```ruby
+class User < ApplicationRecord
+  has_many :images
+end
+
 class MyBatchConsumer < Deimos::ActiveRecordConsumer
 
   record_class User
-  association_list :images
-
-  def build_records(messages)
-    # Initialise bulk_import_id and build ActiveRecord objects out of Kafka message attributes
-    messages.map do |m|
-      u = User.new(first_name: m.first_name, bulk_import_id: SecureRandom.uuid)
-      i = Image.new(attr1: m.image_url)
-      u.images << i
-      u
-    end
+
+  def record_attributes(payload, _key)
+    {
+      first_name: payload.first_name,
+      images: [
+                {
+                  attr1: payload.image_url
+                },
+                {
+                  attr2: payload.other_image_url
+                }
+              ]
+    }
   end
   
-  def key_columns(_records, klass)
+  def key_columns(klass)
     case klass
     when User
-      super
+      nil # use default
     when Image
       ["image_url", "image_name"]
     end
@@ -394,7 +401,7 @@ class MyBatchConsumer < Deimos::ActiveRecordConsumer
   def columns(klass)
     case klass
     when User
-      super
+      nil # use default
     when Image
       klass.columns.map(&:name) - [:created_at, :updated_at, :id]
     end

data/docs/CONFIGURATION.md

@@ -84,6 +84,8 @@ key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../
 disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
 group_id|nil|ID of the consumer group.
 use_schema_classes|nil|Set to true or false to enable or disable using the consumers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
+bulk_import_id_column|:bulk_import_id|Name of the column to use for multi-table imports.
+replace_associations|true|If false, append to associations in multi-table imports rather than replacing them.
 max_db_batch_size|nil|Maximum limit for batching database calls to reduce the load on the db.
 max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
 start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 require 'deimos/active_record_consume/batch_slicer'
+require 'deimos/active_record_consume/batch_record'
+require 'deimos/active_record_consume/batch_record_list'
+require 'deimos/active_record_consume/mass_updater'
+
 require 'deimos/utils/deadlock_retry'
 require 'deimos/message'
 require 'deimos/exceptions'
@@ -40,10 +44,29 @@ module Deimos
         end
       end
 
+    protected
+
+      # Get the set of attribute names that uniquely identify messages in the
+      # batch. Requires at least one record.
+      # The parameters are mutually exclusive. records is used by default implementation.
+      # @param _klass [Class < ActiveRecord::Base] Class Name can be used to fetch columns
+      # @return [Array<String>,nil] List of attribute names.
+      # @raise If records is empty.
+      def key_columns(_klass)
+        nil
+      end
+
+      # Get the list of database table column names that should be saved to the database
+      # @param _klass [Class < ActiveRecord::Base] ActiveRecord class associated to the Entity Object
+      # @return [Array<String>,nil] list of table columns
+      def columns(_klass)
+        nil
+      end
+
       # Get unique key for the ActiveRecord instance from the incoming key.
       # Override this method (with super) to customize the set of attributes that
       # uniquely identifies each record in the database.
-      # @param key [String] The encoded key.
+      # @param key [String,Hash] The encoded key.
       # @return [Hash] The key attributes.
       def record_key(key)
         if key.nil?
@@ -57,7 +80,35 @@ module Deimos
         end
       end
 
-    protected
+      # Create an ActiveRecord relation that matches all of the passed
+      # records. Used for bulk deletion.
+      # @param records [Array<Message>] List of messages.
+      # @return [ActiveRecord::Relation] Matching relation.
+      def deleted_query(records)
+        keys = records.
+          map { |m| record_key(m.key)[@klass.primary_key] }.
+          reject(&:nil?)
+
+        @klass.unscoped.where(@klass.primary_key => keys)
+      end
+
+      # @param _record [ActiveRecord::Base]
+      # @return [Boolean]
+      def should_consume?(_record)
+        true
+      end
+
+    private
+
+      # Compact a batch of messages, taking only the last message for each
+      # unique key.
+      # @param batch [Array<Message>] Batch of messages.
+      # @return [Array<Message>] Compacted batch.
+      def compact_messages(batch)
+        return batch unless batch.first&.key.present?
+
+        batch.reverse.uniq(&:key).reverse!
+      end
 
       # Perform database operations for a batch of messages without compaction.
       # All messages are split into slices containing only unique keys, and
@@ -103,63 +154,44 @@ module Deimos
       # records to either be updated or inserted.
       # @return [void]
       def upsert_records(messages)
-        key_cols = key_columns(messages, @klass)
+        record_list = build_records(messages)
+        record_list.filter!(self.method(:should_consume?).to_proc)
 
-        # Create ActiveRecord Models with payload + key attributes
-        upserts = build_records(messages)
-        # If overridden record_attributes indicated no record, skip
-        upserts.compact!
-        # apply ActiveRecord validations and fetch valid Records
-        valid_upserts = filter_records(upserts)
+        return if record_list.empty?
 
-        return if valid_upserts.empty?
+        key_col_proc = self.method(:key_columns).to_proc
+        col_proc = self.method(:columns).to_proc
 
-        save_records_to_database(@klass, key_cols, valid_upserts)
-        import_associations(valid_upserts) unless @association_list.blank?
+        updater = MassUpdater.new(@klass,
+                                  key_col_proc: key_col_proc,
+                                  col_proc: col_proc,
+                                  replace_associations: self.class.config[:replace_associations])
+        updater.mass_update(record_list)
       end
 
-      def save_records_to_database(record_class, key_cols, records)
-        columns = columns(record_class)
-
-        options = if key_cols.empty?
-                    {} # Can't upsert with no key, just do regular insert
-                  elsif mysql_adapter?
-                    {
-                      on_duplicate_key_update: columns
-                    }
+      # @param messages [Array<Deimos::Message>]
+      # @return [BatchRecordList]
+      def build_records(messages)
+        records = messages.map do |m|
+          attrs = if self.method(:record_attributes).parameters.size == 2
+                    record_attributes(m.payload, m.key)
                   else
-                    {
-                      on_duplicate_key_update: {
-                        conflict_target: key_cols,
-                        columns: columns
-                      }
-                    }
+                    record_attributes(m.payload)
                   end
-        record_class.import!(columns, records, options)
-      end
+          next nil if attrs.nil?
 
-      # Imports associated objects and import them to database table
-      # The base table is expected to contain bulk_import_id column for indexing associated objects with id
-      # @association_list configured on the consumer helps identify the ones required to be saved.
-      def import_associations(entities)
-        _validate_associations(entities)
-        _fill_primary_key_on_entities(entities)
+          attrs = attrs.merge(record_key(m.key))
+          next unless attrs
 
-        # Select associations from config parameter association_list and
-        # fill id to associated_objects foreign_key column
-        @klass.reflect_on_all_associations.select { |assoc| @association_list.include?(assoc.name) }.
-          each do |assoc|
-            sub_records = entities.map { |entity|
-              # Get associated `has_one` or `has_many` records for each entity
-              sub_records = Array(entity.send(assoc.name))
-              # Set IDS from master to each of the records in `has_one` or `has_many` relation
-              sub_records.each { |d| d.send("#{assoc.foreign_key}=", entity.send(assoc.active_record_primary_key)) }
-              sub_records
-            }.flatten
+          col = if @klass.column_names.include?(self.class.bulk_import_id_column.to_s)
+                  self.class.bulk_import_id_column
+                end
 
-            columns = key_columns(nil, assoc.klass)
-            save_records_to_database(assoc.klass, columns, sub_records) if sub_records.any?
-          end
+          BatchRecord.new(klass: @klass,
+                          attributes: attrs,
+                          bulk_import_column: col)
+        end
+        BatchRecordList.new(records.compact)
       end
 
       # Delete any records with a tombstone.
@@ -171,108 +203,6 @@ module Deimos
 
         clause.delete_all
       end
-
-      # Create an ActiveRecord relation that matches all of the passed
-      # records. Used for bulk deletion.
-      # @param records [Array<Message>] List of messages.
-      # @return [ActiveRecord::Relation] Matching relation.
-      def deleted_query(records)
-        keys = records.
-          map { |m| record_key(m.key)[@klass.primary_key] }.
-          reject(&:nil?)
-
-        @klass.unscoped.where(@klass.primary_key => keys)
-      end
-
-      # Get the set of attribute names that uniquely identify messages in the
-      # batch. Requires at least one record.
-      # The parameters are mutually exclusive. records is used by default implementation.
-      # @param records [Array<Message>] Non-empty list of messages.
-      # @param _klass [ActiveRecord::Class] Class Name can be used to fetch columns
-      # @return [Array<String>] List of attribute names.
-      # @raise If records is empty.
-      def key_columns(records, _klass)
-        raise 'Cannot determine key from empty batch' if records.empty?
-
-        first_key = records.first.key
-        record_key(first_key).keys
-      end
-
-      # Get the list of database table column names that should be saved to the database
-      # @param record_class [Class] ActiveRecord class associated to the Entity Object
-      # @return Array[String] list of table columns
-      def columns(record_class)
-        # In-memory records contain created_at and updated_at as nil
-        # which messes up ActiveRecord-Import bulk_import.
-        # It is necessary to ignore timestamp columns when using ActiveRecord objects
-        ignored_columns = %w(created_at updated_at)
-        record_class.columns.map(&:name) - ignored_columns
-      end
-
-      # Compact a batch of messages, taking only the last message for each
-      # unique key.
-      # @param batch [Array<Message>] Batch of messages.
-      # @return [Array<Message>] Compacted batch.
-      def compact_messages(batch)
-        return batch unless batch.first&.key.present?
-
-        batch.reverse.uniq(&:key).reverse!
-      end
-
-      # Turns Kafka payload into ActiveRecord Objects by mapping relevant fields
-      # Override this method to build object and associations with message payload
-      # @param messages [Array<Deimos::Message>] the array of deimos messages in batch mode
-      # @return [Array<ActiveRecord>] Array of ActiveRecord objects
-      def build_records(messages)
-        messages.map do |m|
-          attrs = if self.method(:record_attributes).parameters.size == 2
-                    record_attributes(m.payload, m.key)
-                  else
-                    record_attributes(m.payload)
-                  end
-
-          attrs = attrs&.merge(record_key(m.key))
-          @klass.new(attrs) unless attrs.nil?
-        end
-      end
-
-      # Filters list of Active Records by applying active record validations.
-      # Tip: Add validates_associated in ActiveRecord model to validate associated models
-      # Optionally inherit this method and apply more filters in the application code
-      # The default implementation throws ActiveRecord::RecordInvalid by default
-      # @param records Array<ActiveRecord> - List of active records which will be subjected to model validations
-      # @return valid Array<ActiveRecord> - Subset of records that passed the model validations
-      def filter_records(records)
-        records.each(&:validate!)
-      end
-
-      # Returns true if MySQL Adapter is currently used
-      def mysql_adapter?
-        ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
-      end
-
-      # Checks whether the entities has necessary columns for `association_list` to work
-      # @return void
-      def _validate_associations(entities)
-        raise Deimos::MissingImplementationError unless mysql_adapter?
-
-        return if entities.first.respond_to?(@bulk_import_id_column)
-
-        raise "Create bulk_import_id on #{entities.first.class} and set it in `build_records` for associations." \
-              ' Run rails g deimos:bulk_import_id {table} to create the migration.'
-      end
-
-      # Fills Primary Key ID on in-memory objects.
-      # Uses @bulk_import_id_column on in-memory records to fetch saved records in database.
-      # @return void
-      def _fill_primary_key_on_entities(entities)
-        table_by_bulk_import_id = @klass.
-          where(@bulk_import_id_column => entities.map { |e| e[@bulk_import_id_column] }).
-          select(:id, @bulk_import_id_column).
-          index_by { |e| e[@bulk_import_id_column] }
-        # update IDs in upsert entity
-        entities.each { |entity| entity.id = table_by_bulk_import_id[entity[@bulk_import_id_column]].id }
-      end
     end
   end
 end

data/lib/deimos/active_record_consume/batch_record.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # Keeps track of both an ActiveRecord instance and more detailed attributes.
+    # The attributes are needed for nested associations.
+    class BatchRecord
+      # @return [ActiveRecord::Base]
+      attr_accessor :record
+      # @return [Hash] a set of association information, represented by a hash of attributes.
+      # For has_one, the format would be e.g. { 'detail' => { 'foo' => 'bar'}}. For has_many, it would
+      # be an array, e.g. { 'details' => [{'foo' => 'bar'}, {'foo' => 'baz'}]}
+      attr_accessor :associations
+      # @return [String] A unique UUID used to associate the auto-increment ID back to
+      # the in-memory record.
+      attr_accessor :bulk_import_id
+      # @return [String] The column name to use for bulk IDs - defaults to `bulk_import_id`.
+      attr_accessor :bulk_import_column
+
+      delegate :valid?, to: :record
+
+      # @param klass [Class < ActiveRecord::Base]
+      # @param attributes [Hash] the full attribute list, including associations.
+      # @param bulk_import_column [String]
+      def initialize(klass:, attributes:, bulk_import_column: nil)
+        @klass = klass
+        if bulk_import_column
+          self.bulk_import_column = bulk_import_column
+          self.bulk_import_id = SecureRandom.uuid
+          attributes[bulk_import_column] = bulk_import_id
+        end
+        attributes = attributes.with_indifferent_access
+        self.record = klass.new(attributes.slice(*klass.column_names))
+        assoc_keys = attributes.keys.select { |k| klass.reflect_on_association(k) }
+        # a hash with just the association keys, removing all actual column information.
+        self.associations = attributes.slice(*assoc_keys)
+        validate_import_id! if self.associations.any?
+      end
+
+      # Checks whether the entities has necessary columns for association saving to work
+      # @return void
+      def validate_import_id!
+        return if @klass.column_names.include?(self.bulk_import_column.to_s)
+
+        raise "Create bulk_import_id on the #{@klass.table_name} table." \
+              ' Run rails g deimos:bulk_import_id {table} to create the migration.'
+      end
+
+      # @return [Class < ActiveRecord::Base]
+      def klass
+        self.record.class
+      end
+
+      # Create a list of BatchRecord instances representing associated objects for the given
+      # association name.
+      # @param assoc_name [String]
+      # @param bulk_import_id [String] A UUID which should be set on *every* sub-record. Unlike the
+      # parent bulk_insert_id, where each record has a unique UUID,
+      # this is used to detect and delete old data, so this is basically a "session ID" for this
+      # bulk upsert.
+      # @return [Array<BatchRecord>]
+      def sub_records(assoc_name, bulk_import_id=nil)
+        attr_list = self.associations[assoc_name.to_s]
+        assoc = self.klass.reflect_on_association(assoc_name)
+        Array.wrap(attr_list).map { |attrs|
+          # Set the ID of the original object, e.g. widgets -> details, this will set widget_id.
+          attrs[assoc.foreign_key] = self.record[assoc.active_record_primary_key]
+          if bulk_import_id
+            attrs[self.bulk_import_column] = bulk_import_id
+          end
+          BatchRecord.new(klass: assoc.klass, attributes: attrs) if attrs
+        }.compact
+      end
+
+      # @return [String,Integer]
+    end
+  end
+end

data/lib/deimos/active_record_consume/batch_record_list.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # A set of BatchRecords which typically are worked with together (hence the batching!)
+    class BatchRecordList
+      # @return [Array<BatchRecord>]
+      attr_accessor :batch_records
+      attr_accessor :klass, :bulk_import_column
+
+      delegate :empty?, :map, to: :batch_records
+
+      # @param records [Array<BatchRecord>]
+      def initialize(records)
+        self.batch_records = records
+        self.klass = records.first&.klass
+        self.bulk_import_column = records.first&.bulk_import_column&.to_sym
+      end
+
+      # Filter out any invalid records.
+      # @param method [Proc]
+      def filter!(method)
+        self.batch_records.delete_if { |record| !method.call(record.record) }
+      end
+
+      # Get the original ActiveRecord objects.
+      # @return [Array<ActiveRecord::Base>]
+      def records
+        self.batch_records.map(&:record)
+      end
+
+      # Get the list of relevant associations, based on the keys of the association hashes of all
+      # records in this list.
+      # @return [Array<ActiveRecord::Reflection::AssociationReflection>]
+      def associations
+        return @associations if @associations
+
+        keys = self.batch_records.map { |r| r.associations.keys }.flatten.uniq.map(&:to_sym)
+        @associations = self.klass.reflect_on_all_associations.select { |assoc| keys.include?(assoc.name) }
+      end
+
+      # Go back to the DB and use the bulk_import_id to set the actual primary key (`id`) of the
+      # records.
+      def fill_primary_keys!
+        primary_col = self.klass.primary_key
+        bulk_import_map = self.klass.
+          where(self.bulk_import_column => self.batch_records.map(&:bulk_import_id)).
+          select(primary_col, self.bulk_import_column).
+          index_by(&self.bulk_import_column).to_h
+        self.batch_records.each do |r|
+          r.record[primary_col] = bulk_import_map[r.bulk_import_id][primary_col]
+        end
+      end
+
+      # @param [String] assoc_name
+      # @return [Array<Integer,String>]
+      def primary_keys(assoc_name)
+        assoc = self.associations.find { |a| a.name == assoc_name }
+        self.records.map do |record|
+          record[assoc.active_record_primary_key]
+        end
+      end
+
+      # @param assoc [ActiveRecord::Reflection::AssociationReflection]
+      # @param import_id [String]
+      def delete_old_records(assoc, import_id)
+        return if self.batch_records.none?
+
+        primary_keys = self.primary_keys(assoc.name)
+        assoc.klass.
+          where(assoc.foreign_key => primary_keys).
+          where("#{self.bulk_import_column} != ?", import_id).
+          delete_all
+      end
+
+    end
+  end
+end

data/lib/deimos/active_record_consume/mass_updater.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # Responsible for updating the database itself.
+    class MassUpdater
+
+      # @param klass [Class < ActiveRecord::Base]
+      def default_keys(klass)
+        [klass.primary_key]
+      end
+
+      # @param klass [Class < ActiveRecord::Base]
+      def default_cols(klass)
+        klass.column_names - %w(created_at updated_at)
+      end
+
+      # @param klass [Class < ActiveRecord::Base]
+      # @param key_col_proc [Proc<Class < ActiveRecord::Base>]
+      # @param col_proc [Proc<Class < ActiveRecord::Base>]
+      # @param replace_associations [Boolean]
+      def initialize(klass, key_col_proc: nil, col_proc: nil, replace_associations: true)
+        @klass = klass
+        @replace_associations = replace_associations
+
+        @key_cols = {}
+        @key_col_proc = key_col_proc
+
+        @columns = {}
+        @col_proc = col_proc
+      end
+
+      # @param klass [Class < ActiveRecord::Base]
+      def columns(klass)
+        @columns[klass] ||= @col_proc&.call(klass) || self.default_cols(klass)
+      end
+
+      # @param klass [Class < ActiveRecord::Base]
+      def key_cols(klass)
+        @key_cols[klass] ||= @key_col_proc&.call(klass) || self.default_keys(klass)
+      end
+
+      # @param record_list [BatchRecordList]
+      def save_records_to_database(record_list)
+        columns = self.columns(record_list.klass)
+        key_cols = self.key_cols(record_list.klass)
+        record_list.records.each(&:validate!)
+
+        options = if @key_cols.empty?
+                    {} # Can't upsert with no key, just do regular insert
+                  elsif ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
+                    {
+                      on_duplicate_key_update: columns
+                    }
+                  else
+                    {
+                      on_duplicate_key_update: {
+                        conflict_target: key_cols,
+                        columns: columns
+                      }
+                    }
+                  end
+        record_list.klass.import!(columns, record_list.records, options)
+      end
+
+      # Imports associated objects and import them to database table
+      # The base table is expected to contain bulk_import_id column for indexing associated objects with id
+      # @param record_list [BatchRecordList]
+      def import_associations(record_list)
+        record_list.fill_primary_keys!
+
+        import_id = @replace_associations ? SecureRandom.uuid : nil
+        record_list.associations.each do |assoc|
+          sub_records = record_list.map { |r| r.sub_records(assoc.name, import_id) }.flatten
+          next unless sub_records.any?
+
+          sub_record_list = BatchRecordList.new(sub_records)
+
+          save_records_to_database(sub_record_list)
+          record_list.delete_old_records(assoc, import_id) if import_id
+        end
+      end
+
+      # @param record_list [BatchRecordList]
+      def mass_update(record_list)
+        save_records_to_database(record_list)
+        import_associations(record_list) if record_list.associations.any?
+      end
+
+    end
+  end
+end

data/lib/deimos/active_record_consumer.rb

@@ -30,16 +30,9 @@ module Deimos
         config[:record_class] = klass
       end
 
-      # @param associations [List<String>] Optional list of associations that the consumer
-      # should save in addition to @klass
-      # @return [void]
-      def association_list(associations)
-        config[:association_list] = Array(associations)
-      end
-
-      # @param
-      def bulk_import_id_column(name)
-        config[:bulk_import_id_column] = name
+      # @return [String,nil]
+      def bulk_import_id_column
+        config[:bulk_import_id_column]
       end
 
       # @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
@@ -59,8 +52,6 @@ module Deimos
     # Setup
     def initialize
       @klass = self.class.config[:record_class]
-      @association_list = self.class.config[:association_list]
-      @bulk_import_id_column = self.class.config[:bulk_import_id_column] || :bulk_import_id
       @converter = ActiveRecordConsume::SchemaModelConverter.new(self.class.decoder, @klass)
 
       if self.class.config[:key_schema]