deimos-ruby 1.19.1.pre.beta1 → 1.19.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58e9bdd29f309b3e911e71ca38b872560dbdfc5245a854bde579e8bb086a51b9
-  data.tar.gz: 1f0689ba41f0cd886c47c65e6876cbdeffd055680c2ef83f8399967c59a8a4fd
+  metadata.gz: 34ab9629daca099d1745e192f379cdd5289c081fc241d2920966485c896611c1
+  data.tar.gz: 9dd093633667f4560312c0cd945083fb605d29cb4074d7807ea5a98c1a54d68a
 SHA512:
-  metadata.gz: 0d6806620fcea3e34d863c70f761c90ebbd469537d43c315caeac74016fca213739ed575892314ae406f92909cd60e9edfc99b0872e326a867c1c0625d6c08ec
-  data.tar.gz: 9c53de460d27f5e59ff3f8b5a68a06c7ea1a05155755096c6be17873278d1b87d099f5beabe7ea192c4800a5b0483f958f9dc4e45fc7b65a6a7963749c1781b2
+  metadata.gz: 02276cba6707533b241b0ea2b0f3c8bf74ebf8120515b7ca883a080f4bdd80a1e15e3120097d3b0627a6519b4dc4ccdf5fc4f4e2950111ae69c12a262ec5ac9b
+  data.tar.gz: 4a6236f8d659c90ef864a3b99b04518c5f045235079f7168540b4d25f862ac2554d228c9ea94d10f93dd1ead0c1edd8476a66c6b360cbb572ccdaf9740212dfa

data/CHANGELOG.md

@@ -7,7 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
-- Feature: Limit amount of transactions in a single database call with `max_db_batch_size`
+# 1.19.2 - 2023-03-20
+
+- Fix for saving associations where the primary key is not `id`.
+
+# 1.19.1 - 2023-03-17
+
+- Fix for `rails g deimos:bulk_import_id` generator.
 
 # 1.19.0 - 2023-02-21
 

data/README.md

@@ -360,7 +360,7 @@ end
 Sometimes, the Kafka message needs to be saved to multiple database tables. For example, if a `User` topic provides you metadata and profile image for users, we might want to save it to multiple tables: `User` and `Image`.
 
 - The `association_list` configuration allows you to achieve this use case.
-- The optional `bulk_import_id_column` config allows you to specify column_name on `record_class` which can be used to retrieve IDs after save. Defaults to `bulk_import_id`
+- The `bulk_import_id_column` config allows you to specify column_name on `record_class` which can be used to retrieve IDs after save. Defaults to `bulk_import_id`. This config is *required* if you have associations but optional if you do not.
 
 You must override the `build_records` and `bulk_import_columns` methods on your ActiveRecord class for this feature to work.
 - `build_records` - This method is required to set the value of the `bulk_import_id` column and map Kafka messages to ActiveRecord model objects.
@@ -374,7 +374,7 @@ class MyBatchConsumer < Deimos::ActiveRecordConsumer
 
   def build_records(messages)
     # Initialise bulk_import_id and build ActiveRecord objects out of Kafka message attributes
-    messages.each do |m|
+    messages.map do |m|
       u = User.new(first_name: m.first_name, bulk_import_id: SecureRandom.uuid)
       i = Image.new(attr1: m.image_url)
       u.images << i

data/docs/CONFIGURATION.md

@@ -83,7 +83,6 @@ key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../
 disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
 group_id|nil|ID of the consumer group.
 use_schema_classes|nil|Set to true or false to enable or disable using the consumers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
-max_db_batch_size|nil|Maximum limit for batching database calls to reduce the load on the db.
 max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
 start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic
 max_bytes_per_partition|512.kilobytes|Maximum amount of data fetched from a single partition at a time.

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -78,21 +78,8 @@ module Deimos
         # deleted record (no payload)
         removed, upserted = messages.partition(&:tombstone?)
 
-        if upserted.any?
-          if @max_db_batch_size
-            upserted.each_slice(@max_db_batch_size) { |group| upsert_records(group) }
-          else
-            upsert_records(upserted)
-          end
-        end
-
-        return unless removed.any?
-
-        if @max_db_batch_size
-          removed.each_slice(@max_db_batch_size) { |group| remove_records(group) }
-        else
-          remove_records(removed)
-        end
+        upsert_records(upserted) if upserted.any?
+        remove_records(removed) if removed.any?
       end
 
       # Upsert any non-deleted records
@@ -150,7 +137,7 @@ module Deimos
               # Get associated `has_one` or `has_many` records for each entity
               sub_records = Array(entity.send(assoc.name))
               # Set IDS from master to each of the records in `has_one` or `has_many` relation
-              sub_records.each { |d| d.send("#{assoc.send(:foreign_key)}=", entity.id) }
+              sub_records.each { |d| d.send("#{assoc.foreign_key}=", entity.send(assoc.active_record_primary_key)) }
               sub_records
             }.flatten
 
@@ -256,7 +243,7 @@ module Deimos
         return if entities.first.respond_to?(@bulk_import_id_column)
 
         raise "Create bulk_import_id on #{entities.first.class} and set it in `build_records` for associations." \
-              ' Run rails g deimos:bulk_import_id:setup to create the migration.'
+              ' Run rails g deimos:bulk_import_id {table} to create the migration.'
       end
 
       # Fills Primary Key ID on in-memory objects.

data/lib/deimos/active_record_consumer.rb

@@ -48,12 +48,6 @@ module Deimos
       def compacted(val)
         config[:compacted] = val
       end
-
-      # @param limit [Integer] Maximum number of transactions in a single database call.
-      # @return [void]
-      def max_db_batch_size(limit)
-        config[:max_db_batch_size] = limit
-      end
     end
 
     # Setup
@@ -68,7 +62,6 @@ module Deimos
       end
 
       @compacted = self.class.config[:compacted] != false
-      @max_db_batch_size = self.class.config[:max_db_batch_size]
     end
 
     # Override this method (with `super`) if you want to add/change the default

data/lib/deimos/config/configuration.rb

@@ -424,9 +424,6 @@ module Deimos
       # Configure the usage of generated schema classes for this consumer
       # @return [Boolean]
       setting :use_schema_classes
-      # Optional maximum limit for batching database calls to reduce the load on the db.
-      # @return [Integer]
-      setting :max_db_batch_size
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.19.1-beta1'
+  VERSION = '1.19.2'
 end

data/lib/generators/deimos/bulk_import_id_generator.rb

@@ -12,8 +12,6 @@ module Deimos
       include Rails::Generators::Migration
       include ActiveRecord::Generators::Migration
 
-      namespace 'deimos:bulk_import_id:setup'
-
       argument :table_name, desc: 'The table to add bulk import column.', required: true
       argument :column_name, desc: 'The bulk import ID column name.', default: 'bulk_import_id'
 

data/spec/active_record_batch_consumer_mysql_spec.rb

@@ -108,7 +108,7 @@ module ActiveRecordBatchConsumerTest
           publish_batch([{ key: 2,
                            payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
         }.to raise_error('Create bulk_import_id on ActiveRecordBatchConsumerTest::Widget'\
-        ' and set it in `build_records` for associations. Run rails g deimos:bulk_import_id:setup'\
+        ' and set it in `build_records` for associations. Run rails g deimos:bulk_import_id {table}'\
         ' to create the migration.')
       end
     end

data/spec/config/configuration_spec.rb

@@ -90,8 +90,7 @@ describe Deimos, 'configuration' do
           offset_retention_time: nil,
           heartbeat_interval: 10,
           handler: 'ConsumerTest::MyConsumer',
-          use_schema_classes: nil,
-          max_db_batch_size: nil
+          use_schema_classes: nil
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
@@ -108,8 +107,7 @@ describe Deimos, 'configuration' do
           offset_retention_time: nil,
           heartbeat_interval: 10,
           handler: 'ConsumerTest::MyBatchConsumer',
-          use_schema_classes: nil,
-          max_db_batch_size: nil
+          use_schema_classes: nil
         }
       ],
       producer: {
@@ -260,8 +258,7 @@ describe Deimos, 'configuration' do
             offset_retention_time: 13,
             heartbeat_interval: 13,
             handler: 'MyConfigConsumer',
-            use_schema_classes: false,
-            max_db_batch_size: nil
+            use_schema_classes: false
           }
         ],
         producer: {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.19.1.pre.beta1
+  version: 1.19.2
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-17 00:00:00.000000000 Z
+date: 2023-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -624,9 +624,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.3.20
 signing_key: