deimos-ruby 1.19.0 → 1.19.1.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b73bb23507a35a1e4cd10af15db00f15f434f9464ed5451fb33ef64c25e48687
-  data.tar.gz: cbaccc72e855b4cd9cbb928f2d7c5409d77dd1c356d209e6081889d6cb6092e8
+  metadata.gz: 58e9bdd29f309b3e911e71ca38b872560dbdfc5245a854bde579e8bb086a51b9
+  data.tar.gz: 1f0689ba41f0cd886c47c65e6876cbdeffd055680c2ef83f8399967c59a8a4fd
 SHA512:
-  metadata.gz: 0fe9445824b00c2c492a1357965408c99f4cd36f40ea7431bbf79727466575d69a7c91cb0709204b57ad49cee8158d4c628e3b300a85b5da56203827ecbf27c9
-  data.tar.gz: c0adb09c939eb8c2f784a58ffdfddf2ae6b11dad3dd1ba1be5572551cd4d97774db7f38ce7265f2b017fa0c5e339bf2a258dc9c21f14f3984a9e8ce1ef87ccad
+  metadata.gz: 0d6806620fcea3e34d863c70f761c90ebbd469537d43c315caeac74016fca213739ed575892314ae406f92909cd60e9edfc99b0872e326a867c1c0625d6c08ec
+  data.tar.gz: 9c53de460d27f5e59ff3f8b5a68a06c7ea1a05155755096c6be17873278d1b87d099f5beabe7ea192c4800a5b0483f958f9dc4e45fc7b65a6a7963749c1781b2

data/CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+- Feature: Limit amount of transactions in a single database call with `max_db_batch_size`
+
 # 1.19.0 - 2023-02-21
 
 - Feature: When consuming data in batch mode, this adds ability to save data to multiple tables with `association_list`.

data/docs/CONFIGURATION.md

@@ -83,6 +83,7 @@ key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../
 disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
 group_id|nil|ID of the consumer group.
 use_schema_classes|nil|Set to true or false to enable or disable using the consumers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
+max_db_batch_size|nil|Maximum limit for batching database calls to reduce the load on the db.
 max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
 start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic
 max_bytes_per_partition|512.kilobytes|Maximum amount of data fetched from a single partition at a time.

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -78,8 +78,21 @@ module Deimos
         # deleted record (no payload)
         removed, upserted = messages.partition(&:tombstone?)
 
-        upsert_records(upserted) if upserted.any?
-        remove_records(removed) if removed.any?
+        if upserted.any?
+          if @max_db_batch_size
+            upserted.each_slice(@max_db_batch_size) { |group| upsert_records(group) }
+          else
+            upsert_records(upserted)
+          end
+        end
+
+        return unless removed.any?
+
+        if @max_db_batch_size
+          removed.each_slice(@max_db_batch_size) { |group| remove_records(group) }
+        else
+          remove_records(removed)
+        end
       end
 
       # Upsert any non-deleted records

data/lib/deimos/active_record_consumer.rb

@@ -48,6 +48,12 @@ module Deimos
       def compacted(val)
         config[:compacted] = val
       end
+
+      # @param limit [Integer] Maximum number of transactions in a single database call.
+      # @return [void]
+      def max_db_batch_size(limit)
+        config[:max_db_batch_size] = limit
+      end
     end
 
     # Setup
@@ -62,6 +68,7 @@ module Deimos
       end
 
       @compacted = self.class.config[:compacted] != false
+      @max_db_batch_size = self.class.config[:max_db_batch_size]
     end
 
     # Override this method (with `super`) if you want to add/change the default

data/lib/deimos/config/configuration.rb

@@ -424,6 +424,9 @@ module Deimos
       # Configure the usage of generated schema classes for this consumer
       # @return [Boolean]
       setting :use_schema_classes
+      # Optional maximum limit for batching database calls to reduce the load on the db.
+      # @return [Integer]
+      setting :max_db_batch_size
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.19.0'
+  VERSION = '1.19.1-beta1'
 end

data/spec/config/configuration_spec.rb

@@ -90,7 +90,8 @@ describe Deimos, 'configuration' do
           offset_retention_time: nil,
           heartbeat_interval: 10,
           handler: 'ConsumerTest::MyConsumer',
-          use_schema_classes: nil
+          use_schema_classes: nil,
+          max_db_batch_size: nil
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
@@ -107,7 +108,8 @@ describe Deimos, 'configuration' do
           offset_retention_time: nil,
           heartbeat_interval: 10,
           handler: 'ConsumerTest::MyBatchConsumer',
-          use_schema_classes: nil
+          use_schema_classes: nil,
+          max_db_batch_size: nil
         }
       ],
       producer: {
@@ -258,7 +260,8 @@ describe Deimos, 'configuration' do
             offset_retention_time: 13,
             heartbeat_interval: 13,
             handler: 'MyConfigConsumer',
-            use_schema_classes: false
+            use_schema_classes: false,
+            max_db_batch_size: nil
           }
         ],
         producer: {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.19.0
+  version: 1.19.1.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-21 00:00:00.000000000 Z
+date: 2023-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -624,9 +624,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.3.20
 signing_key: