sbmt-kafka_consumer 2.2.0 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e14fbca0f8cb5daec037424c3d443f57c68ff8e58e6490fcd265b64d9746ceb
-  data.tar.gz: d3077ea80569c187a3d1fa2046ee92ddd6fcd256fb7ac908dedcee31ca6bf467
+  metadata.gz: 903e47e49b20391d11feb8f0f278b52ff6e765daa3a5fa27b50e7e673612d5db
+  data.tar.gz: 95c9e87e8a1dde1c85d1266166694e0b06d82c645cd5affcb4f06200bb66d4af
 SHA512:
-  metadata.gz: b39990b437807097d4142638166a2f6655bc90b8e1a7c1e2ef4fc86238fcc46482fb318753e2e186f09b95949b943225baf76238656141d4393086bb5be5fa7a
-  data.tar.gz: c6696572ef9a933faa0142cb24a51853e4a8961c4f9646081e3601d709e74df3c9f86fbca31ffa6493132f69ff75f06b715d956bbab0ad1aaecf4398a2b8830d
+  metadata.gz: 40aef0b43f7e39c7f61c32a2bd226893988970ad9d0be02692c12fdfaae06553d56d0464aff307b3a01445c0e683f90eb219768289aedea6423671a2f6e06136
+  data.tar.gz: 6fe8bc839e28521bc8795aca004e3dbc0f79bda8d18c2844c01e3ac9d12bdbccb29be894dff8b5ddeb073ae7bcf7e20cfb189b76a2c349aada1457a4f8f857ae

data/CHANGELOG.md

@@ -13,6 +13,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+## [2.3.1] - 2024-06-05
+
+### Fixed
+
+- Rename from `export_batch` to `process_batch`
+
+## [2.3.0] - 2024-05-30
+
+### Added
+
+- New config option `partition_assignment_strategy`
+
+### Changed
+
+- Raise an exception when using the `partition.assignment.strategy` option within `kafka_options` for topics.
+
 ## [2.2.0] - 2024-05-13
 
 ### Changed

data/Gemfile

@@ -3,3 +3,6 @@
 source "https://rubygems.org"
 
 gemspec
+
+# FIXME: remove this after drop support for Ruby 2.7
+gem "ffi", "< 1.17"

data/README.md

@@ -69,6 +69,7 @@ default: &default
   pause_timeout: 1
   pause_max_timeout: 30
   pause_with_exponential_backoff: true
+  partition_assignment_strategy: cooperative-sticky
   auth:
     kind: plaintext
   kafka:
@@ -155,6 +156,7 @@ auth:
 The `servers` key is required and should be in rdkafka format: without `kafka://` prefix, for example: `srv1:port1,srv2:port2,...`.
 
 The `kafka_config` section may contain any [rdkafka option](https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md). Also, `kafka_options` may be redefined for each topic.
+Please note that the `partition.assignment.strategy` option within kafka_options is not supported for topics; instead, use the global option partition_assignment_strategy. 
 
 ### `consumer_groups` config section
 
@@ -231,14 +233,14 @@ require_relative "config/environment"
 some-extra-configuration
 ```
 
-### `Export batch`
+### `Process batch`
 
-To process messages in batches, you need to add the `export_batch` method in the consumer
+To process messages in batches, you need to add the `process_batch` method in the consumer
 
 ```ruby
 # app/consumers/some_consumer.rb
 class SomeConsumer < Sbmt::KafkaConsumer::BaseConsumer
-  def export_batch(messages)
+  def process_batch(messages)
     # some code 
   end
 end

data/lib/generators/kafka_consumer/install/templates/kafka_consumer.yml

@@ -6,6 +6,8 @@ default: &default
   pause_timeout: 1
   pause_max_timeout: 30
   pause_with_exponential_backoff: true
+## available strategies: range, roundrobin, cooperative-sticky
+#  partition_assignment_strategy: "range,roundrobin"
   auth:
     kind: plaintext
   kafka:

data/lib/sbmt/kafka_consumer/base_consumer.rb

@@ -17,9 +17,9 @@ module Sbmt
 
       def consume
         ::Rails.application.executor.wrap do
-          if export_batch?
+          if process_batch?
             with_batch_instrumentation(messages) do
-              export_batch(messages)
+              process_batch(messages)
               mark_as_consumed!(messages.last)
             end
           else
@@ -30,11 +30,11 @@ module Sbmt
         end
       end
 
-      def export_batch?
-        if @export_batch_memoized.nil?
-          @export_batch_memoized = respond_to?(:export_batch)
+      def process_batch?
+        if @process_batch_memoized.nil?
+          @process_batch_memoized = respond_to?(:process_batch)
         end
-        @export_batch_memoized
+        @process_batch_memoized
       end
 
       private

data/lib/sbmt/kafka_consumer/config.rb

@@ -25,7 +25,7 @@ class Sbmt::KafkaConsumer::Config < Anyway::Config
 
   attr_config :client_id,
     :pause_timeout, :pause_max_timeout, :pause_with_exponential_backoff,
-    :max_wait_time, :shutdown_timeout,
+    :max_wait_time, :shutdown_timeout, :partition_assignment_strategy,
     concurrency: 4, auth: {}, kafka: {}, consumer_groups: {}, probes: {}, metrics: {},
     deserializer_class: "::Sbmt::KafkaConsumer::Serialization::NullDeserializer",
     monitor_class: "::Sbmt::KafkaConsumer::Instrumentation::TracingMonitor",
@@ -45,6 +45,7 @@ class Sbmt::KafkaConsumer::Config < Anyway::Config
     pause_with_exponential_backoff: :boolean,
     max_wait_time: :integer,
     shutdown_timeout: :integer,
+    partition_assignment_strategy: :string,
     concurrency: :integer
 
   coerce_types kafka: coerce_to(Kafka)
@@ -54,7 +55,10 @@ class Sbmt::KafkaConsumer::Config < Anyway::Config
   coerce_types consumer_groups: coerce_to_array_of(ConsumerGroup)
 
   def to_kafka_options
-    kafka.to_kafka_options
+    {
+      "partition.assignment.strategy": partition_assignment_strategy
+    }.compact
+      .merge(kafka.to_kafka_options)
       .merge(auth.to_kafka_options)
   end
 
@@ -64,6 +68,9 @@ class Sbmt::KafkaConsumer::Config < Anyway::Config
     consumer_groups.each do |cg|
       raise_validation_error "consumer group #{cg.id} must have at least one topic defined" if cg.topics.blank?
       cg.topics.each do |t|
+        if t.kafka_options.key?(:"partition.assignment.strategy")
+          raise_validation_error "Using the partition.assignment.strategy option for individual topics is not supported due to consuming issues. Use the global option `partition_assignment_strategy` instead"
+        end
         raise_validation_error "topic #{cg.id}.topics.name[#{t.name}] contains invalid consumer class: no const #{t.consumer.klass} defined" unless t.consumer.klass.safe_constantize
         raise_validation_error "topic #{cg.id}.topics.name[#{t.name}] contains invalid deserializer class: no const #{t.deserializer.klass} defined" unless t.deserializer&.klass&.safe_constantize
       end

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "2.2.0"
+    VERSION = "2.3.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.1
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-04 00:00:00.000000000 Z
+date: 2024-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails