sbmt-kafka_consumer 3.5.1 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 872a886b138e7956051e347be06915f7c9153abd50076ae2ae77e0fc83672e48
-  data.tar.gz: 651678b88fe6f8fde83ec4c086dc7bbfcc5edd5ff1dfd214b151bca76f8b3783
+  metadata.gz: 3acb66e582aa1dc634195368a85f0a9fba8857f1fe319614b7caa05f25ea1614
+  data.tar.gz: 393db4b5c06f1b587aa3c1cd2d61b6dc13b99eeb102b52aa359e865e2e7b78dd
 SHA512:
-  metadata.gz: 2b60b4a2a5f84f9e3f6bc7602e352d14a373f49c9bab3d30784259e8a41e01e4d5d4324257a283cec6f3a23169ff47824fbf5c702acab891214c1ba806f2da90
-  data.tar.gz: 2ad55a703d2a6e264d1ceca17335d0a9a6c38250eba00d7c4b8dc4848df1a83442e6c6de1e59916be7ac0de9f3d99114b3c1b0fc0dda24501b8f05d38c867b8f
+  metadata.gz: 25c52294c777720e106a21e73eb35dd0c06dd1425ad2f0ab0286e046498506e8e7ebf7ba0032159abfd38ef4145d3746fea2dd6d1966445c006d220dc7018e92
+  data.tar.gz: bf2364f20485662e310ba80b45e6c6169d6966c30dc37ce49e60b071831e20a6667eccf747156de26751b2d9cd3320beebfa822cd8c883cf533dd5e950eab9e4

data/.rubocop.yml

@@ -33,3 +33,7 @@ Style/SingleLineMethods:
 
 Style/EmptyMethod:
   Enabled: false
+
+Rails/Exit:
+  Exclude:
+    - spec/rails_helper.rb

data/CHANGELOG.md

@@ -13,6 +13,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+## [3.6.0] - 2025-12-02
+
+### Added
+
+- Allow middleware configuration for all consumers
+- Added option `batch_midlewares` to add middleware before message batch processing
+
+## [3.5.2] - 2025-07-21
+
+### Changed
+
+- Add Rack 3 support: unfreeze headers hash in probes response.
+
 ## [3.5.1] - 2025-04-18
 
 ### Fixed

data/README.md

@@ -185,10 +185,12 @@ consumer_groups:
 
 - `skip_on_error` - optional, default false, omit consumer errors in message processing and commit the offset to Kafka
 - `middlewares` - optional, default [], type String, add middleware before message processing
+- `batch_middlewares` - optional, default [], type String, add middleware before batch message processing
 
 ```yaml
 init_attrs:
   middlewares: ['SomeMiddleware']
+  batch_middlewares: ['SomeBatchMiddleware']
 ```
 
 ```ruby
@@ -197,9 +199,16 @@ class SomeMiddleware
     yield if message.payload.id.to_i % 2 == 0
   end
 end
+
+class SomeBatchMiddleware
+  def call(message, consumer)
+    yield
+  end
+end
 ```
 __CAUTION__:
-- ⚠️ `yield` is mandatory for all middleware, as it returns control to the `process_message` method.
+- ⚠️ `yield` is mandatory for all middleware, as it returns control to the `process_message/process_batch` method.
+- ⚠️ all middlewares must take either 1 or 2 parameters.
 
 #### `consumer.init_attrs` options for `InboxConsumer`
 
@@ -207,10 +216,12 @@ __CAUTION__:
 - `event_name` - optional, default nil, used when the inbox item keep several event types
 - `skip_on_error` - optional, default false, omit consumer errors in message processing and commit the offset to Kafka
 - `middlewares` - optional, default [], type String, add middleware before message processing
+- `batch_middlewares` - optional, default [], type String, add middleware before batch message processing
 
 ```yaml
 init_attrs:
   middlewares: ['SomeMiddleware']
+  batch_middlewares: ['SomeBatchMiddleware']
 ```
 
 ```ruby
@@ -219,15 +230,40 @@ class SomeMiddleware
     yield if message.payload.id.to_i % 2 == 0
   end
 end
+
+class SomeBatchMiddleware
+  def call(message, consumer)
+    yield
+  end
+end
 ```
 __CAUTION__:
 - ⚠️ `yield` is mandatory for all middleware, as it returns control to the `process_message` method.
-- ⚠️ Doesn't work with `process_batch`.
+- ⚠️ all middlewares must take either 1 or 2 parameters.
 
 #### `deserializer.init_attrs` options
 
 - `skip_decoding_error` — don't raise an exception when cannot deserialize the message
 
+### Middlewares for all consumersа
+
+To add middleware for all consumers, define them in the configuration
+
+```ruby
+# config/initializers/kafka_consumer.rb
+
+Sbmt::KafkaConsumer.process_message_middlewares.push(
+  SomeMiddleware
+)
+
+Sbmt::KafkaConsumer.process_batch_middlewares(
+  SomeBatchMiddleware
+)
+```
+__CAUTION__:
+- ⚠️ `yield` is mandatory for all middleware, as it returns control to the `process_message` method.
+- ⚠️ all middlewares must take either 1 or 2 parameters.
+
 ### `probes` config section
 
 In Kubernetes, probes are mechanisms used to assess the health of your application running within a container.

data/lib/sbmt/kafka_consumer/base_consumer.rb

@@ -5,8 +5,9 @@ module Sbmt
     class BaseConsumer < Karafka::BaseConsumer
       class_attribute :skip_on_error, instance_writer: false, default: false
       class_attribute :middlewares, instance_writer: false, default: []
+      class_attribute :batch_middlewares, instance_writer: false, default: []
 
-      def self.consumer_klass(skip_on_error: nil, middlewares: nil)
+      def self.consumer_klass(skip_on_error: nil, middlewares: nil, batch_middlewares: nil)
         klass = Class.new(self) do
           def self.name
             superclass.name
@@ -16,6 +17,7 @@ module Sbmt
         # defaults are set in class_attribute definition
         klass.skip_on_error = skip_on_error if skip_on_error
         klass.middlewares = middlewares.map(&:constantize) if middlewares
+        klass.batch_middlewares = batch_middlewares.map(&:constantize) if batch_middlewares
         klass
       end
 
@@ -23,7 +25,7 @@ module Sbmt
         ::Rails.application.executor.wrap do
           if process_batch?
             with_batch_instrumentation(messages) do
-              process_batch(messages)
+              do_process_batch(messages)
               mark_message(messages.last)
             end
           else
@@ -99,6 +101,11 @@ module Sbmt
         end
       end
 
+      def do_process_batch(messages)
+        middleware_list = process_batch_middlewares + batch_middlewares
+        call_middlewares(messages, middleware_list) { process_batch(messages) }
+      end
+
       def do_consume(message)
         log_message(message) if log_payload?
 
@@ -107,7 +114,8 @@ module Sbmt
         message.payload
 
         with_common_instrumentation("process_message", message) do
-          call_middlewares(message, middlewares) { process_message(message) }
+          middleware_list = process_message_middlewares + middlewares
+          call_middlewares(message, middleware_list) { process_message(message) }
         end
 
         with_common_instrumentation("mark_as_consumed", message) do
@@ -147,7 +155,7 @@ module Sbmt
         message.payload || message.raw_payload
       end
 
-      def call_middlewares(message, middlewares)
+      def call_middlewares(payload, middlewares)
         return yield if middlewares.empty?
 
         chain = middlewares.map { |middleware_class| middleware_class.new }
@@ -156,7 +164,13 @@ module Sbmt
           if chain.empty?
             yield
           else
-            chain.shift.call(message, &traverse_chain)
+            middleware = chain.shift
+
+            if middleware.method(:call).arity == 2
+              middleware.call(payload, self, &traverse_chain)
+            else
+              middleware.call(payload, &traverse_chain)
+            end
           end
         end
         traverse_chain.call
@@ -174,6 +188,14 @@ module Sbmt
         @config ||= Sbmt::KafkaConsumer::Config.new
       end
 
+      def process_message_middlewares
+        @process_message_middlewares ||= config.process_message_middlewares
+      end
+
+      def process_batch_middlewares
+        @process_batch_middlewares ||= config.process_batch_middlewares
+      end
+
       def cooperative_sticky?
         config.partition_assignment_strategy == "cooperative-sticky"
       end

data/lib/sbmt/kafka_consumer/config.rb

@@ -27,6 +27,7 @@ class Sbmt::KafkaConsumer::Config < Anyway::Config
     :pause_timeout, :pause_max_timeout, :pause_with_exponential_backoff,
     :max_wait_time, :shutdown_timeout, :partition_assignment_strategy,
     concurrency: 4, auth: {}, kafka: {}, consumer_groups: {}, probes: {}, metrics: {},
+    process_message_middlewares: [], process_batch_middlewares: [],
     deserializer_class: "::Sbmt::KafkaConsumer::Serialization::NullDeserializer",
     monitor_class: "::Sbmt::KafkaConsumer::Instrumentation::TracingMonitor",
     logger_class: "::Sbmt::KafkaConsumer::Logger",
@@ -38,6 +39,8 @@ class Sbmt::KafkaConsumer::Config < Anyway::Config
 
   on_load :validate_consumer_groups
   on_load :set_default_metrics_port
+  on_load :set_process_message_middlewares
+  on_load :set_process_batch_middlewares
 
   coerce_types client_id: :string,
     pause_timeout: :integer,
@@ -80,4 +83,12 @@ class Sbmt::KafkaConsumer::Config < Anyway::Config
   def set_default_metrics_port
     self.metrics = metrics.new(port: probes.port) unless metrics.port
   end
+
+  def set_process_message_middlewares
+    self.process_message_middlewares = Sbmt::KafkaConsumer.process_message_middlewares || []
+  end
+
+  def set_process_batch_middlewares
+    self.process_batch_middlewares = Sbmt::KafkaConsumer.process_batch_middlewares || []
+  end
 end

data/lib/sbmt/kafka_consumer/inbox_consumer.rb

@@ -9,9 +9,9 @@ module Sbmt
       class_attribute :inbox_item_class, instance_writer: false, default: nil
       class_attribute :event_name, instance_writer: false, default: nil
 
-      def self.consumer_klass(inbox_item:, event_name: nil, skip_on_error: nil, name: nil, middlewares: nil)
+      def self.consumer_klass(inbox_item:, event_name: nil, skip_on_error: nil, name: nil, middlewares: nil, batch_middlewares: nil)
         # defaults are set in class_attribute definition
-        klass = super(skip_on_error: skip_on_error, middlewares: middlewares)
+        klass = super(skip_on_error: skip_on_error, middlewares: middlewares, batch_middlewares: batch_middlewares)
         klass.inbox_item_class = inbox_item.constantize
         klass.event_name = event_name if event_name
         klass

data/lib/sbmt/kafka_consumer/instrumentation/logger_listener.rb

@@ -5,6 +5,7 @@ module Sbmt
     module Instrumentation
       class LoggerListener < Karafka::Instrumentation::LoggerListener
         include ListenerHelper
+
         CUSTOM_ERROR_TYPES = %w[consumer.base.consume_one consumer.inbox.consume_one].freeze
         VALID_LOG_LEVELS = %i[error warn].freeze
 

data/lib/sbmt/kafka_consumer/probes/probe.rb

@@ -4,7 +4,7 @@ module Sbmt
   module KafkaConsumer
     module Probes
       module Probe
-        HEADERS = {"Content-Type" => "application/json"}.freeze
+        HEADERS = {"Content-Type" => "application/json"}
 
         def call(env)
           with_error_handler { probe(env) }

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "3.5.1"
+    VERSION = "3.6.0"
   end
 end

data/lib/sbmt/kafka_consumer.rb

@@ -26,6 +26,14 @@ module Sbmt
       def logger
         @logger ||= Rails.logger
       end
+
+      def process_message_middlewares
+        @process_message_middlewares ||= []
+      end
+
+      def process_batch_middlewares
+        @process_batch_middlewares ||= []
+      end
     end
 
     class Error < StandardError; end

data/sbmt-kafka_consumer.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "rails", ">= 6.1"
   spec.add_dependency "zeitwerk", "~> 2.3"
-  spec.add_dependency "karafka", "~> 2.4"
+  spec.add_dependency "karafka", "~> 2.4.0"
   spec.add_dependency "yabeda", ">= 0.11"
   spec.add_dependency "anyway_config", ">= 2.4.0"
   spec.add_dependency "thor"

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 3.5.1
+  version: 3.6.0
 platform: ruby
 authors:
 - Kuper Ruby-Platform Team
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-04-18 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -44,14 +43,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: 2.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: yabeda
   requirement: !ruby/object:Gem::Requirement
@@ -460,7 +459,6 @@ dependencies:
         version: '0'
 description: This gem is used for consuming Kafka messages. It represents a wrapper
   over Karafka gem and is recommended for using as a transport with sbmt-outbox
-email:
 executables:
 - kafka_consumer
 extensions: []
@@ -550,7 +548,6 @@ metadata:
   source_code_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer
   changelog_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer/blob/master/CHANGELOG.md
   rubygems_mfa_required: 'false'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -565,8 +562,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.21
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Ruby gem for consuming Kafka messages
 test_files: []