karafka 2.4.2 → 2.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5eb2aeb0bd3cd0d50d70b0fd792e8290315f3131222d3befab9b5a05bd478e07
-  data.tar.gz: 27b4d40fe18c3daf87ea3f2b601487d6cad5a766121374772179fc5fb3049f75
+  metadata.gz: 1b7333a7812643bc128c721e411449dfd477adfa52968c7c291900e622cdcc04
+  data.tar.gz: 7062798152f0d4e097e5c4fcd228798103a680441579bbc1918c8bf5a65ce98b
 SHA512:
-  metadata.gz: af5c4030ccd1f881558df342c3936152efc6183ccc6238246a327d6da8b716e65e3e56ec198a708b87f292bd01d9e426fb64b3c4b9a4f5b40111ee68795ec050
-  data.tar.gz: 7d04bb79ea03cc5eedf7a8e9ecd62abc0ff2b48aea2e4df929d691b931b728017adf843aec5c248a6d4e8b42dd1957c798e799b9df82ae009795ae6b414532ac
+  metadata.gz: bd24bd450ac1cda02d18d7d487317139988b943ecc92c606cb713f13c819db044d6feae978873d26445e1ebf8d8823bedfd7efae58bc53c8213be9ef56a84c1d
+  data.tar.gz: 0f2f4b50ea90cd76f6275e0cdc9eeada5e5cd30bf6c18a219209dfe4b717d90ef88d7c6b68797e9eb0615c83ba03ee738d41164014ce74aaf27970ef07ba5974

checksums.yaml.gz.sig

@@ -1,4 +1 @@
-_�l*�}3r�;�z!����O4i�䪐U�m��.�Q}�N�*�<_�l�֑Ó|-n�M����\N��B(�Ot��>ß�(��2E%P�ɟY�l[����r��v�ͤ�ō�ܤN�
-�}�F��Ј-Ó�����{L�&�.��"z:q�
-P�0=�6J��,��٭qF�U��=�)��S�|��\ YW`��FdKT��=��*��	V��L
-�����x�?�>��+g�,S�=�N����0�-��V��0�}��[�������U��	�[_�Hȳ-ൠo��{<�f��)�TI�
�{���2�g��?���{��6T�p�������x�`rg����ZD�}�蕽n�焫X�]�I�Qi}��V�
+zTS���?�D=���,��#,

data/.github/workflows/ci.yml

@@ -73,6 +73,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -113,6 +114,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -146,6 +148,11 @@ jobs:
 
           bundle config set without 'tools benchmarks docs'
 
+      - name: Fix directory permissions for Bundler
+        run: |
+          chmod -R o-w /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
+          chmod +t /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
+
       - name: Bundle install
         run: |
           bundle config set without development
@@ -166,6 +173,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'

data/.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.3

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Karafka framework changelog
 
+## 2.4.4 (2024-07-04)
+- [Enhancement] Allow for offset storing from the Filtering API.
+- [Enhancement] Print more extensive error info on forceful shutdown.
+- [Enhancement] Include `original_key` in the DLQ dispatch headers.
+- [Enhancement] Support embedding mode control management from the trap context.
+- [Enhancement] Make sure, that the listener thread is stopped before restarting.
+- [Fix] Do not block on hanging listener shutdown when invoking forceful shutdown.
+- [Fix] Static membership fencing error is not propagated explicitly enough.
+- [Fix] Make sure DLQ dispatches raw headers and not deserialized headers (same as payload).
+- [Fix] Fix a typo where `ms` in logger listener would not have space before it.
+- [Maintenance] Require `karafka-core` `>=` `2.4.3`.
+- [Maintenance] Allow for usage of `karafka-rdkafka` `~` `0.16` to support librdkafka `2.4.0`.
+- [Maintenance] Lower the precision reporting to 100 microseconds in the logger listener.
+
+## 2.4.3 (2024-06-12)
+- [Enhancement] Allow for customization of Virtual Partitions reducer for enhanced parallelization.
+- [Enhancement] Add more error codes to early report on polling issues (kidlab)
+- [Enhancement] Add `transport`, `network_exception` and `coordinator_load_in_progress` alongside `timed_out` to retryable errors for the proxy.
+- [Enhancement] Improve `strict_topics_namespacing` validation message.
+- [Change] Remove default empty thread name from `Async` since Web has been upgraded.
+- [Fix] Installer doesn't respect directories in `KARAFKA_BOOT_FILE`.
+- [Fix] Fix case where non absolute boot file path would not work as expected.
+- [Fix] Allow for installing Karafka in a non-existing (yet) directory
+- [Maintenance] Require `waterdrop` `>=` `2.7.3` to support idempotent producer detection.
+
 ## 2.4.2 (2024-05-14)
 - [Enhancement] Validate ActiveJob adapter custom producer format.
 - [Fix] Internal seek does not resolve the offset correctly for time based lookup.

data/Gemfile

@@ -19,6 +19,7 @@ end
 group :test do
   gem 'byebug'
   gem 'factory_bot'
+  gem 'ostruct'
   gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.2)
+    karafka (2.4.4)
       base64 (~> 0.2)
-      karafka-core (>= 2.4.0, < 2.5.0)
-      waterdrop (>= 2.7.0, < 3.0.0)
+      karafka-core (>= 2.4.3, < 2.5.0)
+      waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.1.3.2)
-      activesupport (= 7.1.3.2)
+    activejob (7.1.3.4)
+      activesupport (= 7.1.3.4)
       globalid (>= 0.3.6)
-    activesupport (7.1.3.2)
+    activesupport (7.1.3.4)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -26,26 +26,26 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.1)
-    erubi (1.12.0)
+    erubi (1.13.0)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
-    ffi (1.16.3)
+    ffi (1.17.0)
     globalid (1.2.1)
       activesupport (>= 6.1)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.0)
-      karafka-rdkafka (>= 0.15.0, < 0.16.0)
-    karafka-rdkafka (0.15.1)
+    karafka-core (2.4.3)
+      karafka-rdkafka (>= 0.15.0, < 0.17.0)
+    karafka-rdkafka (0.16.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.3)
+    karafka-testing (2.4.4)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
     karafka-web (0.9.1)
@@ -54,12 +54,13 @@ GEM
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
-    mini_portile2 (2.8.6)
-    minitest (5.22.3)
+    mini_portile2 (2.8.7)
+    minitest (5.24.0)
     mutex_m (0.2.0)
-    rack (3.0.11)
+    ostruct (0.6.0)
+    rack (3.1.4)
     rake (13.2.1)
-    roda (3.79.0)
+    roda (3.81.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -67,7 +68,7 @@ GEM
       rspec-mocks (~> 3.13.0)
     rspec-core (3.13.0)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-mocks (3.13.1)
@@ -80,14 +81,14 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    tilt (2.3.0)
+    tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.2)
+    waterdrop (2.7.3)
       karafka-core (>= 2.4.0, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.16)
 
 PLATFORMS
   ruby
@@ -100,8 +101,9 @@ DEPENDENCIES
   karafka!
   karafka-testing (>= 2.4.0)
   karafka-web (>= 0.9.0)
+  ostruct
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.9
+   2.5.14

data/config/locales/errors.yml

@@ -121,8 +121,9 @@ en:
       declaratives.details_format: needs to be a hash with only symbol keys
 
       inconsistent_namespacing: |
-        needs to be consistent namespacing style
-        disable this validation by setting config.strict_topics_namespacing to false
+        needs to follow a consistent namespacing style using either dots (.) or underscores (_), but not both.
+        This ensures proper Kafka metrics reporting and avoids name collisions.
+        To disable this validation, set config.strict_topics_namespacing to false.
 
       deserializers.active_format: 'needs to be true'
       deserializers.payload_format: 'needs to respond to #call'

data/config/locales/pro_errors.yml

@@ -4,6 +4,7 @@ en:
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
       virtual_partitions.offset_metadata_strategy_format: needs to be either :exact or :current
+      virtual_partitions.reducer_format: "needs to respond to `#call`"
 
       long_running_job.active_format: needs to be either true or false
 

data/karafka.gemspec

@@ -22,8 +22,8 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
-  spec.add_dependency 'waterdrop', '>= 2.7.0', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.3', '< 2.5.0'
+  spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.0.0'

data/lib/karafka/cli/install.rb

@@ -21,7 +21,7 @@ module Karafka
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'karafka.rb.erb' => Karafka.boot_file,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
         'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
@@ -51,6 +51,7 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           pathed_target = Karafka.root.join(target)
+          FileUtils.mkdir_p File.dirname(pathed_target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
           render = ::ERB.new(template, trim_mode: '-').result(binding)

data/lib/karafka/connection/client.rb

@@ -553,6 +553,15 @@ module Karafka
           early_report = true
         when :transport # -195
           early_report = true
+        when :topic_authorization_failed # 29
+          early_report = true
+        when :group_authorization_failed # 30
+          early_report = true
+        when :cluster_authorization_failed # 31
+          early_report = true
+        # This can happen for many reasons, including issues with static membership being fenced
+        when :fatal # -150
+          early_report = true
         # @see
         # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
         # This will be raised each time poll detects a non-existing topic. When auto creation is

data/lib/karafka/connection/proxy.rb

@@ -16,8 +16,11 @@ module Karafka
       RETRYABLE_DEFAULT_ERRORS = %i[
         all_brokers_down
         timed_out
+        transport
+        network_exception
         not_coordinator
         not_leader_for_partition
+        coordinator_load_in_progress
       ].freeze
 
       private_constant :RETRYABLE_DEFAULT_ERRORS

data/lib/karafka/embedded.rb

@@ -2,11 +2,28 @@
 
 module Karafka
   # Allows to start and stop Karafka as part of a different process
+  # Following limitations and restrictions apply:
+  #
+  # - `#start` cannot be called from a trap context - non blocking
+  # - `#quiet` - can be called from a trap context - non blocking
+  # - `#stop` - can be called from a trap context - blocking
   module Embedded
     class << self
+      # Lock for ensuring we do not control embedding in parallel
+      MUTEX = Mutex.new
+
+      private_constant :MUTEX
+
       # Starts Karafka without supervision and without ownership of signals in a background thread
       # so it won't interrupt other things running
       def start
+        MUTEX.synchronize do
+          # Prevent from double-starting
+          return if @started
+
+          @started = true
+        end
+
         Thread.new do
           Thread.current.name = 'karafka.embedded'
 
@@ -19,9 +36,32 @@ module Karafka
       #
       # @note This method is blocking because we want to wait until Karafka is stopped with final
       #   process shutdown
+      #
+      # @note This method **is** safe to run from a trap context.
       def stop
-        # Stop needs to be blocking to wait for all the things to finalize
-        Karafka::Server.stop
+        # Prevent from double stopping
+        unless @stopping
+          Thread.new do
+            Thread.current.name = 'karafka.embedded.stopping'
+
+            stop = false
+
+            # We spawn a new thread because `#stop` may be called from a trap context
+            MUTEX.synchronize do
+              break if @stopping
+
+              @stopping = true
+              stop = true
+            end
+
+            next unless stop
+
+            Karafka::Server.stop
+          end
+        end
+
+        # Since we want to have this blocking, we wait for the background thread
+        sleep(0.1) until Karafka::App.terminated?
       end
 
       # Quiets Karafka upon any event
@@ -29,6 +69,8 @@ module Karafka
       # @note This method is not blocking and will not wait for Karafka to fully quiet.
       # It will trigger the quiet procedure but won't wait.
       #
+      # @note This method **can** be called from a trap context.
+      #
       # @note Please keep in mind you need to `#stop` to actually stop the server anyhow.
       def quiet
         Karafka::Server.quiet

data/lib/karafka/helpers/async.rb

@@ -22,13 +22,22 @@ module Karafka
         def included(base)
           base.extend ::Forwardable
 
-          base.def_delegators :@thread, :join, :terminate, :alive?, :name
+          base.def_delegators :@thread, :join, :terminate, :name
+        end
+      end
+
+      # @return [Boolean] true if thread is present and is running, false otherwise
+      def alive?
+        MUTEX.synchronize do
+          return false unless @thread
+
+          @thread.alive?
         end
       end
 
       # Runs the `#call` method in a new thread
       # @param thread_name [String] name that we want to assign to the thread when we start it
-      def async_call(thread_name = '')
+      def async_call(thread_name)
         MUTEX.synchronize do
           return if @thread&.alive?
 

data/lib/karafka/instrumentation/logger_listener.rb

@@ -41,7 +41,7 @@ module Karafka
         return unless log_polling?
 
         listener = event[:caller]
-        time = event[:time]
+        time = event[:time].round(2)
         messages_count = event[:messages_buffer].size
 
         message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
@@ -69,14 +69,14 @@ module Karafka
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_processed(event)
         job = event[:job]
-        time = event[:time]
+        time = event[:time].round(2)
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
         partition = job.executor.partition
         info <<~MSG.tr("\n", ' ').strip!
           [#{job.id}] #{job_type} job for #{consumer}
-          on #{topic}/#{partition} finished in #{time}ms
+          on #{topic}/#{partition} finished in #{time} ms
         MSG
       end
 
@@ -306,7 +306,24 @@ module Karafka
           fatal "Runner crashed due to an error: #{error}"
           fatal details
         when 'app.stopping.error'
-          error 'Forceful Karafka server stop'
+          # Counts number of workers and listeners that were still active when forcing the
+          # shutdown. Please note, that unless all listeners are closed, workers will not finalize
+          # their operations as well.
+          # We need to check if listeners and workers are assigned as during super early stages of
+          # boot they are not.
+          listeners = Server.listeners ? Server.listeners.count(&:active?) : 0
+          workers = Server.workers ? Server.workers.count(&:alive?) : 0
+
+          message = <<~MSG.tr("\n", ' ').strip!
+            Forceful Karafka server stop with:
+            #{workers} active workers and
+            #{listeners} active listeners
+          MSG
+
+          error message
+        when 'app.forceful_stopping.error'
+          error "Forceful shutdown error occurred: #{error}"
+          error details
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details

data/lib/karafka/pro/connection/manager.rb

@@ -216,6 +216,10 @@ module Karafka
               next unless multi_part_sgs_families.include?(sg_listener.subscription_group.name)
               # Skip already active connections
               next unless sg_listener.pending? || sg_listener.stopped?
+              # Ensure that the listener thread under which we operate is already stopped and
+              # is not dangling. While not likely to happen, this may protect against a
+              # case where a shutdown critical crash would case a restart of the same listener
+              next if sg_listener.alive?
 
               touch(sg_listener.subscription_group.id)
               sg_listener.start!

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -89,6 +89,23 @@ module Karafka
             applied.map(&:cursor).compact.min_by(&:offset)
           end
 
+          # @return [Boolean] did any of the filters requested offset storage during filter
+          #   application
+          def mark_as_consumed?
+            # We can manage filtering offset only when user wanted that and there is a cursor
+            # to use
+            applied.any?(&:mark_as_consumed?) && cursor
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`
+          def marking_method
+            candidates = applied.map(&:marking_method)
+
+            return :mark_as_consumed! if candidates.include?(:mark_as_consumed!)
+
+            :mark_as_consumed
+          end
+
           private
 
           # @return [Boolean] is filtering active

data/lib/karafka/pro/processing/filters/base.rb

@@ -54,6 +54,18 @@ module Karafka
           def timeout
             0
           end
+
+          # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
+          #   filters returns true, we return lowers applicable cursor value (if any)
+          def mark_as_consumed?
+            false
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Applicable only if
+          #   marking is requested
+          def marking_method
+            :mark_as_consumed
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -33,7 +33,7 @@ module Karafka
 
             # Time on message is in seconds with ms precision, so we need to convert the ttl that
             # is in ms to this format
-            border = ::Time.now.utc - @delay / 1_000.to_f
+            border = ::Time.now.utc - @delay / 1_000.0
 
             messages.delete_if do |message|
               too_young = message.timestamp > border
@@ -52,7 +52,7 @@ module Karafka
           def timeout
             return 0 unless @cursor
 
-            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+            timeout = (@delay / 1_000.0) - (::Time.now.utc - @cursor.timestamp)
 
             timeout <= 0 ? 0 : timeout * 1_000
           end

data/lib/karafka/pro/processing/partitioner.rb

@@ -25,6 +25,8 @@ module Karafka
         def call(topic, messages, coordinator)
           ktopic = @subscription_group.topics.find(topic)
 
+          vps = ktopic.virtual_partitions
+
           # We only partition work if we have:
           # - a virtual partitioner
           # - more than one thread to process the data
@@ -38,21 +40,19 @@ module Karafka
           #
           # This is great because it allows us to run things without the parallelization that adds
           # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
-          if ktopic.virtual_partitions? &&
-             ktopic.virtual_partitions.max_partitions > 1 &&
-             !coordinator.collapsed?
-            # We need to reduce it to the max concurrency, so the group_id is not a direct effect
-            # of the end user action. Otherwise the persistence layer for consumers would cache
-            # it forever and it would cause memory leaks
-            #
-            # This also needs to be consistent because the aggregation here needs to warrant, that
-            # the same partitioned message will always be assigned to the same virtual partition.
-            # Otherwise in case of a window aggregation with VP spanning across several polls, the
-            # data could not be complete.
+          if vps.active? && vps.max_partitions > 1 && !coordinator.collapsed?
             groupings = messages.group_by do |msg|
-              key = ktopic.virtual_partitions.partitioner.call(msg).to_s.sum
-
-              key % ktopic.virtual_partitions.max_partitions
+              # We need to reduce it to the max concurrency, so the group_id is not a direct effect
+              # of the end user action. Otherwise the persistence layer for consumers would cache
+              # it forever and it would cause memory leaks
+              #
+              # This also needs to be consistent because the aggregation here needs to warrant,
+              # that the same partitioned message will always be assigned to the same virtual
+              # partition. Otherwise in case of a window aggregation with VP spanning across
+              # several polls, the data could not be complete.
+              vps.reducer.call(
+                vps.partitioner.call(msg)
+              )
             end
 
             groupings.each do |key, messages_group|

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -154,11 +154,12 @@ module Karafka
                 topic: topic.dead_letter_queue.topic,
                 key: original_partition,
                 payload: skippable_message.raw_payload,
-                headers: skippable_message.headers.merge(
+                headers: skippable_message.raw_headers.merge(
                   'original_topic' => topic.name,
                   'original_partition' => original_partition,
                   'original_offset' => skippable_message.offset.to_s,
                   'original_consumer_group' => topic.consumer_group.id,
+                  'original_key' => skippable_message.raw_key.to_s,
                   'original_attempts' => attempt.to_s
                 )
               }

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -66,6 +66,19 @@ module Karafka
               # and this should not happen
               throttle_timeout = filter.timeout
 
+              # If user requested marking when applying filter, we mark. We may be in the user
+              # flow but even then this is not a problem. Older offsets will be ignored since
+              # we do not force the offset update (expected) and newer are on the user to control.
+              # This can be primarily used when filtering large quantities of data to mark on the
+              # idle runs, so lag reporting is aware that those messages were not consumed but also
+              # are no longer relevant
+              if filter.mark_as_consumed?
+                send(
+                  filter.marking_method,
+                  filter.cursor
+                )
+              end
+
               case filter.action
               when :skip
                 nil

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -22,6 +22,7 @@ module Karafka
             :partitioner,
             :max_partitions,
             :offset_metadata_strategy,
+            :reducer,
             keyword_init: true
           ) { alias_method :active?, :active }
         end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -31,6 +31,7 @@ module Karafka
               nested(:virtual_partitions) do
                 required(:active) { |val| [true, false].include?(val) }
                 required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                required(:reducer) { |val| val.respond_to?(:call) }
                 required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
                 required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
               end

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -26,18 +26,24 @@ module Karafka
             # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
             #   offset. `:exact` will match the offset matching metadata and `:current` will select
             #   the most recently reported metadata
+            # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
+            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
             # @return [VirtualPartitions] method that allows to set the virtual partitions details
             #   during the routing configuration and then allows to retrieve it
             def virtual_partitions(
               max_partitions: Karafka::App.config.concurrency,
               partitioner: nil,
-              offset_metadata_strategy: :current
+              offset_metadata_strategy: :current,
+              reducer: nil
             )
               @virtual_partitions ||= Config.new(
                 active: !partitioner.nil?,
                 max_partitions: max_partitions,
                 partitioner: partitioner,
-                offset_metadata_strategy: offset_metadata_strategy
+                offset_metadata_strategy: offset_metadata_strategy,
+                # If no reducer provided, we use this one. It just runs a modulo on the sum of
+                # a stringified version, providing fairly good distribution.
+                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions }
               )
             end
 

data/lib/karafka/runner.rb

@@ -3,17 +3,18 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
-    def initialize
-      @manager = App.config.internal.connection.manager
-      @conductor = App.config.internal.connection.conductor
-    end
+    include Helpers::ConfigImporter.new(
+      manager: %i[internal connection manager],
+      conductor: %i[internal connection conductor],
+      jobs_queue_class: %i[internal processing jobs_queue_class]
+    )
 
     # Starts listening on all the listeners asynchronously and handles the jobs queue closing
     # after listeners are done with their work.
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
-      jobs_queue = App.config.internal.processing.jobs_queue_class.new
+      jobs_queue = jobs_queue_class.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
@@ -23,7 +24,7 @@ module Karafka
       Karafka::App.run!
 
       # Register all the listeners so they can be started and managed
-      @manager.register(listeners)
+      manager.register(listeners)
 
       workers.each_with_index { |worker, i| worker.async_call("karafka.worker##{i}") }
 
@@ -32,10 +33,10 @@ module Karafka
       Karafka::Server.listeners = listeners
       Karafka::Server.jobs_queue = jobs_queue
 
-      until @manager.done?
-        @conductor.wait
+      until manager.done?
+        conductor.wait
 
-        @manager.control
+        manager.control
       end
 
       # We close the jobs queue only when no listener threads are working.

data/lib/karafka/server.rb

@@ -3,6 +3,12 @@
 module Karafka
   # Karafka consuming server class
   class Server
+    # How long should we wait on the listeners forceful shutdown when they are stuck beyond the
+    # shutdown timeout before forcing a bypass
+    FORCEFUL_SHUTDOWN_WAIT = 5
+
+    private_constant :FORCEFUL_SHUTDOWN_WAIT
+
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :listeners
@@ -105,9 +111,23 @@ module Karafka
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
         listeners.active.each(&:terminate)
+
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
         # This can cause memory leaks and crashes.
-        listeners.each(&:shutdown)
+        # We run it in a separate thread in case this would hang and we ignore it after the time
+        # we assigned to it and force shutdown as we prefer to stop the process rather than wait
+        # indefinitely even with risk of VM crash as this is a last resort.
+        Thread.new do
+          listeners.each(&:shutdown)
+        rescue StandardError => e
+          # If anything wrong happened during shutdown, we also want to record it
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'app.forceful_stopping.error'
+          )
+        end.join(FORCEFUL_SHUTDOWN_WAIT)
 
         # We also do not forcefully terminate everything when running in the embedded mode,
         # otherwise we would overwrite the shutdown process of the process that started Karafka

data/lib/karafka/setup/attributes_map.rb

@@ -52,7 +52,9 @@ module Karafka
         fetch.wait.max.ms
         group.id
         group.instance.id
+        group.protocol
         group.protocol.type
+        group.remote.assignor
         heartbeat.interval.ms
         interceptors
         internal.termination.signal

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.2'
+  VERSION = '2.4.4'
 end

data/lib/karafka.rb

@@ -92,7 +92,17 @@ module Karafka
     #   KARAFKA_BOOT_FILE='/home/app_path/app.rb'
     #   Karafka.boot_file #=> '/home/app_path/app.rb'
     def boot_file
-      Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+      boot_file = Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+
+      return boot_file if boot_file.absolute?
+      return boot_file if boot_file.to_s == 'false'
+
+      Pathname.new(
+        File.expand_path(
+          boot_file,
+          Karafka.root
+        )
+      )
     end
 
     # We need to be able to overwrite both monitor and logger after the configuration in case they

data.tar.gz.sig

@@ -1 +1,2 @@
-~А+4+1���<�&���_1��ߗ���w��
�R}D{w:}y��!�`���"�,�t�xyޥ��0q�L[X"u�:�����BU�]�qu�μ�:Re�cdVS}<�0�n�m����@Gnqzݝb9c<@���)&`���
+�-Mnj��c��Q0��g�1�����W5��5���ҟ"����J�������w�0r趛"|�ȸ_)
nD�ȩ�5޷x��	/5iZ?�3��FDJ
Q�s&m���BH�U+`��/p�c�G�%Q�gP�lØU�}	��6*�(Vۨ���i��E���3�Z����I.ӂ���Ԣ�:�c�^��dz�3Iv�R�M+�8��@v�TS���U�~��xl�Fr��o�	h����%ц8\��$��dCy���*6�)o�e9qsDY9i����ĸ�"s�S
+v@���+C�y�lY>(A����o��P6c�Z��}�v皂9i
�41eLg/d�\GDk����6]���?d���~�0B=��%�WE9<�96��A�

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.2
+  version: 2.4.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-05-14 00:00:00.000000000 Z
+date: 2024-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 2.7.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 2.7.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -549,7 +549,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
+rubygems_version: 3.5.11
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.