karafka 2.4.18 → 2.5.0.beta1

data/examples/payloads/json/ingestion_event.json

@@ -0,0 +1,30 @@
+{
+  "metadata": {
+    "request_id": "0bb22210-4a7b-4df3-a0a2-4fed464f4296",
+    "tenant_id": "tenant_2",
+    "event": "service_2.file_ingested",
+    "raw_tenant_file_id": "64eef122-39a4-4ba4-a0eb-2115bb3be545",
+    "row_number": 4,
+    "batch_item_run_id": "61eb1d69-88ad-421a-a57d-543b1d674a86"
+  },
+  "data": {
+    "user": {
+      "partner_user_id": "111111111166666666_555555522222222222",
+      "email": "email@example.com",
+      "status": "active",
+      "first_name": "first name",
+      "last_name": "last name",
+      "birthdate": "1999-09-22",
+      "country_code": "US"
+    },
+    "product": {
+      "custom_fields": {
+        "membership_id": "tttttttttt"
+      },
+      "reference_id": "yyyyyyyyyyy",
+      "code": "404",
+      "start_time": "2024-05-12 0:00:00",
+      "partner_status": "active"
+    }
+  }
+}

data/examples/payloads/json/transaction_event.json

@@ -0,0 +1,17 @@
+{
+  "event_type": "points_transaction",
+  "source": "service_4",
+  "data": {
+    "points_transaction_id": "6d853550-86c0-4878-a234-32ad3b80e7d7",
+    "user_id": "435092f6-a1e3-4c3b-8f22-bc809d04d9f5",
+    "tenant_id": "tenant_1",
+    "category": "redemption",
+    "product_type": "shopping_cart",
+    "product_sub_type": "cashback",
+    "points": 1000,
+    "transaction_time": "2025-05-05T13:13:57Z",
+    "product_name": "Tenant 1 Cashback",
+    "order_item_type": "CashRedemptionOrderItem",
+    "reward_amount": 10.0
+  }
+}

data/examples/payloads/json/user_event.json

@@ -0,0 +1,11 @@
+{
+  "data": {
+    "user_id": "686eeceb-c763-46fc-b946-17663bb827b4",
+    "tenant_id": "tenant_1",
+    "points_account_id": "95fa8ca3-f89e-4d9b-8aa7-a9ba3b460b49"
+  },
+  "metadata": {
+    "event": "service_1.user_activated",
+    "request_id": "ee3b35f4-68c0-42ab-aa8c-6ca146d3ca89"
+  }
+}

data/karafka.gemspec

@@ -22,9 +22,9 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.4.4', '< 2.5.0'
-  spec.add_dependency 'karafka-rdkafka', '>= 0.17.2'
-  spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.5.0', '< 2.6.0'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.19.2'
+  spec.add_dependency 'waterdrop', '>= 2.8.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.0.0'

data/lib/karafka/active_job/current_attributes.rb

@@ -29,7 +29,7 @@ module Karafka
           # Prevent registering same klass multiple times
           next if Dispatcher._cattr_klasses.value?(stringified_klass)
 
-          key = "cattr_#{Dispatcher._cattr_klasses.count}"
+          key = "cattr_#{Dispatcher._cattr_klasses.size}"
 
           Dispatcher._cattr_klasses[key] = stringified_klass
           Consumer._cattr_klasses[key] = stringified_klass

data/lib/karafka/admin/acl.rb

@@ -11,6 +11,10 @@ module Karafka
     # This API works based on ability to create a `Karafka:Admin::Acl` object that can be then used
     # using `#create`, `#delete` and `#describe` class API.
     class Acl
+      extend Helpers::ConfigImporter.new(
+        max_wait_time: %i[admin max_wait_time]
+      )
+
       # Types of resources for which we can assign permissions.
       #
       # Resource refers to any entity within the Kafka ecosystem for which access control can be
@@ -162,7 +166,7 @@ module Karafka
         # Makes sure that admin is closed afterwards.
         def with_admin_wait
           Admin.with_admin do |admin|
-            yield(admin).wait(max_wait_timeout: Karafka::App.config.admin.max_wait_time)
+            yield(admin).wait(max_wait_timeout: max_wait_time)
           end
         end
 

data/lib/karafka/admin.rb

@@ -10,11 +10,16 @@ module Karafka
   #   Cluster on which operations are performed can be changed via `admin.kafka` config, however
   #   there is no multi-cluster runtime support.
   module Admin
-    # More or less number of seconds of 1 hundred years
-    # Used for time referencing that does not have to be accurate but needs to be big
-    HUNDRED_YEARS = 100 * 365.25 * 24 * 60 * 60
+    # 2010-01-01 00:00:00 - way before Kafka was released so no messages should exist prior to
+    # this date
+    # We do not use the explicit -2 librdkafka value here because we resolve this offset without
+    # consuming data
+    LONG_TIME_AGO = Time.at(1_262_300_400)
 
-    private_constant :HUNDRED_YEARS
+    # one day in seconds for future time reference
+    DAY_IN_SECONDS = 60 * 60 * 24
+
+    private_constant :LONG_TIME_AGO, :DAY_IN_SECONDS
 
     class << self
       # Allows us to read messages from the topic
@@ -55,7 +60,7 @@ module Karafka
           possible_range = requested_range.select { |offset| available_range.include?(offset) }
 
           start_offset = possible_range.first
-          count = possible_range.count
+          count = possible_range.size
 
           tpl.add_topic_and_partitions_with_offsets(name, partition => start_offset)
           consumer.assign(tpl)
@@ -203,14 +208,14 @@ module Karafka
             # Earliest is not always 0. When compacting/deleting it can be much later, that's why
             # we fetch the oldest possible offset
             when 'earliest'
-              Time.now - HUNDRED_YEARS
+              LONG_TIME_AGO
             # Latest will always be the high-watermark offset and we can get it just by getting
             # a future position
             when 'latest'
-              Time.now + HUNDRED_YEARS
-            # Same as `'latest'`
+              Time.now + DAY_IN_SECONDS
+            # Same as `'earliest'`
             when false
-              Time.now - HUNDRED_YEARS
+              LONG_TIME_AGO
             # Regular offset case
             else
               position
@@ -274,27 +279,24 @@ module Karafka
         end
       end
 
-      # Takes consumer group and its topics and migrates all the offsets to a new named group
+      # Takes consumer group and its topics and copies all the offsets to a new named group
       #
       # @param previous_name [String] old consumer group name
       # @param new_name [String] new consumer group name
       # @param topics [Array<String>] topics for which we want to migrate offsets during rename
-      # @param delete_previous [Boolean] should we delete previous consumer group after rename.
-      #   Defaults to true.
+      # @return [Boolean] true if anything was migrated, otherwise false
       #
       # @note This method should **not** be executed on a running consumer group as it creates a
       #   "fake" consumer and uses it to move offsets.
       #
-      # @note After migration unless `delete_previous` is set to `false`, old group will be
-      #   removed.
-      #
       # @note If new consumer group exists, old offsets will be added to it.
-      def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
+      def copy_consumer_group(previous_name, new_name, topics)
         remap = Hash.new { |h, k| h[k] = {} }
 
         old_lags = read_lags_with_offsets({ previous_name => topics })
 
-        return if old_lags.empty?
+        return false if old_lags.empty?
+        return false if old_lags.values.all? { |topic_data| topic_data.values.all?(&:empty?) }
 
         read_lags_with_offsets({ previous_name => topics })
           .fetch(previous_name)
@@ -311,9 +313,35 @@ module Karafka
 
         seek_consumer_group(new_name, remap)
 
-        return unless delete_previous
+        true
+      end
+
+      # Takes consumer group and its topics and migrates all the offsets to a new named group
+      #
+      # @param previous_name [String] old consumer group name
+      # @param new_name [String] new consumer group name
+      # @param topics [Array<String>] topics for which we want to migrate offsets during rename
+      # @param delete_previous [Boolean] should we delete previous consumer group after rename.
+      #   Defaults to true.
+      # @return [Boolean] true if rename (and optionally removal) was ok or false if there was
+      #   nothing really to rename
+      #
+      # @note This method should **not** be executed on a running consumer group as it creates a
+      #   "fake" consumer and uses it to move offsets.
+      #
+      # @note After migration unless `delete_previous` is set to `false`, old group will be
+      #   removed.
+      #
+      # @note If new consumer group exists, old offsets will be added to it.
+      def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
+        copy_result = copy_consumer_group(previous_name, new_name, topics)
+
+        return false unless copy_result
+        return copy_result unless delete_previous
 
         delete_consumer_group(previous_name)
+
+        true
       end
 
       # Removes given consumer group (if exists)
@@ -509,7 +537,11 @@ module Karafka
       def with_admin
         bind_id = SecureRandom.uuid
 
-        admin = config(:producer, {}).admin(native_kafka_auto_start: false)
+        admin = config(:producer, {}).admin(
+          native_kafka_auto_start: false,
+          native_kafka_poll_timeout_ms: app_config.admin.poll_timeout
+        )
+
         bind_oauth(bind_id, admin)
 
         admin.start

data/lib/karafka/base_consumer.rb

@@ -21,7 +21,7 @@ module Karafka
 
     # @return [String] id of the current consumer
     attr_reader :id
-    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    # @return [Karafka::Messages::Messages] current messages batch
     attr_accessor :messages
     # @return [Karafka::Connection::Client] kafka connection client
     attr_accessor :client
@@ -304,7 +304,12 @@ module Karafka
 
       offset = nil if offset == :consecutive
 
-      client.pause(topic.name, partition, offset)
+      client.pause(
+        topic.name,
+        partition,
+        offset,
+        coordinator.pause_tracker.current_timeout
+      )
 
       # Indicate, that user took a manual action of pausing
       coordinator.manual_pause if manual_pause
@@ -333,17 +338,21 @@ module Karafka
 
     # Seeks in the context of current topic and partition
     #
-    # @param offset [Integer, Time] offset where we want to seek or time of the offset where we
-    #   want to seek.
+    # @param offset [Integer, Time, Symbol, String] one of:
+    #   - offset where we want to seek
+    #   - time of the offset where we want to seek
+    #   - :earliest (or as a string) to move to earliest message
+    #   - :latest (or as a string) to move to latest (high-watermark)
+    #
     # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
     #   based seek. User seek operations should take precedence over system actions, hence we need
     #   to know who invoked it.
-    # @param reset_offset [Boolean] should we reset offset when seeking backwards. It is false by
-    #   default to prevent marking in the offset that was earlier than the highest marked offset
-    #   for given consumer group. It can be set to true if we want to reprocess data once again and
+    # @param reset_offset [Boolean] should we reset offset when seeking backwards. It is false
+    #   it prevents marking in the offset that was earlier than the highest marked offset
+    #   for given consumer group. It is set to true by default to reprocess data once again and
     #   want to make sure that the marking starts from where we moved to.
     # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
-    def seek(offset, manual_seek = true, reset_offset: false)
+    def seek(offset, manual_seek = true, reset_offset: true)
       coordinator.manual_seek if manual_seek
       self.seek_offset = nil if reset_offset
 

data/lib/karafka/cli/base.rb

@@ -112,7 +112,7 @@ module Karafka
                 *[names, option[2], option[1]].flatten
               ) { |value| options[option[0]] = value }
             end
-          end.parse(ARGV)
+          end.parse(ARGV.dup)
 
           options
         end
@@ -130,8 +130,14 @@ module Karafka
         #   given Cli command
         # @example for Karafka::Cli::Install
         #   name #=> 'install'
+        # @example for Karafka::Cli::TestMe
+        #   name => 'test_me'
         def name
-          to_s.split('::').last.downcase
+          to_s
+            .split('::')
+            .last
+            .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+            .downcase
         end
 
         # @return [Array<String>] names and aliases for command matching

data/lib/karafka/connection/client.rb

@@ -9,6 +9,11 @@ module Karafka
     # closed consumer instance as it causes Ruby VM process to crash.
     class Client
       include ::Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        logger: %i[logger],
+        tick_interval: %i[internal tick_interval],
+        shutdown_timeout: %i[shutdown_timeout]
+      )
 
       attr_reader :rebalance_manager
 
@@ -65,9 +70,8 @@ module Karafka
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
-        @tick_interval = ::Karafka::App.config.internal.tick_interval
         @rebalance_manager = RebalanceManager.new(@subscription_group.id, @buffer)
-        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group)
+        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group, id)
 
         @interval_runner = Helpers::IntervalRunner.new do
           events_poll
@@ -221,10 +225,14 @@ module Karafka
       # @param offset [Integer, nil] offset of the message on which we want to pause (this message
       #   will be reprocessed after getting back to processing) or nil if we want to pause and
       #   resume from the consecutive offset (+1 from the last message passed to us by librdkafka)
+      # @param timeout [Integer] number of ms timeout of pause. It is used only for
+      #   instrumentation and not in the pause itself as pausing on this level is infinite always.
       # @note This will pause indefinitely and requires manual `#resume`
       # @note When `#internal_seek` is not involved (when offset is `nil`) we will not purge the
       #   librdkafka buffers and continue from the last cursor offset
-      def pause(topic, partition, offset = nil)
+      # @note We accept the timeout value on this layer to have a cohesive pause/resume
+      #   instrumentation, where all the details are available. It is especially needed, when
+      def pause(topic, partition, offset = nil, timeout = 0)
         @mutex.synchronize do
           # Do not pause if the client got closed, would not change anything
           return if @closed
@@ -243,7 +251,8 @@ module Karafka
             subscription_group: @subscription_group,
             topic: topic,
             partition: partition,
-            offset: offset
+            offset: offset,
+            timeout: timeout
           )
 
           @paused_tpls[topic][partition] = tpl
@@ -313,7 +322,7 @@ module Karafka
           @unsubscribing = true
 
           # Give 50% of time for the final close before we reach the forceful
-          max_wait = ::Karafka::App.config.shutdown_timeout * COOP_UNSUBSCRIBE_FACTOR
+          max_wait = shutdown_timeout * COOP_UNSUBSCRIBE_FACTOR
           used = 0
           stopped_at = monotonic_now
 
@@ -472,6 +481,10 @@ module Karafka
           message.offset = detected_partition&.offset || raise(Errors::InvalidTimeBasedOffsetError)
         end
 
+        # Those two are librdkafka hardcoded values
+        message.offset = -1 if message.offset.to_s == 'latest'
+        message.offset = -2 if message.offset.to_s == 'earliest'
+
         # Never seek if we would get the same location as we would get without seeking
         # This prevents us from the expensive buffer purges that can lead to increased network
         # traffic and can cost a lot of money
@@ -573,7 +586,7 @@ module Karafka
         # We should not run a single poll longer than the tick frequency. Otherwise during a single
         # `#batch_poll` we would not be able to run `#events_poll` often enough effectively
         # blocking events from being handled.
-        poll_tick = timeout > @tick_interval ? @tick_interval : timeout
+        poll_tick = timeout > tick_interval ? tick_interval : timeout
 
         result = kafka.poll(poll_tick)
 
@@ -655,7 +668,7 @@ module Karafka
       # Builds a new rdkafka consumer instance based on the subscription group configuration
       # @return [Rdkafka::Consumer]
       def build_consumer
-        ::Rdkafka::Config.logger = ::Karafka::App.config.logger
+        ::Rdkafka::Config.logger = logger
 
         # We need to refresh the setup of this subscription group in case we started running in a
         # swarm. The initial configuration for validation comes from the parent node, but it needs

data/lib/karafka/connection/listener.rb

@@ -12,6 +12,13 @@ module Karafka
     class Listener
       include Helpers::Async
 
+      include Helpers::ConfigImporter.new(
+        jobs_builder: %i[internal processing jobs_builder],
+        partitioner_class: %i[internal processing partitioner_class],
+        reset_backoff: %i[internal connection reset_backoff],
+        listener_thread_priority: %i[internal connection listener_thread_priority]
+      )
+
       # Can be useful for logging
       # @return [String] id of this listener
       attr_reader :id
@@ -19,6 +26,11 @@ module Karafka
       # @return [Karafka::Routing::SubscriptionGroup] subscription group that this listener handles
       attr_reader :subscription_group
 
+      # @return [Processing::CoordinatorsBuffer] coordinator buffers that can be used directly in
+      #   advanced cases of changes to the polling flow (like triggered seek back without messages
+      #   ahead in the topic)
+      attr_reader :coordinators
+
       # How long to wait in the initial events poll. Increases chances of having the initial events
       # immediately available
       INITIAL_EVENTS_POLL_TIMEOUT = 100
@@ -30,16 +42,13 @@ module Karafka
       # @param scheduler [Karafka::Processing::Scheduler] scheduler we want to use
       # @return [Karafka::Connection::Listener] listener instance
       def initialize(subscription_group, jobs_queue, scheduler)
-        proc_config = ::Karafka::App.config.internal.processing
-
         @id = SecureRandom.hex(6)
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
         @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
         @client = Client.new(@subscription_group, -> { running? })
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
-        @jobs_builder = proc_config.jobs_builder
-        @partitioner = proc_config.partitioner_class.new(subscription_group)
+        @partitioner = partitioner_class.new(subscription_group)
         @scheduler = scheduler
         @events_poller = Helpers::IntervalRunner.new { @client.events_poll }
         # We keep one buffer for messages to preserve memory and not allocate extra objects
@@ -111,7 +120,10 @@ module Karafka
 
         @status.start!
 
-        async_call("karafka.listener##{@subscription_group.id}")
+        async_call(
+          "karafka.listener##{@subscription_group.id}",
+          listener_thread_priority
+        )
       end
 
       # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
@@ -254,7 +266,7 @@ module Karafka
         reset
 
         # Ruby sleep is in seconds
-        sleep_time = ::Karafka::App.config.internal.connection.reset_backoff / 10_000.0
+        sleep_time = reset_backoff / 10_000.0
         sleep(sleep_time) && retry
       end
 
@@ -294,7 +306,7 @@ module Karafka
             # here. In cases like this, we do not run a revocation job
             @executors.find_all(topic, partition).each do |executor|
               executor.coordinator.increment(:revoked)
-              jobs << @jobs_builder.revoked(executor)
+              jobs << jobs_builder.revoked(executor)
             end
 
             # We need to remove all the executors of a given topic partition that we have lost, so
@@ -318,7 +330,7 @@ module Karafka
 
         @executors.each do |executor|
           executor.coordinator.increment(:shutdown)
-          job = @jobs_builder.shutdown(executor)
+          job = jobs_builder.shutdown(executor)
           jobs << job
         end
 
@@ -355,7 +367,7 @@ module Karafka
             if coordinator.topic.eofed?
               @executors.find_all_or_create(topic, partition, coordinator).each do |executor|
                 coordinator.increment(:eofed)
-                eofed_jobs << @jobs_builder.eofed(executor)
+                eofed_jobs << jobs_builder.eofed(executor)
               end
             end
 
@@ -372,7 +384,7 @@ module Karafka
             # Start work coordination for this topic partition
             coordinator.increment(:idle)
             executor = @executors.find_or_create(topic, partition, 0, coordinator)
-            idle_jobs << @jobs_builder.idle(executor)
+            idle_jobs << jobs_builder.idle(executor)
 
             next
           end
@@ -383,7 +395,7 @@ module Karafka
           @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
             coordinator.increment(:consume)
             executor = @executors.find_or_create(topic, partition, group_id, coordinator)
-            consume_jobs << @jobs_builder.consume(executor, partition_messages)
+            consume_jobs << jobs_builder.consume(executor, partition_messages)
           end
         end
 
@@ -451,7 +463,7 @@ module Karafka
 
             @executors.find_all_or_create(topic_name, partition, coordinator).each do |executor|
               coordinator.increment(:periodic)
-              jobs << @jobs_builder.periodic(executor)
+              jobs << jobs_builder.periodic(executor)
             end
           end
         end

data/lib/karafka/connection/messages_buffer.rb

@@ -45,7 +45,7 @@ module Karafka
         last_polled_at = raw_messages_buffer.last_polled_at
 
         raw_messages_buffer.each do |topic, partition, messages, eof|
-          @size += messages.count
+          @size += messages.size
 
           ktopic = @subscription_group.topics.find(topic)
 

data/lib/karafka/connection/proxy.rb

@@ -108,6 +108,7 @@ module Karafka
       rescue Rdkafka::RdkafkaError => e
         return false if e.code == :assignment_lost
         return false if e.code == :state
+        return false if e.code == :illegal_generation
 
         raise e
       end
@@ -136,6 +137,8 @@ module Karafka
           return false
         when :unknown_member_id
           return false
+        when :illegal_generation
+          return false
         when :no_offset
           return true
         when :coordinator_load_in_progress

data/lib/karafka/contracts/config.rb

@@ -35,6 +35,7 @@ module Karafka
       required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
       required(:strict_declarative_topics) { |val| [true, false].include?(val) }
+      required(:worker_thread_priority) { |val| (-3..3).to_a.include?(val) }
 
       nested(:swarm) do
         required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
@@ -81,6 +82,7 @@ module Karafka
           required(:manager) { |val| !val.nil? }
           required(:conductor) { |val| !val.nil? }
           required(:reset_backoff) { |val| val.is_a?(Integer) && val >= 1_000 }
+          required(:listener_thread_priority) { |val| (-3..3).to_a.include?(val) }
 
           nested(:proxy) do
             nested(:commit) do
@@ -114,6 +116,7 @@ module Karafka
           required(:jobs_queue_class) { |val| !val.nil? }
           required(:scheduler_class) { |val| !val.nil? }
           required(:coordinator_class) { |val| !val.nil? }
+          required(:errors_tracker_class) { |val| val.nil? || val.is_a?(Class) }
           required(:partitioner_class) { |val| !val.nil? }
           required(:strategy_selector) { |val| !val.nil? }
           required(:expansions_selector) { |val| !val.nil? }

data/lib/karafka/contracts/topic.rb

@@ -70,7 +70,7 @@ module Karafka
         next unless ::Karafka::App.config.strict_topics_namespacing
 
         value = data.fetch(:name)
-        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.count
+        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.size
 
         next if namespacing_chars_count <= 1
 

data/lib/karafka/errors.rb

@@ -35,6 +35,9 @@ module Karafka
     # Raised when given topic is not found while expected
     TopicNotFoundError = Class.new(BaseError)
 
+    # Raised when given consumer group is not found while expected
+    ConsumerGroupNotFoundError = Class.new(BaseError)
+
     # This should never happen. Please open an issue if it does.
     UnsupportedCaseError = Class.new(BaseError)
 
@@ -64,6 +67,10 @@ module Karafka
     # Raised when there is an attempt to run an unrecognized CLI command
     UnrecognizedCommandError = Class.new(BaseError)
 
+    # Raised when you were executing a command and it could not finish successfully because of
+    # a setup state or parameters configuration
+    CommandValidationError = Class.new(BaseError)
+
     # Raised when we attempt to perform operation that is only allowed inside of a transaction and
     # there is no transaction around us
     TransactionRequiredError = Class.new(BaseError)
@@ -71,6 +78,10 @@ module Karafka
     # Raised in case user would want to perform nested transactions.
     TransactionAlreadyInitializedError = Class.new(BaseError)
 
+    # Raised when user used transactional offset marking but after that tried to use
+    # non-transactional marking, effectively mixing both. This is not allowed.
+    NonTransactionalMarkingAttemptError = Class.new(BaseError)
+
     # Raised in case a listener that was paused is being resumed
     InvalidListenerResumeError = Class.new(BaseError)
 

data/lib/karafka/helpers/async.rb

@@ -37,12 +37,14 @@ module Karafka
 
       # Runs the `#call` method in a new thread
       # @param thread_name [String] name that we want to assign to the thread when we start it
-      def async_call(thread_name)
+      # @param thread_priority [Integer] Ruby thread priority
+      def async_call(thread_name, thread_priority = 0)
         MUTEX.synchronize do
           return if @thread&.alive?
 
           @thread = Thread.new do
             Thread.current.name = thread_name
+            Thread.current.priority = thread_priority
 
             Thread.current.abort_on_exception = true
 

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -12,8 +12,10 @@ module Karafka
 
         # @param subscription_group [Karafka::Routes::SubscriptionGroup] subscription group for
         #   which we want to manage rebalances
-        def initialize(subscription_group)
+        # @param client_id [String] id of the client managing this rebalance
+        def initialize(subscription_group, client_id)
           @subscription_group = subscription_group
+          @client_id = client_id
         end
 
         # Publishes an event that partitions are going to be revoked.
@@ -62,6 +64,7 @@ module Karafka
             subscription_group: @subscription_group,
             consumer_group_id: @subscription_group.consumer_group.id,
             consumer_group: @subscription_group.consumer_group,
+            client_id: @client_id,
             tpl: tpl
           )
         rescue StandardError => e
@@ -71,6 +74,7 @@ module Karafka
             subscription_group_id: @subscription_group.id,
             consumer_group_id: @subscription_group.consumer_group.id,
             type: "callbacks.rebalance.#{name}.error",
+            client_id: @client_id,
             error: e
           )
         end