karafka 2.4.18 → 2.5.0.beta1

data/lib/karafka/processing/coordinators_buffer.rb

@@ -22,6 +22,7 @@ module Karafka
 
       # @param topic_name [String] topic name
       # @param partition [Integer] partition number
+      # @return [Karafka::Processing::Coordinator] found or created coordinator
       def find_or_create(topic_name, partition)
         @coordinators[topic_name][partition] ||= begin
           routing_topic = @topics.find(topic_name)

data/lib/karafka/processing/strategies/default.rb

@@ -55,8 +55,8 @@ module Karafka
           # seek offset can be nil only in case `#seek` was invoked with offset reset request
           # In case like this we ignore marking
           return true if seek_offset.nil?
-          # Ignore earlier offsets than the one we already committed
-          return true if seek_offset > message.offset
+          # Ignore double markings of the same offset
+          return true if (seek_offset - 1) == message.offset
           return false if revoked?
           return revoked? unless client.mark_as_consumed(message)
 
@@ -74,8 +74,8 @@ module Karafka
           # seek offset can be nil only in case `#seek` was invoked with offset reset request
           # In case like this we ignore marking
           return true if seek_offset.nil?
-          # Ignore earlier offsets than the one we already committed
-          return true if seek_offset > message.offset
+          # Ignore double markings of the same offset
+          return true if (seek_offset - 1) == message.offset
           return false if revoked?
 
           return revoked? unless client.mark_as_consumed!(message)

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -46,6 +46,7 @@ module Karafka
               # When topic is set to false, it means we just want to skip dispatch on DLQ
               next if topic == false
               next if topic.is_a?(String) && topic_regexp.match?(topic)
+              next if topic == :strategy
 
               [[%i[dead_letter_queue topic], :format]]
             end

data/lib/karafka/routing/subscription_group.rb

@@ -30,7 +30,7 @@ module Karafka
             @group_counter ||= 0
             @group_counter += 1
 
-            ::Digest::MD5.hexdigest(
+            ::Digest::SHA256.hexdigest(
               @group_counter.to_s
             )[0..11]
           end

data/lib/karafka/runner.rb

@@ -4,6 +4,7 @@ module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
     include Helpers::ConfigImporter.new(
+      worker_thread_priority: %i[worker_thread_priority],
       manager: %i[internal connection manager],
       conductor: %i[internal connection conductor],
       jobs_queue_class: %i[internal processing jobs_queue_class]
@@ -26,7 +27,12 @@ module Karafka
       # Register all the listeners so they can be started and managed
       manager.register(listeners)
 
-      workers.each_with_index { |worker, i| worker.async_call("karafka.worker##{i}") }
+      workers.each_with_index do |worker, i|
+        worker.async_call(
+          "karafka.worker##{i}",
+          worker_thread_priority
+        )
+      end
 
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers

data/lib/karafka/server.rb

@@ -30,6 +30,9 @@ module Karafka
       # as not everything is possible when operating in non-standalone mode, etc.
       attr_accessor :execution_mode
 
+      # id of the server. Useful for logging when we want to reference things issued by the server.
+      attr_accessor :id
+
       # Method which runs app
       def run
         self.listeners = []
@@ -187,5 +190,7 @@ module Karafka
     # This is overwritten quickly during boot, but just in case someone would reach it prior to
     # booting, we want to have the default value.
     self.execution_mode = :standalone
+
+    self.id = SecureRandom.hex(6)
   end
 end

data/lib/karafka/setup/attributes_map.rb

@@ -73,6 +73,7 @@ module Karafka
         message.max.bytes
         metadata.broker.list
         metadata.max.age.ms
+        metadata.recovery.strategy
         oauthbearer_token_refresh_cb
         offset.store.method
         offset.store.path
@@ -207,6 +208,7 @@ module Karafka
         message.timeout.ms
         metadata.broker.list
         metadata.max.age.ms
+        metadata.recovery.strategy
         msg_order_cmp
         oauthbearer_token_refresh_cb
         opaque

data/lib/karafka/setup/config.rb

@@ -73,6 +73,9 @@ module Karafka
       # Really useful when you want to ensure that all topics in routing are managed via
       # declaratives.
       setting :strict_declarative_topics, default: false
+      # Defaults to the CPU thread priority slice to -1 (50ms) to ensure that CPU intense
+      # processing does not affect other threads and prevents starvation
+      setting :worker_thread_priority, default: -1
 
       setting :oauth do
         # option [false, #call] Listener for using oauth bearer. This listener will be able to
@@ -133,6 +136,14 @@ module Karafka
         # How many times should be try. 1 000 ms x 60 => 60 seconds wait in total and then we give
         # up on pending operations
         setting :max_attempts, default: 60
+
+        # option poll_timeout [Integer] time in ms
+        # How long should a poll wait before yielding on no results (rdkafka-ruby setting)
+        # Lower value can be especially useful when working with Web UI, because it allows for
+        # increased responsiveness. Many admin operations do not take 100ms but they wait on poll
+        # until then prior to finishing, blocking the execution. Lowering to 25 ms can
+        # improve responsiveness of the Web UI. 50ms is a good trade-off for admin.
+        setting :poll_timeout, default: 50
       end
 
       # Namespace for internal settings that should not be modified directly
@@ -211,6 +222,10 @@ module Karafka
           # How long should we wait before a critical listener recovery
           # Too short may cause endless rebalance loops
           setting :reset_backoff, default: 60_000
+          # Similar to the `#worker_thread_priority`. Listener threads do not operate for long
+          # time and release GVL on polling but we provide this for API consistency and some
+          # special edge cases.
+          setting :listener_thread_priority, default: 0
 
           # Settings that are altered by our client proxy layer
           setting :proxy do
@@ -282,6 +297,9 @@ module Karafka
           setting :jobs_builder, default: Processing::JobsBuilder.new
           # option coordinator [Class] work coordinator we want to user for processing coordination
           setting :coordinator_class, default: Processing::Coordinator
+          # option errors_tracker_class [Class, nil] errors tracker that is used by the coordinator
+          #   for granular error tracking. `nil` for OSS as it is not in use.
+          setting :errors_tracker_class, default: nil
           # option partitioner_class [Class] partitioner we use against a batch of data
           setting :partitioner_class, default: Processing::Partitioner
           # option strategy_selector [Object] processing strategy selector to be used
@@ -367,7 +385,10 @@ module Karafka
           config.producer ||= ::WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
-            producer_config.kafka = AttributesMap.producer(config.kafka.dup)
+            producer_kafka = AttributesMap.producer(config.kafka.dup)
+            # We inject some defaults (mostly for dev) unless user defined them
+            Setup::DefaultsInjector.producer(producer_kafka)
+            producer_config.kafka = producer_kafka
             # We also propagate same listener to the default producer to make sure, that the
             # listener for oauth is also automatically used by the producer. That way we don't
             # have to configure it manually for the default producer

data/lib/karafka/setup/defaults_injector.rb

@@ -36,7 +36,17 @@ module Karafka
         'topic.metadata.refresh.interval.ms': 5_000
       }.freeze
 
-      private_constant :CONSUMER_KAFKA_DEFAULTS, :CONSUMER_KAFKA_DEV_DEFAULTS
+      # Contains settings that should not be used in production but make life easier in dev
+      # It is applied only to the default producer. If users setup their own producers, then
+      # they have to set this by themselves.
+      PRODUCER_KAFKA_DEV_DEFAULTS = {
+        # For all of those same reasoning as for the consumer
+        'allow.auto.create.topics': 'true',
+        'topic.metadata.refresh.interval.ms': 5_000
+      }.freeze
+
+      private_constant :CONSUMER_KAFKA_DEFAULTS, :CONSUMER_KAFKA_DEV_DEFAULTS,
+                       :PRODUCER_KAFKA_DEV_DEFAULTS
 
       class << self
         # Propagates the kafka setting defaults unless they are already present for consumer config
@@ -58,6 +68,21 @@ module Karafka
             kafka_config[key] = value
           end
         end
+
+        # Propagates the kafka settings defaults unless they are already present for producer
+        # config. This makes it easier to set some values that users usually don't change but still
+        # allows them to overwrite the whole hash.
+        #
+        # @param kafka_config [Hash] kafka scoped config
+        def producer(kafka_config)
+          return if Karafka::App.env.production?
+
+          PRODUCER_KAFKA_DEV_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+        end
       end
     end
   end

data/lib/karafka/status.rb

@@ -66,7 +66,12 @@ module Karafka
             return if initializing?
 
             conductor.signal
-            monitor.instrument("app.#{state}", caller: self)
+
+            monitor.instrument(
+              "app.#{state}",
+              caller: self,
+              server_id: Karafka::Server.id
+            )
           end
         end
       RUBY

data/lib/karafka/swarm/node.rb

@@ -27,6 +27,18 @@ module Karafka
       # @return [Integer] pid of the node
       attr_reader :pid
 
+      # When re-creating a producer in the fork, those are not attributes we want to inherit
+      # from the parent process because they are updated in the fork. If user wants to take those
+      # from the parent process, he should redefine them by overwriting the whole producer.
+      SKIPPABLE_NEW_PRODUCER_ATTRIBUTES = %i[
+        id
+        kafka
+        logger
+        oauth
+      ].freeze
+
+      private_constant :SKIPPABLE_NEW_PRODUCER_ATTRIBUTES
+
       # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
       #   other stuff where we need to know a unique reference of the fork in regards to the rest
       #   of them.
@@ -52,15 +64,32 @@ module Karafka
           # an attempt to close it when finalized, meaning it would be kept in memory.
           config.producer.close
 
+          old_producer = config.producer
+          old_producer_config = old_producer.config
+
           # Supervisor producer is closed, hence we need a new one here
           config.producer = ::WaterDrop::Producer.new do |p_config|
             p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
             p_config.logger = config.logger
+
+            old_producer_config.to_h.each do |key, value|
+              next if SKIPPABLE_NEW_PRODUCER_ATTRIBUTES.include?(key)
+
+              p_config.public_send("#{key}=", value)
+            end
+
+            # Namespaced attributes need to be migrated directly on their config node
+            old_producer_config.oauth.to_h.each do |key, value|
+              p_config.oauth.public_send("#{key}=", value)
+            end
           end
 
           @pid = ::Process.pid
           @reader.close
 
+          # Certain features need to be reconfigured / reinitialized after fork in Pro
+          Pro::Loader.post_fork(config, old_producer) if Karafka.pro?
+
           # Indicate we are alive right after start
           healthy
 
@@ -69,6 +98,8 @@ module Karafka
           monitor.instrument('swarm.node.after_fork', caller: self)
 
           Karafka::Process.tags.add(:execution_mode, 'mode:swarm')
+          Karafka::Process.tags.add(:swarm_nodeid, "node:#{@id}")
+
           Server.execution_mode = :swarm
           Server.run
 

data/lib/karafka/swarm/supervisor.rb

@@ -50,6 +50,10 @@ module Karafka
         # producer (should not be initialized but just in case)
         Karafka.producer.close
 
+        # Ensure rdkafka stuff is loaded into memory pre-fork. This will ensure, that we save
+        # few MB on forking as this will be already in memory.
+        Rdkafka::Bindings.rd_kafka_global_init
+
         Karafka::App.warmup
 
         manager.start

data/lib/karafka/templates/karafka.rb.erb

@@ -25,7 +25,20 @@ APP_LOADER.eager_load
 class KarafkaApp < Karafka::App
   setup do |config|
     config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
-    config.client_id = 'example_app'
+    config.client_id = 'YOUR_APP_NAME'
+
+    # IMPORTANT: Customize this group_id with your application name.
+    # The group_id should be unique per application to properly track message consumption.
+    # Example: config.group_id = 'inventory_service_consumer'
+    #
+    # Note: Advanced features and custom routing configurations may define their own consumer
+    # groups. These should also be uniquely named per application to avoid conflicts.
+    # For the advanced features, subscription groups and consumer groups in your routing
+    # configuration, follow the same uniqueness principle.
+    #
+    # For more details on consumer groups and routing configuration, please refer to the
+    # Karafka documentation: https://karafka.io/docs
+    config.group_id = 'YOUR_APP_NAME_consumer'
 <% if rails? -%>
     # Recreate consumers with each batch. This will allow Rails code reload to work in the
     # development mode. Otherwise Karafka process would not be aware of code changes

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.18'
+  VERSION = '2.5.0.beta1'
 end

data/lib/karafka.rb

@@ -16,6 +16,7 @@
   singleton
   digest
   zeitwerk
+  logger
 ].each(&method(:require))
 
 # Karafka framework main namespace
@@ -65,16 +66,23 @@ module Karafka
         return @root
       end
 
-      # By default we infer the project root from bundler.
-      # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
-      # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
-      # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
-      # from which the project starts may not match the original Gemfile.
-      @root = Pathname.new(
-        File.dirname(
-          Bundler.with_unbundled_env { Bundler.default_gemfile }
+      if defined?(::Bundler)
+        # By default we infer the project root from bundler.
+        # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
+        # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
+        # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
+        # from which the project starts may not match the original Gemfile.
+        @root = Pathname.new(
+          File.dirname(
+            Bundler.with_unbundled_env { Bundler.default_gemfile }
+          )
         )
-      )
+      else
+        # Fallback when Bundler is not available: use current directory
+        @root = Pathname.new(Dir.pwd)
+      end
+
+      @root
     end
 
     # @return [Pathname] path to Karafka gem root core

data/renovate.json

@@ -1,9 +1,21 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-      "config:base"
+    "config:recommended"
   ],
   "ignorePaths": [
-      "spec/integrations"
+    "spec/integrations"
+  ],
+  "github-actions": {
+    "enabled": true,
+    "pinDigests": true
+  },
+  "packageRules": [
+    {
+      "matchManagers": [
+        "github-actions"
+      ],
+      "minimumReleaseAge": "7 days"
+    }
   ]
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.18
+  version: 2.5.0.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2025-04-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -56,41 +56,41 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.4
+        version: 2.5.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.4
+        version: 2.5.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: karafka-rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.2
+        version: 0.19.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.2
+        version: 0.19.2
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.3
+        version: 2.8.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -100,7 +100,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.3
+        version: 2.8.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -133,10 +133,12 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
+- ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
+- ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -152,6 +154,7 @@ files:
 - README.md
 - SECURITY.md
 - bin/benchmarks
+- bin/clean_kafka
 - bin/create_token
 - bin/integrations
 - bin/karafka
@@ -160,13 +163,19 @@ files:
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
+- bin/verify_kafka_warnings
 - bin/verify_license_integrity
+- bin/verify_topics_naming
 - bin/wait_for_kafka
 - certs/cert.pem
 - certs/karafka-pro.pem
 - config/locales/errors.yml
 - config/locales/pro_errors.yml
 - docker-compose.yml
+- examples/payloads/json/enrollment_event.json
+- examples/payloads/json/ingestion_event.json
+- examples/payloads/json/transaction_event.json
+- examples/payloads/json/user_event.json
 - karafka.gemspec
 - lib/active_job/karafka.rb
 - lib/active_job/queue_adapters/karafka_adapter.rb
@@ -275,6 +284,10 @@ files:
 - lib/karafka/pro/cleaner/messages/message.rb
 - lib/karafka/pro/cleaner/messages/messages.rb
 - lib/karafka/pro/cleaner/messages/metadata.rb
+- lib/karafka/pro/cli/parallel_segments.rb
+- lib/karafka/pro/cli/parallel_segments/base.rb
+- lib/karafka/pro/cli/parallel_segments/collapse.rb
+- lib/karafka/pro/cli/parallel_segments/distribute.rb
 - lib/karafka/pro/connection/manager.rb
 - lib/karafka/pro/connection/multiplexing/listener.rb
 - lib/karafka/pro/contracts/base.rb
@@ -316,6 +329,9 @@ files:
 - lib/karafka/pro/processing/offset_metadata/consumer.rb
 - lib/karafka/pro/processing/offset_metadata/fetcher.rb
 - lib/karafka/pro/processing/offset_metadata/listener.rb
+- lib/karafka/pro/processing/parallel_segments/filters/base.rb
+- lib/karafka/pro/processing/parallel_segments/filters/default.rb
+- lib/karafka/pro/processing/parallel_segments/filters/mom.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/periodic_job/consumer.rb
 - lib/karafka/pro/processing/piping/consumer.rb
@@ -373,6 +389,9 @@ files:
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
 - lib/karafka/pro/processing/subscription_groups_coordinator.rb
+- lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb
+- lib/karafka/pro/processing/virtual_partitions/distributors/base.rb
+- lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb
 - lib/karafka/pro/recurring_tasks.rb
 - lib/karafka/pro/recurring_tasks/consumer.rb
 - lib/karafka/pro/recurring_tasks/contracts/config.rb
@@ -436,6 +455,12 @@ files:
 - lib/karafka/pro/routing/features/offset_metadata/config.rb
 - lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb
 - lib/karafka/pro/routing/features/offset_metadata/topic.rb
+- lib/karafka/pro/routing/features/parallel_segments.rb
+- lib/karafka/pro/routing/features/parallel_segments/builder.rb
+- lib/karafka/pro/routing/features/parallel_segments/config.rb
+- lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb
+- lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb
+- lib/karafka/pro/routing/features/parallel_segments/topic.rb
 - lib/karafka/pro/routing/features/patterns.rb
 - lib/karafka/pro/routing/features/patterns/builder.rb
 - lib/karafka/pro/routing/features/patterns/config.rb
@@ -619,7 +644,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.
 test_files: []