karafka 2.4.18 → 2.5.0

data/lib/karafka/swarm/node.rb

@@ -27,6 +27,18 @@ module Karafka
       # @return [Integer] pid of the node
       attr_reader :pid
 
+      # When re-creating a producer in the fork, those are not attributes we want to inherit
+      # from the parent process because they are updated in the fork. If user wants to take those
+      # from the parent process, he should redefine them by overwriting the whole producer.
+      SKIPPABLE_NEW_PRODUCER_ATTRIBUTES = %i[
+        id
+        kafka
+        logger
+        oauth
+      ].freeze
+
+      private_constant :SKIPPABLE_NEW_PRODUCER_ATTRIBUTES
+
       # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
       #   other stuff where we need to know a unique reference of the fork in regards to the rest
       #   of them.
@@ -52,15 +64,32 @@ module Karafka
           # an attempt to close it when finalized, meaning it would be kept in memory.
           config.producer.close
 
+          old_producer = config.producer
+          old_producer_config = old_producer.config
+
           # Supervisor producer is closed, hence we need a new one here
           config.producer = ::WaterDrop::Producer.new do |p_config|
             p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
             p_config.logger = config.logger
+
+            old_producer_config.to_h.each do |key, value|
+              next if SKIPPABLE_NEW_PRODUCER_ATTRIBUTES.include?(key)
+
+              p_config.public_send("#{key}=", value)
+            end
+
+            # Namespaced attributes need to be migrated directly on their config node
+            old_producer_config.oauth.to_h.each do |key, value|
+              p_config.oauth.public_send("#{key}=", value)
+            end
           end
 
           @pid = ::Process.pid
           @reader.close
 
+          # Certain features need to be reconfigured / reinitialized after fork in Pro
+          Pro::Loader.post_fork(config, old_producer) if Karafka.pro?
+
           # Indicate we are alive right after start
           healthy
 
@@ -69,6 +98,8 @@ module Karafka
           monitor.instrument('swarm.node.after_fork', caller: self)
 
           Karafka::Process.tags.add(:execution_mode, 'mode:swarm')
+          Karafka::Process.tags.add(:swarm_nodeid, "node:#{@id}")
+
           Server.execution_mode = :swarm
           Server.run
 

data/lib/karafka/swarm/supervisor.rb

@@ -42,7 +42,10 @@ module Karafka
       # Creates needed number of forks, installs signals and starts supervision
       def run
         # Validate the CLI provided options the same way as we do for the regular server
-        cli_contract.validate!(activity_manager.to_h)
+        cli_contract.validate!(
+          activity_manager.to_h,
+          scope: %w[swarm cli]
+        )
 
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
@@ -50,6 +53,10 @@ module Karafka
         # producer (should not be initialized but just in case)
         Karafka.producer.close
 
+        # Ensure rdkafka stuff is loaded into memory pre-fork. This will ensure, that we save
+        # few MB on forking as this will be already in memory.
+        Rdkafka::Bindings.rd_kafka_global_init
+
         Karafka::App.warmup
 
         manager.start
@@ -150,7 +157,7 @@ module Karafka
         # Run forceful kill
         manager.terminate
         # And wait until linux kills them
-        # This prevents us from existing forcefully with any dead child process still existing
+        # This prevents us from exiting forcefully with any dead child process still existing
         # Since we have sent the `KILL` signal, it must die, so we can wait until all dead
         sleep(supervision_sleep) until manager.stopped?
 

data/lib/karafka/templates/karafka.rb.erb

@@ -25,7 +25,20 @@ APP_LOADER.eager_load
 class KarafkaApp < Karafka::App
   setup do |config|
     config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
-    config.client_id = 'example_app'
+    config.client_id = 'YOUR_APP_NAME'
+
+    # IMPORTANT: Customize this group_id with your application name.
+    # The group_id should be unique per application to properly track message consumption.
+    # Example: config.group_id = 'inventory_service_consumer'
+    #
+    # Note: Advanced features and custom routing configurations may define their own consumer
+    # groups. These should also be uniquely named per application to avoid conflicts.
+    # For the advanced features, subscription groups and consumer groups in your routing
+    # configuration, follow the same uniqueness principle.
+    #
+    # For more details on consumer groups and routing configuration, please refer to the
+    # Karafka documentation: https://karafka.io/docs
+    config.group_id = 'YOUR_APP_NAME_consumer'
 <% if rails? -%>
     # Recreate consumers with each batch. This will allow Rails code reload to work in the
     # development mode. Otherwise Karafka process would not be aware of code changes

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.18'
+  VERSION = '2.5.0'
 end

data/lib/karafka.rb

@@ -16,6 +16,7 @@
   singleton
   digest
   zeitwerk
+  logger
 ].each(&method(:require))
 
 # Karafka framework main namespace
@@ -65,16 +66,23 @@ module Karafka
         return @root
       end
 
-      # By default we infer the project root from bundler.
-      # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
-      # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
-      # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
-      # from which the project starts may not match the original Gemfile.
-      @root = Pathname.new(
-        File.dirname(
-          Bundler.with_unbundled_env { Bundler.default_gemfile }
+      if defined?(::Bundler)
+        # By default we infer the project root from bundler.
+        # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
+        # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
+        # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
+        # from which the project starts may not match the original Gemfile.
+        @root = Pathname.new(
+          File.dirname(
+            Bundler.with_unbundled_env { Bundler.default_gemfile }
+          )
         )
-      )
+      else
+        # Fallback when Bundler is not available: use current directory
+        @root = Pathname.new(Dir.pwd)
+      end
+
+      @root
     end
 
     # @return [Pathname] path to Karafka gem root core

data/renovate.json

@@ -1,9 +1,21 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-      "config:base"
+    "config:recommended"
   ],
   "ignorePaths": [
-      "spec/integrations"
+    "spec/integrations"
+  ],
+  "github-actions": {
+    "enabled": true,
+    "pinDigests": true
+  },
+  "packageRules": [
+    {
+      "matchManagers": [
+        "github-actions"
+      ],
+      "minimumReleaseAge": "7 days"
+    }
   ]
 }

metadata

@@ -1,40 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.18
+  version: 2.5.0
 platform: ruby
 authors:
 - Maciej Mensfeld
 bindir: bin
-cert_chain:
-- |
-  -----BEGIN CERTIFICATE-----
-  MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
-  YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
-  dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
-  PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
-  ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
-  Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
-  JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
-  T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
-  flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
-  vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
-  iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
-  sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
-  eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
-  Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
-  agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
-  TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
-  Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
-  oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
-  i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
-  ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
-  -----END CERTIFICATE-----
-date: 2025-04-09 00:00:00.000000000 Z
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -56,41 +29,41 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.4
+        version: 2.5.2
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.4
+        version: 2.5.2
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: karafka-rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.2
+        version: 0.19.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.2
+        version: 0.19.5
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.3
+        version: 2.8.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -100,7 +73,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.3
+        version: 2.8.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -133,10 +106,13 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
+- ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
+- ".github/workflows/push.yml"
+- ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -150,8 +126,10 @@ files:
 - LICENSE-COMM
 - LICENSE-LGPL
 - README.md
+- Rakefile
 - SECURITY.md
 - bin/benchmarks
+- bin/clean_kafka
 - bin/create_token
 - bin/integrations
 - bin/karafka
@@ -160,13 +138,19 @@ files:
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
+- bin/verify_kafka_warnings
 - bin/verify_license_integrity
+- bin/verify_topics_naming
 - bin/wait_for_kafka
-- certs/cert.pem
 - certs/karafka-pro.pem
 - config/locales/errors.yml
 - config/locales/pro_errors.yml
 - docker-compose.yml
+- examples/payloads/avro/.gitkeep
+- examples/payloads/json/sample_set_01/enrollment_event.json
+- examples/payloads/json/sample_set_01/ingestion_event.json
+- examples/payloads/json/sample_set_01/transaction_event.json
+- examples/payloads/json/sample_set_01/user_event.json
 - karafka.gemspec
 - lib/active_job/karafka.rb
 - lib/active_job/queue_adapters/karafka_adapter.rb
@@ -198,6 +182,7 @@ files:
 - lib/karafka/cli/topics/base.rb
 - lib/karafka/cli/topics/create.rb
 - lib/karafka/cli/topics/delete.rb
+- lib/karafka/cli/topics/help.rb
 - lib/karafka/cli/topics/migrate.rb
 - lib/karafka/cli/topics/plan.rb
 - lib/karafka/cli/topics/repartition.rb
@@ -275,6 +260,10 @@ files:
 - lib/karafka/pro/cleaner/messages/message.rb
 - lib/karafka/pro/cleaner/messages/messages.rb
 - lib/karafka/pro/cleaner/messages/metadata.rb
+- lib/karafka/pro/cli/parallel_segments.rb
+- lib/karafka/pro/cli/parallel_segments/base.rb
+- lib/karafka/pro/cli/parallel_segments/collapse.rb
+- lib/karafka/pro/cli/parallel_segments/distribute.rb
 - lib/karafka/pro/connection/manager.rb
 - lib/karafka/pro/connection/multiplexing/listener.rb
 - lib/karafka/pro/contracts/base.rb
@@ -316,6 +305,9 @@ files:
 - lib/karafka/pro/processing/offset_metadata/consumer.rb
 - lib/karafka/pro/processing/offset_metadata/fetcher.rb
 - lib/karafka/pro/processing/offset_metadata/listener.rb
+- lib/karafka/pro/processing/parallel_segments/filters/base.rb
+- lib/karafka/pro/processing/parallel_segments/filters/default.rb
+- lib/karafka/pro/processing/parallel_segments/filters/mom.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/periodic_job/consumer.rb
 - lib/karafka/pro/processing/piping/consumer.rb
@@ -373,6 +365,9 @@ files:
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
 - lib/karafka/pro/processing/subscription_groups_coordinator.rb
+- lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb
+- lib/karafka/pro/processing/virtual_partitions/distributors/base.rb
+- lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb
 - lib/karafka/pro/recurring_tasks.rb
 - lib/karafka/pro/recurring_tasks/consumer.rb
 - lib/karafka/pro/recurring_tasks/contracts/config.rb
@@ -436,6 +431,12 @@ files:
 - lib/karafka/pro/routing/features/offset_metadata/config.rb
 - lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb
 - lib/karafka/pro/routing/features/offset_metadata/topic.rb
+- lib/karafka/pro/routing/features/parallel_segments.rb
+- lib/karafka/pro/routing/features/parallel_segments/builder.rb
+- lib/karafka/pro/routing/features/parallel_segments/config.rb
+- lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb
+- lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb
+- lib/karafka/pro/routing/features/parallel_segments/topic.rb
 - lib/karafka/pro/routing/features/patterns.rb
 - lib/karafka/pro/routing/features/patterns/builder.rb
 - lib/karafka/pro/routing/features/patterns/config.rb
@@ -619,7 +620,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.
 test_files: []

data/certs/cert.pem

@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
-YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
-dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
-PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
-ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
-Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
-JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
-T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
-flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
-vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
-iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
-sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
-eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
-Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
-agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
-TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
-Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
-oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
-i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
-ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
------END CERTIFICATE-----