karafka 2.4.18 → 2.5.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66dcb47a4936205b3d71838001db5c64e6fab4da332f54fcb8e64521a081c871
-  data.tar.gz: 5f113ec8bc1459a5994acf8935c802b893695fd69002937b4a91b5697423cb81
+  metadata.gz: 4bcac2dd9a093cd85ab76342d9c227d552e9927e681e79f55963a9d13e64f79d
+  data.tar.gz: 8b5e32c1c1099b8e654599c6df25c33816c3a4a315ffe7e0de06c49a2b4e720e
 SHA512:
-  metadata.gz: 1abbfec5c703e755f012d9171934e7486ab0fbfd1295c0fd2b81216156783156415aaa2d547215aa9ec8a62f54db53b269181ed6638644c66529456f897a708a
-  data.tar.gz: 677b50678298d384ea4862fc703327a9e351d1f2bacd82f027f4f43fd60cceaac81a93cec133c005e7f90f917162301ff5725bfe5f0ba6ed1754ce2fa0503c66
+  metadata.gz: d2ffa6709d42103eea487e44b6756d9b449bd762b162e8ce35283b2513b26c2fde8ef559b900c7932d1fe4e3e8c771d9874d1ed54a572e3e94f6df067eb354d8
+  data.tar.gz: 9bfafb2b2c8ec8975e55fe1bddeaaedfb4d7fd1cedae36abdfed45819e8136eb614922d4f52c0b011d97602ad728dc7a860a3025e9ab1694720b0f35af00429d

data/.github/CODEOWNERS

@@ -0,0 +1,3 @@
+/.github @mensfeld
+/.github/workflows/ @mensfeld
+/.github/actions/ @mensfeld

data/.github/workflows/ci.yml

@@ -6,10 +6,15 @@ concurrency:
 
 on:
   pull_request:
+    branches: [ main, master ]
   push:
+    branches: [ main, master ]
   schedule:
     - cron:  '0 1 * * *'
 
+permissions:
+  contents: read
+
 env:
   BUNDLE_RETRY: 6
   BUNDLE_JOBS: 4
@@ -17,15 +22,16 @@ env:
 jobs:
   diffend:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
         with:
           ruby-version: 3.4
           bundler-cache: true
@@ -38,10 +44,11 @@ jobs:
 
   karafka-checksum:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
       - name: Run Karafka license checksum verification
@@ -54,19 +61,33 @@ jobs:
 
   coditsu:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
+      - name: Download Coditsu script
+        run: |
+          curl -sSL https://api.coditsu.io/run/ci -o coditsu_script.sh
+          chmod +x coditsu_script.sh
+      - name: Verify Coditsu script checksum
+        run: |
+          EXPECTED_SHA256="0aecc5aa010f53fca264548a41467a2b0a1208d750ce1da3e98a217304cacbbc"
+
+          ACTUAL_SHA256=$(sha256sum coditsu_script.sh | awk '{ print $1 }')
+          if [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then
+            echo "::error::Checksum verification failed. Expected $EXPECTED_SHA256 but got $ACTUAL_SHA256."
+            exit 1
+          fi
       - name: Run Coditsu
-        run: \curl -sSL https://api.coditsu.io/run/ci | bash
+        run: ./coditsu_script.sh
 
   # We do not split RSpec specs to OSS and Pro like integrations because they do not overload
   # Kafka heavily, compute total coverage for specs and are fast enough
   specs:
-    timeout-minutes: 10
+    timeout-minutes: 15
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
@@ -81,7 +102,7 @@ jobs:
           - ruby: '3.4'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
@@ -97,7 +118,7 @@ jobs:
         run: rm -f Gemfile.lock
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
@@ -112,8 +133,14 @@ jobs:
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bin/rspecs
 
+      - name: Check Kafka logs for unexpected warnings
+        run: bin/verify_kafka_warnings
+
+      - name: Check test topics naming convention
+        run: bin/verify_topics_naming
+
   integrations_oss:
-    timeout-minutes: 20
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
@@ -125,7 +152,7 @@ jobs:
           - '3.2'
           - '3.1'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
@@ -137,7 +164,7 @@ jobs:
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
         with:
           # Do not use cache here as we run bundle install also later in some of the integration
           # tests and we need to be able to run it without cache
@@ -170,6 +197,12 @@ jobs:
       - name: Run OSS integration tests
         run: bin/integrations --exclude '/pro'
 
+      - name: Check Kafka logs for unexpected warnings
+        run: bin/verify_kafka_warnings
+
+      - name: Check test topics naming convention
+        run: bin/verify_topics_naming
+
   integrations_pro:
     timeout-minutes: 45
     runs-on: ubuntu-latest
@@ -182,8 +215,11 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
+        parallel_group:
+          - '0'
+          - '1'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
@@ -192,7 +228,7 @@ jobs:
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler: 'latest'
@@ -220,5 +256,13 @@ jobs:
           KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
           KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
           KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
+          SPECS_SEED: ${{ github.run_id }}
+          SPECS_GROUP: ${{ matrix.parallel_group }}
+        run: |
+          bin/integrations '/pro'
+
+      - name: Check Kafka logs for unexpected warnings
+        run: bin/verify_kafka_warnings
 
-        run: bin/integrations '/pro'
+      - name: Check test topics naming convention
+        run: bin/verify_topics_naming

data/.github/workflows/verify-action-pins.yml

@@ -0,0 +1,16 @@
+name: Verify Action Pins
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+jobs:
+  verify_action_pins:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Check SHA pins
+        run: |
+          if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
+            echo "::error::Actions should use SHA pins, not tags or branch names"
+            exit 1
+          fi

data/.ruby-version

@@ -1 +1 @@
-3.4.1
+3.4.4

data/CHANGELOG.md

@@ -1,5 +1,58 @@
 # Karafka Framework Changelog
 
+## 2.4.19 (Unreleased)
+- **[Breaking]** Use DLQ and Piping prefix `source_` instead of `original_` to align with naming convention of Kafka Streams and Apache Flink for future usage.
+- **[Breaking]** Rename scheduled jobs topics names in their config (Pro).
+- **[Feature]** Parallel Segments for concurrent processing of the same partition with more than partition count of processes (Pro).
+- [Enhancement] Support KIP-82 (header values of arrays).
+- [Enhancement] Enhance errors tracker with `#counts` that contains per-error class specific counters for granular flow handling.
+- [Enhancement] Provide explicit `Karafka::Admin.copy_consumer_group` API.
+- [Enhancement] Return explicit value from `Karafka::Admin.copy_consumer_group` and `Karafka::Admin.rename_consumer_group` APIs.
+- [Enhancement] Introduce balanced non-consistent VP distributor improving the utilization up to 50% (Pro).
+- [Enhancement] Make the error tracker for advanced DLQ strategies respond to `#topic` and `#partition` for context aware dispatches.
+- [Enhancement] Allow setting the workers thread priority and set it to -1 (50ms) by default.
+- [Enhancement] Enhance low-level `client.pause` event with timeout value (if provided).
+- [Enhancement] Introduce `#marking_cursor` API (defaults to `#cursor`) in the filtering API (Pro).
+- [Enhancement] Support multiple DLQ target topics via context aware strategies (Pro).
+- [Enhancement] Raise error when post-transactional committing of offset is done outside of the transaction (Pro).
+- [Enhancement] Include info level rebalance logger listener data.
+- [Enhancement] Include info level subscription start info.
+- [Enhancement] Make the generic error handling in the `LoggerListener` more descriptive by logging also the error class.
+- [Enhancement] Allow marking older offsets to support advanced rewind capabilities.
+- [Enhancement] Change optional `#seek` reset offset flag default to `true` as `false` is almost never used and seek by default should move the internal consumer offset position as well.
+- [Enhancement] Include Swarm node ID in the swarm process tags.
+- [Enhancement] Replace internal usage of MD5 with SHA256 for FIPS.
+- [Enhancement] Improve OSS vs. Pro specs execution isolation.
+- [Enhancement] Preload `librdkafka` code prior to forking in the Swarm mode to save memory.
+- [Enhancement] Extract errors tracker class reference into an internal `errors_tracker_class` config option (Pro).
+- [Enhancement] Support rdkafka native kafka polling customization for admin.
+- [Enhancement] Customize the multiplexing scale delay (Pro) per consumer group (Pro).
+- [Enhancement] Set `topic.metadata.refresh.interval.ms` for default producer in dev to 5s to align with consumer setup.
+- [Enhancement] Alias `-2` and `-1` with `latest` and `earliest` for seeking.
+- [Enhancement] Allow for usage of `latest` and `earliest` in the `Karafka::Pro::Iterator`.
+- [Refactor] Introduce a `bin/verify_kafka_warnings` script to clean Kafka from temporary test-suite topics.
+- [Refactor] Introduce a `bin/verify_topics_naming` script to ensure proper test topics naming convention.
+- [Refactor] Make sure all temporary topics have a `it-` prefix in their name.
+- [Refactor] Improve CI specs parallelization.
+- [Maintenance] Lower the `Karafka::Admin` `poll_timeout` to 50 ms to improve responsiveness of admin operations.
+- [Maintenance] Require `karafka-rdkafka` `>=` `0.19.2` due to usage of `#rd_kafka_global_init`, KIP-82 and the new producer caching engine.
+- [Maintenance] Add Deimos routing patch into integration suite not to break it in the future.
+- [Maintenance] Remove Rails `7.0` specs due to upcoming EOL.
+- [Fix] Fix Recurring Tasks and Scheduled Messages not working with Swarm (using closed producer).
+- [Fix] Fix a case where `unknown_topic_or_part` error could leak out of the consumer on consumer shutdown.
+- [Fix] Fix missing `virtual_partitions.partitioner.error` custom error logging in the `LoggerListener`.
+- [Fix] Prevent applied system filters `#timeout` from potentially interacting with user filters.
+- [Fix] Use more sane value in `Admin#seek_consumer_group` for long ago.
+- [Fix] Prevent multiplexing of 1:1 from routing.
+- [Fix] WaterDrop level aborting transaction may cause seek offset to move (Pro).
+- [Fix] Fix inconsistency in the logs where `Karafka::Server` originating logs would not have server id reference.
+- [Fix] Fix inconsistency in the logs where OS signal originating logs would not have server id reference.
+- [Fix] Post-fork WaterDrop instance looses some of the non-kafka settings.
+- [Fix] Max epoch tracking for early cleanup causes messages to be skipped until reload.
+- [Fix] optparse double parse loses ARGV.
+- [Fix] `karafka` cannot be required without Bundler.
+- [Fix] Scheduled Messages re-seek moves to `latest` on inheritance of initial offset when `0` offset is compacted.
+
 ## 2.4.18 (2025-04-09)
 - [Fix] Make sure `Bundler.with_unbundled_env` is not called multiple times.
 

data/Gemfile

@@ -16,9 +16,9 @@ group :integrations, :test do
 end
 
 group :integrations do
-  gem 'activejob', require: false
-  gem 'karafka-testing', '>= 2.4.6', require: false
-  gem 'karafka-web', '>= 0.10.0.rc2', require: false
+  # gem 'activejob', require: false
+  # gem 'karafka-testing', '>= 2.4.6', require: false
+  # gem 'karafka-web', '>= 0.10.4', require: false
 end
 
 group :test do

data/Gemfile.lock

@@ -1,20 +1,17 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.18)
+    karafka (2.5.0.beta1)
       base64 (~> 0.2)
-      karafka-core (>= 2.4.4, < 2.5.0)
-      karafka-rdkafka (>= 0.17.2)
-      waterdrop (>= 2.7.3, < 3.0.0)
+      karafka-core (>= 2.5.0, < 2.6.0)
+      karafka-rdkafka (>= 0.19.2)
+      waterdrop (>= 2.8.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (8.0.1)
-      activesupport (= 8.0.1)
-      globalid (>= 0.3.6)
-    activesupport (8.0.1)
+    activesupport (8.0.2)
       base64
       benchmark (>= 0.3)
       bigdecimal
@@ -28,99 +25,99 @@ GEM
       tzinfo (~> 2.0, >= 2.0.5)
       uri (>= 0.13.1)
     base64 (0.2.0)
-    benchmark (0.3.0)
-    bigdecimal (3.1.8)
-    byebug (11.1.3)
-    concurrent-ruby (1.3.4)
-    connection_pool (2.4.1)
-    diff-lcs (1.5.1)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
+    byebug (12.0.0)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
+    diff-lcs (1.6.2)
     docile (1.4.1)
-    drb (2.2.1)
-    erubi (1.13.0)
+    drb (2.2.3)
     et-orbi (1.2.11)
       tzinfo
-    factory_bot (6.5.0)
-      activesupport (>= 5.0.0)
-    ffi (1.17.0)
+    factory_bot (6.5.1)
+      activesupport (>= 6.1.0)
+    ffi (1.17.2)
+    ffi (1.17.2-aarch64-linux-gnu)
+    ffi (1.17.2-aarch64-linux-musl)
+    ffi (1.17.2-arm-linux-gnu)
+    ffi (1.17.2-arm-linux-musl)
+    ffi (1.17.2-arm64-darwin)
+    ffi (1.17.2-x86-linux-gnu)
+    ffi (1.17.2-x86-linux-musl)
+    ffi (1.17.2-x86_64-darwin)
+    ffi (1.17.2-x86_64-linux-gnu)
+    ffi (1.17.2-x86_64-linux-musl)
     fugit (1.11.1)
       et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
-    globalid (1.2.1)
-      activesupport (>= 6.1)
-    i18n (1.14.6)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.8)
-      karafka-rdkafka (>= 0.17.6, < 0.19.0)
+    karafka-core (2.5.0)
+      karafka-rdkafka (>= 0.19.2, < 0.21.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.18.1)
+    karafka-rdkafka (0.19.2)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.6)
-      karafka (>= 2.4.0, < 2.5.0)
-      waterdrop (>= 2.7.0)
-    karafka-web (0.10.4)
-      erubi (~> 1.4)
-      karafka (>= 2.4.10, < 2.5.0)
-      karafka-core (>= 2.4.0, < 2.5.0)
-      roda (~> 3.68, >= 3.69)
-      tilt (~> 2.0)
-    logger (1.6.3)
-    mini_portile2 (2.8.8)
-    minitest (5.25.4)
+    logger (1.7.0)
+    mini_portile2 (2.8.9)
+    minitest (5.25.5)
     ostruct (0.6.1)
     raabro (1.4.0)
-    rack (3.1.8)
     rake (13.2.1)
-    roda (3.84.0)
-      rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.0)
+    rspec-core (3.13.3)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.1)
+    rspec-expectations (3.13.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.1)
+    rspec-mocks (3.13.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.1)
-    securerandom (0.3.2)
+    rspec-support (3.13.3)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
-    stringio (3.1.2)
-    tilt (2.4.0)
+    stringio (3.1.7)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    uri (1.0.2)
-    waterdrop (2.8.1)
-      karafka-core (>= 2.4.3, < 3.0.0)
-      karafka-rdkafka (>= 0.17.5)
+    uri (1.0.3)
+    waterdrop (2.8.3)
+      karafka-core (>= 2.4.9, < 3.0.0)
+      karafka-rdkafka (>= 0.19.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.7.1)
+    zeitwerk (2.7.3)
 
 PLATFORMS
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
   ruby
-  x86_64-linux
+  x86-linux-gnu
+  x86-linux-musl
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
 
 DEPENDENCIES
-  activejob
   byebug
   factory_bot
   fugit
   karafka!
-  karafka-testing (>= 2.4.6)
-  karafka-web (>= 0.10.0.rc2)
   ostruct
   rspec
   simplecov
   stringio
 
 BUNDLED WITH
-   2.4.22
+   2.6.9

data/LICENSE-COMM

@@ -6,7 +6,7 @@ IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMEN
 
 ------------------------------------------------------------------------------
 
-In order to use the Software under this Agreement, you must receive a "Source URL" to a license package at the time of purchase, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement.
+In order to use the Software under this Agreement, you must either: (a) receive a "Source URL" to a license package at the time of purchase, or (b) for Enterprise customers only, be provided with a fully offline license, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement.
 
 1. License Grant
 
@@ -22,7 +22,7 @@ In order to use the Software under this Agreement, you must receive a "Source UR
 
 3. Restricted Uses.
 
-3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Maciej Mensfeld; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Maciej Mensfeld in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Maciej Mensfeld; (j) use the Software to develop a product which is competitive with any Maciej Mensfeld product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Maciej Mensfeld in writing. If your unique Source URL is ever published, Maciej Mensfeld reserves the right to terminate your access without notice.
+3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Maciej Mensfeld; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Maciej Mensfeld in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Maciej Mensfeld; (j) use the Software to develop a product which is competitive with any Maciej Mensfeld product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Maciej Mensfeld in writing. If your unique Source URL or the offline license is ever published, Maciej Mensfeld reserves the right to terminate your access without notice.
 
 3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
 

data/bin/clean_kafka

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+# A script that removes most of the auto-generated Kafka topics with their data
+# Useful when having long-running Kafka instance that cannot be fully nuked after running specs
+
+# We use the same convention in other framework components (web, waterdrop), so it removes all of
+# them as well.
+
+require_relative '../spec/integrations_helper.rb'
+
+setup_karafka
+
+topics_for_removal = []
+
+Karafka::Admin.cluster_info.topics.each do |topic|
+  topic_name = topic[:topic_name]
+
+  next unless topic_name.start_with?('it-')
+
+  topics_for_removal << topic_name
+end
+
+THREADS_COUNT = 3
+QUEUE = SizedQueue.new(THREADS_COUNT)
+TOPICS_TO_REMOVAL_COUNT = topics_for_removal.size
+
+threads = Array.new(THREADS_COUNT) do
+  Thread.new do
+    while topic_name = QUEUE.pop
+      puts "Removing topic: #{topic_name} (#{topics_for_removal.count} left)"
+      Karafka::Admin.delete_topic(topic_name)
+    end
+  end
+end
+
+while topics_for_removal.size.positive?
+  topic_name = topics_for_removal.pop
+
+  QUEUE << topic_name
+end
+
+QUEUE.close
+threads.each(&:join)

data/bin/integrations

@@ -53,15 +53,19 @@ class Scenario
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES
 
+  attr_reader :index
+
   # Creates scenario instance and runs in the background process
   #
   # @param path [String] path to the scenarios file
-  def initialize(path)
+  def initialize(path, index)
     @path = path
     # First 1024 characters from stdout
     @stdout_head = ''
     # Last 1024 characters from stdout
     @stdout_tail = ''
+    # Assigns the index for parallel execution in the CI if requested
+    @index = index
   end
 
   # Starts running given scenario in a separate process
@@ -252,16 +256,24 @@ specs.delete_if do |spec|
   false
 end
 
-raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
-
 # Randomize order
-seed = (ENV['SEED'] || rand(0..10_000)).to_i
+seed = (ENV['SPECS_SEED'] || rand(0..10_000)).to_i
+group = (ENV['SPECS_GROUP'] || -1).to_i
+groups = (ENV['SPECS_GROUPS'] || 2).to_i
 
 puts "Random seed: #{seed}"
+puts "Group: #{group}"
+puts "Groups: #{groups}"
 
 scenarios = specs
             .shuffle(random: Random.new(seed))
-            .map { |integration_test| Scenario.new(integration_test) }
+            .map
+            .with_index { |integration, index| Scenario.new(integration, index % groups) }
+            .delete_if { |scenario| scenario.index != group && group != -1 }
+
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if scenarios.empty?
+
+puts "Running #{scenarios.size} scenarios"
 
 regulars = scenarios.reject(&:linear?)
 linears = scenarios - regulars

data/bin/rspecs

@@ -3,8 +3,20 @@
 set -e
 
 # Run only regular non-forking specs first
-SPECS_TYPE=regular bundle exec rspec --tag ~type:pro --tag ~mode:fork
+SPECS_TYPE=regular bundle exec rspec \
+  --tag ~type:pro \
+  --tag ~mode:fork \
+  --exclude-pattern "**/pro/**/*_spec.rb" \
+  spec/lib/
+
 # Run forking specs, they need to run in isolation not to crash because of librdkafka
-SPECS_TYPE=regular bundle exec rspec --tag mode:fork
+SPECS_TYPE=regular bundle exec rspec \
+  --tag mode:fork \
+  --exclude-pattern "**/pro/**/*_spec.rb" \
+  spec/lib/
+
 # Run pro specs at the end
-SPECS_TYPE=pro bundle exec rspec --tag type:pro --tag ~mode:fork
+SPECS_TYPE=pro bundle exec rspec \
+  --tag type:pro \
+  --tag ~mode:fork \
+  spec/lib/

data/bin/verify_kafka_warnings

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Checks Kafka logs for unsupported warning patterns
+# Only specified warnings are allowed, all others should trigger failure
+
+allowed_patterns=(
+   "Performing controller activation"
+   "registered with feature metadata.version"
+   "Replayed TopicRecord for"
+   "Replayed PartitionRecord for"
+   "Previous leader None and previous leader epoch"
+   "Creating new"
+)
+
+# Get all warnings
+warnings=$(docker logs --since=0 kafka | grep WARN)
+exit_code=0
+
+while IFS= read -r line; do
+   allowed=0
+   for pattern in "${allowed_patterns[@]}"; do
+       if echo "$line" | grep -q "$pattern"; then
+           allowed=1
+           break
+       fi
+   done
+
+   if [ $allowed -eq 0 ]; then
+       echo "Unexpected warning found:"
+       echo "$line"
+       exit_code=1
+   fi
+done <<< "$warnings"
+
+exit $exit_code