karafka 2.5.3 → 2.5.4.rc1

data/lib/karafka/pro/processing/jobs_queue.rb

@@ -101,10 +101,8 @@ module Karafka
         end
 
         # Allows for explicit unlocking of locked queue of a group
-        #
         # @param group_id [String] id of the group we want to unlock
         # @param lock_id [Object] unique id we want to use to identify our lock
-        #
         def unlock_async(group_id, lock_id)
           @mutex.synchronize do
             if @locks[group_id].delete(lock_id)

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -81,7 +81,7 @@ module Karafka
             # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
             #   information if this message was from marked offset or figured out via mom flow
             def find_skippable_message
-              skippable_message = messages.find do |msg|
+              skippable_message = messages.raw.find do |msg|
                 coordinator.marked? && msg.offset == seek_offset
               end
 

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -156,7 +156,7 @@ module Karafka
               return if collapsed?
 
               coordinator.virtual_offset_manager.register(
-                messages.map(&:offset)
+                messages.raw.map(&:offset)
               )
             end
           end

data/lib/karafka/processing/strategies/dlq.rb

@@ -94,7 +94,7 @@ module Karafka
         # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
         #   information if this message was from marked offset or figured out via mom flow
         def find_skippable_message
-          skippable_message = messages.find do |msg|
+          skippable_message = messages.raw.find do |msg|
             coordinator.marked? && msg.offset == seek_offset
           end
 

data/lib/karafka/routing/consumer_group.rb

@@ -31,6 +31,9 @@ module Karafka
         # Initialize the subscription group so there's always a value for it, since even if not
         # defined directly, a subscription group will be created
         @current_subscription_group_details = { name: SubscriptionGroup.id }
+        # Track the base position for subscription groups to ensure stable positions when
+        # rebuilding. This is critical for static group membership in swarm mode
+        @subscription_groups_base_position = nil
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -42,6 +45,10 @@ module Karafka
       # @param name [String, Symbol] name of topic to which we want to subscribe
       # @return [Karafka::Routing::Topic] newly built topic instance
       def topic=(name, &)
+        # Clear memoized subscription groups since adding a topic requires rebuilding them
+        # This is critical for consumer group reopening across multiple draw calls
+        @subscription_groups = nil
+
         topic = Topic.new(name, self)
         @topics << Proxy.new(
           topic,
@@ -73,7 +80,18 @@ module Karafka
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        @subscription_groups ||= subscription_groups_builder.call(topics)
+        @subscription_groups ||= begin
+          result = subscription_groups_builder.call(
+            topics,
+            base_position: @subscription_groups_base_position
+          )
+
+          # Store the base position from the first subscription group for future rebuilds.
+          # This ensures stable positions for static group membership.
+          @subscription_groups_base_position ||= result.first&.position
+
+          result
+        end
       end
 
       # Hashed version of consumer group that can be used for validation purposes

data/lib/karafka/routing/subscription_group.rb

@@ -14,7 +14,7 @@ module Karafka
         node: %i[swarm node]
       )
 
-      attr_reader :id, :name, :topics, :kafka, :consumer_group
+      attr_reader :id, :name, :topics, :kafka, :consumer_group, :position
 
       # Lock for generating new ids safely
       ID_MUTEX = Mutex.new

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -31,16 +31,31 @@ module Karafka
 
       # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
       #   subscription groups
+      # @param base_position [Integer, nil] optional starting position for subscription groups.
+      #   When provided, positions will start from this value instead of continuing from the
+      #   global counter. This is used when rebuilding subscription groups to maintain stable
+      #   positions for static group membership.
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
-      def call(topics)
+      def call(topics, base_position: nil)
+        # If base_position is provided, use it for stable rebuilding (consumer group reopening).
+        # Otherwise continue from global counter for new subscription groups.
+        # We subtract 1 from base_position because position is incremented in the map block below.
+        # This ensures the first subscription group gets the correct base_position value.
+        use_base = !base_position.nil?
+        position = use_base ? base_position - 1 : @position
+
         topics
           .map { |topic| [checksum(topic), topic] }
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
           .flat_map { |value| expand(value) }
-          .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
+          .map { |grouped_topics| SubscriptionGroup.new(position += 1, grouped_topics) }
           .tap do |subscription_groups|
+            # Always ensure global counter is at least as high as the highest position used.
+            # This prevents position collisions when new consumer groups are created after
+            # existing ones are rebuilt with base_position.
+            @position = position if position > @position
             subscription_groups.each do |subscription_group|
               subscription_group.topics.each do |topic|
                 topic.subscription_group = subscription_group

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.3'
+  VERSION = '2.5.4.rc1'
 end

data/lib/karafka.rb

@@ -11,7 +11,6 @@ require 'fileutils'
 require 'openssl'
 require 'optparse'
 require 'socket'
-require 'base64'
 require 'date'
 require 'singleton'
 require 'digest'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.3
+  version: 2.5.4.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -9,20 +9,6 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: base64
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: karafka-core
   requirement: !ruby/object:Gem::Requirement
@@ -103,60 +89,15 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".coditsu/ci.yml"
-- ".console_irbrc"
-- ".github/CODEOWNERS"
-- ".github/FUNDING.yml"
-- ".github/ISSUE_TEMPLATE/bug_report.md"
-- ".github/ISSUE_TEMPLATE/feature_request.md"
-- ".github/workflows/ci_linux_ubuntu_x86_64_gnu.yml"
-- ".github/workflows/ci_macos_arm64.yml"
-- ".github/workflows/push.yml"
-- ".github/workflows/trigger-wiki-refresh.yml"
-- ".github/workflows/verify-action-pins.yml"
-- ".gitignore"
-- ".rspec"
-- ".ruby-gemset"
-- ".ruby-version"
-- ".yard-lint.yml"
 - CHANGELOG.md
-- CODE_OF_CONDUCT.md
-- CONTRIBUTING.md
-- Gemfile
-- Gemfile.lock
 - LICENSE
 - LICENSE-COMM
 - LICENSE-LGPL
 - README.md
-- Rakefile
-- SECURITY.md
-- bin/benchmarks
-- bin/clean_kafka
-- bin/create_token
-- bin/integrations
 - bin/karafka
-- bin/record_rss
-- bin/rspecs
-- bin/scenario
-- bin/stress_many
-- bin/stress_one
-- bin/verify_kafka_warnings
-- bin/verify_license_integrity
-- bin/verify_topics_naming
-- bin/wait_for_kafka
 - certs/karafka-pro.pem
 - config/locales/errors.yml
 - config/locales/pro_errors.yml
-- docker-compose.yml
-- examples/payloads/avro/.gitkeep
-- examples/payloads/json/sample_set_01/enrollment_event.json
-- examples/payloads/json/sample_set_01/ingestion_event.json
-- examples/payloads/json/sample_set_01/transaction_event.json
-- examples/payloads/json/sample_set_01/user_event.json
-- examples/payloads/json/sample_set_02/download.json
-- examples/payloads/json/sample_set_03/event_type_1.json
-- examples/payloads/json/sample_set_03/event_type_2.json
-- examples/payloads/json/sample_set_03/event_type_3.json
 - karafka.gemspec
 - lib/active_job/karafka.rb
 - lib/active_job/queue_adapters/karafka_adapter.rb
@@ -176,6 +117,8 @@ files:
 - lib/karafka/admin/configs/config.rb
 - lib/karafka/admin/configs/resource.rb
 - lib/karafka/admin/consumer_groups.rb
+- lib/karafka/admin/contracts/replication.rb
+- lib/karafka/admin/replication.rb
 - lib/karafka/admin/topics.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
@@ -604,8 +547,6 @@ files:
 - lib/karafka/time_trackers/pause.rb
 - lib/karafka/time_trackers/poll.rb
 - lib/karafka/version.rb
-- log/.gitkeep
-- renovate.json
 homepage: https://karafka.io
 licenses:
 - LGPL-3.0-only

data/.coditsu/ci.yml

@@ -1,3 +0,0 @@
-repository_id: 'd4482d42-f6b5-44ba-a5e4-00989ac519ee'
-api_key: <%= ENV['CODITSU_API_KEY'] %>
-api_secret: <%= ENV['CODITSU_API_SECRET'] %>

data/.console_irbrc

@@ -1,11 +0,0 @@
-# irbrc for Karafka console
-
-IRB.conf[:AUTO_INDENT] = true
-IRB.conf[:SAVE_HISTORY] = 1000
-IRB.conf[:USE_READLINE] = true
-IRB.conf[:HISTORY_FILE] = ".irb-history"
-IRB.conf[:LOAD_MODULES] = [] unless IRB.conf.key?(:LOAD_MODULES)
-
-unless IRB.conf[:LOAD_MODULES].include?('irb/completion')
-  IRB.conf[:LOAD_MODULES] << 'irb/completion'
-end

data/.github/CODEOWNERS

@@ -1,3 +0,0 @@
-/.github @mensfeld
-/.github/workflows/ @mensfeld
-/.github/actions/ @mensfeld

data/.github/FUNDING.yml

@@ -1 +0,0 @@
-custom: ['https://karafka.io/#become-pro']

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,43 +0,0 @@
----
-name: Bug Report
-about: Report an issue within the Karafka ecosystem you've discovered.
----
-
-To make this process smoother for everyone involved, please read the following information before filling out the template.
-
-Scope of the OSS Support
-===========
-
-We do not provide OSS support for outdated versions of Karafka and its components.
-
-Please ensure that you are using a version that is still actively supported. We cannot assist with any no longer maintained versions unless you support us with our Pro offering (https://karafka.io/docs/Pro-Support/).
-
-We acknowledge that understanding the specifics of your application and its configuration can be essential for resolving certain issues. However, due to the extensive time and resources such analysis can require, this may fall beyond our Open Source Support scope.
-
-If Karafka or its components are critical to your infrastructure, we encourage you to consider our Pro Offering.
-
-By backing us up, you can gain direct assistance and ensure your use case receives the dedicated attention it deserves.
-
-
-Important Links to Read
-===========
-
-Please take a moment to review the following resources before submitting your report:
-
-- Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
-- Support Policy: https://karafka.io/docs/Support/
-- Versions, Lifecycle, and EOL: https://karafka.io/docs/Versions-Lifecycle-and-EOL/
-
-
-Bug Report Details
-===========
-
-Please provide all the details per our Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
-
-Failing to provide the required details may result in the issue being closed. Please include all necessary information to help us understand and resolve your issue effectively.
-
-
-Additional Context
-===========
-
-Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature Request
-about: Suggest new Karafka features or improvements to existing features.
----
-
-## Is your feature request related to a problem? Please describe.
-
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-## Describe the solution you'd like
-
-A clear and concise description of what you want to happen.
-
-## Describe alternatives you've considered
-
-A clear and concise description of any alternative solutions or features you've considered.
-
-## Additional context
-
-Add any other context or screenshots about the feature request here.

data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml

@@ -1,296 +0,0 @@
-name: CI Linux x86_64 GNU
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-on:
-  pull_request:
-    branches: [ master ]
-  schedule:
-    - cron:  '0 1 * * *'
-
-permissions:
-  contents: read
-
-env:
-  BUNDLE_RETRY: 6
-  BUNDLE_JOBS: 4
-
-jobs:
-  karafka-checksum:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-        with:
-          fetch-depth: 0
-      - name: Run Karafka license checksum verification
-        env:
-          KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
-          KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
-          KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
-          KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
-        run: bin/verify_license_integrity
-
-  coditsu:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-        with:
-          fetch-depth: 0
-      - name: Download Coditsu script
-        run: |
-          curl -sSL https://api.coditsu.io/run/ci -o coditsu_script.sh
-          chmod +x coditsu_script.sh
-      - name: Verify Coditsu script checksum
-        run: |
-          EXPECTED_SHA256="0aecc5aa010f53fca264548a41467a2b0a1208d750ce1da3e98a217304cacbbc"
-
-          ACTUAL_SHA256=$(sha256sum coditsu_script.sh | awk '{ print $1 }')
-          if [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then
-            echo "::error::Checksum verification failed. Expected $EXPECTED_SHA256 but got $ACTUAL_SHA256."
-            exit 1
-          fi
-      - name: Run Coditsu
-        run: ./coditsu_script.sh
-
-  yard-lint:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-        with:
-          fetch-depth: 0
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
-        with:
-          ruby-version: '3.4.7'
-          bundler-cache: true
-      - name: Run yard-lint
-        run: bundle exec yard-lint lib/
-
-  # We do not split RSpec specs to OSS and Pro like integrations because they do not overload
-  # Kafka heavily, compute total coverage for specs and are fast enough
-  specs:
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
-    env:
-      BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby:
-          - '3.5.0-preview1'
-          - '3.4'
-          - '3.3'
-          - '3.2'
-        force_ruby_platform:
-          - true
-          - false
-        include:
-          - ruby: '3.4'
-            coverage: 'true'
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - name: Install package dependencies
-        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
-
-      - name: Start Kafka with docker compose
-        run: |
-          docker compose up -d || (sleep 5 && docker compose up -d)
-
-      - name: Remove Gemfile.lock for Ruby dev/preview versions
-        if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
-        run: rm -f Gemfile.lock
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
-        with:
-          ruby-version: ${{matrix.ruby}}
-          bundler-cache: true
-          bundler: 'latest'
-
-      - name: Wait for Kafka
-        run: |
-          bundle exec bin/wait_for_kafka
-
-      - name: Run all specs
-        env:
-          GITHUB_COVERAGE: ${{matrix.coverage}}
-        run: bin/rspecs
-
-      - name: Check Kafka logs for unexpected warnings
-        run: bin/verify_kafka_warnings
-
-      - name: Check test topics naming convention
-        run: bundle exec bin/verify_topics_naming
-
-  integrations_oss:
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    env:
-      BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby:
-          - '3.5.0-preview1'
-          - '3.4'
-          - '3.3'
-          - '3.2'
-        force_ruby_platform:
-          - true
-          - false
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - name: Install package dependencies
-        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
-
-      - name: Remove libzstd-dev to check no supported compressions
-        run: sudo apt-get -y remove libzstd-dev
-
-      - name: Start Kafka with docker compose
-        run: |
-          docker compose up -d || (sleep 5 && docker compose up -d)
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
-        with:
-          # Do not use cache here as we run bundle install also later in some of the integration
-          # tests and we need to be able to run it without cache
-          #
-          # We also want to check that librdkafka is compiling as expected on all versions of Ruby
-          ruby-version: ${{matrix.ruby}}
-          bundler: 'latest'
-
-      - name: Install latest Bundler
-        run: |
-          gem install bundler --no-document
-          gem update --system
-          bundle config set without 'tools benchmarks docs'
-
-      - name: Fix directory permissions for Bundler
-        run: |
-          chmod -R o-w /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
-          chmod +t /opt/hostedtoolcache/Ruby/3*/x64/lib/ruby/gems/3*/gems
-
-      - name: Force Ruby platform for ffi gem on dev/preview Ruby
-        if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
-        run: bundle config set force_ruby_platform ffi
-
-      - name: Bundle install
-        run: |
-          bundle config set without development
-          bundle install --jobs 4 --retry 3
-
-      - name: Wait for Kafka
-        run: |
-          bundle exec bin/wait_for_kafka
-
-      - name: Run OSS integration tests
-        run: bin/integrations --exclude '/pro'
-
-      - name: Check Kafka logs for unexpected warnings
-        run: bin/verify_kafka_warnings
-
-      - name: Check test topics naming convention
-        run: bundle exec bin/verify_topics_naming
-
-  integrations_pro:
-    timeout-minutes: 45
-    runs-on: ubuntu-latest
-    env:
-      BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby:
-          - '3.5.0-preview1'
-          - '3.4'
-          - '3.3'
-          - '3.2'
-        force_ruby_platform:
-          - true
-          - false
-        parallel_group:
-          - '0'
-          - '1'
-    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - name: Install package dependencies
-        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
-
-      - name: Start Kafka with docker compose
-        run: |
-          docker compose up -d || (sleep 5 && docker compose up -d)
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
-        with:
-          ruby-version: ${{matrix.ruby}}
-          bundler: 'latest'
-
-      - name: Install latest Bundler
-        run: |
-          gem install bundler --no-document
-          gem update --system
-          bundle config set without 'tools benchmarks docs'
-
-      - name: Force Ruby platform for ffi gem on dev/preview Ruby
-        if: contains(matrix.ruby, 'dev') || contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
-        run: bundle config set force_ruby_platform ffi
-
-      - name: Bundle install
-        run: |
-          bundle config set without development
-          bundle install --jobs 4 --retry 3
-
-      - name: Wait for Kafka
-        run: |
-          bundle exec bin/wait_for_kafka
-
-      - name: Run Pro integration tests
-        env:
-          KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
-          KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
-          KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
-          KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
-          KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
-          SPECS_SEED: ${{ github.run_id }}
-          SPECS_GROUP: ${{ matrix.parallel_group }}
-        run: |
-          bin/integrations '/pro'
-
-      - name: Check Kafka logs for unexpected warnings
-        run: bin/verify_kafka_warnings
-
-      - name: Check test topics naming convention
-        run: bin/verify_topics_naming
-
-  ci-success:
-    name: CI Linux Ubuntu x86_64 GNU Success
-    runs-on: ubuntu-latest
-    if: always()
-    needs:
-      - karafka-checksum
-      - coditsu
-      - yard-lint
-      - specs
-      - integrations_oss
-      - integrations_pro
-    steps:
-      - name: Check all jobs passed
-        if: |
-          contains(needs.*.result, 'failure') ||
-          contains(needs.*.result, 'cancelled') ||
-          contains(needs.*.result, 'skipped')
-        run: exit 1
-      - run: echo "All CI checks passed!"