karafka 2.3.2 → 2.4.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb47082224d857f3029f9bb8e1b04a35e6b8ed2f7ae75bbe52bf1b778ff56226
-  data.tar.gz: 53d59fd7e140f5b3e9b89dd3e4af28469bc534074110e2e93fae24c59bf81b88
+  metadata.gz: 7c1493774aae6fa07d816d8e76b9d5577f5483e9d93301dc10a8e7d7a86c7045
+  data.tar.gz: 32251813e9e3b9a9f7b75554a7b0175c2e1cf30456acb3207464a36587073259
 SHA512:
-  metadata.gz: 00e09a345122ad2facaf8adcbb52fae3ce87374083d9e6785a1f07a74c87e53c6b0b3dd32b82566e846d160569aafc55c614013d8c9f95664612150fb51d07b1
-  data.tar.gz: 283e50a6b3b25579419bdc9b947e5ba802e22a1cd6d0097ab8929c5394d3858461bda15c3e70a0d2d466a8705466ef1d5b24e1a8bfe5ac8e356c64790049c7b0
+  metadata.gz: 614cc7507dd865a8efde56e40a77f694efb268c4a877917834a7e23d78175cb9ddb4574ccaac75db86fea1e1de1eae17f4df694251268d90f2822a4868cddf0d
+  data.tar.gz: f6ec801b2e47e531bcd4be2fd6e9e8bc01837b6218ffea1b3e187edc450fa519d2fb5f0fa8a69f5b0e67aabff8496aac1dbbc156bb45c2dd02b1ea495fb5710f

data/.github/workflows/ci.yml

@@ -80,8 +80,6 @@ jobs:
           - '3.1'
           - '3.1.0'
           - '3.0'
-          - '3.0.0'
-          - '2.7'
         include:
           - ruby: '3.3'
             coverage: 'true'
@@ -90,9 +88,9 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Start Kafka with docker-compose
+      - name: Start Kafka with docker compose
         run: |
-          docker-compose up -d || (sleep 5 && docker-compose up -d)
+          docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -122,7 +120,6 @@ jobs:
           - '3.2'
           - '3.1'
           - '3.0'
-          - '2.7'
     steps:
       - uses: actions/checkout@v4
       - name: Install package dependencies
@@ -131,9 +128,9 @@ jobs:
       - name: Remove libzstd-dev to check no supported compressions
         run: sudo apt-get -y remove libzstd-dev
 
-      - name: Start Kafka with docker-compose
+      - name: Start Kafka with docker compose
         run: |
-          docker-compose up -d || (sleep 5 && docker-compose up -d)
+          docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -147,26 +144,15 @@ jobs:
 
       - name: Install latest Bundler
         run: |
-          if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
-            gem install bundler -v 2.4.22 --no-document
-            bundle config set version 2.4.22
-            gem update --system 3.4.22 --no-document
-          else
-            gem install bundler --no-document
-            gem update --system --no-document
-          fi
+          gem install bundler --no-document
+          gem update --system --no-document
 
           bundle config set without 'tools benchmarks docs'
 
       - name: Bundle install
         run: |
           bundle config set without development
-
-          if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
-            BUNDLER_VERSION=2.4.22 bundle install --jobs 4 --retry 3
-          else
-            bundle install --jobs 4 --retry 3
-          fi
+          bundle install --jobs 4 --retry 3
 
       - name: Wait for Kafka
         run: |
@@ -187,15 +173,14 @@ jobs:
           - '3.2'
           - '3.1'
           - '3.0'
-          - '2.7'
     steps:
       - uses: actions/checkout@v4
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Start Kafka with docker-compose
+      - name: Start Kafka with docker compose
         run: |
-          docker-compose up -d || (sleep 5 && docker-compose up -d)
+          docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -205,26 +190,15 @@ jobs:
 
       - name: Install latest Bundler
         run: |
-          if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
-            gem install bundler -v 2.4.22 --no-document
-            bundle config set version 2.4.22
-            gem update --system 3.4.22 --no-document
-          else
-            gem install bundler --no-document
-            gem update --system --no-document
-          fi
+          gem install bundler --no-document
+          gem update --system --no-document
 
           bundle config set without 'tools benchmarks docs'
 
       - name: Bundle install
         run: |
           bundle config set without development
-
-          if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
-            BUNDLER_VERSION=2.4.22 bundle install --jobs 4 --retry 3
-          else
-            bundle install --jobs 4 --retry 3
-          fi
+          bundle install --jobs 4 --retry 3
 
       - name: Wait for Kafka
         run: |

data/CHANGELOG.md

@@ -1,5 +1,70 @@
 # Karafka framework changelog
 
+## 2.4.0 (Unreleased)
+
+This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.
+
+- **[Breaking]** Drop Ruby `2.7` support.
+- **[Breaking]** Drop the concept of consumer group mapping.
+- **[Breaking]** `karafka topics migrate` will now perform declarative topics configuration alignment.
+- **[Breaking]** Replace `deserializer` config with `#deserializers` in routing to support key and lazy header deserializers.
+- **[Breaking]** Rename `Karafka::Serializers::JSON::Deserializer` to `Karafka::Deserializers::Payload` to reflect its role.
+- **[Feature]** Support custom OAuth providers (with a lot of help from bruce-szalwinski-he and hotelengine.com).
+- **[Feature]** Provide `karafka topics alter` for declarative topics alignment.
+- **[Feature]** Introduce ability to use direct assignments (Pro).
+- **[Feature]** Provide consumer piping API (Pro).
+- **[Feature]** Introduce `karafka topics plan` to describe changes that will be applied when migrating.
+- **[Feature]** Introduce ability to use custom message key deserializers.
+- **[Feature]** Introduce ability to use custom message headers deserializers.
+- [Enhancement] Assign names to internal threads for better debuggability when on `TTIN`.
+- [Enhancement] Provide `log_polling` setting to the `Karafka::Instrumentation::LoggerListener` to silence polling in any non-debug mode.
+- [Enhancement] Provide `metadata#message` to be able to retrieve message from metadata.
+- [Enhancement] Include number of attempts prior to DLQ message being dispatched including the dispatch one (Pro).
+- [Enhancement] Provide ability to decide how to dispatch from DLQ (sync / async).
+- [Enhancement] Provide ability to decide how to mark as consumed from DLQ (sync / async).
+- [Enhancement] Allow for usage of a custom Appsignal namespace when logging.
+- [Enhancement] Do not run periodic jobs when LRJ job is running despite polling (LRJ can still start when Periodic runs).
+- [Enhancement] Improve accuracy of periodic jobs and make sure they do not run too early after saturated work.
+- [Enhancement] Introduce ability to async lock other subscription groups polling.
+- [Enhancement] Improve shutdown when using long polling setup (high `max_wait_time`).
+- [Enhancement] Provide `Karafka::Admin#read_lags_with_offsets` for ability to query lags and offsets of a given CG.
+- [Enhancement] Allow direct assignments granular distribution in the Swarm (Pro).
+- [Enhancement] Add a buffer to the supervisor supervision on shutdown to prevent a potential race condition when signal pass lags.
+- [Enhancement] Provide ability to automatically generate and validate fingerprints of encrypted payload.
+- [Enhancement] Support `enable.partition.eof` fast yielding.
+- [Enhancement] Provide `#mark_as_consumed` and `#mark_as_consumed!` to the iterator.
+- [Enhancement] Introduce graceful `#stop` to the iterator instead of recommending of usage of `break`.
+- [Enhancement] Do not run jobs schedulers and other interval based operations on each job queue unlock.
+- [Enhancement] Publish listeners status lifecycle events.
+- [Enhancement] Use proxy wrapper for Admin metadata requests.
+- [Enhancement] Use limited scope topic info data when operating on direct topics instead of full cluster queries.
+- [Enhancement] No longer raise `Karafka::UnsupportedCaseError` for not recognized error types to support dynamic errors reporting.
+- [Change] Do not create new proxy object to Rdkafka with certain low-level operations and re-use existing.
+- [Change] Update `karafka.erb` template with a placeholder for waterdrop and karafka error instrumentation.
+- [Fix] Pro Swarm liveness listener can report incorrect failure when dynamic multiplexing scales down.
+- [Fix] K8s liveness listener can report incorrect failure when dynamic multiplexing scales down.
+- [Fix] Fix a case where connection conductor would not be released during manager state changes.
+- [Fix] Make sure, that all `Admin` operations go through stabilization proxy.
+- [Fix] Fix an issue where coordinator running jobs would not count periodic jobs and revocations.
+- [Fix] Fix a case where critically crashed supervisor would raise incorrect error.
+- [Fix] Re-raise critical supervisor errors before shutdown.
+- [Fix] Fix a case when right-open (infinite) swarm matching would not pass validations.
+- [Fix] Make `#enqueue_all` output compatible with `ActiveJob.perform_all_later` (oozzal)
+
+### Upgrade Notes
+
+**PLEASE MAKE SURE TO READ AND APPLY THEM!**
+
+Available [here](https://karafka.io/docs/Upgrades-2.4/).
+
+## 2.3.3 (2024-02-26)
+- [Enhancement] Routing based topics allocation for swarm (Pro)
+- [Enhancement] Publish the `-1` shutdown reason status for a non-responding node in swarm.
+- [Enhancement] Allow for using the `distribution` mode for DataDog listener histogram reporting (Aerdayne).
+- [Change] Change `internal.swarm.node_report_timeout` to 60 seconds from 30 seconds to compensate for long pollings.
+- [Fix] Static membership routing evaluation happens too early in swarm.
+- [Fix] Close producer in supervisor prior to forking and warmup to prevent invalid memory states.
+
 ## 2.3.2 (2024-02-16)
 - **[Feature]** Provide swarm capabilities to OSS and Pro.
 - **[Feature]** Provide ability to use complex strategies in DLQ (Pro).

data/Gemfile

@@ -6,11 +6,14 @@ plugin 'diffend'
 
 gemspec
 
-# Karafka gem does not require activejob nor karafka-web  to work
+# Karafka gem does not require activejob nor karafka-web to work
 # They are added here because they are part of the integration suite
+# Since some of those are only needed for some specs, they should never be required automatically
 group :integrations do
-  gem 'activejob'
-  gem 'karafka-web', '>= 0.8.0'
+  gem 'activejob', require: false
+  gem 'karafka-testing', '>= 2.4.0.beta1', require: false
+  gem 'karafka-web', require: false
+  gem 'rspec', require: false
 end
 
 group :test do

data/Gemfile.lock

@@ -1,18 +1,18 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.2)
-      karafka-core (>= 2.3.0, < 2.4.0)
-      waterdrop (>= 2.6.12, < 3.0.0)
+    karafka (2.4.0.beta1)
+      karafka-core (>= 2.4.0.rc1, < 2.5.0)
+      waterdrop (>= 2.7.0.rc1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.1.3)
-      activesupport (= 7.1.3)
+    activejob (7.1.3.2)
+      activesupport (= 7.1.3.2)
       globalid (>= 0.3.6)
-    activesupport (7.1.3)
+    activesupport (7.1.3.2)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -23,28 +23,30 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.6)
+    bigdecimal (3.1.7)
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
-    drb (2.2.0)
-      ruby2_keywords
+    drb (2.2.1)
     erubi (1.12.0)
-    factory_bot (6.4.5)
+    factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     ffi (1.16.3)
     globalid (1.2.1)
       activesupport (>= 6.1)
-    i18n (1.14.1)
+    i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.3.0)
-      karafka-rdkafka (>= 0.14.8, < 0.15.0)
-    karafka-rdkafka (0.14.10)
+    karafka-core (2.4.0.rc1)
+      karafka-rdkafka (>= 0.15.0.rc1, < 0.16.0)
+    karafka-rdkafka (0.15.0.rc1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
+    karafka-testing (2.4.0.beta1)
+      karafka (>= 2.3.0, < 2.5.0)
+      waterdrop (>= 2.7.0.beta2)
     karafka-web (0.8.2)
       erubi (~> 1.4)
       karafka (>= 2.3.0, < 2.4.0)
@@ -52,11 +54,11 @@ GEM
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.5)
-    minitest (5.21.2)
+    minitest (5.22.3)
     mutex_m (0.2.0)
-    rack (3.0.9)
+    rack (3.0.10)
     rake (13.1.0)
-    roda (3.77.0)
+    roda (3.78.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -70,8 +72,7 @@ GEM
     rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.0)
-    ruby2_keywords (0.0.5)
+    rspec-support (3.13.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -81,8 +82,8 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.14)
-      karafka-core (>= 2.2.3, < 3.0.0)
+    waterdrop (2.7.0.rc1)
+      karafka-core (>= 2.4.0.rc1, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.13)
 
@@ -95,9 +96,10 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
-  karafka-web (>= 0.8.0)
+  karafka-testing (>= 2.4.0.beta1)
+  karafka-web
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.4
+   2.5.6

data/README.md

@@ -9,13 +9,13 @@
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
 
 - Has a built-in [Web UI](https://karafka.io/docs/Web-UI-Features/) providing a convenient way to monitor and manage Karafka-based applications.
-- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
+- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work) and [processes](https://karafka.io/docs/Swarm-Multi-Process).
 - [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
 - Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
 - Has a seamless [Dead Letter Queue](https://karafka.io/docs/Dead-Letter-Queue/) functionality built-in
 - Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
-- Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.
+- Has an out-of the box [AppSignal](https://karafka.io/docs/Monitoring-and-Logging/#appsignal-metrics-and-error-tracking) and [StatsD/DataDog](https://karafka.io/docs/Monitoring-and-Logging/#datadog-and-statsd-integration) monitoring with dashboard templates.
 
 ```ruby
 # Define what topics you want to consume with which consumers in karafka.rb

data/bin/integrations

@@ -28,7 +28,7 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 3
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
-MAX_BUFFER_OUTPUT = 102_400
+MAX_BUFFER_OUTPUT = 307_200
 
 # Abstraction around a single test scenario execution process
 class Scenario

data/config/locales/errors.yml

@@ -7,19 +7,25 @@ en:
 
       missing: needs to be present
       client_id_format: 'needs to be a string with a Kafka accepted format'
+      group_id_format: 'needs to be a string with a Kafka accepted format'
       concurrency_format: needs to be an integer bigger than 0
-      consumer_mapper_format: needs to be present
       consumer_persistence_format: needs to be either true or false
       pause_timeout_format: needs to be an integer bigger than 0
       pause_max_timeout_format: needs to be an integer bigger than 0
       pause_with_exponential_backoff_format: needs to be either true or false
+      strict_topics_namespacing_format: needs to be either true or false
       shutdown_timeout_format: needs to be an integer bigger than 0
       max_wait_time_format: needs to be an integer bigger than 0
+      max_wait_time_max_wait_time_vs_swarm_node_report_timeout: >
+        cannot be more than 80% of internal.swarm.node_report_timeout.
+        Decrease max_wait_time or increase node_report_timeout
       kafka_format: needs to be a filled hash
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
 
+      oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
+
       internal.processing.jobs_builder_format: cannot be nil
       internal.processing.jobs_queue_class_format: cannot be nil
       internal.processing.scheduler_class_format: cannot be nil
@@ -52,6 +58,11 @@ en:
       internal.connection.proxy.committed.timeout_format: needs to be an integer bigger than 0
       internal.connection.proxy.committed.max_attempts_format: needs to be an integer bigger than 0
       internal.connection.proxy.committed.wait_time_format: needs to be an integer bigger than 0
+      internal.connection.proxy.commit.max_attempts_format: needs to be an integer bigger than 0
+      internal.connection.proxy.commit.wait_time_format: needs to be an integer bigger than 0
+      internal.connection.proxy.metadata.timeout_format: needs to be an integer bigger than 0
+      internal.connection.proxy.metadata.max_attempts_format: needs to be an integer bigger than 0
+      internal.connection.proxy.metadata.wait_time_format: needs to be an integer bigger than 0
 
       internal.swarm.manager_format: cannot be nil
       internal.swarm.orphaned_exit_code_format: needs to be an integer bigger or equal to 0
@@ -84,7 +95,7 @@ en:
       max_messages_format: 'needs to be an integer bigger than 0'
       max_wait_time_format: 'needs to be an integer bigger than 0'
       name_format: 'needs to be a string with a Kafka accepted format'
-      deserializer_format: needs to be present
+      deserializers_format: needs to be present
       consumer_format: needs to be present
       id_format: 'needs to be a string with a Kafka accepted format'
       initial_offset_format: needs to be either earliest or latest
@@ -93,20 +104,31 @@ en:
       manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
       inline_insights.active_format: needs to be either true or false
       consumer_active_job_missing: ActiveJob needs to be available
+
       dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       dead_letter_queue.independent_format: needs to be either true or false
       dead_letter_queue.transactional_format: needs to be either true or false
+      dead_letter_queue.dispatch_method_format: 'needs to be either #produce_sync or #produce_async'
+      dead_letter_queue.marking_method_format: 'needs to be either #mark_as_consumed or #mark_as_consumed!'
+
       active_format: needs to be either true or false
+
       declaratives.partitions_format: needs to be more or equal to 1
       declaratives.active_format: needs to be true
       declaratives.replication_factor_format: needs to be more or equal to 1
       declaratives.details_format: needs to be a hash with only symbol keys
+
       inconsistent_namespacing: |
         needs to be consistent namespacing style
         disable this validation by setting config.strict_topics_namespacing to false
 
+      deserializers.active_format: 'needs to be true'
+      deserializers.payload_format: 'needs to respond to #call'
+      deserializers.key_format: 'needs to respond to #call'
+      deserializers.headers_format: 'needs to respond to #call'
+
     consumer_group:
       missing: needs to be present
       topics_names_not_unique: all topic names within a single consumer group must be unique

data/config/locales/pro_errors.yml

@@ -58,10 +58,24 @@ en:
       subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
       subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
 
+      swarm.active_format: needs to be true
+      swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
+      swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
+
+      direct_assignments.active_missing: needs to be present
+      direct_assignments.active_format: 'needs to be boolean'
+      direct_assignments.partitions_missing: 'needs to be present'
+      direct_assignments.partitions_format: 'needs to be true, list of partitions or a range of partitions (finite)'
+      direct_assignments_active_but_empty: 'cannot be empty and active at the same time'
+      direct_assignments_swarm_not_complete: 'cannot have partitions that are assigned but not allocated'
+      direct_assignments_swarm_overbooked: 'cannot allocate partitions in swarm that were not assigned'
+      direct_assignments_patterns_active: 'patterns cannot be used with direct assignments'
+
     consumer_group:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present
       patterns_regexps_not_unique: 'must be unique within consumer group'
+      direct_assignments_homogenous: 'single consumer group cannot mix regular and direct assignments'
 
     pattern:
       regexp_format: must be a regular expression
@@ -78,6 +92,11 @@ en:
       encryption.version_format: must be a non-empty string
       encryption.public_key_format: 'is not a valid public RSA key'
       encryption.private_keys_invalid: 'contains an invalid private RSA key string'
+      encryption.fingerprinter_missing: 'needs to be false or respond to #hexdigest method'
+      encryption.fingerprinter_format: 'needs to be false or respond to #hexdigest method'
 
       patterns.ttl_format: needs to be an integer bigger than 0
       patterns.ttl_missing: needs to be present
+
+    routing:
+      swarm_nodes_not_used: 'At least one of the nodes has no assignments'

data/karafka.gemspec

@@ -21,10 +21,12 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
-  spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.0.rc1', '< 2.5.0'
+  spec.add_dependency 'waterdrop', '>= 2.7.0.rc1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
+  spec.required_ruby_version = '>= 3.0.0'
+
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -16,8 +16,10 @@ module ActiveJob
 
       # Enqueues multiple jobs in one go
       # @param jobs [Array<Object>] jobs that we want to enqueue
+      # @return [Integer] number of jobs enqueued (required by Rails)
       def enqueue_all(jobs)
         ::Karafka::App.config.internal.active_job.dispatcher.dispatch_many(jobs)
+        jobs.size
       end
 
       # Raises info, that Karafka backend does not support scheduling jobs

data/lib/karafka/admin/configs/config.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Admin
+    module Configs
+      # Represents a single config entry that is related to a resource
+      class Config
+        attr_reader :name, :value, :synonyms
+
+        class << self
+          # Creates a single config entry from the Rdkafka config result entry
+          #
+          # @param rd_kafka_config [Rdkafka::Admin::ConfigBindingResult]
+          # @return [Config]
+          def from_rd_kafka(rd_kafka_config)
+            new(
+              name: rd_kafka_config.name,
+              value: rd_kafka_config.value,
+              read_only: rd_kafka_config.read_only,
+              default: rd_kafka_config.default,
+              sensitive: rd_kafka_config.sensitive,
+              synonym: rd_kafka_config.synonym,
+              synonyms: rd_kafka_config.synonyms.map do |rd_kafka_synonym|
+                from_rd_kafka(rd_kafka_synonym)
+              end
+            )
+          end
+        end
+
+        # Creates new config instance either for reading or as part of altering operation
+        #
+        # @param name [String] config name
+        # @param value [String] config value
+        # @param default [Integer] 1 if default
+        # @param read_only [Integer] 1 if read only
+        # @param sensitive [Integer] 1 if sensitive
+        # @param synonym [Integer] 1 if synonym
+        # @param synonyms [Array] given config synonyms (if any)
+        #
+        # @note For alter operations only `name` and `value` are needed
+        def initialize(
+          name:,
+          value:,
+          default: -1,
+          read_only: -1,
+          sensitive: -1,
+          synonym: -1,
+          synonyms: []
+        )
+          @name = name
+          @value = value
+          @synonyms = []
+          @default = default
+          @read_only = read_only
+          @sensitive = sensitive
+          @synonym = synonym
+          @synonyms = synonyms
+        end
+
+        # @return [Boolean] Is the config property is set to its default value on the broker
+        def default? = @default.positive?
+
+        # @return [Boolean] Is the config property is read-only on the broker
+        def read_only? = @read_only.positive?
+
+        # @return [Boolean] if the config property contains sensitive information (such as
+        #   security configuration
+        def sensitive? = @sensitive.positive?
+
+        # @return [Boolean] is this entry is a synonym
+        def synonym? = @synonym.positive?
+
+        # @return [Hash] hash that we can use to operate with rdkafka
+        def to_native_hash = {
+          name: name,
+          value: value
+        }.freeze
+      end
+    end
+  end
+end