karafka 2.5.5 → 2.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/LICENSE-COMM +4 -0
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +3 -3
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +1 -1
- data/lib/karafka/admin/consumer_groups.rb +8 -8
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +21 -21
- data/lib/karafka/admin/topics.rb +6 -6
- data/lib/karafka/admin.rb +4 -5
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +6 -1
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/interval_runner.rb +4 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +76 -63
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
- data/lib/karafka/pro/cli/contracts/server.rb +12 -12
- data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +14 -14
- data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
- data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
- data/lib/karafka/pro/encryption/setup/config.rb +2 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
- data/lib/karafka/pro/iterator.rb +1 -1
- data/lib/karafka/pro/loader.rb +2 -1
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
- data/lib/karafka/pro/processing/filters/base.rb +1 -1
- data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
- data/lib/karafka/pro/processing/partitioner.rb +3 -3
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
- data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
- data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
- data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
- data/lib/karafka/pro/processing/strategies/default.rb +22 -22
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
- data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
- data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
- data/lib/karafka/pro/recurring_tasks.rb +4 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm.rb +1 -1
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
- data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
- data/lib/karafka/pro/scheduled_messages.rb +2 -2
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/jobs_queue.rb +10 -0
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +14 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +8 -37
- data/lib/karafka/swarm/producer_replacer.rb +110 -0
- data/lib/karafka/swarm/supervisor.rb +9 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b395542efd1d57ac4f9fd89091866a39c23e2c48d446bb20fb3628e7952f762e
|
|
4
|
+
data.tar.gz: 14aeb17e690a257bb0d8d7e57b45e797da19adf22f2e5873bd88e18a9c0a6aea
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e9003ff1411da366cc3f70a77e7b949bdda7fed9d717eb6a2c151e3844af1fb0cdfca9f4c220be8d4a692b750e99c202d2afa05e57ffbac7c274579d065ee54c
|
|
7
|
+
data.tar.gz: 4807ec1df8ed8f8169e6f38b9ce5b47e541485e4f4d551822bd2ea9d578219ee565ca71b27d66a9aeef55917ffa2d16fddd629371f48661afd94271eff9c228d
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Karafka Framework Changelog
|
|
2
2
|
|
|
3
|
+
## 2.5.7 (2026-03-16)
|
|
4
|
+
- [Enhancement] Report detailed blocking information (active listeners, alive workers, and in-processing jobs) during forceful shutdown instead of only aggregate counts.
|
|
5
|
+
- [Enhancement] Improve `ForcefulShutdownError` description to clearly explain when and why it is raised.
|
|
6
|
+
- [Enhancement] Cache `messages.last` in `BatchMetadata` builder to avoid duplicate array traversal.
|
|
7
|
+
- [Enhancement] Optimize `VirtualOffsetManager#mark` to use a single array scan instead of separate `include?` and `index` calls (Pro).
|
|
8
|
+
- [Enhancement] Optimize `VirtualOffsetManager#materialize_real_offset` to use `keys.sort` instead of `to_a.sort_by` with tuple destructuring (Pro).
|
|
9
|
+
- [Enhancement] Optimize `IntervalRunner#call` to use a single `monotonic_now` call instead of two per invocation.
|
|
10
|
+
- [Enhancement] Support WaterDrop `:fd` mode in Swarm.
|
|
11
|
+
- [Maintenance] Use both `:fd` and `:thread` producer backends in CI.
|
|
12
|
+
- [Maintenance] Include spec file hash in integration test topic names for easier traceability in Kafka logs (#3056).
|
|
13
|
+
- [Fix] Remove duplicate topic creation in multi-broker health integration specs (#3056).
|
|
14
|
+
- [Fix] Preserve producer-specific kafka settings (e.g., `enable.idempotence`) when recreating the producer in swarm forks.
|
|
15
|
+
|
|
16
|
+
## 2.5.6 (2026-02-28)
|
|
17
|
+
- **[Feature]** Add `karafka topics health` command to check Kafka topics for replication and durability issues, detecting no redundancy (RF=1), zero fault tolerance (RF≤min.insync), and low durability (min.insync=1) configurations with color-coded severity grouping and actionable recommendations (Pro).
|
|
18
|
+
- [Enhancement] Optimize license loading process by reading license files directly from the gem directory instead of requiring the entire gem, reducing initialization overhead and adding support for user-defined License modules.
|
|
19
|
+
- [Change] Migrate Admin `AbstractHandle#wait` calls from deprecated `max_wait_timeout` (seconds) to `max_wait_timeout_ms` (milliseconds) to align with `karafka-rdkafka` `0.24.0`.
|
|
20
|
+
- [Change] Require `karafka-rdkafka` `>=` `0.24.0` to support the new `max_wait_timeout_ms` API.
|
|
21
|
+
- [Fix] Fix LRJ partition freeze when filter returns `applied? true` with `action :skip`, causing partition to remain paused for ~31,709 years instead of resuming after processing.
|
|
22
|
+
|
|
3
23
|
## 2.5.5 (2026-01-24)
|
|
4
24
|
- [Feature] Add multi-cluster Admin support via `Karafka::Admin.new(kafka: { ... })` allowing operations against different Kafka clusters while maintaining backward compatibility with existing class-method API.
|
|
5
25
|
|
data/LICENSE-COMM
CHANGED
|
@@ -12,6 +12,10 @@ In order to use the Software under this Agreement, you must either: (a) receive
|
|
|
12
12
|
|
|
13
13
|
1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold.
|
|
14
14
|
|
|
15
|
+
1.1.1 Entity Definition. Where you have purchased or otherwise acquired the Software as or for an entity, "you" and "your" refer exclusively to the single legal entity (such as a corporation, limited liability company, partnership, or other legally recognized organization) that is the named licensee. This license does not extend to any parent company, subsidiary, affiliate, related entity, or other member of a corporate group, regardless of ownership structure or control relationships. Each separate legal entity requires its own license.
|
|
16
|
+
|
|
17
|
+
1.1.2 Mergers, Acquisitions, and Change of Control. In the event of a merger, acquisition, consolidation, or other change of control affecting the licensee entity: (a) if the licensee entity ceases to exist as a separate legal entity, this license terminates and the surviving or acquiring entity must purchase a new license or obtain prior written consent from Maciej Mensfeld to continue use of the Software; (b) if the licensee entity continues to exist as a separate legal entity (including as a wholly owned subsidiary), this license remains valid solely for that entity and does not extend to the parent, acquirer, or any affiliated entities unless explicitly agreed in writing by Maciej Mensfeld.
|
|
18
|
+
|
|
15
19
|
1.2 Pro License. If you purchased a Pro License (included with the Karafka Pro Software), you may install the Software on an unlimited number of Hosts. "Host" means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. "Worker" means a thread within a Karafka server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers.
|
|
16
20
|
|
|
17
21
|
1.3 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
|
data/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-

|
|
2
2
|
|
|
3
3
|
[](https://github.com/karafka/karafka/actions/workflows/ci_linux_ubuntu_x86_64_gnu.yml)
|
|
4
4
|
[](http://badge.fury.io/rb/karafka)
|
|
@@ -39,7 +39,7 @@ Karafka **uses** threads to handle many messages simultaneously in the same proc
|
|
|
39
39
|
|
|
40
40
|
## Getting started
|
|
41
41
|
|
|
42
|
-

|
|
43
43
|
|
|
44
44
|
If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
|
|
45
45
|
|
data/certs/expired.txt
ADDED
data/karafka.gemspec
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
lib = File.expand_path(
|
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
|
6
|
-
require
|
|
6
|
+
require "karafka/version"
|
|
7
7
|
|
|
8
8
|
Gem::Specification.new do |spec|
|
|
9
|
-
spec.name
|
|
10
|
-
spec.version
|
|
11
|
-
spec.platform
|
|
12
|
-
spec.authors
|
|
13
|
-
spec.email
|
|
14
|
-
spec.homepage
|
|
15
|
-
spec.licenses
|
|
16
|
-
spec.summary
|
|
9
|
+
spec.name = "karafka"
|
|
10
|
+
spec.version = Karafka::VERSION
|
|
11
|
+
spec.platform = Gem::Platform::RUBY
|
|
12
|
+
spec.authors = ["Maciej Mensfeld"]
|
|
13
|
+
spec.email = %w[contact@karafka.io]
|
|
14
|
+
spec.homepage = "https://karafka.io"
|
|
15
|
+
spec.licenses = %w[LGPL-3.0-only Commercial]
|
|
16
|
+
spec.summary = "Karafka is Ruby and Rails efficient Kafka processing framework."
|
|
17
17
|
spec.description = <<-DESC
|
|
18
18
|
Karafka is Ruby and Rails efficient Kafka processing framework.
|
|
19
19
|
|
|
@@ -21,12 +21,12 @@ Gem::Specification.new do |spec|
|
|
|
21
21
|
without having to focus on things that are not your business domain.
|
|
22
22
|
DESC
|
|
23
23
|
|
|
24
|
-
spec.add_dependency
|
|
25
|
-
spec.add_dependency
|
|
26
|
-
spec.add_dependency
|
|
27
|
-
spec.add_dependency
|
|
24
|
+
spec.add_dependency "karafka-core", ">= 2.5.6", "< 2.6.0"
|
|
25
|
+
spec.add_dependency "karafka-rdkafka", ">= 0.24.0"
|
|
26
|
+
spec.add_dependency "waterdrop", ">= 2.8.14", "< 3.0.0"
|
|
27
|
+
spec.add_dependency "zeitwerk", "~> 2.3"
|
|
28
28
|
|
|
29
|
-
spec.required_ruby_version =
|
|
29
|
+
spec.required_ruby_version = ">= 3.2.0"
|
|
30
30
|
|
|
31
31
|
gem_files = %w[
|
|
32
32
|
lib/**/*
|
|
@@ -41,16 +41,16 @@ Gem::Specification.new do |spec|
|
|
|
41
41
|
|
|
42
42
|
spec.files = Dir.glob(gem_files) & `git ls-files -z`.split("\x0")
|
|
43
43
|
|
|
44
|
-
spec.executables
|
|
44
|
+
spec.executables = %w[karafka]
|
|
45
45
|
spec.require_paths = %w[lib]
|
|
46
46
|
|
|
47
47
|
spec.metadata = {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
48
|
+
"funding_uri" => "https://karafka.io/#become-pro",
|
|
49
|
+
"homepage_uri" => "https://karafka.io",
|
|
50
|
+
"changelog_uri" => "https://karafka.io/docs/Changelog-Karafka",
|
|
51
|
+
"bug_tracker_uri" => "https://github.com/karafka/karafka/issues",
|
|
52
|
+
"source_code_uri" => "https://github.com/karafka/karafka",
|
|
53
|
+
"documentation_uri" => "https://karafka.io/docs",
|
|
54
|
+
"rubygems_mfa_required" => "true"
|
|
55
55
|
}
|
|
56
56
|
end
|
data/lib/active_job/karafka.rb
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
begin
|
|
4
4
|
# Do not load active job if already loaded
|
|
5
|
-
require
|
|
5
|
+
require "active_job" unless Object.const_defined?("ActiveJob")
|
|
6
6
|
|
|
7
|
-
require_relative
|
|
7
|
+
require_relative "queue_adapters/karafka_adapter"
|
|
8
8
|
|
|
9
9
|
module ActiveJob
|
|
10
10
|
# Namespace for usage simplification outside of Rails where Railtie will not kick in.
|
|
@@ -22,11 +22,11 @@ module ActiveJob
|
|
|
22
22
|
#
|
|
23
23
|
# @see https://github.com/sidekiq/sidekiq/issues/6746 Similar issue in Sidekiq
|
|
24
24
|
base = if defined?(Rails::VERSION)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
((Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2) ? Object : AbstractAdapter)
|
|
26
|
+
else
|
|
27
|
+
# Fallback when Rails is not loaded
|
|
28
|
+
Object
|
|
29
|
+
end
|
|
30
30
|
|
|
31
31
|
# Karafka adapter for enqueuing jobs
|
|
32
32
|
# This is here for ease of integration with ActiveJob.
|
|
@@ -29,13 +29,13 @@ module Karafka
|
|
|
29
29
|
# @param job_message [Karafka::Messages::Message] message with active job
|
|
30
30
|
def consume_job(job_message)
|
|
31
31
|
with_deserialized_job(job_message) do |job|
|
|
32
|
-
tags.add(:job_class, job[
|
|
32
|
+
tags.add(:job_class, job["job_class"])
|
|
33
33
|
|
|
34
34
|
payload = { caller: self, job: job, message: job_message }
|
|
35
35
|
|
|
36
36
|
# We publish both to make it consistent with `consumer.x` events
|
|
37
|
-
Karafka.monitor.instrument(
|
|
38
|
-
Karafka.monitor.instrument(
|
|
37
|
+
Karafka.monitor.instrument("active_job.consume", payload)
|
|
38
|
+
Karafka.monitor.instrument("active_job.consumed", payload) do
|
|
39
39
|
::ActiveJob::Base.execute(job)
|
|
40
40
|
end
|
|
41
41
|
end
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
4
|
-
require_relative
|
|
5
|
-
require_relative
|
|
6
|
-
require_relative
|
|
3
|
+
require "active_support/current_attributes"
|
|
4
|
+
require_relative "current_attributes/job_wrapper"
|
|
5
|
+
require_relative "current_attributes/loading"
|
|
6
|
+
require_relative "current_attributes/persistence"
|
|
7
7
|
|
|
8
8
|
# This code is based on Sidekiqs approach to persisting current attributes
|
|
9
9
|
# @see https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/current_attributes.rb
|
|
@@ -9,8 +9,8 @@ module Karafka
|
|
|
9
9
|
class JobOptionsContract < Contracts::Base
|
|
10
10
|
configure do |config|
|
|
11
11
|
config.error_messages = YAML.safe_load_file(
|
|
12
|
-
File.join(Karafka.gem_root,
|
|
13
|
-
).fetch(
|
|
12
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
13
|
+
).fetch("en").fetch("validations").fetch("job_options")
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
optional(:dispatch_method) do |val|
|
data/lib/karafka/admin/acl.rb
CHANGED
|
@@ -179,7 +179,7 @@ module Karafka
|
|
|
179
179
|
principal: nil,
|
|
180
180
|
operation: :any,
|
|
181
181
|
permission_type: :any,
|
|
182
|
-
host:
|
|
182
|
+
host: "*"
|
|
183
183
|
)
|
|
184
184
|
)
|
|
185
185
|
end
|
|
@@ -235,7 +235,7 @@ module Karafka
|
|
|
235
235
|
resource_name: nil,
|
|
236
236
|
resource_pattern_type: nil,
|
|
237
237
|
principal: nil,
|
|
238
|
-
host:
|
|
238
|
+
host: "*",
|
|
239
239
|
operation: nil,
|
|
240
240
|
permission_type: nil
|
|
241
241
|
)
|
|
@@ -276,7 +276,7 @@ module Karafka
|
|
|
276
276
|
# Makes sure that admin is closed afterwards.
|
|
277
277
|
def with_admin_wait
|
|
278
278
|
with_admin do |admin|
|
|
279
|
-
yield(admin).wait(
|
|
279
|
+
yield(admin).wait(max_wait_timeout_ms: self.class.max_wait_time)
|
|
280
280
|
end
|
|
281
281
|
end
|
|
282
282
|
|
|
@@ -45,7 +45,7 @@ module Karafka
|
|
|
45
45
|
# def set(name, value)
|
|
46
46
|
# @operations[0] << Config.new(name: name, value: value.to_s)
|
|
47
47
|
# end
|
|
48
|
-
default_value = op_name == :delete ?
|
|
48
|
+
default_value = (op_name == :delete) ? " = nil" : ""
|
|
49
49
|
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
50
50
|
# @param name [String] name of the config to alter
|
|
51
51
|
# @param value [String] value of the config
|
|
@@ -109,7 +109,7 @@ module Karafka
|
|
|
109
109
|
# Makes sure that admin is closed afterwards.
|
|
110
110
|
def with_admin_wait
|
|
111
111
|
with_admin do |admin|
|
|
112
|
-
yield(admin).wait(
|
|
112
|
+
yield(admin).wait(max_wait_timeout_ms: self.class.max_wait_time)
|
|
113
113
|
end
|
|
114
114
|
end
|
|
115
115
|
end
|
|
@@ -129,11 +129,11 @@ module Karafka
|
|
|
129
129
|
# Earliest is not always 0. When compacting/deleting it can be much later, that's why
|
|
130
130
|
# we fetch the oldest possible offset
|
|
131
131
|
# false is treated the same as 'earliest'
|
|
132
|
-
when
|
|
132
|
+
when "earliest", false
|
|
133
133
|
LONG_TIME_AGO
|
|
134
134
|
# Latest will always be the high-watermark offset and we can get it just by getting
|
|
135
135
|
# a future position
|
|
136
|
-
when
|
|
136
|
+
when "latest"
|
|
137
137
|
Time.now + DAY_IN_SECONDS
|
|
138
138
|
# Regular offset case
|
|
139
139
|
else
|
|
@@ -161,7 +161,7 @@ module Karafka
|
|
|
161
161
|
end
|
|
162
162
|
end
|
|
163
163
|
|
|
164
|
-
settings = {
|
|
164
|
+
settings = { "group.id": consumer_group_id }
|
|
165
165
|
|
|
166
166
|
with_consumer(settings) do |consumer|
|
|
167
167
|
# If we have any time based stuff to resolve, we need to do it prior to commits
|
|
@@ -276,7 +276,7 @@ module Karafka
|
|
|
276
276
|
def delete(consumer_group_id)
|
|
277
277
|
with_admin do |admin|
|
|
278
278
|
handler = admin.delete_group(consumer_group_id)
|
|
279
|
-
handler.wait(
|
|
279
|
+
handler.wait(max_wait_timeout_ms: max_wait_time_ms)
|
|
280
280
|
end
|
|
281
281
|
end
|
|
282
282
|
|
|
@@ -333,9 +333,9 @@ module Karafka
|
|
|
333
333
|
# This ensures we use the same settings as the actual consumers
|
|
334
334
|
# Following the same pattern as in Karafka::Connection::Client#build_kafka
|
|
335
335
|
consumer_settings = Setup::AttributesMap.consumer(first_topic.kafka.dup)
|
|
336
|
-
consumer_settings[:
|
|
337
|
-
consumer_settings[:
|
|
338
|
-
consumer_settings[:
|
|
336
|
+
consumer_settings[:"group.id"] = consumer_group.id
|
|
337
|
+
consumer_settings[:"enable.auto.offset.store"] = false
|
|
338
|
+
consumer_settings[:"auto.offset.reset"] ||= first_topic.initial_offset
|
|
339
339
|
|
|
340
340
|
with_consumer(consumer_settings) do |consumer|
|
|
341
341
|
# Subscribe to the topics - this triggers the first rebalance
|
|
@@ -408,7 +408,7 @@ module Karafka
|
|
|
408
408
|
|
|
409
409
|
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
|
410
410
|
|
|
411
|
-
with_consumer(
|
|
411
|
+
with_consumer("group.id": cg) do |consumer|
|
|
412
412
|
topics.each { |topic| tpl.add_topic(topic, existing_topics[topic]) }
|
|
413
413
|
|
|
414
414
|
commit_offsets = consumer.committed(tpl)
|
|
@@ -8,8 +8,8 @@ module Karafka
|
|
|
8
8
|
class Replication < Karafka::Contracts::Base
|
|
9
9
|
configure do |config|
|
|
10
10
|
config.error_messages = YAML.safe_load_file(
|
|
11
|
-
File.join(Karafka.gem_root,
|
|
12
|
-
).fetch(
|
|
11
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
12
|
+
).fetch("en").fetch("validations").fetch("admin").fetch("replication")
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
required(:topic) { |val| val.is_a?(String) && !val.empty? }
|
|
@@ -262,7 +262,7 @@ module Karafka
|
|
|
262
262
|
change = @target_replication_factor - @current_replication_factor
|
|
263
263
|
broker_nodes = @cluster_info[:brokers].map do |broker_info|
|
|
264
264
|
broker_info[:node_id]
|
|
265
|
-
end.join(
|
|
265
|
+
end.join(", ")
|
|
266
266
|
|
|
267
267
|
<<~SUMMARY
|
|
268
268
|
Replication Increase Plan for Topic: #{@topic}
|
|
@@ -328,10 +328,10 @@ module Karafka
|
|
|
328
328
|
# Handle both :replicas (array of objects) and :replica_brokers (array of IDs)
|
|
329
329
|
replicas = partition_info[:replicas] || partition_info[:replica_brokers] || []
|
|
330
330
|
current_replicas = if replicas.first.respond_to?(:node_id)
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
331
|
+
replicas.map(&:node_id).sort
|
|
332
|
+
else
|
|
333
|
+
replicas.sort
|
|
334
|
+
end
|
|
335
335
|
|
|
336
336
|
if rebalance_only
|
|
337
337
|
# For rebalancing, redistribute current replicas optimally
|
|
@@ -442,22 +442,22 @@ module Karafka
|
|
|
442
442
|
# Builds the kafka-reassign-partitions.sh command for generating reassignment plan
|
|
443
443
|
# @return [String] command template with placeholder for broker addresses
|
|
444
444
|
def build_generate_command
|
|
445
|
-
|
|
446
|
-
|
|
445
|
+
"kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> " \
|
|
446
|
+
"--reassignment-json-file reassignment.json --generate"
|
|
447
447
|
end
|
|
448
448
|
|
|
449
449
|
# Builds the kafka-reassign-partitions.sh command for executing reassignment
|
|
450
450
|
# @return [String] command template with placeholder for broker addresses
|
|
451
451
|
def build_execute_command
|
|
452
|
-
|
|
453
|
-
|
|
452
|
+
"kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> " \
|
|
453
|
+
"--reassignment-json-file reassignment.json --execute"
|
|
454
454
|
end
|
|
455
455
|
|
|
456
456
|
# Builds the kafka-reassign-partitions.sh command for verifying reassignment progress
|
|
457
457
|
# @return [String] command template with placeholder for broker addresses
|
|
458
458
|
def build_verify_command
|
|
459
|
-
|
|
460
|
-
|
|
459
|
+
"kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> " \
|
|
460
|
+
"--reassignment-json-file reassignment.json --verify"
|
|
461
461
|
end
|
|
462
462
|
|
|
463
463
|
# Generates detailed step-by-step instructions for executing the reassignment
|
|
@@ -469,16 +469,16 @@ module Karafka
|
|
|
469
469
|
"2. Validate the plan (optional): #{@execution_commands[:generate]}",
|
|
470
470
|
"3. Execute the reassignment: #{@execution_commands[:execute]}",
|
|
471
471
|
"4. Monitor progress: #{@execution_commands[:verify]}",
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
472
|
+
"5. Verify completion by checking topic metadata",
|
|
473
|
+
"",
|
|
474
|
+
"IMPORTANT NOTES:",
|
|
475
|
+
"- Replace <KAFKA_BROKERS> with your actual Kafka broker addresses",
|
|
476
|
+
"- The reassignment process may take time depending on data size",
|
|
477
|
+
"- Monitor disk space and network I/O during reassignment",
|
|
478
|
+
"- Consider running during low-traffic periods",
|
|
479
|
+
"- For large topics, consider throttling replica transfer rate",
|
|
480
|
+
"- Ensure sufficient disk space on target brokers before starting",
|
|
481
|
+
"- Keep monitoring until all replicas are in-sync (ISR)"
|
|
482
482
|
]
|
|
483
483
|
end
|
|
484
484
|
end
|
data/lib/karafka/admin/topics.rb
CHANGED
|
@@ -147,7 +147,7 @@ module Karafka
|
|
|
147
147
|
handler = admin.create_topic(name, partitions, replication_factor, topic_config)
|
|
148
148
|
|
|
149
149
|
with_re_wait(
|
|
150
|
-
-> { handler.wait(
|
|
150
|
+
-> { handler.wait(max_wait_timeout_ms: max_wait_time_ms) },
|
|
151
151
|
-> { names.include?(name) }
|
|
152
152
|
)
|
|
153
153
|
end
|
|
@@ -163,7 +163,7 @@ module Karafka
|
|
|
163
163
|
handler = admin.delete_topic(name)
|
|
164
164
|
|
|
165
165
|
with_re_wait(
|
|
166
|
-
-> { handler.wait(
|
|
166
|
+
-> { handler.wait(max_wait_timeout_ms: max_wait_time_ms) },
|
|
167
167
|
-> { !names.include?(name) }
|
|
168
168
|
)
|
|
169
169
|
end
|
|
@@ -180,7 +180,7 @@ module Karafka
|
|
|
180
180
|
handler = admin.create_partitions(name, partitions)
|
|
181
181
|
|
|
182
182
|
with_re_wait(
|
|
183
|
-
-> { handler.wait(
|
|
183
|
+
-> { handler.wait(max_wait_timeout_ms: max_wait_time_ms) },
|
|
184
184
|
-> { info(name).fetch(:partition_count) >= partitions }
|
|
185
185
|
)
|
|
186
186
|
end
|
|
@@ -272,9 +272,9 @@ module Karafka
|
|
|
272
272
|
|
|
273
273
|
real_offsets = consumer.offsets_for_times(tpl)
|
|
274
274
|
detected_offset = real_offsets
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
275
|
+
.to_h
|
|
276
|
+
.fetch(name)
|
|
277
|
+
.find { |p_data| p_data.partition == partition }
|
|
278
278
|
|
|
279
279
|
detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
|
|
280
280
|
else
|
data/lib/karafka/admin.rb
CHANGED
|
@@ -400,10 +400,9 @@ module Karafka
|
|
|
400
400
|
|
|
401
401
|
private
|
|
402
402
|
|
|
403
|
-
# @return [Integer]
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
self.class.max_wait_time / 1_000.0
|
|
403
|
+
# @return [Integer] max wait time in ms
|
|
404
|
+
def max_wait_time_ms
|
|
405
|
+
self.class.max_wait_time
|
|
407
406
|
end
|
|
408
407
|
|
|
409
408
|
# Adds a new callback for given rdkafka instance for oauth token refresh (if needed)
|
|
@@ -464,7 +463,7 @@ module Karafka
|
|
|
464
463
|
def config(type, settings)
|
|
465
464
|
kafka_config = self.class.app_kafka.dup
|
|
466
465
|
kafka_config.merge!(self.class.admin_kafka)
|
|
467
|
-
kafka_config[:
|
|
466
|
+
kafka_config[:"group.id"] = self.class.group_id
|
|
468
467
|
# We merge after setting the group id so it can be altered if needed
|
|
469
468
|
# In general in admin we only should alter it when we need to impersonate a given
|
|
470
469
|
# consumer group or do something similar
|
data/lib/karafka/app.rb
CHANGED
|
@@ -13,7 +13,7 @@ module Karafka
|
|
|
13
13
|
# Per recommendation, this should not run in children nodes
|
|
14
14
|
return if Karafka::App.config.swarm.node
|
|
15
15
|
|
|
16
|
-
monitor.instrument(
|
|
16
|
+
monitor.instrument("app.before_warmup", caller: self)
|
|
17
17
|
|
|
18
18
|
return GC.compact unless ::Process.respond_to?(:warmup)
|
|
19
19
|
|
|
@@ -48,7 +48,7 @@ module Karafka
|
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
# Just a nicer name for the consumer groups
|
|
51
|
-
|
|
51
|
+
alias_method :routes, :consumer_groups
|
|
52
52
|
|
|
53
53
|
# Returns current assignments of this process. Both topics and partitions
|
|
54
54
|
#
|
|
@@ -101,7 +101,7 @@ module Karafka
|
|
|
101
101
|
# producing messages.
|
|
102
102
|
#
|
|
103
103
|
# @param contexts [String] librdkafka low level debug contexts for granular debugging
|
|
104
|
-
def debug!(contexts =
|
|
104
|
+
def debug!(contexts = "all")
|
|
105
105
|
logger.level = Logger::DEBUG
|
|
106
106
|
producer.config.logger.level = Logger::DEBUG
|
|
107
107
|
|