karafka 2.5.5 → 2.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +20 -0
  3. data/LICENSE-COMM +4 -0
  4. data/README.md +2 -2
  5. data/certs/expired.txt +2 -0
  6. data/karafka.gemspec +23 -23
  7. data/lib/active_job/karafka.rb +2 -2
  8. data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
  9. data/lib/karafka/active_job/consumer.rb +3 -3
  10. data/lib/karafka/active_job/current_attributes.rb +4 -4
  11. data/lib/karafka/active_job/job_options_contract.rb +2 -2
  12. data/lib/karafka/admin/acl.rb +3 -3
  13. data/lib/karafka/admin/configs/resource.rb +1 -1
  14. data/lib/karafka/admin/configs.rb +1 -1
  15. data/lib/karafka/admin/consumer_groups.rb +8 -8
  16. data/lib/karafka/admin/contracts/replication.rb +2 -2
  17. data/lib/karafka/admin/replication.rb +21 -21
  18. data/lib/karafka/admin/topics.rb +6 -6
  19. data/lib/karafka/admin.rb +4 -5
  20. data/lib/karafka/app.rb +3 -3
  21. data/lib/karafka/base_consumer.rb +34 -30
  22. data/lib/karafka/cli/base.rb +8 -8
  23. data/lib/karafka/cli/console.rb +1 -1
  24. data/lib/karafka/cli/contracts/server.rb +12 -12
  25. data/lib/karafka/cli/help.rb +2 -2
  26. data/lib/karafka/cli/info.rb +4 -4
  27. data/lib/karafka/cli/install.rb +11 -11
  28. data/lib/karafka/cli/server.rb +6 -6
  29. data/lib/karafka/cli/swarm.rb +1 -1
  30. data/lib/karafka/cli/topics/align.rb +4 -4
  31. data/lib/karafka/cli/topics/base.rb +5 -5
  32. data/lib/karafka/cli/topics/create.rb +2 -2
  33. data/lib/karafka/cli/topics/delete.rb +2 -2
  34. data/lib/karafka/cli/topics/help.rb +5 -1
  35. data/lib/karafka/cli/topics/plan.rb +16 -16
  36. data/lib/karafka/cli/topics/repartition.rb +3 -3
  37. data/lib/karafka/cli/topics.rb +22 -22
  38. data/lib/karafka/cli.rb +2 -2
  39. data/lib/karafka/connection/client.rb +17 -17
  40. data/lib/karafka/connection/listener.rb +6 -6
  41. data/lib/karafka/connection/mode.rb +1 -1
  42. data/lib/karafka/connection/proxy.rb +1 -1
  43. data/lib/karafka/connection/status.rb +2 -2
  44. data/lib/karafka/constraints.rb +3 -3
  45. data/lib/karafka/embedded.rb +3 -3
  46. data/lib/karafka/env.rb +4 -4
  47. data/lib/karafka/errors.rb +6 -1
  48. data/lib/karafka/execution_mode.rb +1 -1
  49. data/lib/karafka/helpers/config_importer.rb +2 -2
  50. data/lib/karafka/helpers/interval_runner.rb +4 -2
  51. data/lib/karafka/helpers/multi_delegator.rb +1 -1
  52. data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
  53. data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
  54. data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
  55. data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
  56. data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
  57. data/lib/karafka/instrumentation/logger.rb +7 -7
  58. data/lib/karafka/instrumentation/logger_listener.rb +76 -63
  59. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
  60. data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
  61. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
  62. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
  63. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
  64. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
  65. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
  66. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
  67. data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
  68. data/lib/karafka/licenser.rb +115 -8
  69. data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
  70. data/lib/karafka/messages/messages.rb +1 -1
  71. data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
  72. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
  73. data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
  74. data/lib/karafka/pro/cli/contracts/server.rb +12 -12
  75. data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
  76. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
  77. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
  78. data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
  79. data/lib/karafka/pro/cli/topics/health.rb +162 -0
  80. data/lib/karafka/pro/cli/topics.rb +52 -0
  81. data/lib/karafka/pro/connection/manager.rb +14 -14
  82. data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
  83. data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
  84. data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
  85. data/lib/karafka/pro/encryption/setup/config.rb +2 -2
  86. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
  87. data/lib/karafka/pro/iterator.rb +1 -1
  88. data/lib/karafka/pro/loader.rb +2 -1
  89. data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
  90. data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
  91. data/lib/karafka/pro/processing/filters/base.rb +1 -1
  92. data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
  93. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  94. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
  95. data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
  96. data/lib/karafka/pro/processing/partitioner.rb +3 -3
  97. data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
  98. data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
  99. data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
  100. data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
  101. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
  102. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
  103. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
  104. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
  105. data/lib/karafka/pro/processing/strategies/default.rb +22 -22
  106. data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
  107. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
  108. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
  109. data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
  110. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
  111. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
  112. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
  113. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
  114. data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
  115. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
  116. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
  117. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
  118. data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
  119. data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
  120. data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
  121. data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
  122. data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
  123. data/lib/karafka/pro/recurring_tasks.rb +4 -4
  124. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
  125. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  126. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
  127. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
  128. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
  129. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
  130. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
  131. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
  132. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
  133. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
  134. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
  135. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
  136. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
  137. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  138. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
  139. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
  140. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
  141. data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
  142. data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
  143. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
  144. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
  145. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
  146. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
  147. data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
  148. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
  149. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
  150. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  151. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
  152. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
  153. data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
  154. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
  155. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
  156. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
  157. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  158. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
  159. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
  160. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
  161. data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
  162. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
  163. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
  164. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
  165. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
  166. data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
  167. data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
  168. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
  169. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  170. data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
  171. data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
  172. data/lib/karafka/pro/scheduled_messages.rb +2 -2
  173. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  174. data/lib/karafka/process.rb +1 -1
  175. data/lib/karafka/processing/coordinator.rb +1 -1
  176. data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
  177. data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
  178. data/lib/karafka/processing/jobs/base.rb +6 -4
  179. data/lib/karafka/processing/jobs_queue.rb +10 -0
  180. data/lib/karafka/processing/schedulers/default.rb +4 -4
  181. data/lib/karafka/processing/strategies/base.rb +6 -6
  182. data/lib/karafka/processing/strategies/default.rb +13 -13
  183. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  184. data/lib/karafka/processing/worker.rb +5 -5
  185. data/lib/karafka/railtie.rb +11 -11
  186. data/lib/karafka/routing/builder.rb +3 -3
  187. data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
  188. data/lib/karafka/routing/contracts/routing.rb +2 -2
  189. data/lib/karafka/routing/contracts/topic.rb +4 -4
  190. data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
  191. data/lib/karafka/routing/features/base/expander.rb +4 -4
  192. data/lib/karafka/routing/features/base.rb +8 -8
  193. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  194. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
  195. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
  196. data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
  197. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
  198. data/lib/karafka/routing/features/inline_insights.rb +7 -7
  199. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
  200. data/lib/karafka/routing/subscription_group.rb +9 -9
  201. data/lib/karafka/runner.rb +3 -3
  202. data/lib/karafka/server.rb +14 -5
  203. data/lib/karafka/setup/attributes_map.rb +7 -7
  204. data/lib/karafka/setup/config.rb +11 -11
  205. data/lib/karafka/setup/contracts/config.rb +2 -2
  206. data/lib/karafka/setup/defaults_injector.rb +11 -11
  207. data/lib/karafka/swarm/manager.rb +6 -6
  208. data/lib/karafka/swarm/node.rb +8 -37
  209. data/lib/karafka/swarm/producer_replacer.rb +110 -0
  210. data/lib/karafka/swarm/supervisor.rb +9 -6
  211. data/lib/karafka/swarm.rb +1 -1
  212. data/lib/karafka/time_trackers/pause.rb +1 -1
  213. data/lib/karafka/version.rb +1 -1
  214. data/lib/karafka.rb +36 -36
  215. metadata +7 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 664c8ed7d20d66153422c1b9e2475fbcaa166c7a62c938170a5909e48e1d9f99
4
- data.tar.gz: 973f79843261ef8dd002b2f13aa13e939e1e6b6d2f45ab8358af2a347c357c47
3
+ metadata.gz: b395542efd1d57ac4f9fd89091866a39c23e2c48d446bb20fb3628e7952f762e
4
+ data.tar.gz: 14aeb17e690a257bb0d8d7e57b45e797da19adf22f2e5873bd88e18a9c0a6aea
5
5
  SHA512:
6
- metadata.gz: 27df04323e3b2851c14dc01a9e34c4b6c21b5eec48eeb493e6c5c4941ce419c0cd8adc45c3cb898d377632af4e42750092b81fa0e46ee472556a357c9bbcc67f
7
- data.tar.gz: 6a7d33572e7b3b71cb41dc16a92b4581a964429fc246f82792a7ba74bcb1b56b870225ada7232e46dba3f1ef05923140a0b488b513d3a223b5890a7d4ff77df4
6
+ metadata.gz: e9003ff1411da366cc3f70a77e7b949bdda7fed9d717eb6a2c151e3844af1fb0cdfca9f4c220be8d4a692b750e99c202d2afa05e57ffbac7c274579d065ee54c
7
+ data.tar.gz: 4807ec1df8ed8f8169e6f38b9ce5b47e541485e4f4d551822bd2ea9d578219ee565ca71b27d66a9aeef55917ffa2d16fddd629371f48661afd94271eff9c228d
data/CHANGELOG.md CHANGED
@@ -1,5 +1,25 @@
1
1
  # Karafka Framework Changelog
2
2
 
3
+ ## 2.5.7 (2026-03-16)
4
+ - [Enhancement] Report detailed blocking information (active listeners, alive workers, and in-processing jobs) during forceful shutdown instead of only aggregate counts.
5
+ - [Enhancement] Improve `ForcefulShutdownError` description to clearly explain when and why it is raised.
6
+ - [Enhancement] Cache `messages.last` in `BatchMetadata` builder to avoid duplicate array traversal.
7
+ - [Enhancement] Optimize `VirtualOffsetManager#mark` to use a single array scan instead of separate `include?` and `index` calls (Pro).
8
+ - [Enhancement] Optimize `VirtualOffsetManager#materialize_real_offset` to use `keys.sort` instead of `to_a.sort_by` with tuple destructuring (Pro).
9
+ - [Enhancement] Optimize `IntervalRunner#call` to use a single `monotonic_now` call instead of two per invocation.
10
+ - [Enhancement] Support WaterDrop `:fd` mode in Swarm.
11
+ - [Maintenance] Use both `:fd` and `:thread` producer backends in CI.
12
+ - [Maintenance] Include spec file hash in integration test topic names for easier traceability in Kafka logs (#3056).
13
+ - [Fix] Remove duplicate topic creation in multi-broker health integration specs (#3056).
14
+ - [Fix] Preserve producer-specific kafka settings (e.g., `enable.idempotence`) when recreating the producer in swarm forks.
15
+
16
+ ## 2.5.6 (2026-02-28)
17
+ - **[Feature]** Add `karafka topics health` command to check Kafka topics for replication and durability issues, detecting no redundancy (RF=1), zero fault tolerance (RF≤min.insync), and low durability (min.insync=1) configurations with color-coded severity grouping and actionable recommendations (Pro).
18
+ - [Enhancement] Optimize license loading process by reading license files directly from the gem directory instead of requiring the entire gem, reducing initialization overhead and adding support for user-defined License modules.
19
+ - [Change] Migrate Admin `AbstractHandle#wait` calls from deprecated `max_wait_timeout` (seconds) to `max_wait_timeout_ms` (milliseconds) to align with `karafka-rdkafka` `0.24.0`.
20
+ - [Change] Require `karafka-rdkafka` `>=` `0.24.0` to support the new `max_wait_timeout_ms` API.
21
+ - [Fix] Fix LRJ partition freeze when filter returns `applied? true` with `action :skip`, causing partition to remain paused for ~31,709 years instead of resuming after processing.
22
+
3
23
  ## 2.5.5 (2026-01-24)
4
24
  - [Feature] Add multi-cluster Admin support via `Karafka::Admin.new(kafka: { ... })` allowing operations against different Kafka clusters while maintaining backward compatibility with existing class-method API.
5
25
 
data/LICENSE-COMM CHANGED
@@ -12,6 +12,10 @@ In order to use the Software under this Agreement, you must either: (a) receive
12
12
 
13
13
  1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold.
14
14
 
15
+ 1.1.1 Entity Definition. Where you have purchased or otherwise acquired the Software as or for an entity, "you" and "your" refer exclusively to the single legal entity (such as a corporation, limited liability company, partnership, or other legally recognized organization) that is the named licensee. This license does not extend to any parent company, subsidiary, affiliate, related entity, or other member of a corporate group, regardless of ownership structure or control relationships. Each separate legal entity requires its own license.
16
+
17
+ 1.1.2 Mergers, Acquisitions, and Change of Control. In the event of a merger, acquisition, consolidation, or other change of control affecting the licensee entity: (a) if the licensee entity ceases to exist as a separate legal entity, this license terminates and the surviving or acquiring entity must purchase a new license or obtain prior written consent from Maciej Mensfeld to continue use of the Software; (b) if the licensee entity continues to exist as a separate legal entity (including as a wholly owned subsidiary), this license remains valid solely for that entity and does not extend to the parent, acquirer, or any affiliated entities unless explicitly agreed in writing by Maciej Mensfeld.
18
+
15
19
  1.2 Pro License. If you purchased a Pro License (included with the Karafka Pro Software), you may install the Software on an unlimited number of Hosts. "Host" means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. "Worker" means a thread within a Karafka server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers.
16
20
 
17
21
  1.3 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- ![karafka logo](https://cdn.karafka.io/assets/misc/logo/karafka_logotype_transparent2.png)
1
+ ![karafka logo](https://karafka.io/assets/misc/logo/karafka_logotype_transparent2.png)
2
2
 
3
3
  [![Build Status](https://github.com/karafka/karafka/actions/workflows/ci_linux_ubuntu_x86_64_gnu.yml/badge.svg)](https://github.com/karafka/karafka/actions/workflows/ci_linux_ubuntu_x86_64_gnu.yml)
4
4
  [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
@@ -39,7 +39,7 @@ Karafka **uses** threads to handle many messages simultaneously in the same proc
39
39
 
40
40
  ## Getting started
41
41
 
42
- ![karafka web ui](https://cdn.karafka.io/assets/misc/printscreens/web-ui.png)
42
+ ![karafka web ui](https://karafka.io/assets/misc/printscreens/web-ui.png)
43
43
 
44
44
  If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
45
45
 
data/certs/expired.txt ADDED
@@ -0,0 +1,2 @@
1
+ # Expired/revoked license checksums (SHA256 of license token)
2
+ # One checksum per line, lines starting with # are comments
data/karafka.gemspec CHANGED
@@ -1,19 +1,19 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- lib = File.expand_path('lib', __dir__)
3
+ lib = File.expand_path("lib", __dir__)
4
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
- require 'karafka/version'
6
+ require "karafka/version"
7
7
 
8
8
  Gem::Specification.new do |spec|
9
- spec.name = 'karafka'
10
- spec.version = Karafka::VERSION
11
- spec.platform = Gem::Platform::RUBY
12
- spec.authors = ['Maciej Mensfeld']
13
- spec.email = %w[contact@karafka.io]
14
- spec.homepage = 'https://karafka.io'
15
- spec.licenses = %w[LGPL-3.0-only Commercial]
16
- spec.summary = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
9
+ spec.name = "karafka"
10
+ spec.version = Karafka::VERSION
11
+ spec.platform = Gem::Platform::RUBY
12
+ spec.authors = ["Maciej Mensfeld"]
13
+ spec.email = %w[contact@karafka.io]
14
+ spec.homepage = "https://karafka.io"
15
+ spec.licenses = %w[LGPL-3.0-only Commercial]
16
+ spec.summary = "Karafka is Ruby and Rails efficient Kafka processing framework."
17
17
  spec.description = <<-DESC
18
18
  Karafka is Ruby and Rails efficient Kafka processing framework.
19
19
 
@@ -21,12 +21,12 @@ Gem::Specification.new do |spec|
21
21
  without having to focus on things that are not your business domain.
22
22
  DESC
23
23
 
24
- spec.add_dependency 'karafka-core', '>= 2.5.6', '< 2.6.0'
25
- spec.add_dependency 'karafka-rdkafka', '>= 0.23.1'
26
- spec.add_dependency 'waterdrop', '>= 2.8.14', '< 3.0.0'
27
- spec.add_dependency 'zeitwerk', '~> 2.3'
24
+ spec.add_dependency "karafka-core", ">= 2.5.6", "< 2.6.0"
25
+ spec.add_dependency "karafka-rdkafka", ">= 0.24.0"
26
+ spec.add_dependency "waterdrop", ">= 2.8.14", "< 3.0.0"
27
+ spec.add_dependency "zeitwerk", "~> 2.3"
28
28
 
29
- spec.required_ruby_version = '>= 3.2.0'
29
+ spec.required_ruby_version = ">= 3.2.0"
30
30
 
31
31
  gem_files = %w[
32
32
  lib/**/*
@@ -41,16 +41,16 @@ Gem::Specification.new do |spec|
41
41
 
42
42
  spec.files = Dir.glob(gem_files) & `git ls-files -z`.split("\x0")
43
43
 
44
- spec.executables = %w[karafka]
44
+ spec.executables = %w[karafka]
45
45
  spec.require_paths = %w[lib]
46
46
 
47
47
  spec.metadata = {
48
- 'funding_uri' => 'https://karafka.io/#become-pro',
49
- 'homepage_uri' => 'https://karafka.io',
50
- 'changelog_uri' => 'https://karafka.io/docs/Changelog-Karafka',
51
- 'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
52
- 'source_code_uri' => 'https://github.com/karafka/karafka',
53
- 'documentation_uri' => 'https://karafka.io/docs',
54
- 'rubygems_mfa_required' => 'true'
48
+ "funding_uri" => "https://karafka.io/#become-pro",
49
+ "homepage_uri" => "https://karafka.io",
50
+ "changelog_uri" => "https://karafka.io/docs/Changelog-Karafka",
51
+ "bug_tracker_uri" => "https://github.com/karafka/karafka/issues",
52
+ "source_code_uri" => "https://github.com/karafka/karafka",
53
+ "documentation_uri" => "https://karafka.io/docs",
54
+ "rubygems_mfa_required" => "true"
55
55
  }
56
56
  end
@@ -2,9 +2,9 @@
2
2
 
3
3
  begin
4
4
  # Do not load active job if already loaded
5
- require 'active_job' unless Object.const_defined?('ActiveJob')
5
+ require "active_job" unless Object.const_defined?("ActiveJob")
6
6
 
7
- require_relative 'queue_adapters/karafka_adapter'
7
+ require_relative "queue_adapters/karafka_adapter"
8
8
 
9
9
  module ActiveJob
10
10
  # Namespace for usage simplification outside of Rails where Railtie will not kick in.
@@ -22,11 +22,11 @@ module ActiveJob
22
22
  #
23
23
  # @see https://github.com/sidekiq/sidekiq/issues/6746 Similar issue in Sidekiq
24
24
  base = if defined?(Rails::VERSION)
25
- (Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2 ? Object : AbstractAdapter)
26
- else
27
- # Fallback when Rails is not loaded
28
- Object
29
- end
25
+ ((Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2) ? Object : AbstractAdapter)
26
+ else
27
+ # Fallback when Rails is not loaded
28
+ Object
29
+ end
30
30
 
31
31
  # Karafka adapter for enqueuing jobs
32
32
  # This is here for ease of integration with ActiveJob.
@@ -29,13 +29,13 @@ module Karafka
29
29
  # @param job_message [Karafka::Messages::Message] message with active job
30
30
  def consume_job(job_message)
31
31
  with_deserialized_job(job_message) do |job|
32
- tags.add(:job_class, job['job_class'])
32
+ tags.add(:job_class, job["job_class"])
33
33
 
34
34
  payload = { caller: self, job: job, message: job_message }
35
35
 
36
36
  # We publish both to make it consistent with `consumer.x` events
37
- Karafka.monitor.instrument('active_job.consume', payload)
38
- Karafka.monitor.instrument('active_job.consumed', payload) do
37
+ Karafka.monitor.instrument("active_job.consume", payload)
38
+ Karafka.monitor.instrument("active_job.consumed", payload) do
39
39
  ::ActiveJob::Base.execute(job)
40
40
  end
41
41
  end
@@ -1,9 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_support/current_attributes'
4
- require_relative 'current_attributes/job_wrapper'
5
- require_relative 'current_attributes/loading'
6
- require_relative 'current_attributes/persistence'
3
+ require "active_support/current_attributes"
4
+ require_relative "current_attributes/job_wrapper"
5
+ require_relative "current_attributes/loading"
6
+ require_relative "current_attributes/persistence"
7
7
 
8
8
  # This code is based on Sidekiqs approach to persisting current attributes
9
9
  # @see https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/current_attributes.rb
@@ -9,8 +9,8 @@ module Karafka
9
9
  class JobOptionsContract < Contracts::Base
10
10
  configure do |config|
11
11
  config.error_messages = YAML.safe_load_file(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- ).fetch('en').fetch('validations').fetch('job_options')
12
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
13
+ ).fetch("en").fetch("validations").fetch("job_options")
14
14
  end
15
15
 
16
16
  optional(:dispatch_method) do |val|
@@ -179,7 +179,7 @@ module Karafka
179
179
  principal: nil,
180
180
  operation: :any,
181
181
  permission_type: :any,
182
- host: '*'
182
+ host: "*"
183
183
  )
184
184
  )
185
185
  end
@@ -235,7 +235,7 @@ module Karafka
235
235
  resource_name: nil,
236
236
  resource_pattern_type: nil,
237
237
  principal: nil,
238
- host: '*',
238
+ host: "*",
239
239
  operation: nil,
240
240
  permission_type: nil
241
241
  )
@@ -276,7 +276,7 @@ module Karafka
276
276
  # Makes sure that admin is closed afterwards.
277
277
  def with_admin_wait
278
278
  with_admin do |admin|
279
- yield(admin).wait(max_wait_timeout: self.class.max_wait_time)
279
+ yield(admin).wait(max_wait_timeout_ms: self.class.max_wait_time)
280
280
  end
281
281
  end
282
282
 
@@ -45,7 +45,7 @@ module Karafka
45
45
  # def set(name, value)
46
46
  # @operations[0] << Config.new(name: name, value: value.to_s)
47
47
  # end
48
- default_value = op_name == :delete ? ' = nil' : ''
48
+ default_value = (op_name == :delete) ? " = nil" : ""
49
49
  class_eval <<~RUBY, __FILE__, __LINE__ + 1
50
50
  # @param name [String] name of the config to alter
51
51
  # @param value [String] value of the config
@@ -109,7 +109,7 @@ module Karafka
109
109
  # Makes sure that admin is closed afterwards.
110
110
  def with_admin_wait
111
111
  with_admin do |admin|
112
- yield(admin).wait(max_wait_timeout: self.class.max_wait_time)
112
+ yield(admin).wait(max_wait_timeout_ms: self.class.max_wait_time)
113
113
  end
114
114
  end
115
115
  end
@@ -129,11 +129,11 @@ module Karafka
129
129
  # Earliest is not always 0. When compacting/deleting it can be much later, that's why
130
130
  # we fetch the oldest possible offset
131
131
  # false is treated the same as 'earliest'
132
- when 'earliest', false
132
+ when "earliest", false
133
133
  LONG_TIME_AGO
134
134
  # Latest will always be the high-watermark offset and we can get it just by getting
135
135
  # a future position
136
- when 'latest'
136
+ when "latest"
137
137
  Time.now + DAY_IN_SECONDS
138
138
  # Regular offset case
139
139
  else
@@ -161,7 +161,7 @@ module Karafka
161
161
  end
162
162
  end
163
163
 
164
- settings = { 'group.id': consumer_group_id }
164
+ settings = { "group.id": consumer_group_id }
165
165
 
166
166
  with_consumer(settings) do |consumer|
167
167
  # If we have any time based stuff to resolve, we need to do it prior to commits
@@ -276,7 +276,7 @@ module Karafka
276
276
  def delete(consumer_group_id)
277
277
  with_admin do |admin|
278
278
  handler = admin.delete_group(consumer_group_id)
279
- handler.wait(max_wait_timeout: max_wait_time_seconds)
279
+ handler.wait(max_wait_timeout_ms: max_wait_time_ms)
280
280
  end
281
281
  end
282
282
 
@@ -333,9 +333,9 @@ module Karafka
333
333
  # This ensures we use the same settings as the actual consumers
334
334
  # Following the same pattern as in Karafka::Connection::Client#build_kafka
335
335
  consumer_settings = Setup::AttributesMap.consumer(first_topic.kafka.dup)
336
- consumer_settings[:'group.id'] = consumer_group.id
337
- consumer_settings[:'enable.auto.offset.store'] = false
338
- consumer_settings[:'auto.offset.reset'] ||= first_topic.initial_offset
336
+ consumer_settings[:"group.id"] = consumer_group.id
337
+ consumer_settings[:"enable.auto.offset.store"] = false
338
+ consumer_settings[:"auto.offset.reset"] ||= first_topic.initial_offset
339
339
 
340
340
  with_consumer(consumer_settings) do |consumer|
341
341
  # Subscribe to the topics - this triggers the first rebalance
@@ -408,7 +408,7 @@ module Karafka
408
408
 
409
409
  tpl = Rdkafka::Consumer::TopicPartitionList.new
410
410
 
411
- with_consumer('group.id': cg) do |consumer|
411
+ with_consumer("group.id": cg) do |consumer|
412
412
  topics.each { |topic| tpl.add_topic(topic, existing_topics[topic]) }
413
413
 
414
414
  commit_offsets = consumer.committed(tpl)
@@ -8,8 +8,8 @@ module Karafka
8
8
  class Replication < Karafka::Contracts::Base
9
9
  configure do |config|
10
10
  config.error_messages = YAML.safe_load_file(
11
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
12
- ).fetch('en').fetch('validations').fetch('admin').fetch('replication')
11
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
12
+ ).fetch("en").fetch("validations").fetch("admin").fetch("replication")
13
13
  end
14
14
 
15
15
  required(:topic) { |val| val.is_a?(String) && !val.empty? }
@@ -262,7 +262,7 @@ module Karafka
262
262
  change = @target_replication_factor - @current_replication_factor
263
263
  broker_nodes = @cluster_info[:brokers].map do |broker_info|
264
264
  broker_info[:node_id]
265
- end.join(', ')
265
+ end.join(", ")
266
266
 
267
267
  <<~SUMMARY
268
268
  Replication Increase Plan for Topic: #{@topic}
@@ -328,10 +328,10 @@ module Karafka
328
328
  # Handle both :replicas (array of objects) and :replica_brokers (array of IDs)
329
329
  replicas = partition_info[:replicas] || partition_info[:replica_brokers] || []
330
330
  current_replicas = if replicas.first.respond_to?(:node_id)
331
- replicas.map(&:node_id).sort
332
- else
333
- replicas.sort
334
- end
331
+ replicas.map(&:node_id).sort
332
+ else
333
+ replicas.sort
334
+ end
335
335
 
336
336
  if rebalance_only
337
337
  # For rebalancing, redistribute current replicas optimally
@@ -442,22 +442,22 @@ module Karafka
442
442
  # Builds the kafka-reassign-partitions.sh command for generating reassignment plan
443
443
  # @return [String] command template with placeholder for broker addresses
444
444
  def build_generate_command
445
- 'kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> ' \
446
- '--reassignment-json-file reassignment.json --generate'
445
+ "kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> " \
446
+ "--reassignment-json-file reassignment.json --generate"
447
447
  end
448
448
 
449
449
  # Builds the kafka-reassign-partitions.sh command for executing reassignment
450
450
  # @return [String] command template with placeholder for broker addresses
451
451
  def build_execute_command
452
- 'kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> ' \
453
- '--reassignment-json-file reassignment.json --execute'
452
+ "kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> " \
453
+ "--reassignment-json-file reassignment.json --execute"
454
454
  end
455
455
 
456
456
  # Builds the kafka-reassign-partitions.sh command for verifying reassignment progress
457
457
  # @return [String] command template with placeholder for broker addresses
458
458
  def build_verify_command
459
- 'kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> ' \
460
- '--reassignment-json-file reassignment.json --verify'
459
+ "kafka-reassign-partitions.sh --bootstrap-server <KAFKA_BROKERS> " \
460
+ "--reassignment-json-file reassignment.json --verify"
461
461
  end
462
462
 
463
463
  # Generates detailed step-by-step instructions for executing the reassignment
@@ -469,16 +469,16 @@ module Karafka
469
469
  "2. Validate the plan (optional): #{@execution_commands[:generate]}",
470
470
  "3. Execute the reassignment: #{@execution_commands[:execute]}",
471
471
  "4. Monitor progress: #{@execution_commands[:verify]}",
472
- '5. Verify completion by checking topic metadata',
473
- '',
474
- 'IMPORTANT NOTES:',
475
- '- Replace <KAFKA_BROKERS> with your actual Kafka broker addresses',
476
- '- The reassignment process may take time depending on data size',
477
- '- Monitor disk space and network I/O during reassignment',
478
- '- Consider running during low-traffic periods',
479
- '- For large topics, consider throttling replica transfer rate',
480
- '- Ensure sufficient disk space on target brokers before starting',
481
- '- Keep monitoring until all replicas are in-sync (ISR)'
472
+ "5. Verify completion by checking topic metadata",
473
+ "",
474
+ "IMPORTANT NOTES:",
475
+ "- Replace <KAFKA_BROKERS> with your actual Kafka broker addresses",
476
+ "- The reassignment process may take time depending on data size",
477
+ "- Monitor disk space and network I/O during reassignment",
478
+ "- Consider running during low-traffic periods",
479
+ "- For large topics, consider throttling replica transfer rate",
480
+ "- Ensure sufficient disk space on target brokers before starting",
481
+ "- Keep monitoring until all replicas are in-sync (ISR)"
482
482
  ]
483
483
  end
484
484
  end
@@ -147,7 +147,7 @@ module Karafka
147
147
  handler = admin.create_topic(name, partitions, replication_factor, topic_config)
148
148
 
149
149
  with_re_wait(
150
- -> { handler.wait(max_wait_timeout: max_wait_time_seconds) },
150
+ -> { handler.wait(max_wait_timeout_ms: max_wait_time_ms) },
151
151
  -> { names.include?(name) }
152
152
  )
153
153
  end
@@ -163,7 +163,7 @@ module Karafka
163
163
  handler = admin.delete_topic(name)
164
164
 
165
165
  with_re_wait(
166
- -> { handler.wait(max_wait_timeout: max_wait_time_seconds) },
166
+ -> { handler.wait(max_wait_timeout_ms: max_wait_time_ms) },
167
167
  -> { !names.include?(name) }
168
168
  )
169
169
  end
@@ -180,7 +180,7 @@ module Karafka
180
180
  handler = admin.create_partitions(name, partitions)
181
181
 
182
182
  with_re_wait(
183
- -> { handler.wait(max_wait_timeout: max_wait_time_seconds) },
183
+ -> { handler.wait(max_wait_timeout_ms: max_wait_time_ms) },
184
184
  -> { info(name).fetch(:partition_count) >= partitions }
185
185
  )
186
186
  end
@@ -272,9 +272,9 @@ module Karafka
272
272
 
273
273
  real_offsets = consumer.offsets_for_times(tpl)
274
274
  detected_offset = real_offsets
275
- .to_h
276
- .fetch(name)
277
- .find { |p_data| p_data.partition == partition }
275
+ .to_h
276
+ .fetch(name)
277
+ .find { |p_data| p_data.partition == partition }
278
278
 
279
279
  detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
280
280
  else
data/lib/karafka/admin.rb CHANGED
@@ -400,10 +400,9 @@ module Karafka
400
400
 
401
401
  private
402
402
 
403
- # @return [Integer] number of seconds to wait. `rdkafka` requires this value
404
- # (`max_wait_time`) to be provided in seconds while we define it in ms hence the conversion
405
- def max_wait_time_seconds
406
- self.class.max_wait_time / 1_000.0
403
+ # @return [Integer] max wait time in ms
404
+ def max_wait_time_ms
405
+ self.class.max_wait_time
407
406
  end
408
407
 
409
408
  # Adds a new callback for given rdkafka instance for oauth token refresh (if needed)
@@ -464,7 +463,7 @@ module Karafka
464
463
  def config(type, settings)
465
464
  kafka_config = self.class.app_kafka.dup
466
465
  kafka_config.merge!(self.class.admin_kafka)
467
- kafka_config[:'group.id'] = self.class.group_id
466
+ kafka_config[:"group.id"] = self.class.group_id
468
467
  # We merge after setting the group id so it can be altered if needed
469
468
  # In general in admin we only should alter it when we need to impersonate a given
470
469
  # consumer group or do something similar
data/lib/karafka/app.rb CHANGED
@@ -13,7 +13,7 @@ module Karafka
13
13
  # Per recommendation, this should not run in children nodes
14
14
  return if Karafka::App.config.swarm.node
15
15
 
16
- monitor.instrument('app.before_warmup', caller: self)
16
+ monitor.instrument("app.before_warmup", caller: self)
17
17
 
18
18
  return GC.compact unless ::Process.respond_to?(:warmup)
19
19
 
@@ -48,7 +48,7 @@ module Karafka
48
48
  end
49
49
 
50
50
  # Just a nicer name for the consumer groups
51
- alias routes consumer_groups
51
+ alias_method :routes, :consumer_groups
52
52
 
53
53
  # Returns current assignments of this process. Both topics and partitions
54
54
  #
@@ -101,7 +101,7 @@ module Karafka
101
101
  # producing messages.
102
102
  #
103
103
  # @param contexts [String] librdkafka low level debug contexts for granular debugging
104
- def debug!(contexts = 'all')
104
+ def debug!(contexts = "all")
105
105
  logger.level = Logger::DEBUG
106
106
  producer.config.logger.level = Logger::DEBUG
107
107