karafka 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +40 -16
  5. data/.rspec +4 -0
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +459 -796
  8. data/Gemfile +3 -2
  9. data/Gemfile.lock +44 -32
  10. data/LICENSE +1 -1
  11. data/README.md +22 -17
  12. data/bin/integrations +86 -26
  13. data/bin/karafka +1 -13
  14. data/bin/rspecs +6 -0
  15. data/bin/verify_license_integrity +37 -0
  16. data/certs/cert_chain.pem +26 -0
  17. data/config/{errors.yml → locales/errors.yml} +34 -6
  18. data/config/locales/pro_errors.yml +34 -0
  19. data/docker-compose.yml +4 -30
  20. data/karafka.gemspec +17 -10
  21. data/lib/active_job/karafka.rb +3 -7
  22. data/lib/active_job/queue_adapters/karafka_adapter.rb +7 -1
  23. data/lib/karafka/active_job/consumer.rb +29 -6
  24. data/lib/karafka/active_job/current_attributes/loading.rb +36 -0
  25. data/lib/karafka/active_job/current_attributes/persistence.rb +28 -0
  26. data/lib/karafka/active_job/current_attributes.rb +42 -0
  27. data/lib/karafka/active_job/dispatcher.rb +34 -3
  28. data/lib/karafka/active_job/job_options_contract.rb +13 -2
  29. data/lib/karafka/admin.rb +246 -0
  30. data/lib/karafka/app.rb +30 -7
  31. data/lib/karafka/base_consumer.rb +124 -82
  32. data/lib/karafka/cli/base.rb +20 -0
  33. data/lib/karafka/cli/console.rb +13 -8
  34. data/lib/karafka/cli/info.rb +3 -2
  35. data/lib/karafka/cli/server.rb +55 -8
  36. data/lib/karafka/cli/topics.rb +146 -0
  37. data/lib/karafka/connection/client.rb +118 -61
  38. data/lib/karafka/connection/consumer_group_coordinator.rb +48 -0
  39. data/lib/karafka/connection/listener.rb +78 -30
  40. data/lib/karafka/connection/listeners_batch.rb +18 -2
  41. data/lib/karafka/connection/rebalance_manager.rb +3 -0
  42. data/lib/karafka/contracts/config.rb +1 -1
  43. data/lib/karafka/contracts/consumer_group.rb +22 -9
  44. data/lib/karafka/contracts/server_cli_options.rb +71 -11
  45. data/lib/karafka/contracts/topic.rb +65 -0
  46. data/lib/karafka/contracts.rb +1 -1
  47. data/lib/karafka/embedded.rb +36 -0
  48. data/lib/karafka/errors.rb +6 -4
  49. data/lib/karafka/helpers/colorize.rb +6 -0
  50. data/lib/karafka/instrumentation/callbacks/error.rb +3 -4
  51. data/lib/karafka/instrumentation/callbacks/statistics.rb +14 -4
  52. data/lib/karafka/instrumentation/logger_listener.rb +151 -22
  53. data/lib/karafka/instrumentation/notifications.rb +26 -7
  54. data/lib/karafka/instrumentation/proctitle_listener.rb +7 -16
  55. data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -1
  56. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +153 -0
  57. data/lib/karafka/instrumentation/vendors/datadog/{listener.rb → metrics_listener.rb} +62 -35
  58. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +166 -0
  59. data/lib/karafka/licenser.rb +53 -50
  60. data/lib/karafka/messages/builders/batch_metadata.rb +8 -8
  61. data/lib/karafka/messages/builders/messages.rb +6 -2
  62. data/lib/karafka/messages/message.rb +14 -2
  63. data/lib/karafka/messages/messages.rb +5 -0
  64. data/lib/karafka/messages/parser.rb +14 -0
  65. data/lib/karafka/patches/rdkafka/bindings.rb +132 -0
  66. data/lib/karafka/pro/active_job/consumer.rb +10 -9
  67. data/lib/karafka/pro/active_job/dispatcher.rb +28 -3
  68. data/lib/karafka/pro/active_job/job_options_contract.rb +16 -3
  69. data/lib/karafka/pro/encryption/cipher.rb +58 -0
  70. data/lib/karafka/pro/encryption/contracts/config.rb +79 -0
  71. data/lib/karafka/pro/encryption/errors.rb +24 -0
  72. data/lib/karafka/pro/encryption/messages/middleware.rb +46 -0
  73. data/lib/karafka/pro/encryption/messages/parser.rb +56 -0
  74. data/lib/karafka/pro/encryption/setup/config.rb +48 -0
  75. data/lib/karafka/pro/encryption.rb +47 -0
  76. data/lib/karafka/pro/iterator.rb +253 -0
  77. data/lib/karafka/pro/loader.rb +52 -26
  78. data/lib/karafka/pro/performance_tracker.rb +10 -6
  79. data/lib/karafka/pro/processing/collapser.rb +62 -0
  80. data/lib/karafka/pro/processing/coordinator.rb +92 -16
  81. data/lib/karafka/pro/processing/filters/base.rb +61 -0
  82. data/lib/karafka/pro/processing/filters/delayer.rb +70 -0
  83. data/lib/karafka/pro/processing/filters/expirer.rb +51 -0
  84. data/lib/karafka/pro/processing/filters/throttler.rb +84 -0
  85. data/lib/karafka/pro/processing/filters/virtual_limiter.rb +52 -0
  86. data/lib/karafka/pro/processing/filters_applier.rb +104 -0
  87. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +7 -5
  88. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +37 -0
  89. data/lib/karafka/pro/processing/jobs_builder.rb +22 -4
  90. data/lib/karafka/pro/processing/partitioner.rb +33 -24
  91. data/lib/karafka/pro/processing/scheduler.rb +27 -8
  92. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +68 -0
  93. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +74 -0
  94. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +72 -0
  95. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +76 -0
  96. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +64 -0
  97. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +68 -0
  98. data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +64 -0
  99. data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +69 -0
  100. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +38 -0
  101. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +64 -0
  102. data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +38 -0
  103. data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +58 -0
  104. data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +37 -0
  105. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +79 -0
  106. data/lib/karafka/pro/processing/strategies/aj/mom.rb +36 -0
  107. data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +52 -0
  108. data/lib/karafka/pro/processing/strategies/base.rb +26 -0
  109. data/lib/karafka/pro/processing/strategies/default.rb +105 -0
  110. data/lib/karafka/pro/processing/strategies/dlq/default.rb +131 -0
  111. data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +61 -0
  112. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +75 -0
  113. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +71 -0
  114. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +43 -0
  115. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +41 -0
  116. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +69 -0
  117. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +41 -0
  118. data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +40 -0
  119. data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +63 -0
  120. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +63 -0
  121. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +36 -0
  122. data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +39 -0
  123. data/lib/karafka/pro/processing/strategies/dlq/mom.rb +68 -0
  124. data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +37 -0
  125. data/lib/karafka/pro/processing/strategies/dlq/vp.rb +40 -0
  126. data/lib/karafka/pro/processing/strategies/ftr/default.rb +104 -0
  127. data/lib/karafka/pro/processing/strategies/ftr/vp.rb +40 -0
  128. data/lib/karafka/pro/processing/strategies/lrj/default.rb +87 -0
  129. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +69 -0
  130. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +67 -0
  131. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +40 -0
  132. data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +39 -0
  133. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +80 -0
  134. data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +38 -0
  135. data/lib/karafka/pro/processing/strategies/lrj/vp.rb +36 -0
  136. data/lib/karafka/pro/processing/strategies/mom/default.rb +46 -0
  137. data/lib/karafka/pro/processing/strategies/mom/ftr.rb +53 -0
  138. data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +37 -0
  139. data/lib/karafka/pro/processing/strategies/mom/vp.rb +35 -0
  140. data/lib/karafka/pro/processing/strategies/vp/default.rb +104 -0
  141. data/lib/karafka/pro/{contracts/base.rb → processing/strategies.rb} +6 -5
  142. data/lib/karafka/pro/processing/strategy_selector.rb +84 -0
  143. data/lib/karafka/pro/processing/virtual_offset_manager.rb +147 -0
  144. data/lib/karafka/pro/routing/features/base.rb +24 -0
  145. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +50 -0
  146. data/lib/karafka/pro/routing/features/dead_letter_queue.rb +27 -0
  147. data/lib/karafka/pro/routing/{builder_extensions.rb → features/delaying/config.rb} +8 -11
  148. data/lib/karafka/pro/routing/features/delaying/contract.rb +38 -0
  149. data/lib/karafka/pro/routing/features/delaying/topic.rb +59 -0
  150. data/lib/karafka/pro/routing/features/delaying.rb +29 -0
  151. data/lib/karafka/pro/routing/features/expiring/config.rb +27 -0
  152. data/lib/karafka/pro/routing/features/expiring/contract.rb +38 -0
  153. data/lib/karafka/pro/routing/features/expiring/topic.rb +59 -0
  154. data/lib/karafka/pro/routing/features/expiring.rb +27 -0
  155. data/lib/karafka/pro/routing/features/filtering/config.rb +40 -0
  156. data/lib/karafka/pro/routing/features/filtering/contract.rb +41 -0
  157. data/lib/karafka/pro/routing/features/filtering/topic.rb +51 -0
  158. data/lib/karafka/pro/routing/features/filtering.rb +27 -0
  159. data/lib/karafka/pro/routing/features/long_running_job/config.rb +28 -0
  160. data/lib/karafka/pro/routing/features/long_running_job/contract.rb +37 -0
  161. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +42 -0
  162. data/lib/karafka/pro/routing/features/long_running_job.rb +28 -0
  163. data/lib/karafka/pro/routing/features/throttling/config.rb +32 -0
  164. data/lib/karafka/pro/routing/features/throttling/contract.rb +41 -0
  165. data/lib/karafka/pro/routing/features/throttling/topic.rb +69 -0
  166. data/lib/karafka/pro/routing/features/throttling.rb +30 -0
  167. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +30 -0
  168. data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +52 -0
  169. data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +56 -0
  170. data/lib/karafka/pro/routing/features/virtual_partitions.rb +27 -0
  171. data/lib/karafka/process.rb +23 -8
  172. data/lib/karafka/processing/coordinator.rb +102 -23
  173. data/lib/karafka/processing/coordinators_buffer.rb +5 -1
  174. data/lib/karafka/processing/executor.rb +51 -21
  175. data/lib/karafka/processing/executors_buffer.rb +10 -26
  176. data/lib/karafka/processing/jobs/base.rb +6 -2
  177. data/lib/karafka/processing/jobs/consume.rb +8 -5
  178. data/lib/karafka/processing/jobs/idle.rb +24 -0
  179. data/lib/karafka/processing/jobs_builder.rb +2 -3
  180. data/lib/karafka/processing/jobs_queue.rb +14 -8
  181. data/lib/karafka/processing/partitioner.rb +3 -1
  182. data/lib/karafka/processing/result.rb +14 -1
  183. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +44 -0
  184. data/lib/karafka/processing/strategies/aj_mom.rb +21 -0
  185. data/lib/karafka/processing/strategies/base.rb +52 -0
  186. data/lib/karafka/processing/strategies/default.rb +142 -0
  187. data/lib/karafka/processing/strategies/dlq.rb +88 -0
  188. data/lib/karafka/processing/strategies/dlq_mom.rb +49 -0
  189. data/lib/karafka/processing/strategies/mom.rb +29 -0
  190. data/lib/karafka/processing/strategy_selector.rb +47 -0
  191. data/lib/karafka/processing/worker.rb +8 -3
  192. data/lib/karafka/railtie.rb +24 -12
  193. data/lib/karafka/routing/activity_manager.rb +84 -0
  194. data/lib/karafka/routing/builder.rb +20 -0
  195. data/lib/karafka/routing/consumer_group.rb +38 -4
  196. data/lib/karafka/routing/consumer_mapper.rb +0 -10
  197. data/lib/karafka/routing/features/active_job/builder.rb +33 -0
  198. data/lib/karafka/routing/features/active_job/config.rb +15 -0
  199. data/lib/karafka/routing/features/active_job/contract.rb +41 -0
  200. data/lib/karafka/routing/features/active_job/topic.rb +33 -0
  201. data/lib/karafka/routing/features/active_job.rb +13 -0
  202. data/lib/karafka/routing/features/base/expander.rb +53 -0
  203. data/lib/karafka/routing/features/base.rb +34 -0
  204. data/lib/karafka/routing/features/dead_letter_queue/config.rb +19 -0
  205. data/lib/karafka/routing/features/dead_letter_queue/contract.rb +42 -0
  206. data/lib/karafka/routing/features/dead_letter_queue/topic.rb +41 -0
  207. data/lib/karafka/routing/features/dead_letter_queue.rb +16 -0
  208. data/lib/karafka/routing/features/declaratives/config.rb +18 -0
  209. data/lib/karafka/routing/features/declaratives/contract.rb +30 -0
  210. data/lib/karafka/routing/features/declaratives/topic.rb +44 -0
  211. data/lib/karafka/routing/features/declaratives.rb +14 -0
  212. data/lib/karafka/routing/features/manual_offset_management/config.rb +15 -0
  213. data/lib/karafka/routing/features/manual_offset_management/contract.rb +24 -0
  214. data/lib/karafka/routing/features/manual_offset_management/topic.rb +35 -0
  215. data/lib/karafka/routing/features/manual_offset_management.rb +18 -0
  216. data/lib/karafka/routing/proxy.rb +18 -20
  217. data/lib/karafka/routing/router.rb +27 -2
  218. data/lib/karafka/routing/subscription_group.rb +43 -5
  219. data/lib/karafka/routing/subscription_groups_builder.rb +6 -1
  220. data/lib/karafka/routing/topic.rb +17 -5
  221. data/lib/karafka/routing/topics.rb +8 -0
  222. data/lib/karafka/runner.rb +1 -0
  223. data/lib/karafka/serialization/json/deserializer.rb +1 -0
  224. data/lib/karafka/server.rb +58 -28
  225. data/lib/karafka/setup/attributes_map.rb +344 -0
  226. data/lib/karafka/setup/config.rb +48 -19
  227. data/lib/karafka/status.rb +35 -7
  228. data/lib/karafka/templates/karafka.rb.erb +14 -6
  229. data/lib/karafka/time_trackers/base.rb +1 -6
  230. data/lib/karafka/time_trackers/pause.rb +49 -19
  231. data/lib/karafka/time_trackers/poll.rb +7 -3
  232. data/lib/karafka/version.rb +1 -1
  233. data/lib/karafka.rb +56 -3
  234. data/renovate.json +6 -0
  235. data.tar.gz.sig +0 -0
  236. metadata +189 -61
  237. metadata.gz.sig +0 -0
  238. data/bin/wait_for_kafka +0 -20
  239. data/certs/mensfeld.pem +0 -25
  240. data/lib/karafka/active_job/routing/extensions.rb +0 -31
  241. data/lib/karafka/contracts/consumer_group_topic.rb +0 -42
  242. data/lib/karafka/instrumentation.rb +0 -21
  243. data/lib/karafka/pro/base_consumer.rb +0 -82
  244. data/lib/karafka/pro/contracts/consumer_group.rb +0 -34
  245. data/lib/karafka/pro/contracts/consumer_group_topic.rb +0 -33
  246. data/lib/karafka/pro/routing/topic_extensions.rb +0 -38
@@ -1,34 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component.
4
- # All of the commercial components are present in the lib/karafka/pro directory of this
5
- # repository and their usage requires commercial license agreement.
6
- #
7
- # Karafka has also commercial-friendly license, commercial support and commercial components.
8
- #
9
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
10
- # your code to Maciej Mensfeld.
11
-
12
- module Karafka
13
- module Pro
14
- module Contracts
15
- # Contract for validating correct Pro components setup on a consumer group and topic levels
16
- class ConsumerGroup < Base
17
- virtual do |data, errors|
18
- next unless errors.empty?
19
- next unless data.key?(:topics)
20
-
21
- fetched_errors = []
22
-
23
- data.fetch(:topics).each do |topic|
24
- ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
25
- fetched_errors << [[topic, key].flatten, value]
26
- end
27
- end
28
-
29
- fetched_errors
30
- end
31
- end
32
- end
33
- end
34
- end
@@ -1,33 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component.
4
- # All of the commercial components are present in the lib/karafka/pro directory of this
5
- # repository and their usage requires commercial license agreement.
6
- #
7
- # Karafka has also commercial-friendly license, commercial support and commercial components.
8
- #
9
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
10
- # your code to Maciej Mensfeld.
11
-
12
- module Karafka
13
- module Pro
14
- module Contracts
15
- # Contract for validating correct Pro components setup on a topic levels
16
- class ConsumerGroupTopic < Base
17
- configure do |config|
18
- config.error_messages = YAML.safe_load(
19
- File.read(
20
- File.join(Karafka.gem_root, 'config', 'errors.yml')
21
- )
22
- ).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
23
- end
24
-
25
- virtual do |data|
26
- next if data[:consumer] < Karafka::Pro::BaseConsumer
27
-
28
- [[%i[consumer], :consumer_format]]
29
- end
30
- end
31
- end
32
- end
33
- end
@@ -1,38 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component.
4
- # All of the commercial components are present in the lib/karafka/pro directory of this
5
- # repository and their usage requires commercial license agreement.
6
- #
7
- # Karafka has also commercial-friendly license, commercial support and commercial components.
8
- #
9
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
10
- # your code to Maciej Mensfeld.
11
-
12
- module Karafka
13
- module Pro
14
- # Pro routing components
15
- module Routing
16
- # Routing extensions that allow to configure some extra PRO routing options
17
- module TopicExtensions
18
- class << self
19
- # @param base [Class] class we extend
20
- def included(base)
21
- base.attr_accessor :long_running_job
22
- base.attr_accessor :virtual_partitioner
23
- end
24
- end
25
-
26
- # @return [Boolean] true if virtual partitioner is defined, false otherwise
27
- def virtual_partitioner?
28
- virtual_partitioner != nil
29
- end
30
-
31
- # @return [Boolean] is a given job on a topic a long-running one
32
- def long_running_job?
33
- @long_running_job || false
34
- end
35
- end
36
- end
37
- end
38
- end