karafka 2.4.15 → 2.4.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +5 -5
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +15 -0
  6. data/Gemfile +1 -0
  7. data/Gemfile.lock +17 -14
  8. data/LICENSE +7 -2
  9. data/config/locales/errors.yml +1 -0
  10. data/docker-compose.yml +0 -2
  11. data/lib/karafka/base_consumer.rb +70 -16
  12. data/lib/karafka/connection/client.rb +2 -16
  13. data/lib/karafka/connection/rebalance_manager.rb +24 -13
  14. data/lib/karafka/instrumentation/logger_listener.rb +3 -0
  15. data/lib/karafka/instrumentation/notifications.rb +4 -0
  16. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +1 -26
  17. data/lib/karafka/pro/active_job/consumer.rb +2 -10
  18. data/lib/karafka/pro/active_job/dispatcher.rb +2 -10
  19. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -10
  20. data/lib/karafka/pro/base_consumer.rb +2 -10
  21. data/lib/karafka/pro/cleaner/errors.rb +2 -10
  22. data/lib/karafka/pro/cleaner/messages/message.rb +14 -12
  23. data/lib/karafka/pro/cleaner/messages/messages.rb +2 -10
  24. data/lib/karafka/pro/cleaner/messages/metadata.rb +41 -0
  25. data/lib/karafka/pro/cleaner.rb +3 -10
  26. data/lib/karafka/pro/connection/manager.rb +2 -10
  27. data/lib/karafka/pro/connection/multiplexing/listener.rb +2 -10
  28. data/lib/karafka/pro/contracts/base.rb +2 -10
  29. data/lib/karafka/pro/contracts/server_cli_options.rb +2 -10
  30. data/lib/karafka/pro/encryption/cipher.rb +2 -10
  31. data/lib/karafka/pro/encryption/contracts/config.rb +2 -10
  32. data/lib/karafka/pro/encryption/errors.rb +2 -10
  33. data/lib/karafka/pro/encryption/messages/middleware.rb +2 -10
  34. data/lib/karafka/pro/encryption/messages/parser.rb +2 -10
  35. data/lib/karafka/pro/encryption/setup/config.rb +2 -10
  36. data/lib/karafka/pro/encryption.rb +2 -10
  37. data/lib/karafka/pro/instrumentation/performance_tracker.rb +2 -10
  38. data/lib/karafka/pro/iterator/expander.rb +2 -10
  39. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -10
  40. data/lib/karafka/pro/iterator.rb +2 -10
  41. data/lib/karafka/pro/loader.rb +2 -10
  42. data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +2 -10
  43. data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +2 -10
  44. data/lib/karafka/pro/processing/collapser.rb +2 -10
  45. data/lib/karafka/pro/processing/coordinator.rb +2 -10
  46. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -10
  47. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +2 -10
  48. data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +2 -10
  49. data/lib/karafka/pro/processing/executor.rb +2 -10
  50. data/lib/karafka/pro/processing/expansions_selector.rb +2 -10
  51. data/lib/karafka/pro/processing/filters/base.rb +2 -10
  52. data/lib/karafka/pro/processing/filters/delayer.rb +2 -10
  53. data/lib/karafka/pro/processing/filters/expirer.rb +2 -10
  54. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +2 -10
  55. data/lib/karafka/pro/processing/filters/throttler.rb +2 -10
  56. data/lib/karafka/pro/processing/filters/virtual_limiter.rb +2 -10
  57. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +4 -10
  58. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +4 -10
  59. data/lib/karafka/pro/processing/jobs/periodic.rb +4 -10
  60. data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +4 -10
  61. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +4 -10
  62. data/lib/karafka/pro/processing/jobs_builder.rb +2 -10
  63. data/lib/karafka/pro/processing/jobs_queue.rb +2 -10
  64. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +2 -10
  65. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +2 -10
  66. data/lib/karafka/pro/processing/offset_metadata/listener.rb +2 -10
  67. data/lib/karafka/pro/processing/partitioner.rb +2 -10
  68. data/lib/karafka/pro/processing/periodic_job/consumer.rb +2 -10
  69. data/lib/karafka/pro/processing/piping/consumer.rb +2 -10
  70. data/lib/karafka/pro/processing/schedulers/base.rb +2 -10
  71. data/lib/karafka/pro/processing/schedulers/default.rb +2 -10
  72. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +3 -11
  73. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +3 -11
  74. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +2 -10
  75. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +2 -10
  76. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +3 -11
  77. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +3 -11
  78. data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +2 -10
  79. data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +2 -10
  80. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +2 -10
  81. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +3 -11
  82. data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +2 -10
  83. data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +2 -10
  84. data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +2 -10
  85. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +5 -13
  86. data/lib/karafka/pro/processing/strategies/aj/mom.rb +2 -10
  87. data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +2 -10
  88. data/lib/karafka/pro/processing/strategies/base.rb +2 -10
  89. data/lib/karafka/pro/processing/strategies/default.rb +135 -77
  90. data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -14
  91. data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +2 -10
  92. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +3 -11
  93. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +3 -11
  94. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +2 -10
  95. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +2 -10
  96. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +3 -11
  97. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +2 -10
  98. data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +2 -10
  99. data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +3 -11
  100. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +3 -11
  101. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +2 -10
  102. data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +2 -10
  103. data/lib/karafka/pro/processing/strategies/dlq/mom.rb +3 -11
  104. data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +2 -10
  105. data/lib/karafka/pro/processing/strategies/dlq/vp.rb +2 -10
  106. data/lib/karafka/pro/processing/strategies/ftr/default.rb +4 -12
  107. data/lib/karafka/pro/processing/strategies/ftr/vp.rb +2 -10
  108. data/lib/karafka/pro/processing/strategies/lrj/default.rb +5 -13
  109. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +3 -11
  110. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +2 -10
  111. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +2 -10
  112. data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +2 -10
  113. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +4 -12
  114. data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +2 -10
  115. data/lib/karafka/pro/processing/strategies/lrj/vp.rb +2 -10
  116. data/lib/karafka/pro/processing/strategies/mom/default.rb +2 -10
  117. data/lib/karafka/pro/processing/strategies/mom/ftr.rb +2 -10
  118. data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +2 -10
  119. data/lib/karafka/pro/processing/strategies/mom/vp.rb +2 -10
  120. data/lib/karafka/pro/processing/strategies/vp/default.rb +5 -10
  121. data/lib/karafka/pro/processing/strategies.rb +2 -10
  122. data/lib/karafka/pro/processing/strategy_selector.rb +2 -10
  123. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -10
  124. data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -10
  125. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -10
  126. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -10
  127. data/lib/karafka/pro/recurring_tasks/deserializer.rb +2 -10
  128. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -10
  129. data/lib/karafka/pro/recurring_tasks/errors.rb +2 -10
  130. data/lib/karafka/pro/recurring_tasks/executor.rb +2 -10
  131. data/lib/karafka/pro/recurring_tasks/listener.rb +2 -10
  132. data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -10
  133. data/lib/karafka/pro/recurring_tasks/schedule.rb +2 -10
  134. data/lib/karafka/pro/recurring_tasks/serializer.rb +2 -10
  135. data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -10
  136. data/lib/karafka/pro/recurring_tasks/task.rb +2 -10
  137. data/lib/karafka/pro/recurring_tasks.rb +2 -10
  138. data/lib/karafka/pro/routing/features/active_job/builder.rb +2 -10
  139. data/lib/karafka/pro/routing/features/active_job.rb +2 -10
  140. data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb +2 -10
  141. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -10
  142. data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +2 -10
  143. data/lib/karafka/pro/routing/features/adaptive_iterator.rb +2 -10
  144. data/lib/karafka/pro/routing/features/base.rb +2 -10
  145. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -10
  146. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +2 -10
  147. data/lib/karafka/pro/routing/features/dead_letter_queue.rb +2 -10
  148. data/lib/karafka/pro/routing/features/delaying/config.rb +2 -10
  149. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -10
  150. data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -10
  151. data/lib/karafka/pro/routing/features/delaying.rb +2 -10
  152. data/lib/karafka/pro/routing/features/direct_assignments/config.rb +2 -10
  153. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -10
  154. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -10
  155. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +2 -10
  156. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -10
  157. data/lib/karafka/pro/routing/features/direct_assignments.rb +2 -10
  158. data/lib/karafka/pro/routing/features/expiring/config.rb +2 -10
  159. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -10
  160. data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -10
  161. data/lib/karafka/pro/routing/features/expiring.rb +2 -10
  162. data/lib/karafka/pro/routing/features/filtering/config.rb +2 -10
  163. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -10
  164. data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -10
  165. data/lib/karafka/pro/routing/features/filtering.rb +2 -10
  166. data/lib/karafka/pro/routing/features/inline_insights/config.rb +2 -10
  167. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -10
  168. data/lib/karafka/pro/routing/features/inline_insights/topic.rb +2 -10
  169. data/lib/karafka/pro/routing/features/inline_insights.rb +2 -10
  170. data/lib/karafka/pro/routing/features/long_running_job/config.rb +2 -10
  171. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -10
  172. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +2 -10
  173. data/lib/karafka/pro/routing/features/long_running_job.rb +2 -10
  174. data/lib/karafka/pro/routing/features/multiplexing/config.rb +2 -10
  175. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -10
  176. data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +2 -10
  177. data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +2 -10
  178. data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +2 -10
  179. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +2 -10
  180. data/lib/karafka/pro/routing/features/multiplexing.rb +2 -10
  181. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +2 -10
  182. data/lib/karafka/pro/routing/features/non_blocking_job.rb +2 -10
  183. data/lib/karafka/pro/routing/features/offset_metadata/config.rb +2 -10
  184. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -10
  185. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +2 -10
  186. data/lib/karafka/pro/routing/features/offset_metadata.rb +2 -10
  187. data/lib/karafka/pro/routing/features/patterns/builder.rb +2 -10
  188. data/lib/karafka/pro/routing/features/patterns/config.rb +2 -10
  189. data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +2 -10
  190. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -10
  191. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -10
  192. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -10
  193. data/lib/karafka/pro/routing/features/patterns/detector.rb +2 -10
  194. data/lib/karafka/pro/routing/features/patterns/pattern.rb +2 -10
  195. data/lib/karafka/pro/routing/features/patterns/patterns.rb +2 -10
  196. data/lib/karafka/pro/routing/features/patterns/topic.rb +2 -10
  197. data/lib/karafka/pro/routing/features/patterns/topics.rb +2 -10
  198. data/lib/karafka/pro/routing/features/patterns.rb +2 -10
  199. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -10
  200. data/lib/karafka/pro/routing/features/pausing/topic.rb +2 -10
  201. data/lib/karafka/pro/routing/features/pausing.rb +2 -10
  202. data/lib/karafka/pro/routing/features/periodic_job/config.rb +2 -10
  203. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -10
  204. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +2 -10
  205. data/lib/karafka/pro/routing/features/periodic_job.rb +2 -10
  206. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +2 -10
  207. data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +2 -10
  208. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -10
  209. data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +2 -10
  210. data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +2 -10
  211. data/lib/karafka/pro/routing/features/recurring_tasks.rb +2 -10
  212. data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +2 -10
  213. data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +2 -10
  214. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -10
  215. data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb +2 -10
  216. data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +2 -10
  217. data/lib/karafka/pro/routing/features/scheduled_messages.rb +2 -10
  218. data/lib/karafka/pro/routing/features/swarm/config.rb +2 -10
  219. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -10
  220. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -10
  221. data/lib/karafka/pro/routing/features/swarm/topic.rb +2 -10
  222. data/lib/karafka/pro/routing/features/swarm.rb +2 -10
  223. data/lib/karafka/pro/routing/features/throttling/config.rb +2 -10
  224. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -10
  225. data/lib/karafka/pro/routing/features/throttling/topic.rb +2 -10
  226. data/lib/karafka/pro/routing/features/throttling.rb +2 -10
  227. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +2 -10
  228. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -10
  229. data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +2 -10
  230. data/lib/karafka/pro/routing/features/virtual_partitions.rb +2 -10
  231. data/lib/karafka/pro/scheduled_messages/consumer.rb +2 -10
  232. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -10
  233. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +2 -10
  234. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -10
  235. data/lib/karafka/pro/scheduled_messages/day.rb +2 -10
  236. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +2 -10
  237. data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +2 -10
  238. data/lib/karafka/pro/scheduled_messages/dispatcher.rb +2 -10
  239. data/lib/karafka/pro/scheduled_messages/errors.rb +2 -10
  240. data/lib/karafka/pro/scheduled_messages/max_epoch.rb +2 -10
  241. data/lib/karafka/pro/scheduled_messages/proxy.rb +2 -10
  242. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +2 -10
  243. data/lib/karafka/pro/scheduled_messages/serializer.rb +2 -10
  244. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -10
  245. data/lib/karafka/pro/scheduled_messages/state.rb +2 -10
  246. data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -10
  247. data/lib/karafka/pro/scheduled_messages.rb +2 -10
  248. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -10
  249. data/lib/karafka/processing/executor.rb +7 -0
  250. data/lib/karafka/processing/jobs/base.rb +13 -0
  251. data/lib/karafka/processing/jobs/consume.rb +2 -0
  252. data/lib/karafka/processing/jobs/eofed.rb +2 -0
  253. data/lib/karafka/processing/jobs/idle.rb +2 -0
  254. data/lib/karafka/processing/jobs/revoked.rb +2 -0
  255. data/lib/karafka/processing/jobs/shutdown.rb +2 -0
  256. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
  257. data/lib/karafka/processing/strategies/default.rb +28 -17
  258. data/lib/karafka/processing/strategies/dlq.rb +4 -4
  259. data/lib/karafka/processing/strategies/dlq_mom.rb +2 -2
  260. data/lib/karafka/processing/worker.rb +21 -18
  261. data/lib/karafka/version.rb +1 -1
  262. data.tar.gz.sig +0 -0
  263. metadata +4 -6
  264. metadata.gz.sig +0 -0
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -51,7 +43,7 @@ module Karafka
51
43
 
52
44
  # no need to check for manual seek because AJ consumer is internal and
53
45
  # fully controlled by us
54
- seek(coordinator.seek_offset, false) unless revoked?
46
+ seek(seek_offset, false) unless revoked?
55
47
 
56
48
  resume
57
49
  else
@@ -69,8 +61,8 @@ module Karafka
69
61
  coordinator.revoke
70
62
  end
71
63
 
72
- Karafka.monitor.instrument('consumer.revoke', caller: self)
73
- Karafka.monitor.instrument('consumer.revoked', caller: self) do
64
+ monitor.instrument('consumer.revoke', caller: self)
65
+ monitor.instrument('consumer.revoked', caller: self) do
74
66
  revoked
75
67
  end
76
68
  ensure
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -54,28 +46,30 @@ module Karafka
54
46
  # already processed but rather at the next one. This applies to both sync and async
55
47
  # versions of this method.
56
48
  def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
49
+ # If we are inside a transaction than we can just mark as consumed within it
57
50
  if @_in_transaction
58
51
  mark_in_transaction(message, offset_metadata, true)
52
+ elsif @_in_transaction_marked
53
+ mark_in_memory(message)
59
54
  else
60
55
  # seek offset can be nil only in case `#seek` was invoked with offset reset request
61
56
  # In case like this we ignore marking
62
- return true if coordinator.seek_offset.nil?
57
+ return true if seek_offset.nil?
63
58
  # Ignore earlier offsets than the one we already committed
64
- return true if coordinator.seek_offset > message.offset
59
+ return true if seek_offset > message.offset
65
60
  return false if revoked?
66
61
 
67
- # If we have already marked this successfully in a transaction that was running
68
- # we should not mark it again with the client offset delegation but instead we should
69
- # just align the in-memory state
70
- if @_in_transaction_marked
71
- coordinator.seek_offset = message.offset + 1
62
+ # If we are not inside a transaction but this is a transactional topic, we mark with
63
+ # artificially created transaction
64
+ stored = if producer.transactional?
65
+ mark_with_transaction(message, offset_metadata, true)
66
+ else
67
+ client.mark_as_consumed(message, offset_metadata)
68
+ end
72
69
 
73
- return true
74
- end
75
-
76
- return revoked? unless client.mark_as_consumed(message, offset_metadata)
70
+ return revoked? unless stored
77
71
 
78
- coordinator.seek_offset = message.offset + 1
72
+ self.seek_offset = message.offset + 1
79
73
  end
80
74
 
81
75
  true
@@ -92,23 +86,27 @@ module Karafka
92
86
  def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
93
87
  if @_in_transaction
94
88
  mark_in_transaction(message, offset_metadata, false)
89
+ elsif @_in_transaction_marked
90
+ mark_in_memory(message)
95
91
  else
96
92
  # seek offset can be nil only in case `#seek` was invoked with offset reset request
97
93
  # In case like this we ignore marking
98
- return true if coordinator.seek_offset.nil?
94
+ return true if seek_offset.nil?
99
95
  # Ignore earlier offsets than the one we already committed
100
- return true if coordinator.seek_offset > message.offset
96
+ return true if seek_offset > message.offset
101
97
  return false if revoked?
102
98
 
103
- if @_in_transaction_marked
104
- coordinator.seek_offset = message.offset + 1
105
-
106
- return true
107
- end
99
+ # If we are not inside a transaction but this is a transactional topic, we mark with
100
+ # artificially created transaction
101
+ stored = if producer.transactional?
102
+ mark_with_transaction(message, offset_metadata, false)
103
+ else
104
+ client.mark_as_consumed!(message, offset_metadata)
105
+ end
108
106
 
109
- return revoked? unless client.mark_as_consumed!(message, offset_metadata)
107
+ return revoked? unless stored
110
108
 
111
- coordinator.seek_offset = message.offset + 1
109
+ self.seek_offset = message.offset + 1
112
110
  end
113
111
 
114
112
  true
@@ -128,7 +126,7 @@ module Karafka
128
126
  # managing multiple producers. If not provided, default producer taken from `#producer`
129
127
  # will be used.
130
128
  #
131
- # @param block [Proc] code that we want to run in a transaction
129
+ # @yield code that we want to run in a transaction
132
130
  #
133
131
  # @note Please note, that if you provide the producer, it will reassign the producer of
134
132
  # the consumer for the transaction time. This means, that in case you would even
@@ -136,43 +134,57 @@ module Karafka
136
134
  # reassigned producer and not the initially used/assigned producer. It is done that
137
135
  # way, so the message producing aliases operate from within transactions and since the
138
136
  # producer in transaction is locked, it will prevent other threads from using it.
139
- def transaction(active_producer = producer, &block)
140
- default_producer = producer
141
- self.producer = active_producer
142
-
143
- transaction_started = false
144
-
145
- # Prevent from nested transactions. It would not make any sense
146
- raise Errors::TransactionAlreadyInitializedError if @_in_transaction
147
-
148
- transaction_started = true
149
- @_transaction_marked = []
150
- @_in_transaction = true
151
- @_in_transaction_marked = false
152
-
153
- producer.transaction(&block)
154
-
155
- @_in_transaction = false
156
-
157
- # This offset is already stored in transaction but we set it here anyhow because we
158
- # want to make sure our internal in-memory state is aligned with the transaction
159
- #
160
- # @note We never need to use the blocking `#mark_as_consumed!` here because the offset
161
- # anyhow was already stored during the transaction
162
- #
163
- # @note Since the offset could have been already stored in Kafka (could have because
164
- # you can have transactions without marking), we use the `@_in_transaction_marked`
165
- # state to decide if we need to dispatch the offset via client at all
166
- # (if post transaction, then we do not have to)
167
- #
168
- # @note In theory we could only keep reference to the most recent marking and reject
169
- # others. We however do not do it for two reasons:
170
- # - User may have non standard flow relying on some alternative order and we want to
171
- # mimic this
172
- # - Complex strategies like VPs can use this in VPs to mark in parallel without
173
- # having to redefine the transactional flow completely
174
- @_transaction_marked.each do |marking|
175
- marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
137
+ def transaction(active_producer = producer)
138
+ default_producer = nil
139
+ transaction_started = nil
140
+
141
+ monitor.instrument('consumer.consuming.transaction', caller: self) do
142
+ default_producer = producer
143
+ self.producer = active_producer
144
+
145
+ transaction_started = false
146
+
147
+ # Prevent from nested transactions. It would not make any sense
148
+ raise Errors::TransactionAlreadyInitializedError if @_in_transaction
149
+
150
+ transaction_started = true
151
+ @_transaction_marked = []
152
+ @_in_transaction = true
153
+ @_in_transaction_marked = false
154
+
155
+ producer.transaction do
156
+ yield
157
+
158
+ # Ensure this transaction is rolled back if we have lost the ownership of this
159
+ # transaction. We do it only for transactions that contain offset management as for
160
+ # producer only, this is not relevant.
161
+ raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
162
+ end
163
+
164
+ @_in_transaction = false
165
+
166
+ # This offset is already stored in transaction but we set it here anyhow because we
167
+ # want to make sure our internal in-memory state is aligned with the transaction
168
+ #
169
+ # @note We never need to use the blocking `#mark_as_consumed!` here because the
170
+ # offset anyhow was already stored during the transaction
171
+ #
172
+ # @note Since the offset could have been already stored in Kafka (could have because
173
+ # you can have transactions without marking), we use the `@_in_transaction_marked`
174
+ # state to decide if we need to dispatch the offset via client at all
175
+ # (if post transaction, then we do not have to)
176
+ #
177
+ # @note In theory we could only keep reference to the most recent marking and reject
178
+ # others. We however do not do it for two reasons:
179
+ # - User may have non standard flow relying on some alternative order and we want
180
+ # to mimic this
181
+ # - Complex strategies like VPs can use this in VPs to mark in parallel without
182
+ # having to redefine the transactional flow completely
183
+ @_transaction_marked.each do |marking|
184
+ marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
185
+ end
186
+
187
+ true
176
188
  end
177
189
  ensure
178
190
  self.producer = default_producer
@@ -206,9 +218,55 @@ module Karafka
206
218
  @_transaction_marked << [message, offset_metadata, async]
207
219
  end
208
220
 
221
+ # @private
222
+ # @param message [Messages::Message] message we want to commit inside of a transaction
223
+ # @param offset_metadata [String, nil] offset metadata or nil if none
224
+ # @param async [Boolean] should we mark in async or sync way (applicable only to post
225
+ # transaction state synchronization usage as within transaction it is always sync)
226
+ # @return [Boolean] false if marking failed otherwise true
227
+ def mark_with_transaction(message, offset_metadata, async)
228
+ # This flag is used by VPs to differentiate between user initiated transactions and
229
+ # post-execution system transactions.
230
+ @_transaction_internal = true
231
+
232
+ transaction do
233
+ mark_in_transaction(message, offset_metadata, async)
234
+ end
235
+
236
+ true
237
+ # We handle both cases here because this is a private API for internal usage and we want
238
+ # the post-user code execution marking with transactional producer to result in a
239
+ # boolean state of marking for further framework flow. This is a normalization to make it
240
+ # behave the same way as it would behave with a non-transactional one
241
+ rescue ::Rdkafka::RdkafkaError, Errors::AssignmentLostError
242
+ false
243
+ ensure
244
+ @_transaction_internal = false
245
+ end
246
+
247
+ # Marks the current state only in memory as the offset marking has already happened
248
+ # using the producer transaction
249
+ # @param message [Messages::Message] last successfully processed message.
250
+ # @return [Boolean] true if all good, false if we lost assignment and no point in marking
251
+ def mark_in_memory(message)
252
+ # seek offset can be nil only in case `#seek` was invoked with offset reset request
253
+ # In case like this we ignore marking
254
+ return true if seek_offset.nil?
255
+ # Ignore earlier offsets than the one we already committed
256
+ return true if seek_offset > message.offset
257
+ return false if revoked?
258
+
259
+ # If we have already marked this successfully in a transaction that was running
260
+ # we should not mark it again with the client offset delegation but instead we should
261
+ # just align the in-memory state
262
+ self.seek_offset = message.offset + 1
263
+
264
+ true
265
+ end
266
+
209
267
  # No actions needed for the standard flow here
210
268
  def handle_before_schedule_consume
211
- Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
269
+ monitor.instrument('consumer.before_schedule_consume', caller: self)
212
270
 
213
271
  nil
214
272
  end
@@ -227,8 +285,8 @@ module Karafka
227
285
  # This can happen primarily when an LRJ job gets to the internal worker queue and
228
286
  # this partition is revoked prior processing.
229
287
  unless revoked?
230
- Karafka.monitor.instrument('consumer.consume', caller: self)
231
- Karafka.monitor.instrument('consumer.consumed', caller: self) do
288
+ monitor.instrument('consumer.consume', caller: self)
289
+ monitor.instrument('consumer.consumed', caller: self) do
232
290
  consume
233
291
  end
234
292
  end
@@ -274,8 +332,8 @@ module Karafka
274
332
  coordinator.revoke
275
333
  end
276
334
 
277
- Karafka.monitor.instrument('consumer.revoke', caller: self)
278
- Karafka.monitor.instrument('consumer.revoked', caller: self) do
335
+ monitor.instrument('consumer.revoke', caller: self)
336
+ monitor.instrument('consumer.revoked', caller: self) do
279
337
  revoked
280
338
  end
281
339
  ensure
@@ -284,15 +342,15 @@ module Karafka
284
342
 
285
343
  # No action needed for the tick standard flow
286
344
  def handle_before_schedule_tick
287
- Karafka.monitor.instrument('consumer.before_schedule_tick', caller: self)
345
+ monitor.instrument('consumer.before_schedule_tick', caller: self)
288
346
 
289
347
  nil
290
348
  end
291
349
 
292
350
  # Runs the consumer `#tick` method with reporting
293
351
  def handle_tick
294
- Karafka.monitor.instrument('consumer.tick', caller: self)
295
- Karafka.monitor.instrument('consumer.ticked', caller: self) do
352
+ monitor.instrument('consumer.tick', caller: self)
353
+ monitor.instrument('consumer.ticked', caller: self) do
296
354
  tick
297
355
  end
298
356
  ensure
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -90,7 +82,7 @@ module Karafka
90
82
  # information if this message was from marked offset or figured out via mom flow
91
83
  def find_skippable_message
92
84
  skippable_message = messages.find do |msg|
93
- coordinator.marked? && msg.offset == coordinator.seek_offset
85
+ coordinator.marked? && msg.offset == seek_offset
94
86
  end
95
87
 
96
88
  # If we don't have the message matching the last comitted offset, it means that
@@ -119,7 +111,7 @@ module Karafka
119
111
  )
120
112
 
121
113
  # Notify about dispatch on the events bus
122
- Karafka.monitor.instrument(
114
+ monitor.instrument(
123
115
  'dead_letter_queue.dispatched',
124
116
  caller: self,
125
117
  message: skippable_message
@@ -139,7 +131,7 @@ module Karafka
139
131
  if mark_after_dispatch?
140
132
  mark_dispatched_to_dlq(skippable_message)
141
133
  else
142
- coordinator.seek_offset = skippable_message.offset + 1
134
+ self.seek_offset = skippable_message.offset + 1
143
135
  end
144
136
  end
145
137
 
@@ -234,7 +226,7 @@ module Karafka
234
226
  coordinator.pause_tracker.reset
235
227
 
236
228
  # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
237
- pause(coordinator.seek_offset, nil, false)
229
+ pause(seek_offset, nil, false)
238
230
  end
239
231
 
240
232
  # Marks message that went to DLQ (if applicable) based on the requested method
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -48,7 +40,7 @@ module Karafka
48
40
  if coordinator.filtered? && !revoked?
49
41
  handle_post_filtering
50
42
  elsif !revoked? && !coordinator.manual_seek?
51
- seek(coordinator.seek_offset, false)
43
+ seek(seek_offset, false)
52
44
  resume
53
45
  else
54
46
  resume
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -58,7 +50,7 @@ module Karafka
58
50
  if mark_after_dispatch?
59
51
  mark_dispatched_to_dlq(skippable_message)
60
52
  else
61
- coordinator.seek_offset = skippable_message.offset + 1
53
+ self.seek_offset = skippable_message.offset + 1
62
54
  end
63
55
  end
64
56
  end
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -49,7 +41,7 @@ module Karafka
49
41
  if mark_after_dispatch?
50
42
  mark_dispatched_to_dlq(skippable_message)
51
43
  else
52
- coordinator.seek_offset = skippable_message.offset + 1
44
+ self.seek_offset = skippable_message.offset + 1
53
45
  end
54
46
  end
55
47
  end