karafka 2.4.0 → 2.4.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (326) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +26 -34
  4. data/.github/workflows/ci.yml +18 -6
  5. data/.ruby-version +1 -1
  6. data/CHANGELOG.md +146 -1
  7. data/Gemfile +10 -5
  8. data/Gemfile.lock +60 -39
  9. data/LICENSE +8 -3
  10. data/bin/integrations +13 -1
  11. data/certs/cert.pem +26 -0
  12. data/config/locales/errors.yml +18 -2
  13. data/config/locales/pro_errors.yml +44 -0
  14. data/docker-compose.yml +1 -3
  15. data/karafka.gemspec +6 -4
  16. data/lib/active_job/queue_adapters/karafka_adapter.rb +18 -7
  17. data/lib/karafka/active_job/dispatcher.rb +13 -0
  18. data/lib/karafka/active_job/job_extensions.rb +3 -0
  19. data/lib/karafka/admin.rb +86 -0
  20. data/lib/karafka/app.rb +17 -0
  21. data/lib/karafka/base_consumer.rb +130 -19
  22. data/lib/karafka/cli/base.rb +24 -8
  23. data/lib/karafka/cli/install.rb +2 -1
  24. data/lib/karafka/cli/server.rb +1 -0
  25. data/lib/karafka/cli/swarm.rb +1 -0
  26. data/lib/karafka/cli/topics/align.rb +12 -2
  27. data/lib/karafka/cli/topics/plan.rb +54 -6
  28. data/lib/karafka/cli/topics.rb +45 -18
  29. data/lib/karafka/connection/client.rb +102 -35
  30. data/lib/karafka/connection/listener.rb +48 -11
  31. data/lib/karafka/connection/messages_buffer.rb +19 -6
  32. data/lib/karafka/connection/proxy.rb +3 -0
  33. data/lib/karafka/connection/raw_messages_buffer.rb +43 -9
  34. data/lib/karafka/connection/rebalance_manager.rb +24 -13
  35. data/lib/karafka/contracts/config.rb +4 -0
  36. data/lib/karafka/contracts/consumer_group.rb +17 -0
  37. data/lib/karafka/contracts/routing.rb +59 -0
  38. data/lib/karafka/contracts/topic.rb +14 -0
  39. data/lib/karafka/embedded.rb +46 -3
  40. data/lib/karafka/errors.rb +3 -2
  41. data/lib/karafka/helpers/async.rb +11 -2
  42. data/lib/karafka/helpers/config_importer.rb +13 -0
  43. data/lib/karafka/instrumentation/assignments_tracker.rb +7 -2
  44. data/lib/karafka/instrumentation/logger_listener.rb +45 -4
  45. data/lib/karafka/instrumentation/notifications.rb +12 -0
  46. data/lib/karafka/instrumentation/vendors/appsignal/client.rb +32 -11
  47. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
  48. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +3 -1
  49. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +17 -19
  50. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +27 -18
  51. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +2 -2
  52. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +41 -13
  53. data/lib/karafka/messages/message.rb +9 -9
  54. data/lib/karafka/pro/active_job/consumer.rb +2 -10
  55. data/lib/karafka/pro/active_job/dispatcher.rb +67 -19
  56. data/lib/karafka/pro/active_job/job_options_contract.rb +12 -10
  57. data/lib/karafka/pro/base_consumer.rb +2 -10
  58. data/lib/karafka/pro/cleaner/errors.rb +2 -10
  59. data/lib/karafka/pro/cleaner/messages/message.rb +14 -12
  60. data/lib/karafka/pro/cleaner/messages/messages.rb +2 -10
  61. data/lib/karafka/pro/cleaner/messages/metadata.rb +41 -0
  62. data/lib/karafka/pro/cleaner.rb +3 -10
  63. data/lib/karafka/pro/connection/manager.rb +6 -10
  64. data/lib/karafka/pro/connection/multiplexing/listener.rb +2 -10
  65. data/lib/karafka/pro/contracts/base.rb +2 -10
  66. data/lib/karafka/pro/contracts/server_cli_options.rb +2 -10
  67. data/lib/karafka/pro/encryption/cipher.rb +2 -10
  68. data/lib/karafka/pro/encryption/contracts/config.rb +2 -10
  69. data/lib/karafka/pro/encryption/errors.rb +2 -10
  70. data/lib/karafka/pro/encryption/messages/middleware.rb +2 -10
  71. data/lib/karafka/pro/encryption/messages/parser.rb +2 -10
  72. data/lib/karafka/pro/encryption/setup/config.rb +2 -10
  73. data/lib/karafka/pro/encryption.rb +2 -10
  74. data/lib/karafka/pro/instrumentation/performance_tracker.rb +2 -10
  75. data/lib/karafka/pro/iterator/expander.rb +2 -10
  76. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -10
  77. data/lib/karafka/pro/iterator.rb +2 -10
  78. data/lib/karafka/pro/loader.rb +5 -11
  79. data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +54 -0
  80. data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +67 -0
  81. data/lib/karafka/pro/processing/collapser.rb +2 -10
  82. data/lib/karafka/pro/processing/coordinator.rb +2 -10
  83. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -10
  84. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +19 -10
  85. data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +2 -10
  86. data/lib/karafka/pro/processing/executor.rb +2 -10
  87. data/lib/karafka/pro/processing/expansions_selector.rb +3 -10
  88. data/lib/karafka/pro/processing/filters/base.rb +14 -10
  89. data/lib/karafka/pro/processing/filters/delayer.rb +4 -12
  90. data/lib/karafka/pro/processing/filters/expirer.rb +2 -10
  91. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +2 -10
  92. data/lib/karafka/pro/processing/filters/throttler.rb +2 -10
  93. data/lib/karafka/pro/processing/filters/virtual_limiter.rb +2 -10
  94. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +4 -10
  95. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +26 -0
  96. data/lib/karafka/pro/processing/jobs/periodic.rb +4 -10
  97. data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +4 -10
  98. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +4 -10
  99. data/lib/karafka/pro/processing/jobs_builder.rb +14 -10
  100. data/lib/karafka/pro/processing/jobs_queue.rb +2 -10
  101. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +2 -10
  102. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +2 -10
  103. data/lib/karafka/pro/processing/offset_metadata/listener.rb +2 -10
  104. data/lib/karafka/pro/processing/partitioner.rb +35 -24
  105. data/lib/karafka/pro/processing/periodic_job/consumer.rb +2 -10
  106. data/lib/karafka/pro/processing/piping/consumer.rb +2 -10
  107. data/lib/karafka/pro/processing/schedulers/base.rb +2 -10
  108. data/lib/karafka/pro/processing/schedulers/default.rb +3 -10
  109. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +3 -11
  110. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +3 -11
  111. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +2 -10
  112. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +2 -10
  113. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +3 -11
  114. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +3 -11
  115. data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +2 -10
  116. data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +2 -10
  117. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +2 -10
  118. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +3 -11
  119. data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +2 -10
  120. data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +2 -10
  121. data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +2 -10
  122. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +5 -13
  123. data/lib/karafka/pro/processing/strategies/aj/mom.rb +2 -10
  124. data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +2 -10
  125. data/lib/karafka/pro/processing/strategies/base.rb +2 -10
  126. data/lib/karafka/pro/processing/strategies/default.rb +140 -58
  127. data/lib/karafka/pro/processing/strategies/dlq/default.rb +23 -15
  128. data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +2 -10
  129. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +3 -11
  130. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +7 -11
  131. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +2 -10
  132. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +2 -10
  133. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +19 -11
  134. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +2 -10
  135. data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +2 -10
  136. data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +3 -11
  137. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +19 -11
  138. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +2 -10
  139. data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +2 -10
  140. data/lib/karafka/pro/processing/strategies/dlq/mom.rb +24 -16
  141. data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +2 -10
  142. data/lib/karafka/pro/processing/strategies/dlq/vp.rb +2 -10
  143. data/lib/karafka/pro/processing/strategies/ftr/default.rb +17 -12
  144. data/lib/karafka/pro/processing/strategies/ftr/vp.rb +2 -10
  145. data/lib/karafka/pro/processing/strategies/lrj/default.rb +5 -13
  146. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +3 -11
  147. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +2 -10
  148. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +2 -10
  149. data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +2 -10
  150. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +4 -12
  151. data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +2 -10
  152. data/lib/karafka/pro/processing/strategies/lrj/vp.rb +2 -10
  153. data/lib/karafka/pro/processing/strategies/mom/default.rb +2 -10
  154. data/lib/karafka/pro/processing/strategies/mom/ftr.rb +2 -10
  155. data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +2 -10
  156. data/lib/karafka/pro/processing/strategies/mom/vp.rb +2 -10
  157. data/lib/karafka/pro/processing/strategies/vp/default.rb +5 -10
  158. data/lib/karafka/pro/processing/strategies.rb +2 -10
  159. data/lib/karafka/pro/processing/strategy_selector.rb +2 -10
  160. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -10
  161. data/lib/karafka/pro/recurring_tasks/consumer.rb +97 -0
  162. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +45 -0
  163. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +33 -0
  164. data/lib/karafka/pro/recurring_tasks/deserializer.rb +27 -0
  165. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +79 -0
  166. data/lib/karafka/pro/recurring_tasks/errors.rb +26 -0
  167. data/lib/karafka/pro/recurring_tasks/executor.rb +144 -0
  168. data/lib/karafka/pro/recurring_tasks/listener.rb +30 -0
  169. data/lib/karafka/pro/recurring_tasks/matcher.rb +30 -0
  170. data/lib/karafka/pro/recurring_tasks/schedule.rb +55 -0
  171. data/lib/karafka/pro/recurring_tasks/serializer.rb +105 -0
  172. data/lib/karafka/pro/recurring_tasks/setup/config.rb +44 -0
  173. data/lib/karafka/pro/recurring_tasks/task.rb +143 -0
  174. data/lib/karafka/pro/recurring_tasks.rb +79 -0
  175. data/lib/karafka/pro/routing/features/active_job/builder.rb +2 -10
  176. data/lib/karafka/pro/routing/features/active_job.rb +2 -10
  177. data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb +26 -0
  178. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +66 -0
  179. data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +54 -0
  180. data/lib/karafka/pro/routing/features/adaptive_iterator.rb +23 -0
  181. data/lib/karafka/pro/routing/features/base.rb +2 -10
  182. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -10
  183. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +2 -10
  184. data/lib/karafka/pro/routing/features/dead_letter_queue.rb +2 -10
  185. data/lib/karafka/pro/routing/features/delaying/config.rb +2 -10
  186. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -10
  187. data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -10
  188. data/lib/karafka/pro/routing/features/delaying.rb +2 -10
  189. data/lib/karafka/pro/routing/features/direct_assignments/config.rb +2 -10
  190. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -10
  191. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -10
  192. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +2 -10
  193. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -10
  194. data/lib/karafka/pro/routing/features/direct_assignments.rb +2 -10
  195. data/lib/karafka/pro/routing/features/expiring/config.rb +2 -10
  196. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -10
  197. data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -10
  198. data/lib/karafka/pro/routing/features/expiring.rb +2 -10
  199. data/lib/karafka/pro/routing/features/filtering/config.rb +2 -10
  200. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -10
  201. data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -10
  202. data/lib/karafka/pro/routing/features/filtering.rb +2 -10
  203. data/lib/karafka/pro/routing/features/inline_insights/config.rb +2 -10
  204. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -10
  205. data/lib/karafka/pro/routing/features/inline_insights/topic.rb +2 -10
  206. data/lib/karafka/pro/routing/features/inline_insights.rb +2 -10
  207. data/lib/karafka/pro/routing/features/long_running_job/config.rb +2 -10
  208. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -10
  209. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +2 -10
  210. data/lib/karafka/pro/routing/features/long_running_job.rb +2 -10
  211. data/lib/karafka/pro/routing/features/multiplexing/config.rb +2 -10
  212. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -10
  213. data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +2 -10
  214. data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +2 -10
  215. data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +2 -10
  216. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +2 -10
  217. data/lib/karafka/pro/routing/features/multiplexing.rb +2 -10
  218. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +2 -10
  219. data/lib/karafka/pro/routing/features/non_blocking_job.rb +2 -10
  220. data/lib/karafka/pro/routing/features/offset_metadata/config.rb +2 -10
  221. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -10
  222. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +2 -10
  223. data/lib/karafka/pro/routing/features/offset_metadata.rb +2 -10
  224. data/lib/karafka/pro/routing/features/patterns/builder.rb +2 -10
  225. data/lib/karafka/pro/routing/features/patterns/config.rb +2 -10
  226. data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +2 -10
  227. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -10
  228. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -10
  229. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -10
  230. data/lib/karafka/pro/routing/features/patterns/detector.rb +2 -10
  231. data/lib/karafka/pro/routing/features/patterns/pattern.rb +2 -10
  232. data/lib/karafka/pro/routing/features/patterns/patterns.rb +2 -10
  233. data/lib/karafka/pro/routing/features/patterns/topic.rb +2 -10
  234. data/lib/karafka/pro/routing/features/patterns/topics.rb +2 -10
  235. data/lib/karafka/pro/routing/features/patterns.rb +2 -10
  236. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -10
  237. data/lib/karafka/pro/routing/features/pausing/topic.rb +2 -10
  238. data/lib/karafka/pro/routing/features/pausing.rb +2 -10
  239. data/lib/karafka/pro/routing/features/periodic_job/config.rb +2 -10
  240. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -10
  241. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +2 -10
  242. data/lib/karafka/pro/routing/features/periodic_job.rb +2 -10
  243. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +123 -0
  244. data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +20 -0
  245. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +32 -0
  246. data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +19 -0
  247. data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +36 -0
  248. data/lib/karafka/pro/routing/features/recurring_tasks.rb +17 -0
  249. data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +123 -0
  250. data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +20 -0
  251. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +32 -0
  252. data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb +19 -0
  253. data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +36 -0
  254. data/lib/karafka/pro/routing/features/scheduled_messages.rb +16 -0
  255. data/lib/karafka/pro/routing/features/swarm/config.rb +2 -10
  256. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -10
  257. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -10
  258. data/lib/karafka/pro/routing/features/swarm/topic.rb +2 -10
  259. data/lib/karafka/pro/routing/features/swarm.rb +2 -10
  260. data/lib/karafka/pro/routing/features/throttling/config.rb +2 -10
  261. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -10
  262. data/lib/karafka/pro/routing/features/throttling/topic.rb +2 -10
  263. data/lib/karafka/pro/routing/features/throttling.rb +2 -10
  264. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +3 -10
  265. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +3 -10
  266. data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +10 -12
  267. data/lib/karafka/pro/routing/features/virtual_partitions.rb +2 -10
  268. data/lib/karafka/pro/scheduled_messages/consumer.rb +177 -0
  269. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +48 -0
  270. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +88 -0
  271. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +71 -0
  272. data/lib/karafka/pro/scheduled_messages/day.rb +37 -0
  273. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +38 -0
  274. data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +27 -0
  275. data/lib/karafka/pro/scheduled_messages/dispatcher.rb +114 -0
  276. data/lib/karafka/pro/scheduled_messages/errors.rb +20 -0
  277. data/lib/karafka/pro/scheduled_messages/max_epoch.rb +33 -0
  278. data/lib/karafka/pro/scheduled_messages/proxy.rb +177 -0
  279. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +29 -0
  280. data/lib/karafka/pro/scheduled_messages/serializer.rb +47 -0
  281. data/lib/karafka/pro/scheduled_messages/setup/config.rb +52 -0
  282. data/lib/karafka/pro/scheduled_messages/state.rb +54 -0
  283. data/lib/karafka/pro/scheduled_messages/tracker.rb +56 -0
  284. data/lib/karafka/pro/scheduled_messages.rb +59 -0
  285. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -10
  286. data/lib/karafka/processing/coordinator.rb +14 -0
  287. data/lib/karafka/processing/executor.rb +29 -1
  288. data/lib/karafka/processing/jobs/base.rb +13 -0
  289. data/lib/karafka/processing/jobs/consume.rb +2 -0
  290. data/lib/karafka/processing/jobs/eofed.rb +29 -0
  291. data/lib/karafka/processing/jobs/idle.rb +2 -0
  292. data/lib/karafka/processing/jobs/revoked.rb +2 -0
  293. data/lib/karafka/processing/jobs/shutdown.rb +2 -0
  294. data/lib/karafka/processing/jobs_builder.rb +6 -0
  295. data/lib/karafka/processing/schedulers/default.rb +1 -0
  296. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
  297. data/lib/karafka/processing/strategies/default.rb +45 -13
  298. data/lib/karafka/processing/strategies/dlq.rb +19 -5
  299. data/lib/karafka/processing/strategies/dlq_mom.rb +27 -8
  300. data/lib/karafka/processing/worker.rb +26 -13
  301. data/lib/karafka/railtie.rb +11 -42
  302. data/lib/karafka/routing/builder.rb +19 -1
  303. data/lib/karafka/routing/consumer_group.rb +9 -14
  304. data/lib/karafka/routing/features/dead_letter_queue/config.rb +3 -0
  305. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -0
  306. data/lib/karafka/routing/features/dead_letter_queue/topic.rb +7 -2
  307. data/lib/karafka/routing/features/eofed/config.rb +15 -0
  308. data/lib/karafka/routing/features/eofed/contracts/topic.rb +39 -0
  309. data/lib/karafka/routing/features/eofed/topic.rb +31 -0
  310. data/lib/karafka/routing/features/eofed.rb +14 -0
  311. data/lib/karafka/routing/subscription_group.rb +29 -1
  312. data/lib/karafka/routing/topic.rb +24 -1
  313. data/lib/karafka/runner.rb +10 -9
  314. data/lib/karafka/server.rb +37 -1
  315. data/lib/karafka/setup/attributes_map.rb +11 -4
  316. data/lib/karafka/setup/config.rb +11 -52
  317. data/lib/karafka/setup/defaults_injector.rb +64 -0
  318. data/lib/karafka/swarm/node.rb +2 -0
  319. data/lib/karafka/swarm/supervisor.rb +11 -2
  320. data/lib/karafka/templates/karafka.rb.erb +2 -2
  321. data/lib/karafka/version.rb +1 -1
  322. data/lib/karafka.rb +47 -7
  323. data.tar.gz.sig +0 -0
  324. metadata +116 -33
  325. metadata.gz.sig +0 -0
  326. data/certs/cert_chain.pem +0 -26
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -68,6 +60,7 @@ module Karafka
68
60
  alias on_schedule_shutdown schedule_fifo
69
61
  alias on_schedule_idle schedule_fifo
70
62
  alias on_schedule_periodic schedule_fifo
63
+ alias on_schedule_eofed schedule_fifo
71
64
 
72
65
  # This scheduler does not have anything to manage as it is a pass through and has no
73
66
  # state
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -46,7 +38,7 @@ module Karafka
46
38
  elsif !revoked?
47
39
  # no need to check for manual seek because AJ consumer is internal and
48
40
  # fully controlled by us
49
- seek(coordinator.seek_offset, false)
41
+ seek(seek_offset, false)
50
42
  resume
51
43
  else
52
44
  resume
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -52,7 +44,7 @@ module Karafka
52
44
  elsif !revoked?
53
45
  # no need to check for manual seek because AJ consumer is internal and
54
46
  # fully controlled by us
55
- seek(coordinator.seek_offset, false)
47
+ seek(seek_offset, false)
56
48
  resume
57
49
  else
58
50
  resume
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -44,7 +36,7 @@ module Karafka
44
36
 
45
37
  # no need to check for manual seek because AJ consumer is internal and
46
38
  # fully controlled by us
47
- seek(coordinator.seek_offset, false) unless revoked?
39
+ seek(seek_offset, false) unless revoked?
48
40
 
49
41
  resume
50
42
  else
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -48,7 +40,7 @@ module Karafka
48
40
  mark_as_consumed(last_group_message) unless revoked?
49
41
  # no need to check for manual seek because AJ consumer is internal and
50
42
  # fully controlled by us
51
- seek(coordinator.seek_offset, false) unless revoked?
43
+ seek(seek_offset, false) unless revoked?
52
44
 
53
45
  resume
54
46
  else
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -48,7 +40,7 @@ module Karafka
48
40
  elsif !revoked?
49
41
  # no need to check for manual seek because AJ consumer is internal and
50
42
  # fully controlled by us
51
- seek(coordinator.seek_offset, false)
43
+ seek(seek_offset, false)
52
44
  resume
53
45
  else
54
46
  resume
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -51,7 +43,7 @@ module Karafka
51
43
 
52
44
  # no need to check for manual seek because AJ consumer is internal and
53
45
  # fully controlled by us
54
- seek(coordinator.seek_offset, false) unless revoked?
46
+ seek(seek_offset, false) unless revoked?
55
47
 
56
48
  resume
57
49
  else
@@ -69,8 +61,8 @@ module Karafka
69
61
  coordinator.revoke
70
62
  end
71
63
 
72
- Karafka.monitor.instrument('consumer.revoke', caller: self)
73
- Karafka.monitor.instrument('consumer.revoked', caller: self) do
64
+ monitor.instrument('consumer.revoke', caller: self)
65
+ monitor.instrument('consumer.revoked', caller: self) do
74
66
  revoked
75
67
  end
76
68
  ensure
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Pro