karafka-web 0.10.4 → 0.11.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +54 -176
  4. data/Gemfile +4 -0
  5. data/Gemfile.lock +87 -43
  6. data/LICENSE +6 -2
  7. data/bin/verify_kafka_warnings +35 -0
  8. data/bin/verify_topics_naming +35 -0
  9. data/config/locales/pro_errors.yml +1 -0
  10. data/docker-compose.yml +1 -1
  11. data/gulpfile.js +0 -2
  12. data/karafka-web.gemspec +2 -2
  13. data/lib/karafka/web/config.rb +80 -9
  14. data/lib/karafka/web/contracts/config.rb +44 -5
  15. data/lib/karafka/web/errors.rb +10 -0
  16. data/lib/karafka/web/management/actions/create_initial_states.rb +6 -6
  17. data/lib/karafka/web/management/actions/create_topics.rb +30 -64
  18. data/lib/karafka/web/management/actions/delete_topics.rb +5 -5
  19. data/lib/karafka/web/management/actions/enable.rb +5 -5
  20. data/lib/karafka/web/pro/commanding/commands/base.rb +37 -13
  21. data/lib/karafka/web/pro/commanding/commands/consumers/quiet.rb +33 -0
  22. data/lib/karafka/web/pro/commanding/commands/consumers/stop.rb +32 -0
  23. data/lib/karafka/web/pro/commanding/commands/consumers/trace.rb +37 -0
  24. data/lib/karafka/web/pro/commanding/commands/partitions/pause.rb +30 -0
  25. data/lib/karafka/web/pro/commanding/commands/partitions/resume.rb +30 -0
  26. data/lib/karafka/web/pro/commanding/commands/partitions/seek.rb +30 -0
  27. data/lib/karafka/web/pro/commanding/config.rb +6 -10
  28. data/lib/karafka/web/pro/commanding/contracts/config.rb +2 -10
  29. data/lib/karafka/web/pro/commanding/dispatcher.rb +45 -24
  30. data/lib/karafka/web/pro/commanding/handlers/partitions/commands/base.rb +67 -0
  31. data/lib/karafka/web/pro/commanding/handlers/partitions/commands/pause.rb +44 -0
  32. data/lib/karafka/web/pro/commanding/handlers/partitions/commands/resume.rb +29 -0
  33. data/lib/karafka/web/pro/commanding/handlers/partitions/commands/seek.rb +86 -0
  34. data/lib/karafka/web/pro/commanding/handlers/partitions/executor.rb +56 -0
  35. data/lib/karafka/web/pro/commanding/handlers/partitions/listener.rb +55 -0
  36. data/lib/karafka/web/pro/commanding/handlers/partitions/tracker.rb +62 -0
  37. data/lib/karafka/web/pro/commanding/listener.rb +4 -12
  38. data/lib/karafka/web/pro/commanding/manager.rb +36 -24
  39. data/lib/karafka/web/pro/commanding/matcher.rb +7 -17
  40. data/lib/karafka/web/pro/commanding/request.rb +39 -0
  41. data/lib/karafka/web/pro/commanding.rb +2 -10
  42. data/lib/karafka/web/pro/loader.rb +13 -10
  43. data/lib/karafka/web/pro/ui/app.rb +31 -390
  44. data/lib/karafka/web/pro/ui/controllers/base_controller.rb +8 -10
  45. data/lib/karafka/web/pro/ui/controllers/cluster_controller.rb +2 -10
  46. data/lib/karafka/web/pro/ui/controllers/consumers/base_controller.rb +21 -0
  47. data/lib/karafka/web/pro/ui/controllers/consumers/commanding_controller.rb +148 -0
  48. data/lib/karafka/web/pro/ui/controllers/consumers/commands_controller.rb +96 -0
  49. data/lib/karafka/web/pro/ui/controllers/consumers/consumers_controller.rb +101 -0
  50. data/lib/karafka/web/pro/ui/controllers/consumers/controls_controller.rb +36 -0
  51. data/lib/karafka/web/pro/ui/controllers/consumers/jobs_controller.rb +57 -0
  52. data/lib/karafka/web/pro/ui/controllers/consumers/partitions/base_controller.rb +86 -0
  53. data/lib/karafka/web/pro/ui/controllers/consumers/partitions/offsets_controller.rb +75 -0
  54. data/lib/karafka/web/pro/ui/controllers/consumers/partitions/pauses_controller.rb +110 -0
  55. data/lib/karafka/web/pro/ui/controllers/dashboard_controller.rb +2 -10
  56. data/lib/karafka/web/pro/ui/controllers/dlq_controller.rb +2 -10
  57. data/lib/karafka/web/pro/ui/controllers/errors_controller.rb +3 -11
  58. data/lib/karafka/web/pro/ui/controllers/explorer/base_controller.rb +21 -0
  59. data/lib/karafka/web/pro/ui/controllers/explorer/explorer_controller.rb +215 -0
  60. data/lib/karafka/web/pro/ui/controllers/explorer/messages_controller.rb +145 -0
  61. data/lib/karafka/web/pro/ui/controllers/explorer/search_controller.rb +68 -0
  62. data/lib/karafka/web/pro/ui/controllers/health_controller.rb +2 -10
  63. data/lib/karafka/web/pro/ui/controllers/jobs_controller.rb +2 -10
  64. data/lib/karafka/web/pro/ui/controllers/recurring_tasks_controller.rb +12 -13
  65. data/lib/karafka/web/pro/ui/controllers/routing_controller.rb +2 -10
  66. data/lib/karafka/web/pro/ui/controllers/scheduled_messages/base_controller.rb +2 -10
  67. data/lib/karafka/web/pro/ui/controllers/scheduled_messages/explorer_controller.rb +8 -16
  68. data/lib/karafka/web/pro/ui/controllers/scheduled_messages/messages_controller.rb +9 -15
  69. data/lib/karafka/web/pro/ui/controllers/scheduled_messages/schedules_controller.rb +2 -10
  70. data/lib/karafka/web/pro/ui/controllers/status_controller.rb +2 -10
  71. data/lib/karafka/web/pro/ui/controllers/support_controller.rb +2 -10
  72. data/lib/karafka/web/pro/ui/controllers/topics/base_controller.rb +21 -0
  73. data/lib/karafka/web/pro/ui/controllers/topics/configs_controller.rb +86 -0
  74. data/lib/karafka/web/pro/ui/controllers/topics/distributions_controller.rb +91 -0
  75. data/lib/karafka/web/pro/ui/controllers/topics/offsets_controller.rb +55 -0
  76. data/lib/karafka/web/pro/ui/controllers/topics/replications_controller.rb +37 -0
  77. data/lib/karafka/web/pro/ui/controllers/topics/topics_controller.rb +101 -0
  78. data/lib/karafka/web/pro/ui/controllers/ux_controller.rb +2 -10
  79. data/lib/karafka/web/pro/ui/lib/branding/config.rb +2 -10
  80. data/lib/karafka/web/pro/ui/lib/branding/contracts/config.rb +2 -10
  81. data/lib/karafka/web/pro/ui/lib/branding.rb +2 -10
  82. data/lib/karafka/web/pro/ui/lib/features.rb +53 -0
  83. data/lib/karafka/web/pro/ui/lib/patterns_detector.rb +2 -10
  84. data/lib/karafka/web/pro/ui/lib/policies/config.rb +2 -10
  85. data/lib/karafka/web/pro/ui/lib/policies/contracts/config.rb +2 -10
  86. data/lib/karafka/web/pro/ui/lib/policies/messages.rb +2 -10
  87. data/lib/karafka/web/pro/ui/lib/policies/requests.rb +2 -10
  88. data/lib/karafka/web/pro/ui/lib/policies.rb +2 -10
  89. data/lib/karafka/web/pro/ui/lib/safe_runner.rb +5 -0
  90. data/lib/karafka/web/pro/ui/lib/search/config.rb +2 -10
  91. data/lib/karafka/web/pro/ui/lib/search/contracts/config.rb +2 -10
  92. data/lib/karafka/web/pro/ui/lib/search/contracts/form.rb +2 -10
  93. data/lib/karafka/web/pro/ui/lib/search/matchers/base.rb +2 -10
  94. data/lib/karafka/web/pro/ui/lib/search/matchers/raw_header_includes.rb +10 -11
  95. data/lib/karafka/web/pro/ui/lib/search/matchers/raw_key_includes.rb +2 -10
  96. data/lib/karafka/web/pro/ui/lib/search/matchers/raw_payload_includes.rb +23 -11
  97. data/lib/karafka/web/pro/ui/lib/search/normalizer.rb +2 -10
  98. data/lib/karafka/web/pro/ui/lib/search/runner.rb +3 -11
  99. data/lib/karafka/web/pro/ui/lib/search.rb +2 -10
  100. data/lib/karafka/web/pro/ui/routes/base.rb +19 -0
  101. data/lib/karafka/web/pro/ui/routes/cluster.rb +37 -0
  102. data/lib/karafka/web/pro/ui/routes/consumers.rb +145 -0
  103. data/lib/karafka/web/pro/ui/routes/dashboard.rb +25 -0
  104. data/lib/karafka/web/pro/ui/routes/dlq.rb +24 -0
  105. data/lib/karafka/web/pro/ui/routes/errors.rb +39 -0
  106. data/lib/karafka/web/pro/ui/routes/explorer.rb +118 -0
  107. data/lib/karafka/web/pro/ui/routes/health.rb +47 -0
  108. data/lib/karafka/web/pro/ui/routes/jobs.rb +33 -0
  109. data/lib/karafka/web/pro/ui/routes/recurring_tasks.rb +59 -0
  110. data/lib/karafka/web/pro/ui/routes/routing.rb +31 -0
  111. data/lib/karafka/web/pro/ui/routes/scheduled_messages.rb +75 -0
  112. data/lib/karafka/web/pro/ui/routes/status.rb +24 -0
  113. data/lib/karafka/web/pro/ui/routes/support.rb +24 -0
  114. data/lib/karafka/web/pro/ui/routes/topics.rb +90 -0
  115. data/lib/karafka/web/pro/ui/routes/ux.rb +24 -0
  116. data/lib/karafka/web/pro/ui/views/cluster/_breadcrumbs.erb +3 -0
  117. data/lib/karafka/web/pro/ui/views/cluster/_broker.erb +3 -0
  118. data/lib/karafka/web/pro/ui/views/cluster/_config.erb +3 -0
  119. data/lib/karafka/web/pro/ui/views/cluster/_tabs.erb +3 -0
  120. data/lib/karafka/web/pro/ui/views/cluster/index.erb +4 -1
  121. data/lib/karafka/web/pro/ui/views/cluster/show.erb +3 -0
  122. data/lib/karafka/web/pro/ui/views/{commands → consumers/commands}/_backtrace.erb +3 -0
  123. data/lib/karafka/web/pro/ui/views/consumers/commands/_breadcrumbs.erb +24 -0
  124. data/lib/karafka/web/pro/ui/views/{commands → consumers/commands}/_command.erb +22 -6
  125. data/lib/karafka/web/pro/ui/views/consumers/commands/_command_details.erb +4 -0
  126. data/lib/karafka/web/pro/ui/views/consumers/commands/_empty.erb +6 -0
  127. data/lib/karafka/web/pro/ui/views/{commands → consumers/commands}/_incompatible_schema.erb +3 -0
  128. data/lib/karafka/web/pro/ui/views/{commands → consumers/commands}/_metadata.erb +4 -1
  129. data/lib/karafka/web/pro/ui/views/{commands → consumers/commands}/_table.erb +5 -2
  130. data/lib/karafka/web/pro/ui/views/{commands → consumers/commands}/index.erb +7 -4
  131. data/lib/karafka/web/pro/ui/views/consumers/commands/show.erb +32 -0
  132. data/lib/karafka/web/pro/ui/views/consumers/consumers/_breadcrumbs.erb +46 -0
  133. data/lib/karafka/web/pro/ui/views/consumers/consumers/_consumer.erb +59 -0
  134. data/lib/karafka/web/pro/ui/views/consumers/consumers/_consumer_performance.erb +71 -0
  135. data/lib/karafka/web/pro/ui/views/consumers/consumers/_tabs.erb +38 -0
  136. data/lib/karafka/web/pro/ui/views/consumers/consumers/consumer/_commands.erb +80 -0
  137. data/lib/karafka/web/pro/ui/views/consumers/consumers/consumer/_consumer_group.erb +11 -0
  138. data/lib/karafka/web/pro/ui/views/consumers/{consumer → consumers/consumer}/_metrics.erb +3 -0
  139. data/lib/karafka/web/pro/ui/views/consumers/consumers/consumer/_no_subscriptions.erb +10 -0
  140. data/lib/karafka/web/pro/ui/views/consumers/{consumer → consumers/consumer}/_partition.erb +16 -0
  141. data/lib/karafka/web/pro/ui/views/consumers/consumers/consumer/_partition_edit_options.erb +33 -0
  142. data/lib/karafka/web/pro/ui/views/consumers/{consumer → consumers/consumer}/_stopped.erb +3 -0
  143. data/lib/karafka/web/pro/ui/views/consumers/{consumer → consumers/consumer}/_subscription_group.erb +7 -3
  144. data/lib/karafka/web/pro/ui/views/consumers/{consumer → consumers/consumer}/_tabs.erb +7 -4
  145. data/lib/karafka/web/pro/ui/views/consumers/consumers/details.erb +15 -0
  146. data/lib/karafka/web/pro/ui/views/consumers/{index.erb → consumers/index.erb} +6 -3
  147. data/lib/karafka/web/pro/ui/views/consumers/{performance.erb → consumers/performance.erb} +6 -3
  148. data/lib/karafka/web/pro/ui/views/consumers/consumers/subscriptions.erb +24 -0
  149. data/lib/karafka/web/pro/ui/views/consumers/controls/_breadcrumbs.erb +16 -0
  150. data/lib/karafka/web/pro/ui/views/consumers/controls/_controls.erb +107 -0
  151. data/lib/karafka/web/pro/ui/views/consumers/{controls.erb → controls/index.erb} +8 -5
  152. data/lib/karafka/web/pro/ui/views/consumers/jobs/_breadcrumbs.erb +36 -0
  153. data/lib/karafka/web/pro/ui/views/consumers/{consumer → jobs}/_job.erb +3 -0
  154. data/lib/karafka/web/pro/ui/views/consumers/{consumer → jobs}/_no_jobs.erb +3 -0
  155. data/lib/karafka/web/pro/ui/views/consumers/{pending_jobs.erb → jobs/pending.erb} +7 -8
  156. data/lib/karafka/web/pro/ui/views/consumers/{running_jobs.erb → jobs/running.erb} +7 -8
  157. data/lib/karafka/web/pro/ui/views/consumers/partitions/offsets/_basics.erb +77 -0
  158. data/lib/karafka/web/pro/ui/views/consumers/partitions/offsets/_breadcrumbs.erb +58 -0
  159. data/lib/karafka/web/pro/ui/views/consumers/partitions/offsets/_form.erb +109 -0
  160. data/lib/karafka/web/pro/ui/views/consumers/partitions/offsets/_not_running_error.erb +16 -0
  161. data/lib/karafka/web/pro/ui/views/consumers/partitions/offsets/_running_warning.erb +15 -0
  162. data/lib/karafka/web/pro/ui/views/consumers/partitions/offsets/edit.erb +12 -0
  163. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/_active_not_editable.erb +22 -0
  164. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/_adjusting_warning.erb +27 -0
  165. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/_breadcrumbs.erb +60 -0
  166. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/_edit_form.erb +59 -0
  167. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/_lrj_not_manageable.erb +24 -0
  168. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/_new_form.erb +78 -0
  169. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/_not_running.erb +16 -0
  170. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/edit.erb +20 -0
  171. data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/new.erb +16 -0
  172. data/lib/karafka/web/pro/ui/views/dashboard/index.erb +4 -1
  173. data/lib/karafka/web/pro/ui/views/dlq/_breadcrumbs.erb +3 -0
  174. data/lib/karafka/web/pro/ui/views/dlq/_no_topics.erb +3 -0
  175. data/lib/karafka/web/pro/ui/views/dlq/_topic.erb +4 -1
  176. data/lib/karafka/web/pro/ui/views/dlq/index.erb +3 -0
  177. data/lib/karafka/web/pro/ui/views/errors/_breadcrumbs.erb +3 -0
  178. data/lib/karafka/web/pro/ui/views/errors/_error.erb +3 -0
  179. data/lib/karafka/web/pro/ui/views/errors/_partition_option.erb +3 -0
  180. data/lib/karafka/web/pro/ui/views/errors/_selector.erb +3 -0
  181. data/lib/karafka/web/pro/ui/views/errors/_table.erb +4 -1
  182. data/lib/karafka/web/pro/ui/views/errors/index.erb +6 -3
  183. data/lib/karafka/web/pro/ui/views/errors/partition.erb +5 -2
  184. data/lib/karafka/web/pro/ui/views/errors/show.erb +3 -0
  185. data/lib/karafka/web/pro/ui/views/explorer/{_breadcrumbs.erb → explorer/_breadcrumbs.erb} +7 -4
  186. data/lib/karafka/web/pro/ui/views/explorer/{_failed_deserialization.erb → explorer/_failed_deserialization.erb} +3 -0
  187. data/lib/karafka/web/pro/ui/views/explorer/{_filtered.erb → explorer/_filtered.erb} +3 -0
  188. data/lib/karafka/web/pro/ui/views/explorer/{_message.erb → explorer/_message.erb} +4 -1
  189. data/lib/karafka/web/pro/ui/views/explorer/explorer/_no_topics.erb +4 -0
  190. data/lib/karafka/web/pro/ui/views/explorer/{_partition_option.erb → explorer/_partition_option.erb} +4 -1
  191. data/lib/karafka/web/pro/ui/views/explorer/{_selector.erb → explorer/_selector.erb} +4 -1
  192. data/lib/karafka/web/pro/ui/views/explorer/explorer/_topic.erb +13 -0
  193. data/lib/karafka/web/pro/ui/views/explorer/explorer/index.erb +17 -0
  194. data/lib/karafka/web/pro/ui/views/explorer/{message → explorer/message}/_metadata.erb +10 -7
  195. data/lib/karafka/web/pro/ui/views/explorer/{message → explorer/message}/_payload.erb +6 -3
  196. data/lib/karafka/web/pro/ui/views/explorer/{message → explorer/message}/_resources_utilization.erb +7 -4
  197. data/lib/karafka/web/pro/ui/views/explorer/{message → explorer/message}/_too_big_to_be_displayed.erb +3 -0
  198. data/lib/karafka/web/pro/ui/views/explorer/{messages → explorer/messages}/_detail.erb +3 -0
  199. data/lib/karafka/web/pro/ui/views/explorer/explorer/messages/_headers.erb +51 -0
  200. data/lib/karafka/web/pro/ui/views/explorer/{messages → explorer/messages}/_key.erb +3 -0
  201. data/lib/karafka/web/pro/ui/views/explorer/explorer/partition/_cleaned.erb +6 -0
  202. data/lib/karafka/web/pro/ui/views/explorer/explorer/partition/_empty.erb +6 -0
  203. data/lib/karafka/web/pro/ui/views/explorer/{partition → explorer/partition}/_messages.erb +4 -1
  204. data/lib/karafka/web/pro/ui/views/explorer/explorer/partition/_time_selector.erb +16 -0
  205. data/lib/karafka/web/pro/ui/views/explorer/explorer/partition/_timestamp_selector.erb +33 -0
  206. data/lib/karafka/web/pro/ui/views/explorer/{partition.erb → explorer/partition.erb} +24 -17
  207. data/lib/karafka/web/pro/ui/views/explorer/{show.erb → explorer/show.erb} +17 -19
  208. data/lib/karafka/web/pro/ui/views/explorer/{topic → explorer/topic}/_actions.erb +5 -2
  209. data/lib/karafka/web/pro/ui/views/explorer/explorer/topic/_empty.erb +6 -0
  210. data/lib/karafka/web/pro/ui/views/explorer/{topic → explorer/topic}/_limited.erb +3 -0
  211. data/lib/karafka/web/pro/ui/views/explorer/{topic.erb → explorer/topic.erb} +7 -4
  212. data/lib/karafka/web/pro/ui/views/explorer/messages/_breadcrumbs.erb +32 -0
  213. data/lib/karafka/web/pro/ui/views/explorer/messages/forward.erb +143 -0
  214. data/lib/karafka/web/pro/ui/views/explorer/search/_breadcrumbs.erb +4 -0
  215. data/lib/karafka/web/pro/ui/views/explorer/search/_fix_errors.erb +6 -0
  216. data/lib/karafka/web/pro/ui/views/{search → explorer/search}/_metadata.erb +3 -0
  217. data/lib/karafka/web/pro/ui/views/explorer/search/_no_results.erb +6 -0
  218. data/lib/karafka/web/pro/ui/views/{search → explorer/search}/_no_search_criteria.erb +3 -0
  219. data/lib/karafka/web/pro/ui/views/{search → explorer/search}/_search_criteria.erb +3 -0
  220. data/lib/karafka/web/pro/ui/views/{search → explorer/search}/_search_modal.erb +5 -2
  221. data/lib/karafka/web/pro/ui/views/explorer/search/_timeout.erb +6 -0
  222. data/lib/karafka/web/pro/ui/views/explorer/search/index.erb +32 -0
  223. data/lib/karafka/web/pro/ui/views/health/_breadcrumbs.erb +3 -0
  224. data/lib/karafka/web/pro/ui/views/health/_no_data.erb +3 -0
  225. data/lib/karafka/web/pro/ui/views/health/_partition.erb +16 -1
  226. data/lib/karafka/web/pro/ui/views/health/_partition_lags.erb +3 -0
  227. data/lib/karafka/web/pro/ui/views/health/_partition_offset.erb +3 -0
  228. data/lib/karafka/web/pro/ui/views/health/_partition_times.erb +3 -0
  229. data/lib/karafka/web/pro/ui/views/health/_table_metadata.erb +4 -1
  230. data/lib/karafka/web/pro/ui/views/health/_tabs.erb +3 -0
  231. data/lib/karafka/web/pro/ui/views/health/changes.erb +4 -1
  232. data/lib/karafka/web/pro/ui/views/health/cluster_lags.erb +3 -0
  233. data/lib/karafka/web/pro/ui/views/health/lags.erb +5 -2
  234. data/lib/karafka/web/pro/ui/views/health/offsets.erb +4 -1
  235. data/lib/karafka/web/pro/ui/views/health/overview.erb +8 -3
  236. data/lib/karafka/web/pro/ui/views/jobs/_job.erb +50 -38
  237. data/lib/karafka/web/pro/ui/views/jobs/_no_jobs.erb +3 -0
  238. data/lib/karafka/web/pro/ui/views/jobs/pending.erb +4 -1
  239. data/lib/karafka/web/pro/ui/views/jobs/running.erb +4 -1
  240. data/lib/karafka/web/pro/ui/views/recurring_tasks/_actions.erb +3 -0
  241. data/lib/karafka/web/pro/ui/views/recurring_tasks/_batch_actions.erb +3 -0
  242. data/lib/karafka/web/pro/ui/views/recurring_tasks/_breadcrumbs.erb +3 -0
  243. data/lib/karafka/web/pro/ui/views/recurring_tasks/_log.erb +3 -0
  244. data/lib/karafka/web/pro/ui/views/recurring_tasks/_not_active.erb +3 -0
  245. data/lib/karafka/web/pro/ui/views/recurring_tasks/_tabs.erb +3 -0
  246. data/lib/karafka/web/pro/ui/views/recurring_tasks/_task.erb +3 -0
  247. data/lib/karafka/web/pro/ui/views/recurring_tasks/logs.erb +3 -0
  248. data/lib/karafka/web/pro/ui/views/recurring_tasks/schedule.erb +3 -0
  249. data/lib/karafka/web/pro/ui/views/routing/_consumer_group.erb +3 -0
  250. data/lib/karafka/web/pro/ui/views/routing/_detail.erb +3 -0
  251. data/lib/karafka/web/pro/ui/views/routing/_topic.erb +3 -0
  252. data/lib/karafka/web/pro/ui/views/routing/index.erb +3 -0
  253. data/lib/karafka/web/pro/ui/views/routing/show.erb +3 -0
  254. data/lib/karafka/web/pro/ui/views/scheduled_messages/explorer/_breadcrumbs.erb +6 -3
  255. data/lib/karafka/web/pro/ui/views/scheduled_messages/explorer/_key.erb +3 -0
  256. data/lib/karafka/web/pro/ui/views/scheduled_messages/explorer/_message.erb +4 -1
  257. data/lib/karafka/web/pro/ui/views/scheduled_messages/explorer/_messages.erb +3 -0
  258. data/lib/karafka/web/pro/ui/views/scheduled_messages/explorer/partition.erb +23 -16
  259. data/lib/karafka/web/pro/ui/views/scheduled_messages/explorer/topic.erb +6 -3
  260. data/lib/karafka/web/pro/ui/views/scheduled_messages/schedules/_breadcrumbs.erb +3 -0
  261. data/lib/karafka/web/pro/ui/views/scheduled_messages/schedules/_no_groups.erb +3 -0
  262. data/lib/karafka/web/pro/ui/views/scheduled_messages/schedules/index.erb +4 -1
  263. data/lib/karafka/web/pro/ui/views/scheduled_messages/schedules/show.erb +3 -0
  264. data/lib/karafka/web/pro/ui/views/shared/_navigation.erb +25 -17
  265. data/lib/karafka/web/pro/ui/views/shared/_rdkafka_form_error_alert_box.erb +16 -0
  266. data/lib/karafka/web/pro/ui/views/shared/branding/_label.erb +3 -0
  267. data/lib/karafka/web/pro/ui/views/shared/branding/_notice.erb +3 -0
  268. data/lib/karafka/web/pro/ui/views/topics/configs/_breadcrumbs.erb +34 -0
  269. data/lib/karafka/web/pro/ui/views/topics/configs/_config.erb +26 -0
  270. data/lib/karafka/web/pro/ui/views/topics/configs/_delete_button.erb +13 -0
  271. data/lib/karafka/web/pro/ui/views/topics/configs/_edit_form.erb +50 -0
  272. data/lib/karafka/web/pro/ui/views/topics/configs/_edit_plan.erb +16 -0
  273. data/lib/karafka/web/pro/ui/views/topics/configs/_edit_warning.erb +12 -0
  274. data/lib/karafka/web/pro/ui/views/topics/configs/edit.erb +16 -0
  275. data/lib/karafka/web/pro/ui/views/topics/{config.erb → configs/index.erb} +9 -3
  276. data/lib/karafka/web/pro/ui/views/topics/distributions/_add_partitions_button.erb +13 -0
  277. data/lib/karafka/web/pro/ui/views/topics/{distribution → distributions}/_badges.erb +3 -0
  278. data/lib/karafka/web/pro/ui/views/topics/distributions/_breadcrumbs.erb +28 -0
  279. data/lib/karafka/web/pro/ui/views/topics/{distribution → distributions}/_chart.erb +3 -0
  280. data/lib/karafka/web/pro/ui/views/topics/distributions/_edit_form.erb +47 -0
  281. data/lib/karafka/web/pro/ui/views/topics/distributions/_edit_hints.erb +15 -0
  282. data/lib/karafka/web/pro/ui/views/topics/distributions/_edit_warnings.erb +14 -0
  283. data/lib/karafka/web/pro/ui/views/topics/distributions/_empty_partitions.erb +4 -0
  284. data/lib/karafka/web/pro/ui/views/topics/{distribution → distributions}/_limited.erb +3 -0
  285. data/lib/karafka/web/pro/ui/views/topics/distributions/_partition.erb +13 -0
  286. data/lib/karafka/web/pro/ui/views/topics/distributions/edit.erb +16 -0
  287. data/lib/karafka/web/pro/ui/views/topics/{distribution.erb → distributions/show.erb} +11 -7
  288. data/lib/karafka/web/pro/ui/views/topics/offsets/_breadcrumbs.erb +20 -0
  289. data/lib/karafka/web/pro/ui/views/topics/offsets/_partition.erb +13 -0
  290. data/lib/karafka/web/pro/ui/views/topics/{offsets.erb → offsets/show.erb} +6 -3
  291. data/lib/karafka/web/pro/ui/views/topics/replications/_breadcrumbs.erb +20 -0
  292. data/lib/karafka/web/pro/ui/views/topics/{_partition.erb → replications/_partition.erb} +4 -1
  293. data/lib/karafka/web/pro/ui/views/topics/{replication.erb → replications/show.erb} +6 -3
  294. data/lib/karafka/web/pro/ui/views/topics/topics/_breadcrumbs.erb +32 -0
  295. data/lib/karafka/web/pro/ui/views/topics/topics/_create_button.erb +13 -0
  296. data/lib/karafka/web/pro/ui/views/topics/topics/_create_hints.erb +15 -0
  297. data/lib/karafka/web/pro/ui/views/topics/topics/_delete_form.erb +36 -0
  298. data/lib/karafka/web/pro/ui/views/topics/topics/_delete_hints.erb +15 -0
  299. data/lib/karafka/web/pro/ui/views/topics/topics/_delete_warning.erb +13 -0
  300. data/lib/karafka/web/pro/ui/views/topics/topics/_new_form.erb +80 -0
  301. data/lib/karafka/web/pro/ui/views/topics/{_tabs.erb → topics/_tabs.erb} +7 -4
  302. data/lib/karafka/web/pro/ui/views/topics/topics/_topic.erb +12 -0
  303. data/lib/karafka/web/pro/ui/views/topics/topics/edit.erb +10 -0
  304. data/lib/karafka/web/pro/ui/views/topics/topics/index.erb +19 -0
  305. data/lib/karafka/web/pro/ui/views/topics/topics/new.erb +12 -0
  306. data/lib/karafka/web/processing/consumers/metrics.rb +1 -1
  307. data/lib/karafka/web/processing/consumers/state.rb +1 -1
  308. data/lib/karafka/web/processing/publisher.rb +4 -4
  309. data/lib/karafka/web/tracking/consumers/contracts/partition.rb +1 -0
  310. data/lib/karafka/web/tracking/consumers/listeners/pausing.rb +2 -2
  311. data/lib/karafka/web/tracking/consumers/listeners/transactions.rb +44 -0
  312. data/lib/karafka/web/tracking/consumers/reporter.rb +2 -2
  313. data/lib/karafka/web/tracking/consumers/sampler.rb +81 -14
  314. data/lib/karafka/web/tracking/helpers/sysconf.rb +33 -0
  315. data/lib/karafka/web/tracking/producers/reporter.rb +1 -1
  316. data/lib/karafka/web/ui/app.rb +19 -112
  317. data/lib/karafka/web/ui/base.rb +63 -4
  318. data/lib/karafka/web/ui/controllers/base_controller.rb +43 -1
  319. data/lib/karafka/web/ui/controllers/cluster_controller.rb +5 -2
  320. data/lib/karafka/web/ui/controllers/errors_controller.rb +1 -1
  321. data/lib/karafka/web/ui/controllers/requests/execution_wrapper.rb +52 -0
  322. data/lib/karafka/web/ui/controllers/requests/hookable.rb +99 -0
  323. data/lib/karafka/web/ui/controllers/requests/params.rb +39 -1
  324. data/lib/karafka/web/ui/controllers/responses/redirect.rb +0 -5
  325. data/lib/karafka/web/ui/controllers/status_controller.rb +3 -0
  326. data/lib/karafka/web/ui/helpers/application_helper.rb +10 -1
  327. data/lib/karafka/web/ui/helpers/paths_helper.rb +54 -10
  328. data/lib/karafka/web/ui/lib/admin.rb +1 -1
  329. data/lib/karafka/web/ui/lib/cache.rb +135 -0
  330. data/lib/karafka/web/ui/models/broker.rb +1 -2
  331. data/lib/karafka/web/ui/models/cluster_info.rb +15 -21
  332. data/lib/karafka/web/ui/models/consumers_metrics.rb +1 -1
  333. data/lib/karafka/web/ui/models/consumers_state.rb +1 -1
  334. data/lib/karafka/web/ui/models/counters.rb +1 -1
  335. data/lib/karafka/web/ui/models/health.rb +9 -7
  336. data/lib/karafka/web/ui/models/process.rb +14 -0
  337. data/lib/karafka/web/ui/models/processes.rb +2 -2
  338. data/lib/karafka/web/ui/models/recurring_tasks/schedule.rb +1 -1
  339. data/lib/karafka/web/ui/models/status.rb +27 -8
  340. data/lib/karafka/web/ui/models/topic.rb +1 -2
  341. data/lib/karafka/web/ui/public/javascripts/application.js +8 -98
  342. data/lib/karafka/web/ui/public/javascripts/application.min.js +12 -4
  343. data/lib/karafka/web/ui/public/javascripts/application.min.js.br +0 -0
  344. data/lib/karafka/web/ui/public/javascripts/application.min.js.gz +0 -0
  345. data/lib/karafka/web/ui/public/javascripts/components/action_confirmation_manager.js +30 -0
  346. data/lib/karafka/web/ui/public/javascripts/components/alerts.js +39 -0
  347. data/lib/karafka/web/ui/public/javascripts/components/button_lock_manager.js +50 -0
  348. data/lib/karafka/web/ui/public/javascripts/components/live_poll.js +71 -19
  349. data/lib/karafka/web/ui/public/javascripts/components/message_republish_manager.js +50 -0
  350. data/lib/karafka/web/ui/public/javascripts/components/page_title_tracker.js +21 -0
  351. data/lib/karafka/web/ui/public/javascripts/components/partition_redirect_manager.js +21 -0
  352. data/lib/karafka/web/ui/public/javascripts/components/time_ago_manager.js +25 -0
  353. data/lib/karafka/web/ui/public/javascripts/components/timestamp_selector.js +30 -0
  354. data/lib/karafka/web/ui/public/javascripts/libs/datepicker.js +2 -2
  355. data/lib/karafka/web/ui/public/stylesheets/application.css +30 -0
  356. data/lib/karafka/web/ui/public/stylesheets/application.min.css +5122 -13
  357. data/lib/karafka/web/ui/public/stylesheets/application.min.css.br +0 -0
  358. data/lib/karafka/web/ui/public/stylesheets/application.min.css.gz +0 -0
  359. data/lib/karafka/web/ui/public/stylesheets/libs/highlight_dark.min.css.gz +0 -0
  360. data/lib/karafka/web/ui/public/stylesheets/libs/highlight_light.min.css.gz +0 -0
  361. data/lib/karafka/web/ui/public/stylesheets/libs/tailwind.css +512 -213
  362. data/lib/karafka/web/ui/routes/assets.rb +53 -0
  363. data/lib/karafka/web/ui/routes/base.rb +36 -0
  364. data/lib/karafka/web/ui/routes/cluster.rb +28 -0
  365. data/lib/karafka/web/ui/routes/consumers.rb +35 -0
  366. data/lib/karafka/web/ui/routes/dashboard.rb +20 -0
  367. data/lib/karafka/web/ui/routes/errors.rb +26 -0
  368. data/lib/karafka/web/ui/routes/jobs.rb +28 -0
  369. data/lib/karafka/web/ui/routes/pro_only.rb +27 -0
  370. data/lib/karafka/web/ui/routes/routing.rb +26 -0
  371. data/lib/karafka/web/ui/routes/status.rb +19 -0
  372. data/lib/karafka/web/ui/routes/support.rb +19 -0
  373. data/lib/karafka/web/ui/routes/ux.rb +19 -0
  374. data/lib/karafka/web/ui/views/cluster/_partition.erb +2 -2
  375. data/lib/karafka/web/ui/views/cluster/brokers.erb +1 -1
  376. data/lib/karafka/web/ui/views/consumers/_breadcrumbs.erb +7 -1
  377. data/lib/karafka/web/ui/views/consumers/_consumer.erb +39 -30
  378. data/lib/karafka/web/ui/views/consumers/_incompatible.erb +13 -0
  379. data/lib/karafka/web/ui/views/consumers/_no_consumers.erb +2 -2
  380. data/lib/karafka/web/ui/views/consumers/_tabs.erb +4 -4
  381. data/lib/karafka/web/ui/views/consumers/index.erb +1 -1
  382. data/lib/karafka/web/ui/views/dashboard/_feature_pro.erb +1 -1
  383. data/lib/karafka/web/ui/views/dashboard/_not_enough_data.erb +2 -2
  384. data/lib/karafka/web/ui/views/dashboard/_ranges_selector.erb +1 -1
  385. data/lib/karafka/web/ui/views/dashboard/index.erb +6 -49
  386. data/lib/karafka/web/ui/views/errors/_detail.erb +3 -3
  387. data/lib/karafka/web/ui/views/errors/index.erb +1 -1
  388. data/lib/karafka/web/ui/views/jobs/_job.erb +38 -29
  389. data/lib/karafka/web/ui/views/jobs/pending.erb +1 -1
  390. data/lib/karafka/web/ui/views/jobs/running.erb +1 -1
  391. data/lib/karafka/web/ui/views/layout.erb +7 -5
  392. data/lib/karafka/web/ui/views/shared/_become_pro.erb +1 -1
  393. data/lib/karafka/web/ui/views/shared/_brand.erb +1 -1
  394. data/lib/karafka/web/ui/views/shared/_breadcrumbs.erb +1 -1
  395. data/lib/karafka/web/ui/views/shared/_content.erb +1 -1
  396. data/lib/karafka/web/ui/views/shared/_controls.erb +10 -3
  397. data/lib/karafka/web/ui/views/shared/_custom_nav.erb +9 -0
  398. data/lib/karafka/web/ui/views/shared/_flashes.erb +3 -5
  399. data/lib/karafka/web/ui/views/shared/_header.erb +25 -2
  400. data/lib/karafka/web/ui/views/shared/_navigation.erb +17 -15
  401. data/lib/karafka/web/ui/views/shared/alerts/_error.erb +8 -0
  402. data/lib/karafka/web/ui/views/shared/alerts/_info.erb +8 -0
  403. data/lib/karafka/web/ui/views/shared/alerts/_primary.erb +8 -0
  404. data/lib/karafka/web/ui/views/shared/alerts/_secondary.erb +8 -0
  405. data/lib/karafka/web/ui/views/shared/alerts/_success.erb +8 -0
  406. data/lib/karafka/web/ui/views/shared/alerts/_warning.erb +8 -0
  407. data/lib/karafka/web/ui/views/shared/exceptions/incompatible_schema.erb +34 -0
  408. data/lib/karafka/web/ui/views/shared/exceptions/not_allowed.erb +4 -0
  409. data/lib/karafka/web/ui/views/shared/exceptions/not_found.erb +5 -1
  410. data/lib/karafka/web/ui/views/shared/exceptions/pro_only.erb +4 -0
  411. data/lib/karafka/web/ui/views/shared/icons/_arrow_left.erb +3 -0
  412. data/lib/karafka/web/ui/views/shared/icons/_arrow_up_tray.erb +3 -0
  413. data/lib/karafka/web/ui/views/shared/icons/_clock.erb +3 -0
  414. data/lib/karafka/web/ui/views/shared/icons/_pencil.erb +3 -0
  415. data/lib/karafka/web/ui/views/shared/icons/_pencil_square.erb +3 -0
  416. data/lib/karafka/web/ui/views/shared/icons/_play_pause.erb +3 -0
  417. data/lib/karafka/web/ui/views/shared/icons/_plus.erb +3 -0
  418. data/lib/karafka/web/ui/views/shared/icons/_trash.erb +3 -0
  419. data/lib/karafka/web/ui/views/status/failures/_live_reporting.erb +1 -1
  420. data/lib/karafka/web/ui/views/status/failures/_partitions.erb +3 -3
  421. data/lib/karafka/web/ui/views/status/failures/_state_calculation.erb +2 -2
  422. data/lib/karafka/web/ui/views/status/info/_components.erb +6 -6
  423. data/lib/karafka/web/ui/views/status/show.erb +15 -0
  424. data/lib/karafka/web/ui/views/status/warnings/_consumers_schemas.erb +31 -0
  425. data/lib/karafka/web/ui/views/ux/_icons.erb +1 -1
  426. data/lib/karafka/web/ui/views/ux/_status_rows.erb +6 -0
  427. data/lib/karafka/web/version.rb +1 -1
  428. data/lib/karafka/web.rb +3 -0
  429. data/package-lock.json +776 -1208
  430. data/package.json +3 -4
  431. data/postcss.config.js +1 -2
  432. data/renovate.json +13 -1
  433. data/tailwind.config.js +0 -4
  434. data.tar.gz.sig +0 -0
  435. metadata +232 -108
  436. metadata.gz.sig +0 -0
  437. data/lib/karafka/web/pro/commanding/commands/quiet.rb +0 -34
  438. data/lib/karafka/web/pro/commanding/commands/stop.rb +0 -34
  439. data/lib/karafka/web/pro/commanding/commands/trace.rb +0 -41
  440. data/lib/karafka/web/pro/ui/controllers/commanding_controller.rb +0 -118
  441. data/lib/karafka/web/pro/ui/controllers/commands_controller.rb +0 -96
  442. data/lib/karafka/web/pro/ui/controllers/consumers_controller.rb +0 -138
  443. data/lib/karafka/web/pro/ui/controllers/explorer_controller.rb +0 -220
  444. data/lib/karafka/web/pro/ui/controllers/messages_controller.rb +0 -107
  445. data/lib/karafka/web/pro/ui/controllers/search_controller.rb +0 -73
  446. data/lib/karafka/web/pro/ui/controllers/topics_controller.rb +0 -130
  447. data/lib/karafka/web/pro/ui/views/commands/_breadcrumbs.erb +0 -21
  448. data/lib/karafka/web/pro/ui/views/commands/_command_details.erb +0 -1
  449. data/lib/karafka/web/pro/ui/views/commands/_empty.erb +0 -3
  450. data/lib/karafka/web/pro/ui/views/commands/show.erb +0 -33
  451. data/lib/karafka/web/pro/ui/views/consumers/_breadcrumbs.erb +0 -55
  452. data/lib/karafka/web/pro/ui/views/consumers/_consumer.erb +0 -47
  453. data/lib/karafka/web/pro/ui/views/consumers/_consumer_controls.erb +0 -95
  454. data/lib/karafka/web/pro/ui/views/consumers/_consumer_performance.erb +0 -59
  455. data/lib/karafka/web/pro/ui/views/consumers/_tabs.erb +0 -33
  456. data/lib/karafka/web/pro/ui/views/consumers/consumer/_commands.erb +0 -72
  457. data/lib/karafka/web/pro/ui/views/consumers/consumer/_consumer_group.erb +0 -8
  458. data/lib/karafka/web/pro/ui/views/consumers/consumer/_no_subscriptions.erb +0 -7
  459. data/lib/karafka/web/pro/ui/views/consumers/details.erb +0 -13
  460. data/lib/karafka/web/pro/ui/views/consumers/subscriptions.erb +0 -25
  461. data/lib/karafka/web/pro/ui/views/explorer/_no_topics.erb +0 -1
  462. data/lib/karafka/web/pro/ui/views/explorer/_topic.erb +0 -10
  463. data/lib/karafka/web/pro/ui/views/explorer/index.erb +0 -14
  464. data/lib/karafka/web/pro/ui/views/explorer/messages/_headers.erb +0 -33
  465. data/lib/karafka/web/pro/ui/views/explorer/partition/_cleaned.erb +0 -3
  466. data/lib/karafka/web/pro/ui/views/explorer/partition/_empty.erb +0 -3
  467. data/lib/karafka/web/pro/ui/views/explorer/topic/_empty.erb +0 -3
  468. data/lib/karafka/web/pro/ui/views/search/_breadcrumbs.erb +0 -1
  469. data/lib/karafka/web/pro/ui/views/search/_fix_errors.erb +0 -3
  470. data/lib/karafka/web/pro/ui/views/search/_no_results.erb +0 -3
  471. data/lib/karafka/web/pro/ui/views/search/_timeout.erb +0 -3
  472. data/lib/karafka/web/pro/ui/views/search/index.erb +0 -29
  473. data/lib/karafka/web/pro/ui/views/topics/_breadcrumbs.erb +0 -45
  474. data/lib/karafka/web/pro/ui/views/topics/_partition_offsets.erb +0 -10
  475. data/lib/karafka/web/pro/ui/views/topics/_topic.erb +0 -9
  476. data/lib/karafka/web/pro/ui/views/topics/distribution/_empty_partitions.erb +0 -1
  477. data/lib/karafka/web/pro/ui/views/topics/distribution/_partition.erb +0 -10
  478. data/lib/karafka/web/pro/ui/views/topics/index.erb +0 -14
  479. data/lib/karafka/web/ui/lib/ttl_cache.rb +0 -82
@@ -35,24 +35,79 @@ module Karafka
35
35
  # Topics naming - used for processing and UI
36
36
  setting :topics do
37
37
  # All the errors encountered will be dispatched to this topic for inspection
38
- setting :errors, default: 'karafka_errors'
38
+ setting :errors do
39
+ setting :name, default: 'karafka_errors'
40
+
41
+ # Remove really old errors (older than 3 months just to preserve space)
42
+ setting :config, default: {
43
+ 'cleanup.policy': 'delete',
44
+ 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
45
+ }
46
+ end
39
47
 
40
48
  setting :consumers do
41
49
  # Reports containing particular consumer processes. This topic contains the heartbeat
42
50
  # information sent from each consumer process.
43
- setting :reports, default: 'karafka_consumers_reports'
51
+ setting :reports do
52
+ # Name of the topic
53
+ setting :name, default: 'karafka_consumers_reports'
54
+
55
+ # We do not need to to store this data for longer than 1 day as this data is only
56
+ # used to materialize the end states
57
+ # On the other hand we do not want to have it really short-living because in case
58
+ # of a consumer crash, we may want to use this info to catch up and backfill the
59
+ # state.
60
+ #
61
+ # In case its not consumed because no processes are running, it also usually means
62
+ # there's no data to consume because no karafka servers report
63
+ setting :config, default: {
64
+ 'cleanup.policy': 'delete',
65
+ 'retention.ms': 24 * 60 * 60 * 1_000 # 1 day
66
+ }
67
+ end
44
68
 
45
69
  # Topic for storing states aggregated info
46
- setting :states, default: 'karafka_consumers_states'
70
+ setting :states do
71
+ setting :name, default: 'karafka_consumers_states'
72
+
73
+ # We care only about the most recent state, previous are irrelevant. So we can
74
+ # easily compact after one minute. We do not use this beyond the most recent
75
+ # collective state, hence it all can easily go away. We also limit the segment
76
+ # size to at most 100MB not to use more space ever.
77
+ setting :config, default: {
78
+ 'cleanup.policy': 'compact',
79
+ 'retention.ms': 60 * 60 * 1_000,
80
+ 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
81
+ 'segment.bytes': 104_857_600 # 100MB
82
+ }
83
+ end
47
84
 
48
85
  # Topic for storing consumers historical metrics info
49
- setting :metrics, default: 'karafka_consumers_metrics'
86
+ setting :metrics do
87
+ setting :name, default: 'karafka_consumers_metrics'
88
+
89
+ setting :config, default: {
90
+ 'cleanup.policy': 'compact',
91
+ 'retention.ms': 24 * 60 * 60 * 1_000, # 1 day
92
+ 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
93
+ 'segment.bytes': 104_857_600 # 100MB
94
+ }
95
+ end
50
96
 
51
97
  # Topic for storing commands and their results
52
98
  # This is used only in Pro, however we do setup it in OSS in case of upgrade so the
53
99
  # transition from one to another is smooth. Otherwise upgrade would require changes
54
100
  # to topics (migration) which may be more complex
55
- setting :commands, default: 'karafka_consumers_commands'
101
+ setting :commands do
102
+ setting :name, default: 'karafka_consumers_commands'
103
+
104
+ setting :config, default: {
105
+ 'cleanup.policy': 'delete',
106
+ 'retention.ms': 7 * 24 * 60 * 60 * 1_000, # 7 days
107
+ 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
108
+ 'segment.bytes': 104_857_600 # 100MB
109
+ }
110
+ end
56
111
  end
57
112
  end
58
113
 
@@ -93,7 +148,8 @@ module Karafka
93
148
  Tracking::Consumers::Listeners::Statistics.new,
94
149
  Tracking::Consumers::Listeners::Pausing.new,
95
150
  Tracking::Consumers::Listeners::Processing.new,
96
- Tracking::Consumers::Listeners::Tags.new
151
+ Tracking::Consumers::Listeners::Tags.new,
152
+ Tracking::Consumers::Listeners::Transactions.new
97
153
  ]
98
154
  end
99
155
 
@@ -150,10 +206,13 @@ module Karafka
150
206
  setting :secret, default: SecureRandom.hex(32)
151
207
  end
152
208
 
153
- # UI cache to improve performance of views that reuse states that are not often changed
154
- setting :cache, default: Ui::Lib::TtlCache.new(
209
+ # Process cache with invalidation only
210
+ setting :cache, default: Ui::Lib::Cache.new(
155
211
  # Use the TTL for internal cache in prod but invalidate quickly in other environments,
156
- # as for example in development things may change frequently
212
+ # as for example in development things may change frequently. In dev also the cache
213
+ # is less needed since local clusters are usually smaller than prod.
214
+ # It is set to 5 minutes by default because in prod the cluster chances are not frequent
215
+ # and anyhow we refresh on certain web ui actions so this is as fail-safe
157
216
  Karafka.env.production? ? 60_000 * 5 : 5_000
158
217
  )
159
218
 
@@ -186,6 +245,18 @@ module Karafka
186
245
  # default will not be displayed not to hang the browser. 512KB of serialized data is a lot.
187
246
  setting :max_visible_payload_size, default: 524_288
188
247
 
248
+ setting :custom do
249
+ # Path to a custom CSS, string with the CSS or false if not used
250
+ setting :css, default: false
251
+
252
+ # Path to a custom JS, string with the JS or false if not used
253
+ setting :js, default: false
254
+
255
+ # ERB template (path or string) for injecting extra navigation items below the default
256
+ # menu. Useful for adding "Back to app" links or other quick access actions.
257
+ setting :nav_erb, default: false
258
+ end
259
+
189
260
  # Specific kafka settings that are tuned to operate within the Web UI interface.
190
261
  #
191
262
  # Please do not change them unless you know what you are doing as their misconfiguration
@@ -15,12 +15,46 @@ module Karafka
15
15
  required(:group_id) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
16
16
 
17
17
  nested(:topics) do
18
- required(:errors) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
18
+ nested(:errors) do
19
+ required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
20
+
21
+ required(:config) do |val|
22
+ val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
23
+ end
24
+ end
19
25
 
20
26
  nested(:consumers) do
21
- required(:reports) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
22
- required(:states) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
23
- required(:metrics) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
27
+ nested(:reports) do
28
+ required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
29
+
30
+ required(:config) do |val|
31
+ val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
32
+ end
33
+ end
34
+
35
+ nested(:states) do
36
+ required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
37
+
38
+ required(:config) do |val|
39
+ val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
40
+ end
41
+ end
42
+
43
+ nested(:metrics) do
44
+ required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
45
+
46
+ required(:config) do |val|
47
+ val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
48
+ end
49
+ end
50
+
51
+ nested(:commands) do
52
+ required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
53
+
54
+ required(:config) do |val|
55
+ val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
56
+ end
57
+ end
24
58
  end
25
59
  end
26
60
 
@@ -60,11 +94,16 @@ module Karafka
60
94
  required(:secret) { |val| val.is_a?(String) && val.length >= 64 }
61
95
  end
62
96
 
63
- required(:cache) { |val| !val.nil? }
64
97
  required(:per_page) { |val| val.is_a?(Integer) && val >= 1 && val <= 100 }
65
98
  required(:max_visible_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
66
99
  required(:kafka) { |val| val.is_a?(Hash) }
67
100
 
101
+ nested(:custom) do
102
+ required(:css) { |val| val == false || (val.is_a?(String) && !val.empty?) }
103
+ required(:js) { |val| val == false || (val.is_a?(String) && !val.empty?) }
104
+ required(:nav_erb) { |val| val == false || (val.is_a?(String) && !val.empty?) }
105
+ end
106
+
68
107
  required(:dlq_patterns) do |val|
69
108
  val.is_a?(Array) &&
70
109
  val.all? { |attr| attr.is_a?(String) || attr.is_a?(Regexp) }
@@ -20,6 +20,11 @@ module Karafka
20
20
  # configures Karafka
21
21
  KarafkaNotInitializedError = Class.new(BaseError)
22
22
 
23
+ # Raised when you try to configure Web UI after it was enabled. It is not allowed because
24
+ # Karafka Web UI uses the setup values during the enablement and their later change may not
25
+ # be fully reflected. Always run `#setup` before `#enable!`.
26
+ LateSetupError = Class.new(BaseError)
27
+
23
28
  # Errors specific to management
24
29
  module Management
25
30
  # Similar to processing error with the same name, it is raised when a critical
@@ -74,6 +79,11 @@ module Karafka
74
79
 
75
80
  # Raised when we want to stop the flow and render 403
76
81
  ForbiddenError = Class.new(BaseError)
82
+
83
+ # Raised when trying to get info about a consumer that has incompatible schema in its
84
+ # report. It usually means you are running different version of the Web UI in the consumer
85
+ # and in the Web server
86
+ IncompatibleSchemaError = Class.new(BaseError)
77
87
  end
78
88
  end
79
89
  end
@@ -20,25 +20,25 @@ module Karafka
20
20
 
21
21
  # Creates the initial states for the Web-UI if needed (if they don't exist)
22
22
  def call
23
- if exists?(Karafka::Web.config.topics.consumers.states)
23
+ if exists?(Karafka::Web.config.topics.consumers.states.name)
24
24
  exists('consumers state')
25
25
  else
26
26
  creating('consumers state')
27
27
  ::Karafka::Web.producer.produce_sync(
28
- topic: Karafka::Web.config.topics.consumers.states,
29
- key: Karafka::Web.config.topics.consumers.states,
28
+ topic: Karafka::Web.config.topics.consumers.states.name,
29
+ key: Karafka::Web.config.topics.consumers.states.name,
30
30
  payload: DEFAULT_STATE.to_json
31
31
  )
32
32
  created('consumers state')
33
33
  end
34
34
 
35
- if exists?(Karafka::Web.config.topics.consumers.metrics)
35
+ if exists?(Karafka::Web.config.topics.consumers.metrics.name)
36
36
  exists('consumers metrics')
37
37
  else
38
38
  creating('consumers metrics')
39
39
  ::Karafka::Web.producer.produce_sync(
40
- topic: Karafka::Web.config.topics.consumers.metrics,
41
- key: Karafka::Web.config.topics.consumers.metrics,
40
+ topic: Karafka::Web.config.topics.consumers.metrics.name,
41
+ key: Karafka::Web.config.topics.consumers.metrics.name,
42
42
  payload: DEFAULT_METRICS.to_json
43
43
  )
44
44
  created('consumers metrics')
@@ -23,114 +23,80 @@ module Karafka
23
23
  consumers_commands_topic = ::Karafka::Web.config.topics.consumers.commands
24
24
  errors_topic = ::Karafka::Web.config.topics.errors
25
25
 
26
- if existing_topics_names.include?(errors_topic)
27
- exists(errors_topic)
26
+ if existing_topics_names.include?(errors_topic.name)
27
+ exists(errors_topic.name)
28
28
  else
29
- creating(errors_topic)
29
+ creating(errors_topic.name)
30
30
  # All the errors will be dispatched here
31
31
  # This topic can have multiple partitions but we go with one by default. A single
32
32
  # Ruby process should not crash that often and if there is an expectation of a higher
33
33
  # volume of errors, this can be changed by the end user
34
34
  ::Karafka::Admin.create_topic(
35
- errors_topic,
35
+ errors_topic.name,
36
36
  1,
37
37
  replication_factor,
38
- # Remove really old errors (older than 3 months just to preserve space)
39
- {
40
- 'cleanup.policy': 'delete',
41
- 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
42
- }
38
+ errors_topic.config
43
39
  )
44
- created(errors_topic)
40
+ created(errors_topic.name)
45
41
  end
46
42
 
47
- if existing_topics_names.include?(consumers_reports_topic)
48
- exists(consumers_reports_topic)
43
+ if existing_topics_names.include?(consumers_reports_topic.name)
44
+ exists(consumers_reports_topic.name)
49
45
  else
50
- creating(consumers_reports_topic)
46
+ creating(consumers_reports_topic.name)
51
47
  # This topic needs to have one partition
52
48
  ::Karafka::Admin.create_topic(
53
- consumers_reports_topic,
49
+ consumers_reports_topic.name,
54
50
  1,
55
51
  replication_factor,
56
- # We do not need to to store this data for longer than 1 day as this data is only
57
- # used to materialize the end states
58
- # On the other hand we do not want to have it really short-living because in case
59
- # of a consumer crash, we may want to use this info to catch up and backfill the
60
- # state.
61
- #
62
- # In case its not consumed because no processes are running, it also usually means
63
- # there's no data to consume because no karafka servers report
64
- {
65
- 'cleanup.policy': 'delete',
66
- 'retention.ms': 24 * 60 * 60 * 1_000 # 1 day
67
- }
52
+ consumers_reports_topic.config
68
53
  )
69
- created(consumers_reports_topic)
54
+ created(consumers_reports_topic.name)
70
55
  end
71
56
 
72
- if existing_topics_names.include?(consumers_metrics_topic)
73
- exists(consumers_metrics_topic)
57
+ if existing_topics_names.include?(consumers_metrics_topic.name)
58
+ exists(consumers_metrics_topic.name)
74
59
  else
75
- creating(consumers_metrics_topic)
60
+ creating(consumers_metrics_topic.name)
76
61
  # This topic needs to have one partition
77
62
  # Same as states - only most recent is relevant as it is a materialized state
78
63
  ::Karafka::Admin.create_topic(
79
- consumers_metrics_topic,
64
+ consumers_metrics_topic.name,
80
65
  1,
81
66
  replication_factor,
82
- {
83
- 'cleanup.policy': 'compact',
84
- 'retention.ms': 24 * 60 * 60 * 1_000, # 1 day
85
- 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
86
- 'segment.bytes': 104_857_600 # 100MB
87
- }
67
+ consumers_metrics_topic.config
88
68
  )
89
- created(consumers_metrics_topic)
69
+ created(consumers_metrics_topic.name)
90
70
  end
91
71
 
92
- if existing_topics_names.include?(consumers_commands_topic)
93
- exists(consumers_commands_topic)
72
+ if existing_topics_names.include?(consumers_commands_topic.name)
73
+ exists(consumers_commands_topic.name)
94
74
  else
95
- creating(consumers_commands_topic)
75
+ creating(consumers_commands_topic.name)
96
76
  # Commands are suppose to live short and be used for controlling processes and some
97
77
  # debug. Their data can be removed safely fast.
98
78
  ::Karafka::Admin.create_topic(
99
- consumers_commands_topic,
79
+ consumers_commands_topic.name,
100
80
  1,
101
81
  replication_factor,
102
- {
103
- 'cleanup.policy': 'delete',
104
- 'retention.ms': 7 * 24 * 60 * 60 * 1_000, # 7 days
105
- 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
106
- 'segment.bytes': 104_857_600 # 100MB
107
- }
82
+ consumers_commands_topic.config
108
83
  )
109
- created(consumers_commands_topic)
84
+ created(consumers_commands_topic.name)
110
85
  end
111
86
 
112
87
  # Create only if needed
113
- if existing_topics_names.include?(consumers_states_topic)
114
- exists(consumers_states_topic)
88
+ if existing_topics_names.include?(consumers_states_topic.name)
89
+ exists(consumers_states_topic.name)
115
90
  else
116
- creating(consumers_states_topic)
91
+ creating(consumers_states_topic.name)
117
92
  # This topic needs to have one partition
118
93
  ::Karafka::Admin.create_topic(
119
- consumers_states_topic,
94
+ consumers_states_topic.name,
120
95
  1,
121
96
  replication_factor,
122
- # We care only about the most recent state, previous are irrelevant. So we can
123
- # easily compact after one minute. We do not use this beyond the most recent
124
- # collective state, hence it all can easily go away. We also limit the segment
125
- # size to at most 100MB not to use more space ever.
126
- {
127
- 'cleanup.policy': 'compact',
128
- 'retention.ms': 60 * 60 * 1_000,
129
- 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
130
- 'segment.bytes': 104_857_600 # 100MB
131
- }
97
+ consumers_states_topic.config
132
98
  )
133
- created(consumers_states_topic)
99
+ created(consumers_states_topic.name)
134
100
  end
135
101
  end
136
102
 
@@ -9,11 +9,11 @@ module Karafka
9
9
  # Removes the Web-UI topics
10
10
  def call
11
11
  [
12
- ::Karafka::Web.config.topics.consumers.states,
13
- ::Karafka::Web.config.topics.consumers.reports,
14
- ::Karafka::Web.config.topics.consumers.metrics,
15
- ::Karafka::Web.config.topics.consumers.commands,
16
- ::Karafka::Web.config.topics.errors
12
+ ::Karafka::Web.config.topics.consumers.states.name,
13
+ ::Karafka::Web.config.topics.consumers.reports.name,
14
+ ::Karafka::Web.config.topics.consumers.metrics.name,
15
+ ::Karafka::Web.config.topics.consumers.commands.name,
16
+ ::Karafka::Web.config.topics.errors.name
17
17
  ].each do |topic_name|
18
18
  if existing_topics_names.include?(topic_name.to_s)
19
19
  puts "Removing #{topic_name}..."
@@ -55,7 +55,7 @@ module Karafka
55
55
 
56
56
  consumer_group ::Karafka::Web.config.group_id do
57
57
  # Topic we listen on to materialize the states
58
- topic ::Karafka::Web.config.topics.consumers.reports do
58
+ topic ::Karafka::Web.config.topics.consumers.reports.name do
59
59
  config(active: false)
60
60
  active ::Karafka::Web.config.processing.active
61
61
  # Since we materialize state in intervals, we can poll for half of this time
@@ -80,25 +80,25 @@ module Karafka
80
80
 
81
81
  # We define those three here without consumption, so Web understands how to
82
82
  # deserialize them when used / viewed
83
- topic ::Karafka::Web.config.topics.consumers.states do
83
+ topic ::Karafka::Web.config.topics.consumers.states.name do
84
84
  config(active: false)
85
85
  active false
86
86
  deserializers(payload: payload_deserializer)
87
87
  end
88
88
 
89
- topic ::Karafka::Web.config.topics.consumers.metrics do
89
+ topic ::Karafka::Web.config.topics.consumers.metrics.name do
90
90
  config(active: false)
91
91
  active false
92
92
  deserializers(payload: payload_deserializer)
93
93
  end
94
94
 
95
- topic ::Karafka::Web.config.topics.consumers.commands do
95
+ topic ::Karafka::Web.config.topics.consumers.commands.name do
96
96
  config(active: false)
97
97
  active false
98
98
  deserializers(payload: payload_deserializer)
99
99
  end
100
100
 
101
- topic ::Karafka::Web.config.topics.errors do
101
+ topic ::Karafka::Web.config.topics.errors.name do
102
102
  config(active: false)
103
103
  active false
104
104
  deserializers(payload: payload_deserializer)
@@ -1,15 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
13
5
 
14
6
  module Karafka
15
7
  module Web
@@ -19,11 +11,38 @@ module Karafka
19
11
  module Commands
20
12
  # Base for all the commands
21
13
  class Base
14
+ class << self
15
+ attr_accessor :name
16
+ end
17
+
18
+ # @return [Hash]
19
+ attr_reader :command
20
+
21
+ # @param command [Command] command details (if any). Some commands may require extra
22
+ # details to work. They can be obtained from here.
23
+ def initialize(command)
24
+ @command = command
25
+ end
26
+
27
+ # Executes the command after receiving it.
28
+ def call
29
+ raise NotImlementedError, 'Please implement in a subclass'
30
+ end
31
+
22
32
  private
23
33
 
24
- # @return [String] current process id
25
- def process_id
26
- @process_id ||= ::Karafka::Web.config.tracking.consumers.sampler.process_id
34
+ # Dispatches the acceptance message back to Kafka as a confirmation
35
+ #
36
+ # @param params [Hash] hash with the acceptance message details
37
+ def acceptance(params)
38
+ Dispatcher.acceptance(self.class.name, process_id, params)
39
+ end
40
+
41
+ # Dispatches the result message back to Kafka with execution details
42
+ #
43
+ # @param params [Hash] hash with the result message details
44
+ def result(params)
45
+ Dispatcher.result(self.class.name, process_id, params)
27
46
  end
28
47
 
29
48
  # @return [Boolean] Is given process to which a command was sent operating in an
@@ -33,6 +52,11 @@ module Karafka
33
52
  def standalone?
34
53
  Karafka::Server.execution_mode == :standalone
35
54
  end
55
+
56
+ # @return [String] id of the current consumer process
57
+ def process_id
58
+ ::Karafka::Web.config.tracking.consumers.sampler.process_id
59
+ end
36
60
  end
37
61
  end
38
62
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
5
+
6
+ module Karafka
7
+ module Web
8
+ module Pro
9
+ module Commanding
10
+ # Namespace for commands the process can react to
11
+ module Commands
12
+ # Namespace for commands related to consumers themselves
13
+ module Consumers
14
+ # Sends a signal to quiet the consumer
15
+ # @note Does not work in an embedded mode because we do not own the Ruby process.
16
+ class Quiet < Base
17
+ self.name = 'consumers.quiet'
18
+
19
+ # Performs the command if not in embedded mode
20
+ def call
21
+ return unless standalone?
22
+
23
+ ::Process.kill('TSTP', ::Process.pid)
24
+
25
+ result(status: 'applied')
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
5
+
6
+ module Karafka
7
+ module Web
8
+ module Pro
9
+ module Commanding
10
+ # Namespace for commands the process can react to
11
+ module Commands
12
+ module Consumers
13
+ # Sends a signal to stop the process
14
+ # @note Does not work in an embedded mode because we do not own the Ruby process.
15
+ class Stop < Base
16
+ self.name = 'consumers.stop'
17
+
18
+ # Performs the command if not in embedded mode
19
+ def call
20
+ return unless standalone?
21
+
22
+ ::Process.kill('QUIT', ::Process.pid)
23
+
24
+ result(status: 'applied')
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
5
+
6
+ module Karafka
7
+ module Web
8
+ module Pro
9
+ module Commanding
10
+ # Namespace for commands the process can react to
11
+ module Commands
12
+ module Consumers
13
+ # Collects all backtraces from the available Ruby threads and publishes their details
14
+ # back to Kafka for debug.
15
+ class Trace < Base
16
+ self.name = 'consumers.trace'
17
+
18
+ # Runs tracing and publishes result back to Kafka
19
+ def call
20
+ threads = {}
21
+
22
+ Thread.list.each do |thread|
23
+ tid = (thread.object_id ^ ::Process.pid).to_s(36)
24
+ t_d = threads[tid] = {}
25
+ t_d[:label] = "Thread TID-#{tid} #{thread.name}"
26
+ t_d[:backtrace] = (thread.backtrace || ['<no backtrace available>']).join("\n")
27
+ end
28
+
29
+ result(threads)
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
5
+
6
+ module Karafka
7
+ module Web
8
+ module Pro
9
+ module Commanding
10
+ module Commands
11
+ # Namespace for post-fetch command execution
12
+ module Partitions
13
+ # Delegates the pause request into the partition changes tracker and dispatches the
14
+ # acceptance message back to Kafka
15
+ class Pause < Base
16
+ self.name = 'partitions.pause'
17
+
18
+ # Delegates the pause request to async handling
19
+ def call
20
+ Handlers::Partitions::Tracker.instance << command
21
+
22
+ acceptance(command.to_h)
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end