karafka-web 0.7.9 → 0.8.0.rc1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (163) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +21 -6
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +66 -0
  6. data/Gemfile.lock +22 -22
  7. data/docker-compose.yml +3 -1
  8. data/karafka-web.gemspec +2 -2
  9. data/lib/karafka/web/config.rb +16 -3
  10. data/lib/karafka/web/contracts/config.rb +7 -2
  11. data/lib/karafka/web/errors.rb +12 -0
  12. data/lib/karafka/web/inflector.rb +33 -0
  13. data/lib/karafka/web/installer.rb +20 -11
  14. data/lib/karafka/web/management/actions/base.rb +36 -0
  15. data/lib/karafka/web/management/actions/clean_boot_file.rb +33 -0
  16. data/lib/karafka/web/management/actions/create_initial_states.rb +77 -0
  17. data/lib/karafka/web/management/actions/create_topics.rb +139 -0
  18. data/lib/karafka/web/management/actions/delete_topics.rb +30 -0
  19. data/lib/karafka/web/management/actions/enable.rb +117 -0
  20. data/lib/karafka/web/management/actions/extend_boot_file.rb +39 -0
  21. data/lib/karafka/web/management/actions/migrate_states_data.rb +18 -0
  22. data/lib/karafka/web/management/migrations/0_base.rb +58 -0
  23. data/lib/karafka/web/management/migrations/0_set_initial_consumers_metrics.rb +36 -0
  24. data/lib/karafka/web/management/migrations/0_set_initial_consumers_state.rb +43 -0
  25. data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_metrics.rb +26 -0
  26. data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_state.rb +23 -0
  27. data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_metrics.rb +24 -0
  28. data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_state.rb +20 -0
  29. data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_metrics.rb +24 -0
  30. data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_state.rb +20 -0
  31. data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_metrics.rb +36 -0
  32. data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_states.rb +32 -0
  33. data/lib/karafka/web/management/migrator.rb +117 -0
  34. data/lib/karafka/web/processing/consumer.rb +39 -38
  35. data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +15 -7
  36. data/lib/karafka/web/processing/consumers/aggregators/state.rb +8 -3
  37. data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb +5 -1
  38. data/lib/karafka/web/processing/publisher.rb +59 -0
  39. data/lib/karafka/web/tracking/consumers/contracts/job.rb +3 -2
  40. data/lib/karafka/web/tracking/consumers/contracts/partition.rb +1 -0
  41. data/lib/karafka/web/tracking/consumers/contracts/report.rb +6 -1
  42. data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb +10 -1
  43. data/lib/karafka/web/tracking/consumers/listeners/connections.rb +49 -0
  44. data/lib/karafka/web/tracking/consumers/listeners/pausing.rb +7 -4
  45. data/lib/karafka/web/tracking/consumers/listeners/processing.rb +78 -70
  46. data/lib/karafka/web/tracking/consumers/listeners/statistics.rb +40 -13
  47. data/lib/karafka/web/tracking/consumers/sampler.rb +82 -25
  48. data/lib/karafka/web/tracking/helpers/ttls/array.rb +72 -0
  49. data/lib/karafka/web/tracking/helpers/ttls/hash.rb +34 -0
  50. data/lib/karafka/web/tracking/helpers/ttls/stats.rb +49 -0
  51. data/lib/karafka/web/tracking/helpers/ttls/windows.rb +32 -0
  52. data/lib/karafka/web/tracking/reporter.rb +1 -0
  53. data/lib/karafka/web/ui/app.rb +22 -4
  54. data/lib/karafka/web/ui/base.rb +18 -2
  55. data/lib/karafka/web/ui/controllers/base.rb +34 -4
  56. data/lib/karafka/web/ui/controllers/become_pro.rb +1 -1
  57. data/lib/karafka/web/ui/controllers/cluster.rb +33 -9
  58. data/lib/karafka/web/ui/controllers/consumers.rb +8 -2
  59. data/lib/karafka/web/ui/controllers/dashboard.rb +2 -2
  60. data/lib/karafka/web/ui/controllers/errors.rb +2 -2
  61. data/lib/karafka/web/ui/controllers/jobs.rb +55 -5
  62. data/lib/karafka/web/ui/controllers/requests/params.rb +5 -0
  63. data/lib/karafka/web/ui/controllers/responses/deny.rb +15 -0
  64. data/lib/karafka/web/ui/controllers/responses/file.rb +23 -0
  65. data/lib/karafka/web/ui/controllers/responses/{data.rb → render.rb} +3 -3
  66. data/lib/karafka/web/ui/controllers/routing.rb +11 -2
  67. data/lib/karafka/web/ui/controllers/status.rb +1 -1
  68. data/lib/karafka/web/ui/helpers/application_helper.rb +70 -0
  69. data/lib/karafka/web/ui/lib/hash_proxy.rb +29 -14
  70. data/lib/karafka/web/ui/lib/sorter.rb +170 -0
  71. data/lib/karafka/web/ui/models/counters.rb +6 -0
  72. data/lib/karafka/web/ui/models/health.rb +23 -2
  73. data/lib/karafka/web/ui/models/jobs.rb +48 -0
  74. data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb +33 -0
  75. data/lib/karafka/web/ui/models/metrics/charts/topics.rb +1 -10
  76. data/lib/karafka/web/ui/models/process.rb +2 -1
  77. data/lib/karafka/web/ui/models/status.rb +23 -7
  78. data/lib/karafka/web/ui/models/topic.rb +3 -1
  79. data/lib/karafka/web/ui/models/visibility_filter.rb +16 -0
  80. data/lib/karafka/web/ui/pro/app.rb +44 -6
  81. data/lib/karafka/web/ui/pro/controllers/cluster.rb +1 -0
  82. data/lib/karafka/web/ui/pro/controllers/consumers.rb +52 -6
  83. data/lib/karafka/web/ui/pro/controllers/dashboard.rb +1 -1
  84. data/lib/karafka/web/ui/pro/controllers/dlq.rb +1 -1
  85. data/lib/karafka/web/ui/pro/controllers/errors.rb +3 -3
  86. data/lib/karafka/web/ui/pro/controllers/explorer.rb +8 -8
  87. data/lib/karafka/web/ui/pro/controllers/health.rb +34 -2
  88. data/lib/karafka/web/ui/pro/controllers/jobs.rb +11 -0
  89. data/lib/karafka/web/ui/pro/controllers/messages.rb +42 -0
  90. data/lib/karafka/web/ui/pro/controllers/routing.rb +11 -2
  91. data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb +8 -2
  92. data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb +14 -8
  93. data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +8 -6
  94. data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb +4 -1
  95. data/lib/karafka/web/ui/pro/views/consumers/consumer/_no_jobs.erb +1 -1
  96. data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +1 -3
  97. data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +28 -11
  98. data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb +10 -3
  99. data/lib/karafka/web/ui/pro/views/consumers/index.erb +3 -3
  100. data/lib/karafka/web/ui/pro/views/consumers/pending_jobs.erb +43 -0
  101. data/lib/karafka/web/ui/pro/views/consumers/{jobs.erb → running_jobs.erb} +11 -10
  102. data/lib/karafka/web/ui/pro/views/dashboard/index.erb +7 -1
  103. data/lib/karafka/web/ui/pro/views/explorer/message/_message_actions.erb +18 -0
  104. data/lib/karafka/web/ui/pro/views/explorer/message/_metadata.erb +43 -0
  105. data/lib/karafka/web/ui/pro/views/explorer/message/_payload.erb +21 -0
  106. data/lib/karafka/web/ui/pro/views/explorer/message/_payload_actions.erb +19 -0
  107. data/lib/karafka/web/ui/pro/views/explorer/show.erb +9 -84
  108. data/lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb +8 -0
  109. data/lib/karafka/web/ui/pro/views/health/_partition.erb +1 -3
  110. data/lib/karafka/web/ui/pro/views/health/_partition_offset.erb +4 -4
  111. data/lib/karafka/web/ui/pro/views/health/_partition_times.erb +32 -0
  112. data/lib/karafka/web/ui/pro/views/health/_tabs.erb +9 -0
  113. data/lib/karafka/web/ui/pro/views/health/changes.erb +66 -0
  114. data/lib/karafka/web/ui/pro/views/health/offsets.erb +14 -14
  115. data/lib/karafka/web/ui/pro/views/health/overview.erb +11 -11
  116. data/lib/karafka/web/ui/pro/views/jobs/_job.erb +1 -1
  117. data/lib/karafka/web/ui/pro/views/jobs/_no_jobs.erb +1 -1
  118. data/lib/karafka/web/ui/pro/views/jobs/pending.erb +39 -0
  119. data/lib/karafka/web/ui/pro/views/jobs/running.erb +39 -0
  120. data/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb +2 -2
  121. data/lib/karafka/web/ui/pro/views/routing/_topic.erb +9 -0
  122. data/lib/karafka/web/ui/pro/views/routing/show.erb +12 -0
  123. data/lib/karafka/web/ui/pro/views/shared/_navigation.erb +1 -1
  124. data/lib/karafka/web/ui/public/javascripts/application.js +10 -0
  125. data/lib/karafka/web/ui/public/stylesheets/application.css +4 -0
  126. data/lib/karafka/web/ui/views/cluster/_breadcrumbs.erb +16 -0
  127. data/lib/karafka/web/ui/views/cluster/_tabs.erb +27 -0
  128. data/lib/karafka/web/ui/views/cluster/brokers.erb +27 -0
  129. data/lib/karafka/web/ui/views/cluster/topics.erb +35 -0
  130. data/lib/karafka/web/ui/views/consumers/_counters.erb +8 -6
  131. data/lib/karafka/web/ui/views/consumers/_summary.erb +2 -2
  132. data/lib/karafka/web/ui/views/consumers/index.erb +3 -3
  133. data/lib/karafka/web/ui/views/dashboard/_ranges_selector.erb +23 -7
  134. data/lib/karafka/web/ui/views/dashboard/index.erb +19 -8
  135. data/lib/karafka/web/ui/views/errors/show.erb +2 -23
  136. data/lib/karafka/web/ui/views/jobs/_breadcrumbs.erb +17 -1
  137. data/lib/karafka/web/ui/views/jobs/_job.erb +1 -1
  138. data/lib/karafka/web/ui/views/jobs/_no_jobs.erb +1 -1
  139. data/lib/karafka/web/ui/views/jobs/_tabs.erb +27 -0
  140. data/lib/karafka/web/ui/views/jobs/{index.erb → pending.erb} +9 -7
  141. data/lib/karafka/web/ui/{pro/views/jobs/index.erb → views/jobs/running.erb} +9 -11
  142. data/lib/karafka/web/ui/views/routing/_consumer_group.erb +14 -12
  143. data/lib/karafka/web/ui/views/shared/_navigation.erb +1 -1
  144. data/lib/karafka/web/ui/views/shared/_pagination.erb +1 -1
  145. data/lib/karafka/web/ui/views/shared/exceptions/not_allowed.erb +37 -0
  146. data/lib/karafka/web/ui/views/status/show.erb +17 -2
  147. data/lib/karafka/web/ui/views/status/warnings/_routing_topics_presence.erb +15 -0
  148. data/lib/karafka/web/version.rb +1 -1
  149. data/lib/karafka/web.rb +6 -2
  150. data.tar.gz.sig +0 -0
  151. metadata +61 -26
  152. metadata.gz.sig +0 -0
  153. data/lib/karafka/web/management/base.rb +0 -34
  154. data/lib/karafka/web/management/clean_boot_file.rb +0 -31
  155. data/lib/karafka/web/management/create_initial_states.rb +0 -101
  156. data/lib/karafka/web/management/create_topics.rb +0 -133
  157. data/lib/karafka/web/management/delete_topics.rb +0 -28
  158. data/lib/karafka/web/management/enable.rb +0 -102
  159. data/lib/karafka/web/management/extend_boot_file.rb +0 -37
  160. data/lib/karafka/web/tracking/ttl_array.rb +0 -59
  161. data/lib/karafka/web/tracking/ttl_hash.rb +0 -16
  162. data/lib/karafka/web/ui/pro/views/dashboard/_ranges_selector.erb +0 -39
  163. data/lib/karafka/web/ui/views/cluster/index.erb +0 -74
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 24bc5fe488a1304886f3f7f697a8662463f4c21fae60daf52d317852f9f43669
4
- data.tar.gz: eaa8e4b7d9b9d70200c8929e8fa65ed7906a1b4085465b15525bfee670ebbcc9
3
+ metadata.gz: d3e890ec145255109e1f29ea742e6e0ed2887b0b4ca1de6047973816307744c1
4
+ data.tar.gz: 55766edeb8f40c68c1200b577697728037882e2af3dd0dfb9723741be0343b7e
5
5
  SHA512:
6
- metadata.gz: 0ba2c668103a3b25169d86f2c0624fae04d144694040694ff79c4932bf81fe648e5ba11223be5a5f9e8d8ba0bbbac9da84f2ec5312e81d07b35873707b7b754c
7
- data.tar.gz: b0ec51820b50dfde4f88e942e3b0dcc2400170d0fbe256503294f366e2f4fe0057249bb525528b789bed254ba6ec035d5502e515042bd094999663155f8a221d
6
+ metadata.gz: 29dccc30c89888815b33a703317ca44bcf1b0a5f37228e9db7ba26eeba2d3b5ce5a66f5246121c4b9c1bb9d248e9027f3e8992f324109fd807d62a94c70c47ff
7
+ data.tar.gz: 32ed3012d866810f85db46c8c82f20888693e1c533edb9bcdc54cf098c2bece4478e8fe82f8a3c62c313f219d622f8b42e25bd4f38793d82951dec311f164c5c
checksums.yaml.gz.sig CHANGED
Binary file
@@ -1,6 +1,8 @@
1
1
  name: ci
2
2
 
3
- concurrency: ci-${{ github.ref }}
3
+ concurrency:
4
+ group: ${{ github.workflow }}-${{ github.ref }}
5
+ cancel-in-progress: true
4
6
 
5
7
  on:
6
8
  pull_request:
@@ -20,13 +22,13 @@ jobs:
20
22
  fail-fast: false
21
23
  matrix:
22
24
  ruby:
23
- - '3.3.0-preview2'
25
+ - '3.3'
24
26
  - '3.2'
25
27
  - '3.1'
26
28
  - '3.0'
27
29
  - '2.7'
28
30
  include:
29
- - ruby: '3.2'
31
+ - ruby: '3.3'
30
32
  coverage: 'true'
31
33
  steps:
32
34
  - uses: actions/checkout@v4
@@ -42,16 +44,29 @@ jobs:
42
44
  with:
43
45
  ruby-version: ${{matrix.ruby}}
44
46
  bundler-cache: true
47
+ bundler: 'latest'
45
48
 
46
49
  - name: Install latest bundler
47
50
  run: |
48
- gem install bundler --no-document
51
+ if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
52
+ gem install bundler -v 2.4.22 --no-document
53
+ gem update --system 3.4.22 --no-document
54
+ else
55
+ gem install bundler --no-document
56
+ gem update --system --no-document
57
+ fi
58
+
49
59
  bundle config set without 'tools benchmarks docs'
50
60
 
51
61
  - name: Bundle install
52
62
  run: |
53
63
  bundle config set without development
54
- bundle install --jobs 4 --retry 3
64
+
65
+ if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
66
+ BUNDLER_VERSION=2.4.22 bundle install --jobs 4 --retry 3
67
+ else
68
+ bundle install --jobs 4 --retry 3
69
+ fi
55
70
 
56
71
  - name: Wait for Kafka
57
72
  run: |
@@ -75,7 +90,7 @@ jobs:
75
90
  - name: Set up Ruby
76
91
  uses: ruby/setup-ruby@v1
77
92
  with:
78
- ruby-version: 3.2
93
+ ruby-version: 3.3
79
94
  bundler-cache: true
80
95
 
81
96
  - name: Install Diffend plugin
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 3.2.2
1
+ 3.3.0
data/CHANGELOG.md CHANGED
@@ -1,5 +1,62 @@
1
1
  # Karafka Web changelog
2
2
 
3
+ ## 0.8.0 (Unreleased)
4
+ - **[Feature]** Provide ability to sort table data for part of the views (note: not all attributes can be sorted due to technical limitations of sub-components fetching from Kafka).
5
+ - **[Feature]** Track and report pause timeouts via "Changes" view in Health.
6
+ - **[Feature]** Introduce pending jobs visibility alongside of running jobs both in total and per process.
7
+ - **[Feature]** Introduce states migrations for seamless upgrades.
8
+ - **[Feature]** Introduce "Data transfers" chart with data received and data sent to the cluster.
9
+ - **[Feature]** Introduce ability to download raw payloads.
10
+ - **[Feature]** Introduce ability to download deserialized message payload as JSON.
11
+ - [Enhancement] Support reporting of standby and active listeners for connection multiplexed subscription groups.
12
+ - [Enhancement] Support Periodic Jobs reporting.
13
+ - [Enhancement] Support multiplexed subscription groups.
14
+ - [Enhancement] Split cluster info into two tabs, one for brokers and one for topics with partitions.
15
+ - [Enhancement] Track pending jobs. Pending jobs are jobs that are not yet scheduled for execution by advanced schedulers.
16
+ - [Enhancement] Rename "Enqueued" to "Pending" to support jobs that are not yet enqueued but within a scheduler.
17
+ - [Enhancement] Make sure only running jobs are displayed in running jobs
18
+ - [Enhancement] Improve jobs related breadcrumbs
19
+ - [Enhancement] Display errors backtraces in OSS.
20
+ - [Enhancement] Display concurrency graph in OSS.
21
+ - [Enhancement] Support time ranges for graphs in OSS.
22
+ - [Enhancement] Report last poll time for each subscription group.
23
+ - [Enhancement] Show last poll time per consumer instance.
24
+ - [Enhancement] Display number of jobs in a particular process jobs view.
25
+ - [Enhancement] Promote "Batches" chart to OSS.
26
+ - [Enhancement] Promote "Utilization" chart to OSS.
27
+ - [Enhancement] Allow for explicit disabling of the Web UI tracking.
28
+ - [Fix] Web UI will keep reporting status even when not activated as long as required and in routes.
29
+ - [Fix] Fix times precisions that could be incorrectly reported by 1 second in few places.
30
+ - [Fix] Fix random order in Consumers groups Health view.
31
+ - [Change] Rename "Busy" to "Running" to align with "Running Jobs".
32
+ - [Change] Rename "Active subscriptions" to "Subscriptions" as process subscriptions are always active.
33
+ - [Maintenance] Introduce granular subscription group contracts.
34
+
35
+ ### Upgrade Notes
36
+
37
+ This is a **major** release that brings many things to the table.
38
+
39
+ #### Configuration
40
+
41
+ **No** configuration changes are needed.
42
+
43
+ #### Deployment
44
+
45
+ Because of the reporting schema update, it is recommended to:
46
+
47
+ 0. Make sure you have upgraded to `0.7.10` before and that it was fully deployed.
48
+ 1. Test the upgrade on a staging or dev environment.
49
+ 2. Starting from `0.7.0` Karafka Web UI supports rolling deploys, so there is no need to "stop the world".
50
+ 3. The Web UI interface may throw 500 errors during the upgrade because of schema incompatibility (until Puma is deployed). This will have no long-term effects and can be ignored.
51
+ 4. `Karafka::Web::Errors::Processing::IncompatibleSchemaError` **is expected**. It is part of the Karafka Web UI zero-downtime deployment strategy. This error allows the Web UI materialization consumer to back off and wait for it to be replaced with a new one.
52
+ 5. Perform a rolling deployment (or a regular one) and replace all consumer processes.
53
+ 6. Update the Web UI Puma.
54
+ 7. **No** CLI command execution is required. Starting from this release (`0.8.0`), the Karafka Web UI contains an automatic schema migrator that allows it to automatically adjust internal topic data formats.
55
+ 8. Enjoy.
56
+
57
+ ## 0.7.10 (2023-10-31)
58
+ - [Fix] Max LSO chart does not work as expected (#201)
59
+
3
60
  ## 0.7.9 (2023-10-25)
4
61
  - [Enhancement] Allow for `Karafka::Web.producer` reconfiguration from the default (`Karafka.producer`).
5
62
  - [Change] Rely on `karafka-core` `>=` `2.2.4` to support lazy loaded custom web producer.
@@ -26,6 +83,15 @@
26
83
  - [Fix] Cache assets for 1 year instead of 7 days.
27
84
  - [Fix] Remove source maps pointing to non-existing locations.
28
85
  - [Maintenance] Include license and copyrights notice for `timeago.js` that was missing in the JS min file.
86
+ - [Refactor] Rename `ui.show_internal_topics` to `ui.visibility.internal_topics_display`
87
+
88
+ ### Upgrade Notes
89
+
90
+ **NO** rolling upgrade needed. Just configuration update.
91
+
92
+ 1. If you are using `ui.visibility_filter` this option is now `ui.visibility.filter` (yes, only `.` difference).
93
+ 2. If you are using a custom visibility filter, it requires now two extra methods: `#download?` and `#export?`. The default visibility filter allows both actions unless message is encrypted.
94
+ 3. `ui.show_internal_topics` config option has been moved and renamed to `ui.visibility.internal_topics`.
29
95
 
30
96
  ## 0.7.4 (2023-09-19)
31
97
  - [Improvement] Skip aggregations on older schemas during upgrades. This only skips process-reports (that are going to be rolled) on the 5s window in case of an upgrade that should not be a rolling one anyhow. This simplifies the operations and minimizes the risk on breaking upgrades.
data/Gemfile.lock CHANGED
@@ -1,17 +1,17 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka-web (0.7.9)
4
+ karafka-web (0.8.0.rc1)
5
5
  erubi (~> 1.4)
6
- karafka (>= 2.2.9, < 3.0.0)
7
- karafka-core (>= 2.2.4, < 3.0.0)
6
+ karafka (>= 2.3.0.rc1, < 2.4.0)
7
+ karafka-core (>= 2.3.0.rc1, < 2.4.0)
8
8
  roda (~> 3.68, >= 3.69)
9
9
  tilt (~> 2.0)
10
10
 
11
11
  GEM
12
12
  remote: https://rubygems.org/
13
13
  specs:
14
- activesupport (7.1.1)
14
+ activesupport (7.1.3)
15
15
  base64
16
16
  bigdecimal
17
17
  concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -21,43 +21,42 @@ GEM
21
21
  minitest (>= 5.1)
22
22
  mutex_m
23
23
  tzinfo (~> 2.0)
24
- base64 (0.1.1)
25
- bigdecimal (3.1.4)
24
+ base64 (0.2.0)
25
+ bigdecimal (3.1.6)
26
26
  byebug (11.1.3)
27
- concurrent-ruby (1.2.2)
27
+ concurrent-ruby (1.2.3)
28
28
  connection_pool (2.4.1)
29
29
  diff-lcs (1.5.0)
30
30
  docile (1.4.0)
31
- drb (2.1.1)
31
+ drb (2.2.0)
32
32
  ruby2_keywords
33
33
  erubi (1.12.0)
34
- factory_bot (6.3.0)
34
+ factory_bot (6.4.5)
35
35
  activesupport (>= 5.0.0)
36
36
  ffi (1.16.3)
37
37
  i18n (1.14.1)
38
38
  concurrent-ruby (~> 1.0)
39
- karafka (2.2.9)
40
- karafka-core (>= 2.2.2, < 2.3.0)
41
- waterdrop (>= 2.6.10, < 3.0.0)
39
+ karafka (2.3.0.rc1)
40
+ karafka-core (>= 2.3.0.rc1, < 2.4.0)
41
+ waterdrop (>= 2.6.12, < 3.0.0)
42
42
  zeitwerk (~> 2.3)
43
- karafka-core (2.2.4)
44
- concurrent-ruby (>= 1.1)
45
- karafka-rdkafka (>= 0.13.6, < 0.14.0)
46
- karafka-rdkafka (0.13.6)
43
+ karafka-core (2.3.0.rc1)
44
+ karafka-rdkafka (>= 0.14.7, < 0.15.0)
45
+ karafka-rdkafka (0.14.7)
47
46
  ffi (~> 1.15)
48
47
  mini_portile2 (~> 2.6)
49
48
  rake (> 12)
50
49
  mini_portile2 (2.8.5)
51
- minitest (5.20.0)
52
- mutex_m (0.1.2)
50
+ minitest (5.21.2)
51
+ mutex_m (0.2.0)
53
52
  rack (3.0.8)
54
53
  rack-test (2.1.0)
55
54
  rack (>= 1.3)
56
55
  rackup (0.2.3)
57
56
  rack (>= 3.0.0.beta1)
58
57
  webrick
59
- rake (13.0.6)
60
- roda (3.73.0)
58
+ rake (13.1.0)
59
+ roda (3.76.0)
61
60
  rack
62
61
  rspec (3.12.0)
63
62
  rspec-core (~> 3.12.0)
@@ -82,13 +81,14 @@ GEM
82
81
  tilt (2.3.0)
83
82
  tzinfo (2.0.6)
84
83
  concurrent-ruby (~> 1.0)
85
- waterdrop (2.6.10)
84
+ waterdrop (2.6.12)
86
85
  karafka-core (>= 2.2.3, < 3.0.0)
87
86
  zeitwerk (~> 2.3)
88
87
  webrick (1.8.1)
89
88
  zeitwerk (2.6.12)
90
89
 
91
90
  PLATFORMS
91
+ ruby
92
92
  x86_64-linux
93
93
 
94
94
  DEPENDENCIES
@@ -101,4 +101,4 @@ DEPENDENCIES
101
101
  simplecov
102
102
 
103
103
  BUNDLED WITH
104
- 2.4.19
104
+ 2.5.4
data/docker-compose.yml CHANGED
@@ -3,7 +3,7 @@ version: '2'
3
3
  services:
4
4
  kafka:
5
5
  container_name: kafka
6
- image: confluentinc/cp-kafka:7.5.1
6
+ image: confluentinc/cp-kafka:7.5.3
7
7
 
8
8
  ports:
9
9
  - 9092:9092
@@ -23,3 +23,5 @@ services:
23
23
  KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
24
24
  KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
25
25
  KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
26
+ KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
27
+ KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
data/karafka-web.gemspec CHANGED
@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
17
17
  spec.licenses = %w[LGPL-3.0 Commercial]
18
18
 
19
19
  spec.add_dependency 'erubi', '~> 1.4'
20
- spec.add_dependency 'karafka', '>= 2.2.9', '< 3.0.0'
21
- spec.add_dependency 'karafka-core', '>= 2.2.4', '< 3.0.0'
20
+ spec.add_dependency 'karafka', '>= 2.3.0.rc1', '< 2.4.0'
21
+ spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
22
22
  spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
23
23
  spec.add_dependency 'tilt', '~> 2.0'
24
24
 
@@ -47,6 +47,11 @@ module Karafka
47
47
  # 5 seconds should be enough
48
48
  setting :interval, default: 5_000
49
49
 
50
+ # Should we be tracking anything at all. If this is set to false, no reporting will happen
51
+ # even if Web-UI is configured. When set to `nil` (default) it will be switched to true
52
+ # during the Web UI initialization.
53
+ setting :active, default: nil
54
+
50
55
  # Main Web UI reporting scheduler that runs a background thread and reports periodically
51
56
  # from the consumer reporter and producer reporter
52
57
  setting :scheduler, default: Tracking::Scheduler.new
@@ -62,6 +67,7 @@ module Karafka
62
67
  setting :listeners, default: [
63
68
  Tracking::Consumers::Listeners::Status.new,
64
69
  Tracking::Consumers::Listeners::Errors.new,
70
+ Tracking::Consumers::Listeners::Connections.new,
65
71
  Tracking::Consumers::Listeners::Statistics.new,
66
72
  Tracking::Consumers::Listeners::Pausing.new,
67
73
  Tracking::Consumers::Listeners::Processing.new,
@@ -117,9 +123,16 @@ module Karafka
117
123
  Karafka.env.production? ? 60_000 * 5 : 5_000
118
124
  )
119
125
 
120
- # Should we display internal topics of Kafka. The once starting with `__`
121
- # By default we do not display them as they are not usable from regular users perspective
122
- setting :show_internal_topics, default: false
126
+ setting :visibility do
127
+ # Allows to manage visibility of payload, headers and message key in the UI
128
+ # In some cases you may want to limit what is being displayed due to the type of data you
129
+ # are dealing with
130
+ setting :filter, default: Ui::Models::VisibilityFilter.new
131
+
132
+ # Should we display internal topics of Kafka. The once starting with `__`
133
+ # By default we do not display them as they are not usable from regular users perspective
134
+ setting :internal_topics, default: false
135
+ end
123
136
 
124
137
  # How many elements should we display on pages that support pagination
125
138
  setting :per_page, default: 25
@@ -23,6 +23,8 @@ module Karafka
23
23
  end
24
24
 
25
25
  nested(:tracking) do
26
+ # If set to nil, it is up to us to initialize
27
+ required(:active) { |val| [true, false, nil].include?(val) }
26
28
  # Do not report more often then every second, this could overload the system
27
29
  required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
28
30
 
@@ -52,10 +54,13 @@ module Karafka
52
54
  required(:secret) { |val| val.is_a?(String) && val.length >= 64 }
53
55
  end
54
56
 
55
- required(:show_internal_topics) { |val| [true, false].include?(val) }
56
57
  required(:cache) { |val| !val.nil? }
57
58
  required(:per_page) { |val| val.is_a?(Integer) && val >= 1 && val <= 100 }
58
- required(:visibility_filter) { |val| !val.nil? }
59
+
60
+ nested(:visibility) do
61
+ required(:filter) { |val| !val.nil? }
62
+ required(:internal_topics) { |val| [true, false].include?(val) }
63
+ end
59
64
  end
60
65
  end
61
66
  end
@@ -11,6 +11,18 @@ module Karafka
11
11
  # This should never happen and if you see this, please open an issue.
12
12
  ContractError = Class.new(BaseError)
13
13
 
14
+ # Errors specific to management
15
+ module Management
16
+ # Similar to processing error with the same name, it is raised when a critical
17
+ # incompatibility is detected.
18
+ #
19
+ # This error is raised when there was an attempt to operate on aggregated Web UI states
20
+ # that are already in a newer version that the one in the current process. We prevent
21
+ # this from happening not to corrupt the data. Please upgrade all the Web UI consumers to
22
+ # the same version
23
+ IncompatibleSchemaError = Class.new(BaseError)
24
+ end
25
+
14
26
  # Processing related errors namespace
15
27
  module Processing
16
28
  # Raised when we try to process reports but we do not have the current state bootstrapped
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Web
5
+ # Web UI Zeitwerk Inflector that allows us to have time prefixed files with migrations, similar
6
+ # to how Rails does that.
7
+ class Inflector < Zeitwerk::GemInflector
8
+ # Checks if given path is a migration one
9
+ MIGRATION_ABSPATH_REGEXP = /migrations\/[0-9]+_(.*)/
10
+
11
+ # Checks if it is a migration file
12
+ MIGRATION_BASENAME_REGEXP = /\A[0-9]+_(.*)/
13
+
14
+ private_constant :MIGRATION_ABSPATH_REGEXP, :MIGRATION_BASENAME_REGEXP
15
+
16
+ # @param [String] basename of the file to be loaded
17
+ # @param abspath [String] absolute path of the file to be loaded
18
+ # @return [String] Constant name to be used for given file
19
+ def camelize(basename, abspath)
20
+ # If not migration directory with proper migration files, use defaults
21
+ return super unless abspath.match?(MIGRATION_ABSPATH_REGEXP)
22
+ # If base name is not of a proper name in migrations, use defaults
23
+ return super unless basename.match?(MIGRATION_BASENAME_REGEXP)
24
+
25
+ super(
26
+ # Extract only the name without the timestamp
27
+ basename.match(MIGRATION_BASENAME_REGEXP).to_a.last,
28
+ abspath
29
+ )
30
+ end
31
+ end
32
+ end
33
+ end
@@ -17,11 +17,14 @@ module Karafka
17
17
  puts
18
18
  puts 'Creating necessary topics and populating state data...'
19
19
  puts
20
- Management::CreateTopics.new.call(replication_factor)
20
+ Management::Actions::CreateTopics.new.call(replication_factor)
21
21
  wait_for_topics
22
- Management::CreateInitialStates.new.call
22
+ Management::Actions::CreateInitialStates.new.call
23
23
  puts
24
- Management::ExtendBootFile.new.call
24
+ puts 'Running data migrations...'
25
+ Management::Actions::MigrateStatesData.new.call
26
+ puts
27
+ Management::Actions::ExtendBootFile.new.call
25
28
  puts
26
29
  puts("Installation #{green('completed')}. Have fun!")
27
30
  puts
@@ -35,9 +38,12 @@ module Karafka
35
38
  puts
36
39
  puts 'Creating necessary topics and populating state data...'
37
40
  puts
38
- Management::CreateTopics.new.call(replication_factor)
41
+ Management::Actions::CreateTopics.new.call(replication_factor)
39
42
  wait_for_topics
40
- Management::CreateInitialStates.new.call
43
+ Management::Actions::CreateInitialStates.new.call
44
+ puts
45
+ puts 'Running data migrations...'
46
+ Management::Actions::MigrateStatesData.new.call
41
47
  puts
42
48
  puts("Migration #{green('completed')}. Have fun!")
43
49
  puts
@@ -49,11 +55,14 @@ module Karafka
49
55
  puts
50
56
  puts 'Resetting Karafka Web UI...'
51
57
  puts
52
- Management::DeleteTopics.new.call
58
+ Management::Actions::DeleteTopics.new.call
53
59
  puts
54
- Management::CreateTopics.new.call(replication_factor)
60
+ Management::Actions::CreateTopics.new.call(replication_factor)
55
61
  wait_for_topics
56
- Management::CreateInitialStates.new.call
62
+ Management::Actions::CreateInitialStates.new.call
63
+ puts
64
+ puts 'Running data migrations...'
65
+ Management::Actions::MigrateStatesData.new.call
57
66
  puts
58
67
  puts("Resetting #{green('completed')}. Have fun!")
59
68
  puts
@@ -64,8 +73,8 @@ module Karafka
64
73
  puts
65
74
  puts 'Uninstalling Karafka Web UI...'
66
75
  puts
67
- Management::DeleteTopics.new.call
68
- Management::CleanBootFile.new.call
76
+ Management::Actions::DeleteTopics.new.call
77
+ Management::Actions::CleanBootFile.new.call
69
78
  puts
70
79
  puts("Uninstalling #{green('completed')}. Goodbye!")
71
80
  puts
@@ -73,7 +82,7 @@ module Karafka
73
82
 
74
83
  # Enables the Web-UI in the karafka app. Sets up needed routes and listeners.
75
84
  def enable!
76
- Management::Enable.new.call
85
+ Management::Actions::Enable.new.call
77
86
  end
78
87
 
79
88
  private
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Web
5
+ module Management
6
+ # Namespace for all the commands related to management of the Web-UI in the context of
7
+ # Karafka. It includes things like installing, creating needed topics, etc.
8
+ module Actions
9
+ # Base class for all the commands that we use to manage
10
+ class Base
11
+ include ::Karafka::Helpers::Colorize
12
+
13
+ private
14
+
15
+ # @return [String] green colored word "successfully"
16
+ def successfully
17
+ green('successfully')
18
+ end
19
+
20
+ # @return [String] green colored word "already"
21
+ def already
22
+ green('already')
23
+ end
24
+
25
+ # @return [Array<String>] topics available in the cluster
26
+ def existing_topics_names
27
+ @existing_topics_names ||= ::Karafka::Admin
28
+ .cluster_info
29
+ .topics
30
+ .map { |topic| topic[:topic_name] }
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Web
5
+ module Management
6
+ module Actions
7
+ # Cleans the boot file from Karafka Web-UI details.
8
+ class CleanBootFile < Base
9
+ # Web-UI enabled code
10
+ ENABLER_CODE = ExtendBootFile::ENABLER_CODE
11
+
12
+ private_constant :ENABLER_CODE
13
+
14
+ # Removes the Web-UI boot file data
15
+ def call
16
+ karafka_rb = File.readlines(Karafka.boot_file)
17
+
18
+ if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
19
+ puts 'Updating the Karafka boot file...'
20
+ karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
21
+
22
+ File.write(Karafka.boot_file, karafka_rb.join)
23
+ puts "Karafka boot file #{successfully} updated."
24
+ puts 'Make sure to remove configuration and other customizations as well.'
25
+ else
26
+ puts 'Karafka Web UI components not found in the boot file.'
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Web
5
+ module Management
6
+ module Actions
7
+ # Creates the records needed for the Web-UI to operate.
8
+ # It creates "almost" empty states because the rest is handled via migrations
9
+ class CreateInitialStates < Base
10
+ # Whole default empty state
11
+ # This will be further migrated by the migrator
12
+ DEFAULT_STATE = {
13
+ schema_version: '0.0.0'
14
+ }.freeze
15
+
16
+ # Default metrics state
17
+ DEFAULT_METRICS = {
18
+ schema_version: '0.0.0'
19
+ }.freeze
20
+
21
+ # Creates the initial states for the Web-UI if needed (if they don't exist)
22
+ def call
23
+ if exists?(Karafka::Web.config.topics.consumers.states)
24
+ exists('consumers state')
25
+ else
26
+ creating('consumers state')
27
+ ::Karafka::Web.producer.produce_sync(
28
+ topic: Karafka::Web.config.topics.consumers.states,
29
+ key: Karafka::Web.config.topics.consumers.states,
30
+ payload: DEFAULT_STATE.to_json
31
+ )
32
+ created('consumers state')
33
+ end
34
+
35
+ if exists?(Karafka::Web.config.topics.consumers.metrics)
36
+ exists('consumers metrics')
37
+ else
38
+ creating('consumers metrics')
39
+ ::Karafka::Web.producer.produce_sync(
40
+ topic: Karafka::Web.config.topics.consumers.metrics,
41
+ key: Karafka::Web.config.topics.consumers.metrics,
42
+ payload: DEFAULT_METRICS.to_json
43
+ )
44
+ created('consumers metrics')
45
+ end
46
+ end
47
+
48
+ private
49
+
50
+ # @param topic [String] topic name
51
+ # @return [Boolean] true if there is already an initial record in a given topic
52
+ def exists?(topic)
53
+ !::Karafka::Admin.read_topic(topic, 0, 5).last.nil?
54
+ end
55
+
56
+ # @param type [String] type of state
57
+ # @return [String] exists message
58
+ def exists(type)
59
+ puts "Initial #{type} #{already} exists."
60
+ end
61
+
62
+ # @param type [String] type of state
63
+ # @return [String] message that the state is being created
64
+ def creating(type)
65
+ puts "Creating #{type} initial record..."
66
+ end
67
+
68
+ # @param type [String] type of state
69
+ # @return [String] message that the state was created
70
+ def created(type)
71
+ puts "Initial #{type} record #{successfully} created."
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end