karafka-web 0.7.9 → 0.8.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +21 -6
- data/.ruby-version +1 -1
- data/CHANGELOG.md +66 -0
- data/Gemfile.lock +22 -22
- data/docker-compose.yml +3 -1
- data/karafka-web.gemspec +2 -2
- data/lib/karafka/web/config.rb +16 -3
- data/lib/karafka/web/contracts/config.rb +7 -2
- data/lib/karafka/web/errors.rb +12 -0
- data/lib/karafka/web/inflector.rb +33 -0
- data/lib/karafka/web/installer.rb +20 -11
- data/lib/karafka/web/management/actions/base.rb +36 -0
- data/lib/karafka/web/management/actions/clean_boot_file.rb +33 -0
- data/lib/karafka/web/management/actions/create_initial_states.rb +77 -0
- data/lib/karafka/web/management/actions/create_topics.rb +139 -0
- data/lib/karafka/web/management/actions/delete_topics.rb +30 -0
- data/lib/karafka/web/management/actions/enable.rb +117 -0
- data/lib/karafka/web/management/actions/extend_boot_file.rb +39 -0
- data/lib/karafka/web/management/actions/migrate_states_data.rb +18 -0
- data/lib/karafka/web/management/migrations/0_base.rb +58 -0
- data/lib/karafka/web/management/migrations/0_set_initial_consumers_metrics.rb +36 -0
- data/lib/karafka/web/management/migrations/0_set_initial_consumers_state.rb +43 -0
- data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_metrics.rb +26 -0
- data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_state.rb +23 -0
- data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_metrics.rb +24 -0
- data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_state.rb +20 -0
- data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_metrics.rb +24 -0
- data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_state.rb +20 -0
- data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_metrics.rb +36 -0
- data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_states.rb +32 -0
- data/lib/karafka/web/management/migrator.rb +117 -0
- data/lib/karafka/web/processing/consumer.rb +39 -38
- data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +15 -7
- data/lib/karafka/web/processing/consumers/aggregators/state.rb +8 -3
- data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb +5 -1
- data/lib/karafka/web/processing/publisher.rb +59 -0
- data/lib/karafka/web/tracking/consumers/contracts/job.rb +3 -2
- data/lib/karafka/web/tracking/consumers/contracts/partition.rb +1 -0
- data/lib/karafka/web/tracking/consumers/contracts/report.rb +6 -1
- data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb +10 -1
- data/lib/karafka/web/tracking/consumers/listeners/connections.rb +49 -0
- data/lib/karafka/web/tracking/consumers/listeners/pausing.rb +7 -4
- data/lib/karafka/web/tracking/consumers/listeners/processing.rb +78 -70
- data/lib/karafka/web/tracking/consumers/listeners/statistics.rb +40 -13
- data/lib/karafka/web/tracking/consumers/sampler.rb +82 -25
- data/lib/karafka/web/tracking/helpers/ttls/array.rb +72 -0
- data/lib/karafka/web/tracking/helpers/ttls/hash.rb +34 -0
- data/lib/karafka/web/tracking/helpers/ttls/stats.rb +49 -0
- data/lib/karafka/web/tracking/helpers/ttls/windows.rb +32 -0
- data/lib/karafka/web/tracking/reporter.rb +1 -0
- data/lib/karafka/web/ui/app.rb +22 -4
- data/lib/karafka/web/ui/base.rb +18 -2
- data/lib/karafka/web/ui/controllers/base.rb +34 -4
- data/lib/karafka/web/ui/controllers/become_pro.rb +1 -1
- data/lib/karafka/web/ui/controllers/cluster.rb +33 -9
- data/lib/karafka/web/ui/controllers/consumers.rb +8 -2
- data/lib/karafka/web/ui/controllers/dashboard.rb +2 -2
- data/lib/karafka/web/ui/controllers/errors.rb +2 -2
- data/lib/karafka/web/ui/controllers/jobs.rb +55 -5
- data/lib/karafka/web/ui/controllers/requests/params.rb +5 -0
- data/lib/karafka/web/ui/controllers/responses/deny.rb +15 -0
- data/lib/karafka/web/ui/controllers/responses/file.rb +23 -0
- data/lib/karafka/web/ui/controllers/responses/{data.rb → render.rb} +3 -3
- data/lib/karafka/web/ui/controllers/routing.rb +11 -2
- data/lib/karafka/web/ui/controllers/status.rb +1 -1
- data/lib/karafka/web/ui/helpers/application_helper.rb +70 -0
- data/lib/karafka/web/ui/lib/hash_proxy.rb +29 -14
- data/lib/karafka/web/ui/lib/sorter.rb +170 -0
- data/lib/karafka/web/ui/models/counters.rb +6 -0
- data/lib/karafka/web/ui/models/health.rb +23 -2
- data/lib/karafka/web/ui/models/jobs.rb +48 -0
- data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb +33 -0
- data/lib/karafka/web/ui/models/metrics/charts/topics.rb +1 -10
- data/lib/karafka/web/ui/models/process.rb +2 -1
- data/lib/karafka/web/ui/models/status.rb +23 -7
- data/lib/karafka/web/ui/models/topic.rb +3 -1
- data/lib/karafka/web/ui/models/visibility_filter.rb +16 -0
- data/lib/karafka/web/ui/pro/app.rb +44 -6
- data/lib/karafka/web/ui/pro/controllers/cluster.rb +1 -0
- data/lib/karafka/web/ui/pro/controllers/consumers.rb +52 -6
- data/lib/karafka/web/ui/pro/controllers/dashboard.rb +1 -1
- data/lib/karafka/web/ui/pro/controllers/dlq.rb +1 -1
- data/lib/karafka/web/ui/pro/controllers/errors.rb +3 -3
- data/lib/karafka/web/ui/pro/controllers/explorer.rb +8 -8
- data/lib/karafka/web/ui/pro/controllers/health.rb +34 -2
- data/lib/karafka/web/ui/pro/controllers/jobs.rb +11 -0
- data/lib/karafka/web/ui/pro/controllers/messages.rb +42 -0
- data/lib/karafka/web/ui/pro/controllers/routing.rb +11 -2
- data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb +8 -2
- data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb +14 -8
- data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +8 -6
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb +4 -1
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_no_jobs.erb +1 -1
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +1 -3
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +28 -11
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb +10 -3
- data/lib/karafka/web/ui/pro/views/consumers/index.erb +3 -3
- data/lib/karafka/web/ui/pro/views/consumers/pending_jobs.erb +43 -0
- data/lib/karafka/web/ui/pro/views/consumers/{jobs.erb → running_jobs.erb} +11 -10
- data/lib/karafka/web/ui/pro/views/dashboard/index.erb +7 -1
- data/lib/karafka/web/ui/pro/views/explorer/message/_message_actions.erb +18 -0
- data/lib/karafka/web/ui/pro/views/explorer/message/_metadata.erb +43 -0
- data/lib/karafka/web/ui/pro/views/explorer/message/_payload.erb +21 -0
- data/lib/karafka/web/ui/pro/views/explorer/message/_payload_actions.erb +19 -0
- data/lib/karafka/web/ui/pro/views/explorer/show.erb +9 -84
- data/lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb +8 -0
- data/lib/karafka/web/ui/pro/views/health/_partition.erb +1 -3
- data/lib/karafka/web/ui/pro/views/health/_partition_offset.erb +4 -4
- data/lib/karafka/web/ui/pro/views/health/_partition_times.erb +32 -0
- data/lib/karafka/web/ui/pro/views/health/_tabs.erb +9 -0
- data/lib/karafka/web/ui/pro/views/health/changes.erb +66 -0
- data/lib/karafka/web/ui/pro/views/health/offsets.erb +14 -14
- data/lib/karafka/web/ui/pro/views/health/overview.erb +11 -11
- data/lib/karafka/web/ui/pro/views/jobs/_job.erb +1 -1
- data/lib/karafka/web/ui/pro/views/jobs/_no_jobs.erb +1 -1
- data/lib/karafka/web/ui/pro/views/jobs/pending.erb +39 -0
- data/lib/karafka/web/ui/pro/views/jobs/running.erb +39 -0
- data/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb +2 -2
- data/lib/karafka/web/ui/pro/views/routing/_topic.erb +9 -0
- data/lib/karafka/web/ui/pro/views/routing/show.erb +12 -0
- data/lib/karafka/web/ui/pro/views/shared/_navigation.erb +1 -1
- data/lib/karafka/web/ui/public/javascripts/application.js +10 -0
- data/lib/karafka/web/ui/public/stylesheets/application.css +4 -0
- data/lib/karafka/web/ui/views/cluster/_breadcrumbs.erb +16 -0
- data/lib/karafka/web/ui/views/cluster/_tabs.erb +27 -0
- data/lib/karafka/web/ui/views/cluster/brokers.erb +27 -0
- data/lib/karafka/web/ui/views/cluster/topics.erb +35 -0
- data/lib/karafka/web/ui/views/consumers/_counters.erb +8 -6
- data/lib/karafka/web/ui/views/consumers/_summary.erb +2 -2
- data/lib/karafka/web/ui/views/consumers/index.erb +3 -3
- data/lib/karafka/web/ui/views/dashboard/_ranges_selector.erb +23 -7
- data/lib/karafka/web/ui/views/dashboard/index.erb +19 -8
- data/lib/karafka/web/ui/views/errors/show.erb +2 -23
- data/lib/karafka/web/ui/views/jobs/_breadcrumbs.erb +17 -1
- data/lib/karafka/web/ui/views/jobs/_job.erb +1 -1
- data/lib/karafka/web/ui/views/jobs/_no_jobs.erb +1 -1
- data/lib/karafka/web/ui/views/jobs/_tabs.erb +27 -0
- data/lib/karafka/web/ui/views/jobs/{index.erb → pending.erb} +9 -7
- data/lib/karafka/web/ui/{pro/views/jobs/index.erb → views/jobs/running.erb} +9 -11
- data/lib/karafka/web/ui/views/routing/_consumer_group.erb +14 -12
- data/lib/karafka/web/ui/views/shared/_navigation.erb +1 -1
- data/lib/karafka/web/ui/views/shared/_pagination.erb +1 -1
- data/lib/karafka/web/ui/views/shared/exceptions/not_allowed.erb +37 -0
- data/lib/karafka/web/ui/views/status/show.erb +17 -2
- data/lib/karafka/web/ui/views/status/warnings/_routing_topics_presence.erb +15 -0
- data/lib/karafka/web/version.rb +1 -1
- data/lib/karafka/web.rb +6 -2
- data.tar.gz.sig +0 -0
- metadata +61 -26
- metadata.gz.sig +0 -0
- data/lib/karafka/web/management/base.rb +0 -34
- data/lib/karafka/web/management/clean_boot_file.rb +0 -31
- data/lib/karafka/web/management/create_initial_states.rb +0 -101
- data/lib/karafka/web/management/create_topics.rb +0 -133
- data/lib/karafka/web/management/delete_topics.rb +0 -28
- data/lib/karafka/web/management/enable.rb +0 -102
- data/lib/karafka/web/management/extend_boot_file.rb +0 -37
- data/lib/karafka/web/tracking/ttl_array.rb +0 -59
- data/lib/karafka/web/tracking/ttl_hash.rb +0 -16
- data/lib/karafka/web/ui/pro/views/dashboard/_ranges_selector.erb +0 -39
- data/lib/karafka/web/ui/views/cluster/index.erb +0 -74
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d3e890ec145255109e1f29ea742e6e0ed2887b0b4ca1de6047973816307744c1
|
4
|
+
data.tar.gz: 55766edeb8f40c68c1200b577697728037882e2af3dd0dfb9723741be0343b7e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 29dccc30c89888815b33a703317ca44bcf1b0a5f37228e9db7ba26eeba2d3b5ce5a66f5246121c4b9c1bb9d248e9027f3e8992f324109fd807d62a94c70c47ff
|
7
|
+
data.tar.gz: 32ed3012d866810f85db46c8c82f20888693e1c533edb9bcdc54cf098c2bece4478e8fe82f8a3c62c313f219d622f8b42e25bd4f38793d82951dec311f164c5c
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.github/workflows/ci.yml
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
name: ci
|
2
2
|
|
3
|
-
concurrency:
|
3
|
+
concurrency:
|
4
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
5
|
+
cancel-in-progress: true
|
4
6
|
|
5
7
|
on:
|
6
8
|
pull_request:
|
@@ -20,13 +22,13 @@ jobs:
|
|
20
22
|
fail-fast: false
|
21
23
|
matrix:
|
22
24
|
ruby:
|
23
|
-
- '3.3
|
25
|
+
- '3.3'
|
24
26
|
- '3.2'
|
25
27
|
- '3.1'
|
26
28
|
- '3.0'
|
27
29
|
- '2.7'
|
28
30
|
include:
|
29
|
-
- ruby: '3.
|
31
|
+
- ruby: '3.3'
|
30
32
|
coverage: 'true'
|
31
33
|
steps:
|
32
34
|
- uses: actions/checkout@v4
|
@@ -42,16 +44,29 @@ jobs:
|
|
42
44
|
with:
|
43
45
|
ruby-version: ${{matrix.ruby}}
|
44
46
|
bundler-cache: true
|
47
|
+
bundler: 'latest'
|
45
48
|
|
46
49
|
- name: Install latest bundler
|
47
50
|
run: |
|
48
|
-
|
51
|
+
if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
|
52
|
+
gem install bundler -v 2.4.22 --no-document
|
53
|
+
gem update --system 3.4.22 --no-document
|
54
|
+
else
|
55
|
+
gem install bundler --no-document
|
56
|
+
gem update --system --no-document
|
57
|
+
fi
|
58
|
+
|
49
59
|
bundle config set without 'tools benchmarks docs'
|
50
60
|
|
51
61
|
- name: Bundle install
|
52
62
|
run: |
|
53
63
|
bundle config set without development
|
54
|
-
|
64
|
+
|
65
|
+
if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
|
66
|
+
BUNDLER_VERSION=2.4.22 bundle install --jobs 4 --retry 3
|
67
|
+
else
|
68
|
+
bundle install --jobs 4 --retry 3
|
69
|
+
fi
|
55
70
|
|
56
71
|
- name: Wait for Kafka
|
57
72
|
run: |
|
@@ -75,7 +90,7 @@ jobs:
|
|
75
90
|
- name: Set up Ruby
|
76
91
|
uses: ruby/setup-ruby@v1
|
77
92
|
with:
|
78
|
-
ruby-version: 3.
|
93
|
+
ruby-version: 3.3
|
79
94
|
bundler-cache: true
|
80
95
|
|
81
96
|
- name: Install Diffend plugin
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.
|
1
|
+
3.3.0
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,62 @@
|
|
1
1
|
# Karafka Web changelog
|
2
2
|
|
3
|
+
## 0.8.0 (Unreleased)
|
4
|
+
- **[Feature]** Provide ability to sort table data for part of the views (note: not all attributes can be sorted due to technical limitations of sub-components fetching from Kafka).
|
5
|
+
- **[Feature]** Track and report pause timeouts via "Changes" view in Health.
|
6
|
+
- **[Feature]** Introduce pending jobs visibility alongside of running jobs both in total and per process.
|
7
|
+
- **[Feature]** Introduce states migrations for seamless upgrades.
|
8
|
+
- **[Feature]** Introduce "Data transfers" chart with data received and data sent to the cluster.
|
9
|
+
- **[Feature]** Introduce ability to download raw payloads.
|
10
|
+
- **[Feature]** Introduce ability to download deserialized message payload as JSON.
|
11
|
+
- [Enhancement] Support reporting of standby and active listeners for connection multiplexed subscription groups.
|
12
|
+
- [Enhancement] Support Periodic Jobs reporting.
|
13
|
+
- [Enhancement] Support multiplexed subscription groups.
|
14
|
+
- [Enhancement] Split cluster info into two tabs, one for brokers and one for topics with partitions.
|
15
|
+
- [Enhancement] Track pending jobs. Pending jobs are jobs that are not yet scheduled for execution by advanced schedulers.
|
16
|
+
- [Enhancement] Rename "Enqueued" to "Pending" to support jobs that are not yet enqueued but within a scheduler.
|
17
|
+
- [Enhancement] Make sure only running jobs are displayed in running jobs
|
18
|
+
- [Enhancement] Improve jobs related breadcrumbs
|
19
|
+
- [Enhancement] Display errors backtraces in OSS.
|
20
|
+
- [Enhancement] Display concurrency graph in OSS.
|
21
|
+
- [Enhancement] Support time ranges for graphs in OSS.
|
22
|
+
- [Enhancement] Report last poll time for each subscription group.
|
23
|
+
- [Enhancement] Show last poll time per consumer instance.
|
24
|
+
- [Enhancement] Display number of jobs in a particular process jobs view.
|
25
|
+
- [Enhancement] Promote "Batches" chart to OSS.
|
26
|
+
- [Enhancement] Promote "Utilization" chart to OSS.
|
27
|
+
- [Enhancement] Allow for explicit disabling of the Web UI tracking.
|
28
|
+
- [Fix] Web UI will keep reporting status even when not activated as long as required and in routes.
|
29
|
+
- [Fix] Fix times precisions that could be incorrectly reported by 1 second in few places.
|
30
|
+
- [Fix] Fix random order in Consumers groups Health view.
|
31
|
+
- [Change] Rename "Busy" to "Running" to align with "Running Jobs".
|
32
|
+
- [Change] Rename "Active subscriptions" to "Subscriptions" as process subscriptions are always active.
|
33
|
+
- [Maintenance] Introduce granular subscription group contracts.
|
34
|
+
|
35
|
+
### Upgrade Notes
|
36
|
+
|
37
|
+
This is a **major** release that brings many things to the table.
|
38
|
+
|
39
|
+
#### Configuration
|
40
|
+
|
41
|
+
**No** configuration changes are needed.
|
42
|
+
|
43
|
+
#### Deployment
|
44
|
+
|
45
|
+
Because of the reporting schema update, it is recommended to:
|
46
|
+
|
47
|
+
0. Make sure you have upgraded to `0.7.10` before and that it was fully deployed.
|
48
|
+
1. Test the upgrade on a staging or dev environment.
|
49
|
+
2. Starting from `0.7.0` Karafka Web UI supports rolling deploys, so there is no need to "stop the world".
|
50
|
+
3. The Web UI interface may throw 500 errors during the upgrade because of schema incompatibility (until Puma is deployed). This will have no long-term effects and can be ignored.
|
51
|
+
4. `Karafka::Web::Errors::Processing::IncompatibleSchemaError` **is expected**. It is part of the Karafka Web UI zero-downtime deployment strategy. This error allows the Web UI materialization consumer to back off and wait for it to be replaced with a new one.
|
52
|
+
5. Perform a rolling deployment (or a regular one) and replace all consumer processes.
|
53
|
+
6. Update the Web UI Puma.
|
54
|
+
7. **No** CLI command execution is required. Starting from this release (`0.8.0`), the Karafka Web UI contains an automatic schema migrator that allows it to automatically adjust internal topic data formats.
|
55
|
+
8. Enjoy.
|
56
|
+
|
57
|
+
## 0.7.10 (2023-10-31)
|
58
|
+
- [Fix] Max LSO chart does not work as expected (#201)
|
59
|
+
|
3
60
|
## 0.7.9 (2023-10-25)
|
4
61
|
- [Enhancement] Allow for `Karafka::Web.producer` reconfiguration from the default (`Karafka.producer`).
|
5
62
|
- [Change] Rely on `karafka-core` `>=` `2.2.4` to support lazy loaded custom web producer.
|
@@ -26,6 +83,15 @@
|
|
26
83
|
- [Fix] Cache assets for 1 year instead of 7 days.
|
27
84
|
- [Fix] Remove source maps pointing to non-existing locations.
|
28
85
|
- [Maintenance] Include license and copyrights notice for `timeago.js` that was missing in the JS min file.
|
86
|
+
- [Refactor] Rename `ui.show_internal_topics` to `ui.visibility.internal_topics_display`
|
87
|
+
|
88
|
+
### Upgrade Notes
|
89
|
+
|
90
|
+
**NO** rolling upgrade needed. Just configuration update.
|
91
|
+
|
92
|
+
1. If you are using `ui.visibility_filter` this option is now `ui.visibility.filter` (yes, only `.` difference).
|
93
|
+
2. If you are using a custom visibility filter, it requires now two extra methods: `#download?` and `#export?`. The default visibility filter allows both actions unless message is encrypted.
|
94
|
+
3. `ui.show_internal_topics` config option has been moved and renamed to `ui.visibility.internal_topics`.
|
29
95
|
|
30
96
|
## 0.7.4 (2023-09-19)
|
31
97
|
- [Improvement] Skip aggregations on older schemas during upgrades. This only skips process-reports (that are going to be rolled) on the 5s window in case of an upgrade that should not be a rolling one anyhow. This simplifies the operations and minimizes the risk on breaking upgrades.
|
data/Gemfile.lock
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka-web (0.
|
4
|
+
karafka-web (0.8.0.rc1)
|
5
5
|
erubi (~> 1.4)
|
6
|
-
karafka (>= 2.
|
7
|
-
karafka-core (>= 2.
|
6
|
+
karafka (>= 2.3.0.rc1, < 2.4.0)
|
7
|
+
karafka-core (>= 2.3.0.rc1, < 2.4.0)
|
8
8
|
roda (~> 3.68, >= 3.69)
|
9
9
|
tilt (~> 2.0)
|
10
10
|
|
11
11
|
GEM
|
12
12
|
remote: https://rubygems.org/
|
13
13
|
specs:
|
14
|
-
activesupport (7.1.
|
14
|
+
activesupport (7.1.3)
|
15
15
|
base64
|
16
16
|
bigdecimal
|
17
17
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
@@ -21,43 +21,42 @@ GEM
|
|
21
21
|
minitest (>= 5.1)
|
22
22
|
mutex_m
|
23
23
|
tzinfo (~> 2.0)
|
24
|
-
base64 (0.
|
25
|
-
bigdecimal (3.1.
|
24
|
+
base64 (0.2.0)
|
25
|
+
bigdecimal (3.1.6)
|
26
26
|
byebug (11.1.3)
|
27
|
-
concurrent-ruby (1.2.
|
27
|
+
concurrent-ruby (1.2.3)
|
28
28
|
connection_pool (2.4.1)
|
29
29
|
diff-lcs (1.5.0)
|
30
30
|
docile (1.4.0)
|
31
|
-
drb (2.
|
31
|
+
drb (2.2.0)
|
32
32
|
ruby2_keywords
|
33
33
|
erubi (1.12.0)
|
34
|
-
factory_bot (6.
|
34
|
+
factory_bot (6.4.5)
|
35
35
|
activesupport (>= 5.0.0)
|
36
36
|
ffi (1.16.3)
|
37
37
|
i18n (1.14.1)
|
38
38
|
concurrent-ruby (~> 1.0)
|
39
|
-
karafka (2.
|
40
|
-
karafka-core (>= 2.
|
41
|
-
waterdrop (>= 2.6.
|
39
|
+
karafka (2.3.0.rc1)
|
40
|
+
karafka-core (>= 2.3.0.rc1, < 2.4.0)
|
41
|
+
waterdrop (>= 2.6.12, < 3.0.0)
|
42
42
|
zeitwerk (~> 2.3)
|
43
|
-
karafka-core (2.
|
44
|
-
|
45
|
-
|
46
|
-
karafka-rdkafka (0.13.6)
|
43
|
+
karafka-core (2.3.0.rc1)
|
44
|
+
karafka-rdkafka (>= 0.14.7, < 0.15.0)
|
45
|
+
karafka-rdkafka (0.14.7)
|
47
46
|
ffi (~> 1.15)
|
48
47
|
mini_portile2 (~> 2.6)
|
49
48
|
rake (> 12)
|
50
49
|
mini_portile2 (2.8.5)
|
51
|
-
minitest (5.
|
52
|
-
mutex_m (0.
|
50
|
+
minitest (5.21.2)
|
51
|
+
mutex_m (0.2.0)
|
53
52
|
rack (3.0.8)
|
54
53
|
rack-test (2.1.0)
|
55
54
|
rack (>= 1.3)
|
56
55
|
rackup (0.2.3)
|
57
56
|
rack (>= 3.0.0.beta1)
|
58
57
|
webrick
|
59
|
-
rake (13.0
|
60
|
-
roda (3.
|
58
|
+
rake (13.1.0)
|
59
|
+
roda (3.76.0)
|
61
60
|
rack
|
62
61
|
rspec (3.12.0)
|
63
62
|
rspec-core (~> 3.12.0)
|
@@ -82,13 +81,14 @@ GEM
|
|
82
81
|
tilt (2.3.0)
|
83
82
|
tzinfo (2.0.6)
|
84
83
|
concurrent-ruby (~> 1.0)
|
85
|
-
waterdrop (2.6.
|
84
|
+
waterdrop (2.6.12)
|
86
85
|
karafka-core (>= 2.2.3, < 3.0.0)
|
87
86
|
zeitwerk (~> 2.3)
|
88
87
|
webrick (1.8.1)
|
89
88
|
zeitwerk (2.6.12)
|
90
89
|
|
91
90
|
PLATFORMS
|
91
|
+
ruby
|
92
92
|
x86_64-linux
|
93
93
|
|
94
94
|
DEPENDENCIES
|
@@ -101,4 +101,4 @@ DEPENDENCIES
|
|
101
101
|
simplecov
|
102
102
|
|
103
103
|
BUNDLED WITH
|
104
|
-
2.4
|
104
|
+
2.5.4
|
data/docker-compose.yml
CHANGED
@@ -3,7 +3,7 @@ version: '2'
|
|
3
3
|
services:
|
4
4
|
kafka:
|
5
5
|
container_name: kafka
|
6
|
-
image: confluentinc/cp-kafka:7.5.
|
6
|
+
image: confluentinc/cp-kafka:7.5.3
|
7
7
|
|
8
8
|
ports:
|
9
9
|
- 9092:9092
|
@@ -23,3 +23,5 @@ services:
|
|
23
23
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
24
24
|
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
25
25
|
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
26
|
+
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
|
27
|
+
KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
|
data/karafka-web.gemspec
CHANGED
@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.licenses = %w[LGPL-3.0 Commercial]
|
18
18
|
|
19
19
|
spec.add_dependency 'erubi', '~> 1.4'
|
20
|
-
spec.add_dependency 'karafka', '>= 2.
|
21
|
-
spec.add_dependency 'karafka-core', '>= 2.
|
20
|
+
spec.add_dependency 'karafka', '>= 2.3.0.rc1', '< 2.4.0'
|
21
|
+
spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
|
22
22
|
spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
|
23
23
|
spec.add_dependency 'tilt', '~> 2.0'
|
24
24
|
|
data/lib/karafka/web/config.rb
CHANGED
@@ -47,6 +47,11 @@ module Karafka
|
|
47
47
|
# 5 seconds should be enough
|
48
48
|
setting :interval, default: 5_000
|
49
49
|
|
50
|
+
# Should we be tracking anything at all. If this is set to false, no reporting will happen
|
51
|
+
# even if Web-UI is configured. When set to `nil` (default) it will be switched to true
|
52
|
+
# during the Web UI initialization.
|
53
|
+
setting :active, default: nil
|
54
|
+
|
50
55
|
# Main Web UI reporting scheduler that runs a background thread and reports periodically
|
51
56
|
# from the consumer reporter and producer reporter
|
52
57
|
setting :scheduler, default: Tracking::Scheduler.new
|
@@ -62,6 +67,7 @@ module Karafka
|
|
62
67
|
setting :listeners, default: [
|
63
68
|
Tracking::Consumers::Listeners::Status.new,
|
64
69
|
Tracking::Consumers::Listeners::Errors.new,
|
70
|
+
Tracking::Consumers::Listeners::Connections.new,
|
65
71
|
Tracking::Consumers::Listeners::Statistics.new,
|
66
72
|
Tracking::Consumers::Listeners::Pausing.new,
|
67
73
|
Tracking::Consumers::Listeners::Processing.new,
|
@@ -117,9 +123,16 @@ module Karafka
|
|
117
123
|
Karafka.env.production? ? 60_000 * 5 : 5_000
|
118
124
|
)
|
119
125
|
|
120
|
-
|
121
|
-
|
122
|
-
|
126
|
+
setting :visibility do
|
127
|
+
# Allows to manage visibility of payload, headers and message key in the UI
|
128
|
+
# In some cases you may want to limit what is being displayed due to the type of data you
|
129
|
+
# are dealing with
|
130
|
+
setting :filter, default: Ui::Models::VisibilityFilter.new
|
131
|
+
|
132
|
+
# Should we display internal topics of Kafka. The once starting with `__`
|
133
|
+
# By default we do not display them as they are not usable from regular users perspective
|
134
|
+
setting :internal_topics, default: false
|
135
|
+
end
|
123
136
|
|
124
137
|
# How many elements should we display on pages that support pagination
|
125
138
|
setting :per_page, default: 25
|
@@ -23,6 +23,8 @@ module Karafka
|
|
23
23
|
end
|
24
24
|
|
25
25
|
nested(:tracking) do
|
26
|
+
# If set to nil, it is up to us to initialize
|
27
|
+
required(:active) { |val| [true, false, nil].include?(val) }
|
26
28
|
# Do not report more often then every second, this could overload the system
|
27
29
|
required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
|
28
30
|
|
@@ -52,10 +54,13 @@ module Karafka
|
|
52
54
|
required(:secret) { |val| val.is_a?(String) && val.length >= 64 }
|
53
55
|
end
|
54
56
|
|
55
|
-
required(:show_internal_topics) { |val| [true, false].include?(val) }
|
56
57
|
required(:cache) { |val| !val.nil? }
|
57
58
|
required(:per_page) { |val| val.is_a?(Integer) && val >= 1 && val <= 100 }
|
58
|
-
|
59
|
+
|
60
|
+
nested(:visibility) do
|
61
|
+
required(:filter) { |val| !val.nil? }
|
62
|
+
required(:internal_topics) { |val| [true, false].include?(val) }
|
63
|
+
end
|
59
64
|
end
|
60
65
|
end
|
61
66
|
end
|
data/lib/karafka/web/errors.rb
CHANGED
@@ -11,6 +11,18 @@ module Karafka
|
|
11
11
|
# This should never happen and if you see this, please open an issue.
|
12
12
|
ContractError = Class.new(BaseError)
|
13
13
|
|
14
|
+
# Errors specific to management
|
15
|
+
module Management
|
16
|
+
# Similar to processing error with the same name, it is raised when a critical
|
17
|
+
# incompatibility is detected.
|
18
|
+
#
|
19
|
+
# This error is raised when there was an attempt to operate on aggregated Web UI states
|
20
|
+
# that are already in a newer version that the one in the current process. We prevent
|
21
|
+
# this from happening not to corrupt the data. Please upgrade all the Web UI consumers to
|
22
|
+
# the same version
|
23
|
+
IncompatibleSchemaError = Class.new(BaseError)
|
24
|
+
end
|
25
|
+
|
14
26
|
# Processing related errors namespace
|
15
27
|
module Processing
|
16
28
|
# Raised when we try to process reports but we do not have the current state bootstrapped
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
# Web UI Zeitwerk Inflector that allows us to have time prefixed files with migrations, similar
|
6
|
+
# to how Rails does that.
|
7
|
+
class Inflector < Zeitwerk::GemInflector
|
8
|
+
# Checks if given path is a migration one
|
9
|
+
MIGRATION_ABSPATH_REGEXP = /migrations\/[0-9]+_(.*)/
|
10
|
+
|
11
|
+
# Checks if it is a migration file
|
12
|
+
MIGRATION_BASENAME_REGEXP = /\A[0-9]+_(.*)/
|
13
|
+
|
14
|
+
private_constant :MIGRATION_ABSPATH_REGEXP, :MIGRATION_BASENAME_REGEXP
|
15
|
+
|
16
|
+
# @param [String] basename of the file to be loaded
|
17
|
+
# @param abspath [String] absolute path of the file to be loaded
|
18
|
+
# @return [String] Constant name to be used for given file
|
19
|
+
def camelize(basename, abspath)
|
20
|
+
# If not migration directory with proper migration files, use defaults
|
21
|
+
return super unless abspath.match?(MIGRATION_ABSPATH_REGEXP)
|
22
|
+
# If base name is not of a proper name in migrations, use defaults
|
23
|
+
return super unless basename.match?(MIGRATION_BASENAME_REGEXP)
|
24
|
+
|
25
|
+
super(
|
26
|
+
# Extract only the name without the timestamp
|
27
|
+
basename.match(MIGRATION_BASENAME_REGEXP).to_a.last,
|
28
|
+
abspath
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -17,11 +17,14 @@ module Karafka
|
|
17
17
|
puts
|
18
18
|
puts 'Creating necessary topics and populating state data...'
|
19
19
|
puts
|
20
|
-
Management::CreateTopics.new.call(replication_factor)
|
20
|
+
Management::Actions::CreateTopics.new.call(replication_factor)
|
21
21
|
wait_for_topics
|
22
|
-
Management::CreateInitialStates.new.call
|
22
|
+
Management::Actions::CreateInitialStates.new.call
|
23
23
|
puts
|
24
|
-
|
24
|
+
puts 'Running data migrations...'
|
25
|
+
Management::Actions::MigrateStatesData.new.call
|
26
|
+
puts
|
27
|
+
Management::Actions::ExtendBootFile.new.call
|
25
28
|
puts
|
26
29
|
puts("Installation #{green('completed')}. Have fun!")
|
27
30
|
puts
|
@@ -35,9 +38,12 @@ module Karafka
|
|
35
38
|
puts
|
36
39
|
puts 'Creating necessary topics and populating state data...'
|
37
40
|
puts
|
38
|
-
Management::CreateTopics.new.call(replication_factor)
|
41
|
+
Management::Actions::CreateTopics.new.call(replication_factor)
|
39
42
|
wait_for_topics
|
40
|
-
Management::CreateInitialStates.new.call
|
43
|
+
Management::Actions::CreateInitialStates.new.call
|
44
|
+
puts
|
45
|
+
puts 'Running data migrations...'
|
46
|
+
Management::Actions::MigrateStatesData.new.call
|
41
47
|
puts
|
42
48
|
puts("Migration #{green('completed')}. Have fun!")
|
43
49
|
puts
|
@@ -49,11 +55,14 @@ module Karafka
|
|
49
55
|
puts
|
50
56
|
puts 'Resetting Karafka Web UI...'
|
51
57
|
puts
|
52
|
-
Management::DeleteTopics.new.call
|
58
|
+
Management::Actions::DeleteTopics.new.call
|
53
59
|
puts
|
54
|
-
Management::CreateTopics.new.call(replication_factor)
|
60
|
+
Management::Actions::CreateTopics.new.call(replication_factor)
|
55
61
|
wait_for_topics
|
56
|
-
Management::CreateInitialStates.new.call
|
62
|
+
Management::Actions::CreateInitialStates.new.call
|
63
|
+
puts
|
64
|
+
puts 'Running data migrations...'
|
65
|
+
Management::Actions::MigrateStatesData.new.call
|
57
66
|
puts
|
58
67
|
puts("Resetting #{green('completed')}. Have fun!")
|
59
68
|
puts
|
@@ -64,8 +73,8 @@ module Karafka
|
|
64
73
|
puts
|
65
74
|
puts 'Uninstalling Karafka Web UI...'
|
66
75
|
puts
|
67
|
-
Management::DeleteTopics.new.call
|
68
|
-
Management::CleanBootFile.new.call
|
76
|
+
Management::Actions::DeleteTopics.new.call
|
77
|
+
Management::Actions::CleanBootFile.new.call
|
69
78
|
puts
|
70
79
|
puts("Uninstalling #{green('completed')}. Goodbye!")
|
71
80
|
puts
|
@@ -73,7 +82,7 @@ module Karafka
|
|
73
82
|
|
74
83
|
# Enables the Web-UI in the karafka app. Sets up needed routes and listeners.
|
75
84
|
def enable!
|
76
|
-
Management::Enable.new.call
|
85
|
+
Management::Actions::Enable.new.call
|
77
86
|
end
|
78
87
|
|
79
88
|
private
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Management
|
6
|
+
# Namespace for all the commands related to management of the Web-UI in the context of
|
7
|
+
# Karafka. It includes things like installing, creating needed topics, etc.
|
8
|
+
module Actions
|
9
|
+
# Base class for all the commands that we use to manage
|
10
|
+
class Base
|
11
|
+
include ::Karafka::Helpers::Colorize
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
# @return [String] green colored word "successfully"
|
16
|
+
def successfully
|
17
|
+
green('successfully')
|
18
|
+
end
|
19
|
+
|
20
|
+
# @return [String] green colored word "already"
|
21
|
+
def already
|
22
|
+
green('already')
|
23
|
+
end
|
24
|
+
|
25
|
+
# @return [Array<String>] topics available in the cluster
|
26
|
+
def existing_topics_names
|
27
|
+
@existing_topics_names ||= ::Karafka::Admin
|
28
|
+
.cluster_info
|
29
|
+
.topics
|
30
|
+
.map { |topic| topic[:topic_name] }
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Management
|
6
|
+
module Actions
|
7
|
+
# Cleans the boot file from Karafka Web-UI details.
|
8
|
+
class CleanBootFile < Base
|
9
|
+
# Web-UI enabled code
|
10
|
+
ENABLER_CODE = ExtendBootFile::ENABLER_CODE
|
11
|
+
|
12
|
+
private_constant :ENABLER_CODE
|
13
|
+
|
14
|
+
# Removes the Web-UI boot file data
|
15
|
+
def call
|
16
|
+
karafka_rb = File.readlines(Karafka.boot_file)
|
17
|
+
|
18
|
+
if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
|
19
|
+
puts 'Updating the Karafka boot file...'
|
20
|
+
karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
|
21
|
+
|
22
|
+
File.write(Karafka.boot_file, karafka_rb.join)
|
23
|
+
puts "Karafka boot file #{successfully} updated."
|
24
|
+
puts 'Make sure to remove configuration and other customizations as well.'
|
25
|
+
else
|
26
|
+
puts 'Karafka Web UI components not found in the boot file.'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Management
|
6
|
+
module Actions
|
7
|
+
# Creates the records needed for the Web-UI to operate.
|
8
|
+
# It creates "almost" empty states because the rest is handled via migrations
|
9
|
+
class CreateInitialStates < Base
|
10
|
+
# Whole default empty state
|
11
|
+
# This will be further migrated by the migrator
|
12
|
+
DEFAULT_STATE = {
|
13
|
+
schema_version: '0.0.0'
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
# Default metrics state
|
17
|
+
DEFAULT_METRICS = {
|
18
|
+
schema_version: '0.0.0'
|
19
|
+
}.freeze
|
20
|
+
|
21
|
+
# Creates the initial states for the Web-UI if needed (if they don't exist)
|
22
|
+
def call
|
23
|
+
if exists?(Karafka::Web.config.topics.consumers.states)
|
24
|
+
exists('consumers state')
|
25
|
+
else
|
26
|
+
creating('consumers state')
|
27
|
+
::Karafka::Web.producer.produce_sync(
|
28
|
+
topic: Karafka::Web.config.topics.consumers.states,
|
29
|
+
key: Karafka::Web.config.topics.consumers.states,
|
30
|
+
payload: DEFAULT_STATE.to_json
|
31
|
+
)
|
32
|
+
created('consumers state')
|
33
|
+
end
|
34
|
+
|
35
|
+
if exists?(Karafka::Web.config.topics.consumers.metrics)
|
36
|
+
exists('consumers metrics')
|
37
|
+
else
|
38
|
+
creating('consumers metrics')
|
39
|
+
::Karafka::Web.producer.produce_sync(
|
40
|
+
topic: Karafka::Web.config.topics.consumers.metrics,
|
41
|
+
key: Karafka::Web.config.topics.consumers.metrics,
|
42
|
+
payload: DEFAULT_METRICS.to_json
|
43
|
+
)
|
44
|
+
created('consumers metrics')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
# @param topic [String] topic name
|
51
|
+
# @return [Boolean] true if there is already an initial record in a given topic
|
52
|
+
def exists?(topic)
|
53
|
+
!::Karafka::Admin.read_topic(topic, 0, 5).last.nil?
|
54
|
+
end
|
55
|
+
|
56
|
+
# @param type [String] type of state
|
57
|
+
# @return [String] exists message
|
58
|
+
def exists(type)
|
59
|
+
puts "Initial #{type} #{already} exists."
|
60
|
+
end
|
61
|
+
|
62
|
+
# @param type [String] type of state
|
63
|
+
# @return [String] message that the state is being created
|
64
|
+
def creating(type)
|
65
|
+
puts "Creating #{type} initial record..."
|
66
|
+
end
|
67
|
+
|
68
|
+
# @param type [String] type of state
|
69
|
+
# @return [String] message that the state was created
|
70
|
+
def created(type)
|
71
|
+
puts "Initial #{type} record #{successfully} created."
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|