karafka-web 0.6.3 → 0.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +13 -4
- data/CHANGELOG.md +126 -5
- data/Gemfile +1 -0
- data/Gemfile.lock +27 -24
- data/README.md +2 -0
- data/bin/rspecs +6 -0
- data/certs/cert_chain.pem +21 -21
- data/docker-compose.yml +22 -0
- data/karafka-web.gemspec +3 -3
- data/lib/karafka/web/app.rb +6 -2
- data/lib/karafka/web/cli.rb +51 -47
- data/lib/karafka/web/config.rb +33 -9
- data/lib/karafka/web/contracts/base.rb +32 -0
- data/lib/karafka/web/contracts/config.rb +63 -0
- data/lib/karafka/web/deserializer.rb +10 -1
- data/lib/karafka/web/errors.rb +29 -7
- data/lib/karafka/web/installer.rb +58 -148
- data/lib/karafka/web/management/base.rb +34 -0
- data/lib/karafka/web/management/clean_boot_file.rb +31 -0
- data/lib/karafka/web/management/create_initial_states.rb +101 -0
- data/lib/karafka/web/management/create_topics.rb +127 -0
- data/lib/karafka/web/management/delete_topics.rb +28 -0
- data/lib/karafka/web/management/enable.rb +82 -0
- data/lib/karafka/web/management/extend_boot_file.rb +37 -0
- data/lib/karafka/web/processing/consumer.rb +73 -17
- data/lib/karafka/web/processing/consumers/aggregators/base.rb +56 -0
- data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +154 -0
- data/lib/karafka/web/processing/consumers/aggregators/state.rb +180 -0
- data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb +32 -0
- data/lib/karafka/web/processing/consumers/contracts/metrics.rb +53 -0
- data/lib/karafka/web/processing/consumers/contracts/process.rb +19 -0
- data/lib/karafka/web/processing/consumers/contracts/state.rb +49 -0
- data/lib/karafka/web/processing/consumers/contracts/topic_stats.rb +21 -0
- data/lib/karafka/web/processing/consumers/metrics.rb +29 -0
- data/lib/karafka/web/processing/consumers/schema_manager.rb +56 -0
- data/lib/karafka/web/processing/consumers/state.rb +6 -9
- data/lib/karafka/web/processing/time_series_tracker.rb +130 -0
- data/lib/karafka/web/tracking/consumers/contracts/consumer_group.rb +2 -2
- data/lib/karafka/web/tracking/consumers/contracts/job.rb +2 -1
- data/lib/karafka/web/tracking/consumers/contracts/partition.rb +14 -1
- data/lib/karafka/web/tracking/consumers/contracts/report.rb +10 -8
- data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb +2 -2
- data/lib/karafka/web/tracking/consumers/contracts/topic.rb +2 -2
- data/lib/karafka/web/tracking/consumers/listeners/processing.rb +6 -2
- data/lib/karafka/web/tracking/consumers/listeners/statistics.rb +15 -1
- data/lib/karafka/web/tracking/consumers/reporter.rb +14 -6
- data/lib/karafka/web/tracking/consumers/sampler.rb +80 -39
- data/lib/karafka/web/tracking/contracts/error.rb +2 -1
- data/lib/karafka/web/ui/app.rb +20 -10
- data/lib/karafka/web/ui/base.rb +56 -6
- data/lib/karafka/web/ui/controllers/base.rb +28 -0
- data/lib/karafka/web/ui/controllers/become_pro.rb +1 -1
- data/lib/karafka/web/ui/controllers/cluster.rb +12 -6
- data/lib/karafka/web/ui/controllers/consumers.rb +4 -2
- data/lib/karafka/web/ui/controllers/dashboard.rb +32 -0
- data/lib/karafka/web/ui/controllers/errors.rb +19 -6
- data/lib/karafka/web/ui/controllers/jobs.rb +4 -2
- data/lib/karafka/web/ui/controllers/requests/params.rb +28 -0
- data/lib/karafka/web/ui/controllers/responses/redirect.rb +29 -0
- data/lib/karafka/web/ui/helpers/application_helper.rb +57 -14
- data/lib/karafka/web/ui/helpers/paths_helper.rb +48 -0
- data/lib/karafka/web/ui/lib/hash_proxy.rb +18 -6
- data/lib/karafka/web/ui/lib/paginations/base.rb +61 -0
- data/lib/karafka/web/ui/lib/paginations/offset_based.rb +96 -0
- data/lib/karafka/web/ui/lib/paginations/page_based.rb +70 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/arrays.rb +33 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/base.rb +23 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/partitions.rb +52 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/sets.rb +85 -0
- data/lib/karafka/web/ui/lib/paginations/watermark_offsets_based.rb +75 -0
- data/lib/karafka/web/ui/lib/ttl_cache.rb +82 -0
- data/lib/karafka/web/ui/models/cluster_info.rb +59 -0
- data/lib/karafka/web/ui/models/consumers_metrics.rb +46 -0
- data/lib/karafka/web/ui/models/{state.rb → consumers_state.rb} +6 -2
- data/lib/karafka/web/ui/models/health.rb +37 -7
- data/lib/karafka/web/ui/models/message.rb +123 -39
- data/lib/karafka/web/ui/models/metrics/aggregated.rb +196 -0
- data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb +50 -0
- data/lib/karafka/web/ui/models/metrics/charts/topics.rb +109 -0
- data/lib/karafka/web/ui/models/metrics/topics.rb +101 -0
- data/lib/karafka/web/ui/models/partition.rb +27 -0
- data/lib/karafka/web/ui/models/process.rb +12 -1
- data/lib/karafka/web/ui/models/status.rb +110 -22
- data/lib/karafka/web/ui/models/visibility_filter.rb +33 -0
- data/lib/karafka/web/ui/pro/app.rb +87 -19
- data/lib/karafka/web/ui/pro/controllers/cluster.rb +11 -0
- data/lib/karafka/web/ui/pro/controllers/consumers.rb +13 -7
- data/lib/karafka/web/ui/pro/controllers/dashboard.rb +54 -0
- data/lib/karafka/web/ui/pro/controllers/dlq.rb +1 -2
- data/lib/karafka/web/ui/pro/controllers/errors.rb +46 -10
- data/lib/karafka/web/ui/pro/controllers/explorer.rb +145 -15
- data/lib/karafka/web/ui/pro/controllers/health.rb +10 -2
- data/lib/karafka/web/ui/pro/controllers/messages.rb +62 -0
- data/lib/karafka/web/ui/pro/controllers/routing.rb +44 -0
- data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb +7 -1
- data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb +6 -2
- data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +7 -5
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb +3 -3
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb +5 -4
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +13 -4
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +3 -2
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb +7 -0
- data/lib/karafka/web/ui/pro/views/consumers/details.erb +21 -0
- data/lib/karafka/web/ui/pro/views/consumers/index.erb +4 -2
- data/lib/karafka/web/ui/pro/views/dashboard/_ranges_selector.erb +39 -0
- data/lib/karafka/web/ui/pro/views/dashboard/index.erb +82 -0
- data/lib/karafka/web/ui/pro/views/dlq/_topic.erb +1 -1
- data/lib/karafka/web/ui/pro/views/errors/_breadcrumbs.erb +8 -6
- data/lib/karafka/web/ui/pro/views/errors/_error.erb +2 -2
- data/lib/karafka/web/ui/pro/views/errors/_partition_option.erb +1 -1
- data/lib/karafka/web/ui/pro/views/errors/_table.erb +21 -0
- data/lib/karafka/web/ui/pro/views/errors/_title_with_select.erb +31 -0
- data/lib/karafka/web/ui/pro/views/errors/index.erb +9 -56
- data/lib/karafka/web/ui/pro/views/errors/partition.erb +17 -0
- data/lib/karafka/web/ui/pro/views/errors/show.erb +1 -1
- data/lib/karafka/web/ui/pro/views/explorer/_breadcrumbs.erb +6 -4
- data/lib/karafka/web/ui/pro/views/explorer/_filtered.erb +16 -0
- data/lib/karafka/web/ui/pro/views/explorer/_message.erb +14 -4
- data/lib/karafka/web/ui/pro/views/explorer/_no_topics.erb +7 -0
- data/lib/karafka/web/ui/pro/views/explorer/_partition_option.erb +3 -3
- data/lib/karafka/web/ui/pro/views/explorer/_topic.erb +1 -1
- data/lib/karafka/web/ui/pro/views/explorer/index.erb +12 -8
- data/lib/karafka/web/ui/pro/views/explorer/messages/_headers.erb +15 -0
- data/lib/karafka/web/ui/pro/views/explorer/messages/_key.erb +12 -0
- data/lib/karafka/web/ui/pro/views/explorer/partition/_details.erb +35 -0
- data/lib/karafka/web/ui/pro/views/explorer/partition/_messages.erb +1 -0
- data/lib/karafka/web/ui/pro/views/explorer/partition.erb +6 -4
- data/lib/karafka/web/ui/pro/views/explorer/show.erb +48 -5
- data/lib/karafka/web/ui/pro/views/explorer/topic/_details.erb +23 -0
- data/lib/karafka/web/ui/pro/views/explorer/topic/_empty.erb +3 -0
- data/lib/karafka/web/ui/pro/views/explorer/topic/_limited.erb +4 -0
- data/lib/karafka/web/ui/pro/views/explorer/topic.erb +51 -0
- data/lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb +16 -0
- data/lib/karafka/web/ui/pro/views/health/_no_data.erb +9 -0
- data/lib/karafka/web/ui/pro/views/health/_partition.erb +17 -15
- data/lib/karafka/web/ui/pro/views/health/_partition_offset.erb +40 -0
- data/lib/karafka/web/ui/pro/views/health/_tabs.erb +27 -0
- data/lib/karafka/web/ui/pro/views/health/offsets.erb +71 -0
- data/lib/karafka/web/ui/pro/views/health/overview.erb +68 -0
- data/lib/karafka/web/ui/pro/views/jobs/_job.erb +6 -3
- data/lib/karafka/web/ui/pro/views/jobs/index.erb +4 -1
- data/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb +37 -0
- data/lib/karafka/web/ui/pro/views/routing/_detail.erb +25 -0
- data/lib/karafka/web/ui/pro/views/routing/_topic.erb +23 -0
- data/lib/karafka/web/ui/pro/views/routing/index.erb +10 -0
- data/lib/karafka/web/ui/pro/views/routing/show.erb +26 -0
- data/lib/karafka/web/ui/pro/views/shared/_navigation.erb +7 -10
- data/lib/karafka/web/ui/public/images/logo-gray.svg +28 -0
- data/lib/karafka/web/ui/public/javascripts/application.js +30 -0
- data/lib/karafka/web/ui/public/javascripts/chart.min.js +14 -0
- data/lib/karafka/web/ui/public/javascripts/charts.js +330 -0
- data/lib/karafka/web/ui/public/javascripts/datepicker.js +6 -0
- data/lib/karafka/web/ui/public/javascripts/live_poll.js +48 -12
- data/lib/karafka/web/ui/public/javascripts/offset_datetime.js +74 -0
- data/lib/karafka/web/ui/public/javascripts/tabs.js +59 -0
- data/lib/karafka/web/ui/public/stylesheets/application.css +11 -0
- data/lib/karafka/web/ui/public/stylesheets/datepicker.min.css +12 -0
- data/lib/karafka/web/ui/views/cluster/_no_partitions.erb +3 -0
- data/lib/karafka/web/ui/views/cluster/_partition.erb +20 -22
- data/lib/karafka/web/ui/views/cluster/index.erb +6 -1
- data/lib/karafka/web/ui/views/consumers/_consumer.erb +1 -1
- data/lib/karafka/web/ui/views/consumers/_counters.erb +6 -4
- data/lib/karafka/web/ui/views/consumers/_summary.erb +3 -3
- data/lib/karafka/web/ui/views/consumers/index.erb +3 -1
- data/lib/karafka/web/ui/views/dashboard/_feature_pro.erb +3 -0
- data/lib/karafka/web/ui/views/dashboard/_not_enough_data.erb +15 -0
- data/lib/karafka/web/ui/views/dashboard/_ranges_selector.erb +23 -0
- data/lib/karafka/web/ui/views/dashboard/index.erb +95 -0
- data/lib/karafka/web/ui/views/errors/_detail.erb +12 -0
- data/lib/karafka/web/ui/views/errors/_error.erb +2 -2
- data/lib/karafka/web/ui/views/errors/show.erb +1 -1
- data/lib/karafka/web/ui/views/jobs/index.erb +3 -1
- data/lib/karafka/web/ui/views/layout.erb +10 -3
- data/lib/karafka/web/ui/views/routing/_consumer_group.erb +8 -6
- data/lib/karafka/web/ui/views/routing/_detail.erb +2 -2
- data/lib/karafka/web/ui/views/routing/_topic.erb +1 -1
- data/lib/karafka/web/ui/views/routing/show.erb +1 -1
- data/lib/karafka/web/ui/views/shared/_brand.erb +2 -2
- data/lib/karafka/web/ui/views/shared/_chart.erb +14 -0
- data/lib/karafka/web/ui/views/shared/_content.erb +2 -2
- data/lib/karafka/web/ui/views/shared/_feature_pro.erb +1 -1
- data/lib/karafka/web/ui/views/shared/_flashes.erb +9 -0
- data/lib/karafka/web/ui/views/shared/_footer.erb +22 -0
- data/lib/karafka/web/ui/views/shared/_header.erb +15 -9
- data/lib/karafka/web/ui/views/shared/_live_poll.erb +7 -0
- data/lib/karafka/web/ui/views/shared/_navigation.erb +5 -8
- data/lib/karafka/web/ui/views/shared/_no_paginated_data.erb +9 -0
- data/lib/karafka/web/ui/views/shared/_pagination.erb +17 -13
- data/lib/karafka/web/ui/views/shared/_tab_nav.erb +7 -0
- data/lib/karafka/web/ui/views/shared/exceptions/not_found.erb +34 -32
- data/lib/karafka/web/ui/views/shared/exceptions/pro_only.erb +45 -43
- data/lib/karafka/web/ui/views/status/failures/_consumers_reports_schema_state.erb +15 -0
- data/lib/karafka/web/ui/views/status/failures/_enabled.erb +8 -0
- data/lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb +11 -0
- data/lib/karafka/web/ui/views/status/failures/{_initial_state.erb → _initial_consumers_state.erb} +3 -3
- data/lib/karafka/web/ui/views/status/failures/_partitions.erb +14 -6
- data/lib/karafka/web/ui/views/status/info/_components.erb +21 -1
- data/lib/karafka/web/ui/views/status/show.erb +62 -5
- data/lib/karafka/web/ui/views/status/successes/_enabled.erb +1 -0
- data/lib/karafka/web/ui/views/status/warnings/_replication.erb +19 -0
- data/lib/karafka/web/version.rb +1 -1
- data/lib/karafka/web.rb +11 -0
- data.tar.gz.sig +0 -0
- metadata +124 -39
- metadata.gz.sig +0 -0
- data/lib/karafka/web/processing/consumers/aggregator.rb +0 -130
- data/lib/karafka/web/tracking/contracts/base.rb +0 -34
- data/lib/karafka/web/ui/lib/paginate_array.rb +0 -38
- data/lib/karafka/web/ui/pro/views/explorer/_encryption_enabled.erb +0 -18
- data/lib/karafka/web/ui/pro/views/explorer/partition/_watermark_offsets.erb +0 -10
- data/lib/karafka/web/ui/pro/views/health/index.erb +0 -60
- /data/lib/karafka/web/ui/pro/views/explorer/{_detail.erb → messages/_detail.erb} +0 -0
@@ -18,18 +18,45 @@ module Karafka
|
|
18
18
|
module Controllers
|
19
19
|
# Errors details controller
|
20
20
|
class Errors < Ui::Controllers::Base
|
21
|
+
include Ui::Lib::Paginations
|
22
|
+
|
23
|
+
# Lists all the errors from all the partitions
|
24
|
+
def index
|
25
|
+
@topic_id = errors_topic
|
26
|
+
@partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
|
27
|
+
|
28
|
+
@active_partitions, materialized_page, @limited = Paginators::Partitions.call(
|
29
|
+
@partitions_count, @params.current_page
|
30
|
+
)
|
31
|
+
|
32
|
+
@error_messages, next_page = Models::Message.topic_page(
|
33
|
+
errors_topic, @active_partitions, materialized_page
|
34
|
+
)
|
35
|
+
|
36
|
+
paginate(@params.current_page, next_page)
|
37
|
+
|
38
|
+
respond
|
39
|
+
end
|
40
|
+
|
21
41
|
# @param partition_id [Integer] id of the partition of errors we are interested in
|
22
|
-
def
|
23
|
-
errors_topic = ::Karafka::Web.config.topics.errors
|
42
|
+
def partition(partition_id)
|
24
43
|
@partition_id = partition_id
|
25
|
-
@previous_page, @error_messages, @next_page, @partitions_count = \
|
26
|
-
Models::Message.page(
|
27
|
-
errors_topic,
|
28
|
-
@partition_id,
|
29
|
-
@params.current_page
|
30
|
-
)
|
31
|
-
|
32
44
|
@watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, @partition_id)
|
45
|
+
@partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
|
46
|
+
|
47
|
+
previous_offset, @error_messages, next_offset = Models::Message.offset_page(
|
48
|
+
errors_topic,
|
49
|
+
@partition_id,
|
50
|
+
@params.current_offset,
|
51
|
+
@watermark_offsets
|
52
|
+
)
|
53
|
+
|
54
|
+
paginate(
|
55
|
+
previous_offset,
|
56
|
+
@params.current_offset,
|
57
|
+
next_offset,
|
58
|
+
@error_messages.map(&:offset)
|
59
|
+
)
|
33
60
|
|
34
61
|
respond
|
35
62
|
end
|
@@ -39,7 +66,6 @@ module Karafka
|
|
39
66
|
# @param partition_id [Integer]
|
40
67
|
# @param offset [Integer]
|
41
68
|
def show(partition_id, offset)
|
42
|
-
errors_topic = ::Karafka::Web.config.topics.errors
|
43
69
|
@partition_id = partition_id
|
44
70
|
@offset = offset
|
45
71
|
@error_message = Models::Message.find(
|
@@ -48,8 +74,18 @@ module Karafka
|
|
48
74
|
@offset
|
49
75
|
)
|
50
76
|
|
77
|
+
watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, partition_id)
|
78
|
+
paginate(offset, watermark_offsets.low, watermark_offsets.high)
|
79
|
+
|
51
80
|
respond
|
52
81
|
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
# @return [String] errors topic
|
86
|
+
def errors_topic
|
87
|
+
::Karafka::Web.config.topics.errors
|
88
|
+
end
|
53
89
|
end
|
54
90
|
end
|
55
91
|
end
|
@@ -18,14 +18,48 @@ module Karafka
|
|
18
18
|
module Controllers
|
19
19
|
# Data explorer controller
|
20
20
|
class Explorer < Ui::Controllers::Base
|
21
|
+
include Ui::Lib::Paginations
|
22
|
+
|
21
23
|
# Lists all the topics we can explore
|
22
24
|
def index
|
23
|
-
@topics =
|
24
|
-
.cluster_info
|
25
|
+
@topics = Models::ClusterInfo
|
25
26
|
.topics
|
26
|
-
.reject { |topic| topic[:topic_name] == '__consumer_offsets' }
|
27
27
|
.sort_by { |topic| topic[:topic_name] }
|
28
28
|
|
29
|
+
unless ::Karafka::Web.config.ui.show_internal_topics
|
30
|
+
@topics.reject! { |topic| topic[:topic_name].start_with?('__') }
|
31
|
+
end
|
32
|
+
|
33
|
+
respond
|
34
|
+
end
|
35
|
+
|
36
|
+
# Displays aggregated messages from (potentially) all partitions of a topic
|
37
|
+
#
|
38
|
+
# @param topic_id [String]
|
39
|
+
#
|
40
|
+
# @note This view may not be 100% accurate because we merge multiple partitions data
|
41
|
+
# into a single view and this is never accurate. It can be used however to quickly
|
42
|
+
# look at most recent data flowing, etc, hence it is still useful for aggregated
|
43
|
+
# metrics information
|
44
|
+
#
|
45
|
+
# @note We cannot use offset references here because each of the partitions may have
|
46
|
+
# completely different values
|
47
|
+
def topic(topic_id)
|
48
|
+
@visibility_filter = ::Karafka::Web.config.ui.visibility_filter
|
49
|
+
|
50
|
+
@topic_id = topic_id
|
51
|
+
@partitions_count = Models::ClusterInfo.partitions_count(topic_id)
|
52
|
+
|
53
|
+
@active_partitions, materialized_page, @limited = Paginators::Partitions.call(
|
54
|
+
@partitions_count, @params.current_page
|
55
|
+
)
|
56
|
+
|
57
|
+
@messages, next_page = Models::Message.topic_page(
|
58
|
+
topic_id, @active_partitions, materialized_page
|
59
|
+
)
|
60
|
+
|
61
|
+
paginate(@params.current_page, next_page)
|
62
|
+
|
29
63
|
respond
|
30
64
|
end
|
31
65
|
|
@@ -34,15 +68,20 @@ module Karafka
|
|
34
68
|
# @param topic_id [String]
|
35
69
|
# @param partition_id [Integer]
|
36
70
|
def partition(topic_id, partition_id)
|
71
|
+
@visibility_filter = ::Karafka::Web.config.ui.visibility_filter
|
37
72
|
@topic_id = topic_id
|
38
73
|
@partition_id = partition_id
|
39
|
-
|
40
74
|
@watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
|
75
|
+
@partitions_count = Models::ClusterInfo.partitions_count(topic_id)
|
41
76
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
77
|
+
previous_offset, @messages, next_offset = current_partition_data
|
78
|
+
|
79
|
+
paginate(
|
80
|
+
previous_offset,
|
81
|
+
@params.current_offset,
|
82
|
+
next_offset,
|
83
|
+
# If message is an array, it means it's a compacted dummy offset representation
|
84
|
+
@messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
|
46
85
|
)
|
47
86
|
|
48
87
|
respond
|
@@ -53,27 +92,118 @@ module Karafka
|
|
53
92
|
# @param topic_id [String]
|
54
93
|
# @param partition_id [Integer]
|
55
94
|
# @param offset [Integer] offset of the message we want to display
|
56
|
-
|
95
|
+
# @param paginate [Boolean] do we want to have pagination
|
96
|
+
def show(topic_id, partition_id, offset, paginate: true)
|
97
|
+
@visibility_filter = ::Karafka::Web.config.ui.visibility_filter
|
57
98
|
@topic_id = topic_id
|
58
99
|
@partition_id = partition_id
|
59
100
|
@offset = offset
|
60
101
|
@message = Ui::Models::Message.find(@topic_id, @partition_id, @offset)
|
61
102
|
@payload_error = false
|
62
103
|
|
63
|
-
@decrypt = if ::Karafka::App.config.encryption.active
|
64
|
-
::Karafka::Web.config.ui.decrypt
|
65
|
-
else
|
66
|
-
true
|
67
|
-
end
|
68
|
-
|
69
104
|
begin
|
70
105
|
@pretty_payload = JSON.pretty_generate(@message.payload)
|
71
106
|
rescue StandardError => e
|
72
107
|
@payload_error = e
|
73
108
|
end
|
74
109
|
|
110
|
+
# This may be off for certain views like recent view where we are interested only
|
111
|
+
# in the most recent all the time. It does not make any sense to display pagination
|
112
|
+
# there
|
113
|
+
if paginate
|
114
|
+
# We need watermark offsets to decide if we can paginate left and right
|
115
|
+
watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
|
116
|
+
paginate(offset, watermark_offsets.low, watermark_offsets.high)
|
117
|
+
end
|
118
|
+
|
75
119
|
respond
|
76
120
|
end
|
121
|
+
|
122
|
+
# Displays the most recent message on a topic/partition
|
123
|
+
#
|
124
|
+
# @param topic_id [String]
|
125
|
+
# @param partition_id [Integer, nil] partition we're interested in or nil if we are
|
126
|
+
# interested in the most recent message from all the partitions
|
127
|
+
def recent(topic_id, partition_id)
|
128
|
+
if partition_id
|
129
|
+
active_partitions = [partition_id]
|
130
|
+
else
|
131
|
+
partitions_count = Models::ClusterInfo.partitions_count(topic_id)
|
132
|
+
active_partitions, = Paginators::Partitions.call(partitions_count, 1)
|
133
|
+
end
|
134
|
+
|
135
|
+
# This selects first page with most recent messages
|
136
|
+
messages, = Models::Message.topic_page(topic_id, active_partitions, 1)
|
137
|
+
|
138
|
+
# Selects newest out of all partitions
|
139
|
+
recent = messages.max_by(&:timestamp)
|
140
|
+
|
141
|
+
recent || raise(::Karafka::Web::Errors::Ui::NotFoundError)
|
142
|
+
|
143
|
+
show(topic_id, recent.partition, recent.offset, paginate: false)
|
144
|
+
end
|
145
|
+
|
146
|
+
# Computes a page on which the given offset is in the middle of the page (if possible)
|
147
|
+
# Useful often when debugging to be able to quickly jump to the historical location
|
148
|
+
# of message and its surrounding to understand failure
|
149
|
+
#
|
150
|
+
# @param topic_id [String]
|
151
|
+
# @param partition_id [Integer]
|
152
|
+
# @param offset [Integer] offset of the message we want to display
|
153
|
+
def surrounding(topic_id, partition_id, offset)
|
154
|
+
watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
|
155
|
+
|
156
|
+
raise ::Karafka::Web::Errors::Ui::NotFoundError if offset < watermark_offsets.low
|
157
|
+
raise ::Karafka::Web::Errors::Ui::NotFoundError if offset >= watermark_offsets.high
|
158
|
+
|
159
|
+
# Assume we start from this offset
|
160
|
+
shift = 0
|
161
|
+
elements = 0
|
162
|
+
|
163
|
+
# Position the offset as close to the middle of offset based page as possible
|
164
|
+
::Karafka::Web.config.ui.per_page.times do
|
165
|
+
break if elements >= ::Karafka::Web.config.ui.per_page
|
166
|
+
|
167
|
+
elements += 1 if offset + shift < watermark_offsets.high
|
168
|
+
|
169
|
+
if offset - shift > watermark_offsets.low
|
170
|
+
shift += 1
|
171
|
+
elements += 1
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
target = offset - shift
|
176
|
+
|
177
|
+
redirect("explorer/#{topic_id}/#{partition_id}?offset=#{target}")
|
178
|
+
end
|
179
|
+
|
180
|
+
# Finds the closest offset matching the requested time and redirects to this location
|
181
|
+
# Note, that it redirects to closest but always younger.
|
182
|
+
#
|
183
|
+
# @param topic_id [String]
|
184
|
+
# @param partition_id [Integer]
|
185
|
+
# @param time [Time] time of the message
|
186
|
+
def closest(topic_id, partition_id, time)
|
187
|
+
target = ::Karafka::Admin.read_topic(topic_id, partition_id, 1, time).first
|
188
|
+
|
189
|
+
partition_path = "explorer/#{topic_id}/#{partition_id}"
|
190
|
+
partition_path += "?offset=#{target.offset}" if target
|
191
|
+
|
192
|
+
redirect(partition_path)
|
193
|
+
end
|
194
|
+
|
195
|
+
private
|
196
|
+
|
197
|
+
# Fetches current page data
|
198
|
+
# @return [Array] fetched data with pagination information for the requested partition
|
199
|
+
def current_partition_data
|
200
|
+
Ui::Models::Message.offset_page(
|
201
|
+
@topic_id,
|
202
|
+
@partition_id,
|
203
|
+
@params.current_offset,
|
204
|
+
@watermark_offsets
|
205
|
+
)
|
206
|
+
end
|
77
207
|
end
|
78
208
|
end
|
79
209
|
end
|
@@ -19,12 +19,20 @@ module Karafka
|
|
19
19
|
# Health state controller
|
20
20
|
class Health < Ui::Controllers::Base
|
21
21
|
# Displays the current system state
|
22
|
-
def
|
23
|
-
current_state = Models::
|
22
|
+
def overview
|
23
|
+
current_state = Models::ConsumersState.current!
|
24
24
|
@stats = Models::Health.current(current_state)
|
25
25
|
|
26
26
|
respond
|
27
27
|
end
|
28
|
+
|
29
|
+
# Displays details about offsets and their progression/statuses
|
30
|
+
def offsets
|
31
|
+
# Same data as overview but presented differently
|
32
|
+
overview
|
33
|
+
|
34
|
+
respond
|
35
|
+
end
|
28
36
|
end
|
29
37
|
end
|
30
38
|
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Web
|
16
|
+
module Ui
|
17
|
+
module Pro
|
18
|
+
module Controllers
|
19
|
+
# Controller for working with messages
|
20
|
+
# While part of messages operations is done via explorer (exploring), this controller
|
21
|
+
# handles other cases not related to viewing data
|
22
|
+
class Messages < Ui::Controllers::Base
|
23
|
+
# Takes a requested message content and republishes it again
|
24
|
+
#
|
25
|
+
# @param topic_id [String]
|
26
|
+
# @param partition_id [Integer]
|
27
|
+
# @param offset [Integer] offset of the message we want to republish
|
28
|
+
def republish(topic_id, partition_id, offset)
|
29
|
+
message = Ui::Models::Message.find(topic_id, partition_id, offset)
|
30
|
+
|
31
|
+
delivery = ::Karafka.producer.produce_sync(
|
32
|
+
topic: topic_id,
|
33
|
+
partition: partition_id,
|
34
|
+
payload: message.raw_payload,
|
35
|
+
headers: message.headers,
|
36
|
+
key: message.key
|
37
|
+
)
|
38
|
+
|
39
|
+
redirect(
|
40
|
+
:back,
|
41
|
+
success: reproduced(message, delivery)
|
42
|
+
)
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
# @param message [Karafka::Messages::Message]
|
48
|
+
# @param delivery [Rdkafka::Producer::DeliveryReport]
|
49
|
+
# @return [String] flash message about message reproducing
|
50
|
+
def reproduced(message, delivery)
|
51
|
+
<<~MSG
|
52
|
+
Message with offset #{message.offset}
|
53
|
+
has been sent again to #{message.topic}##{message.partition}
|
54
|
+
and received offset #{delivery.offset}.
|
55
|
+
MSG
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -18,6 +18,50 @@ module Karafka
|
|
18
18
|
module Controllers
|
19
19
|
# Routing details - same as in OSS
|
20
20
|
class Routing < Ui::Controllers::Routing
|
21
|
+
# Routing list
|
22
|
+
def index
|
23
|
+
detect_patterns_routes
|
24
|
+
|
25
|
+
@routes = Karafka::App.routes
|
26
|
+
|
27
|
+
respond
|
28
|
+
end
|
29
|
+
|
30
|
+
# Given route details
|
31
|
+
#
|
32
|
+
# @param topic_id [String] topic id
|
33
|
+
def show(topic_id)
|
34
|
+
detect_patterns_routes
|
35
|
+
|
36
|
+
@topic = Karafka::Routing::Router.find_by(id: topic_id)
|
37
|
+
|
38
|
+
@topic || raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_id)
|
39
|
+
|
40
|
+
respond
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
# Checks list of topics and tries to match them against the available patterns
|
46
|
+
# Uses the Pro detector to expand routes in the Web-UI so we include topics that are
|
47
|
+
# or will be matched using our regular expressions
|
48
|
+
def detect_patterns_routes
|
49
|
+
detector = ::Karafka::Pro::Routing::Features::Patterns::Detector.new
|
50
|
+
topics_names = Models::ClusterInfo.topics.map(&:topic_name)
|
51
|
+
|
52
|
+
Karafka::App
|
53
|
+
.routes
|
54
|
+
.flat_map(&:subscription_groups)
|
55
|
+
.each do |subscription_group|
|
56
|
+
sg_topics = subscription_group.topics
|
57
|
+
|
58
|
+
# Reject topics that are already part of routing for given subscription groups
|
59
|
+
# and then for remaining try to apply patterns and expand routes
|
60
|
+
topics_names
|
61
|
+
.reject { |t_name| sg_topics.any? { |rtopic| rtopic.name == t_name } }
|
62
|
+
.each { |t_name| detector.expand(sg_topics, t_name) }
|
63
|
+
end
|
64
|
+
end
|
21
65
|
end
|
22
66
|
end
|
23
67
|
end
|
@@ -17,11 +17,17 @@
|
|
17
17
|
Running jobs
|
18
18
|
</a>
|
19
19
|
</li>
|
20
|
-
<%
|
20
|
+
<% elsif current_path.include?('/subscriptions') %>
|
21
21
|
<li class="breadcrumb-item">
|
22
22
|
<a href="<%= root_path('consumers', @process.id, 'subscriptions') %>">
|
23
23
|
Active subscriptions
|
24
24
|
</a>
|
25
25
|
</li>
|
26
|
+
<% else %>
|
27
|
+
<li class="breadcrumb-item">
|
28
|
+
<a href="<%= root_path('consumers', @process.id, 'details') %>">
|
29
|
+
Details
|
30
|
+
</a>
|
31
|
+
</li>
|
26
32
|
<% end %>
|
27
33
|
<% end %>
|
@@ -14,7 +14,11 @@
|
|
14
14
|
<% subscription_group.topics.each do |topic| %>
|
15
15
|
<span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
|
16
16
|
<%= topic.name %>:
|
17
|
-
|
17
|
+
<% if topic.partitions.size > 10 %>
|
18
|
+
<%= "#{topic.partitions.first(10).map(&:id).join(',')}..." %>
|
19
|
+
<% else %>
|
20
|
+
<%= topic.partitions.map(&:id).join(',') %>
|
21
|
+
<% end %>
|
18
22
|
</span>
|
19
23
|
<% end %>
|
20
24
|
<% end %>
|
@@ -47,7 +51,7 @@
|
|
47
51
|
<%= process.utilization.round(1) %>%
|
48
52
|
</span>
|
49
53
|
<span class="badge bg-primary badge-topic">
|
50
|
-
<%= process.
|
54
|
+
<%= process.workers %> /
|
51
55
|
<%= process.busy %>
|
52
56
|
</span>
|
53
57
|
</td>
|
@@ -14,19 +14,21 @@
|
|
14
14
|
<div class="desc">Batches</div>
|
15
15
|
</li>
|
16
16
|
<li class="col-sm">
|
17
|
-
<div class="count mb-1"><%= @
|
18
|
-
<div class="desc">Lag</div>
|
17
|
+
<div class="count mb-1"><%= @counters.lag_stored %></div>
|
18
|
+
<div class="desc">Lag stored</div>
|
19
19
|
</li>
|
20
20
|
<li class="col-sm">
|
21
|
-
<
|
22
|
-
|
21
|
+
<a href="<%= root_path('jobs') %>">
|
22
|
+
<div class="count mb-1"><%= @counters.busy %></div>
|
23
|
+
<div class="desc">Busy</div>
|
24
|
+
</a>
|
23
25
|
</li>
|
24
26
|
<li class="col-sm">
|
25
27
|
<div class="count mb-1"><%= @counters.enqueued %></div>
|
26
28
|
<div class="desc">Enqueued</div>
|
27
29
|
</li>
|
28
30
|
<li class="col-sm">
|
29
|
-
<a href="<%= root_path('errors
|
31
|
+
<a href="<%= root_path('errors') %>">
|
30
32
|
<div class="count mb-1"><%= @counters.errors %></div>
|
31
33
|
<div class="desc">Errors</div>
|
32
34
|
</a>
|
@@ -17,13 +17,13 @@
|
|
17
17
|
<code>#<%= job.type %></code>
|
18
18
|
</td>
|
19
19
|
<td>
|
20
|
-
<%== offset_with_label job.first_offset %>
|
20
|
+
<%== offset_with_label job.topic, job.partition, job.first_offset, explore: true %>
|
21
21
|
</td>
|
22
22
|
<td>
|
23
|
-
<%== offset_with_label job.last_offset %>
|
23
|
+
<%== offset_with_label job.topic, job.partition, job.last_offset, explore: true %>
|
24
24
|
</td>
|
25
25
|
<td>
|
26
|
-
<%== offset_with_label job.committed_offset %>
|
26
|
+
<%== offset_with_label job.topic, job.partition, job.committed_offset, explore: true %>
|
27
27
|
</td>
|
28
28
|
<td>
|
29
29
|
<%== relative_time job.started_at %>
|
@@ -19,6 +19,7 @@
|
|
19
19
|
<%== relative_time @process.started_at %>
|
20
20
|
</span>
|
21
21
|
</li>
|
22
|
+
|
22
23
|
<li class="align-items-center d-flex justify-content-between">
|
23
24
|
State from:
|
24
25
|
<span class="badge bg-secondary">
|
@@ -41,13 +42,13 @@
|
|
41
42
|
<p class="card-text">
|
42
43
|
<ul style="list-style: square !important;">
|
43
44
|
<li class="align-items-center d-flex justify-content-between">
|
44
|
-
|
45
|
+
Workers:
|
45
46
|
<span class="badge bg-primary">
|
46
|
-
<%= @process.
|
47
|
+
<%= @process.workers %>
|
47
48
|
</span>
|
48
49
|
</li>
|
49
50
|
<li class="align-items-center d-flex justify-content-between">
|
50
|
-
|
51
|
+
Utilization:
|
51
52
|
<span class="badge bg-primary">
|
52
53
|
<%= @process.utilization.round(2) %>%
|
53
54
|
</span>
|
@@ -55,7 +56,7 @@
|
|
55
56
|
<li class="align-items-center d-flex justify-content-between">
|
56
57
|
CPUs:
|
57
58
|
<span class="badge bg-primary">
|
58
|
-
<%= @process.
|
59
|
+
<%= @process.cpus %>
|
59
60
|
</span>
|
60
61
|
</li>
|
61
62
|
<li class="align-items-center d-flex justify-content-between">
|
@@ -1,4 +1,4 @@
|
|
1
|
-
<tr>
|
1
|
+
<tr class="align-middle <%= lso_risk_state_bg(partition) %>">
|
2
2
|
<td>
|
3
3
|
<%= topic.name %>
|
4
4
|
</td>
|
@@ -6,7 +6,7 @@
|
|
6
6
|
<%= partition.id %>
|
7
7
|
</td>
|
8
8
|
<td>
|
9
|
-
<%==
|
9
|
+
<%== lag_with_label partition.lag_stored %>
|
10
10
|
</td>
|
11
11
|
<td>
|
12
12
|
<span class="badge <%= lag_trend_bg(partition.lag_stored_d) %>">
|
@@ -14,10 +14,14 @@
|
|
14
14
|
</span>
|
15
15
|
</td>
|
16
16
|
<td>
|
17
|
-
|
17
|
+
<% if partition.stored_offset.negative? %>
|
18
|
+
<%== offset_with_label topic.name, partition.id, partition.committed_offset - 1 %>
|
19
|
+
<% else %>
|
20
|
+
<%== offset_with_label topic.name, partition.id, partition.committed_offset %>
|
21
|
+
<% end %>
|
18
22
|
</td>
|
19
23
|
<td>
|
20
|
-
<%== offset_with_label partition.stored_offset
|
24
|
+
<%== offset_with_label topic.name, partition.id, partition.stored_offset %>
|
21
25
|
</td>
|
22
26
|
<td>
|
23
27
|
<span class="badge <%= kafka_state_bg(partition.fetch_state) %> mt-1 mb-1">
|
@@ -29,4 +33,9 @@
|
|
29
33
|
<%= partition.poll_state %>
|
30
34
|
</span>
|
31
35
|
</td>
|
36
|
+
<td>
|
37
|
+
<span class="badge bg-success <%= lso_risk_state_bg(partition) %> bg-opacity-100">
|
38
|
+
<%= partition.lso_risk_state %>
|
39
|
+
</span>
|
40
|
+
</td>
|
32
41
|
</tr>
|
@@ -65,7 +65,7 @@
|
|
65
65
|
</div>
|
66
66
|
|
67
67
|
<% if subscription_group.topics.empty? %>
|
68
|
-
<div class="row">
|
68
|
+
<div class="row mb-4">
|
69
69
|
<div class="col-lg-12">
|
70
70
|
<div class="alert alert-info" role="alert">
|
71
71
|
This process does not consume any messages from any topics of this consumer group.
|
@@ -81,11 +81,12 @@
|
|
81
81
|
<th>Topic</th>
|
82
82
|
<th>Partition</th>
|
83
83
|
<th>Lag stored</th>
|
84
|
-
<th>Lag trend</th>
|
84
|
+
<th>Lag stored trend</th>
|
85
85
|
<th>Committed offset</th>
|
86
86
|
<th>Stored offset</th>
|
87
87
|
<th>Fetch state</th>
|
88
88
|
<th>Poll state</th>
|
89
|
+
<th>LSO state</th>
|
89
90
|
</tr>
|
90
91
|
</thead>
|
91
92
|
<tbody>
|
@@ -9,12 +9,19 @@
|
|
9
9
|
(<%= @process.subscribed_partitions_count %>)
|
10
10
|
</a>
|
11
11
|
</li>
|
12
|
+
|
12
13
|
<li class="nav-item">
|
13
14
|
<a class="nav-link <%= nav_class(include: 'jobs') %>" href="<%= root_path('consumers', @process.id, 'jobs') %>">
|
14
15
|
Running jobs
|
15
16
|
(<%= @process.jobs.count %>)
|
16
17
|
</a>
|
17
18
|
</li>
|
19
|
+
|
20
|
+
<li class="nav-item">
|
21
|
+
<a class="nav-link <%= nav_class(include: 'details') %>" href="<%= root_path('consumers', @process.id, 'details') %>">
|
22
|
+
Details
|
23
|
+
</a>
|
24
|
+
</li>
|
18
25
|
</ul>
|
19
26
|
|
20
27
|
</div>
|
@@ -0,0 +1,21 @@
|
|
1
|
+
<%== view_title(@process.name) %>
|
2
|
+
|
3
|
+
<% if @process.status == 'stopped' %>
|
4
|
+
<%== partial 'consumers/consumer/stopped' %>
|
5
|
+
<% end %>
|
6
|
+
|
7
|
+
<%== partial 'consumers/consumer/metrics' %>
|
8
|
+
|
9
|
+
<%== partial 'consumers/consumer/tabs' %>
|
10
|
+
|
11
|
+
<div class="container">
|
12
|
+
<div class="row">
|
13
|
+
<div class="col-sm-12">
|
14
|
+
<div class="card">
|
15
|
+
<div class="card-body">
|
16
|
+
<pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= JSON.pretty_generate(@process.to_h) %></code></pre>
|
17
|
+
</div>
|
18
|
+
</div>
|
19
|
+
</div>
|
20
|
+
</div>
|
21
|
+
</div>
|
@@ -4,8 +4,10 @@
|
|
4
4
|
<div class="container">
|
5
5
|
<div class="row">
|
6
6
|
<div class="col-sm-12">
|
7
|
-
<% if @processes.empty? %>
|
7
|
+
<% if @processes.empty? && params.current_page <= 1 %>
|
8
8
|
<%== partial 'consumers/no_consumers' %>
|
9
|
+
<% elsif @processes.empty? %>
|
10
|
+
<%== partial 'shared/no_paginated_data' %>
|
9
11
|
<% else %>
|
10
12
|
<table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
|
11
13
|
<thead>
|
@@ -15,7 +17,7 @@
|
|
15
17
|
<th class="col-sm-1">Memory</th>
|
16
18
|
<th class="col-sm-1">Performance</th>
|
17
19
|
<th class="col-sm-1">Load</th>
|
18
|
-
<th class="col-sm-1">
|
20
|
+
<th class="col-sm-1">Lag stored</th>
|
19
21
|
</tr>
|
20
22
|
</thead>
|
21
23
|
<tbody>
|