karafka-web 0.6.3 → 0.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +13 -4
- data/CHANGELOG.md +126 -5
- data/Gemfile +1 -0
- data/Gemfile.lock +27 -24
- data/README.md +2 -0
- data/bin/rspecs +6 -0
- data/certs/cert_chain.pem +21 -21
- data/docker-compose.yml +22 -0
- data/karafka-web.gemspec +3 -3
- data/lib/karafka/web/app.rb +6 -2
- data/lib/karafka/web/cli.rb +51 -47
- data/lib/karafka/web/config.rb +33 -9
- data/lib/karafka/web/contracts/base.rb +32 -0
- data/lib/karafka/web/contracts/config.rb +63 -0
- data/lib/karafka/web/deserializer.rb +10 -1
- data/lib/karafka/web/errors.rb +29 -7
- data/lib/karafka/web/installer.rb +58 -148
- data/lib/karafka/web/management/base.rb +34 -0
- data/lib/karafka/web/management/clean_boot_file.rb +31 -0
- data/lib/karafka/web/management/create_initial_states.rb +101 -0
- data/lib/karafka/web/management/create_topics.rb +127 -0
- data/lib/karafka/web/management/delete_topics.rb +28 -0
- data/lib/karafka/web/management/enable.rb +82 -0
- data/lib/karafka/web/management/extend_boot_file.rb +37 -0
- data/lib/karafka/web/processing/consumer.rb +73 -17
- data/lib/karafka/web/processing/consumers/aggregators/base.rb +56 -0
- data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +154 -0
- data/lib/karafka/web/processing/consumers/aggregators/state.rb +180 -0
- data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb +32 -0
- data/lib/karafka/web/processing/consumers/contracts/metrics.rb +53 -0
- data/lib/karafka/web/processing/consumers/contracts/process.rb +19 -0
- data/lib/karafka/web/processing/consumers/contracts/state.rb +49 -0
- data/lib/karafka/web/processing/consumers/contracts/topic_stats.rb +21 -0
- data/lib/karafka/web/processing/consumers/metrics.rb +29 -0
- data/lib/karafka/web/processing/consumers/schema_manager.rb +56 -0
- data/lib/karafka/web/processing/consumers/state.rb +6 -9
- data/lib/karafka/web/processing/time_series_tracker.rb +130 -0
- data/lib/karafka/web/tracking/consumers/contracts/consumer_group.rb +2 -2
- data/lib/karafka/web/tracking/consumers/contracts/job.rb +2 -1
- data/lib/karafka/web/tracking/consumers/contracts/partition.rb +14 -1
- data/lib/karafka/web/tracking/consumers/contracts/report.rb +10 -8
- data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb +2 -2
- data/lib/karafka/web/tracking/consumers/contracts/topic.rb +2 -2
- data/lib/karafka/web/tracking/consumers/listeners/processing.rb +6 -2
- data/lib/karafka/web/tracking/consumers/listeners/statistics.rb +15 -1
- data/lib/karafka/web/tracking/consumers/reporter.rb +14 -6
- data/lib/karafka/web/tracking/consumers/sampler.rb +80 -39
- data/lib/karafka/web/tracking/contracts/error.rb +2 -1
- data/lib/karafka/web/ui/app.rb +20 -10
- data/lib/karafka/web/ui/base.rb +56 -6
- data/lib/karafka/web/ui/controllers/base.rb +28 -0
- data/lib/karafka/web/ui/controllers/become_pro.rb +1 -1
- data/lib/karafka/web/ui/controllers/cluster.rb +12 -6
- data/lib/karafka/web/ui/controllers/consumers.rb +4 -2
- data/lib/karafka/web/ui/controllers/dashboard.rb +32 -0
- data/lib/karafka/web/ui/controllers/errors.rb +19 -6
- data/lib/karafka/web/ui/controllers/jobs.rb +4 -2
- data/lib/karafka/web/ui/controllers/requests/params.rb +28 -0
- data/lib/karafka/web/ui/controllers/responses/redirect.rb +29 -0
- data/lib/karafka/web/ui/helpers/application_helper.rb +57 -14
- data/lib/karafka/web/ui/helpers/paths_helper.rb +48 -0
- data/lib/karafka/web/ui/lib/hash_proxy.rb +18 -6
- data/lib/karafka/web/ui/lib/paginations/base.rb +61 -0
- data/lib/karafka/web/ui/lib/paginations/offset_based.rb +96 -0
- data/lib/karafka/web/ui/lib/paginations/page_based.rb +70 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/arrays.rb +33 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/base.rb +23 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/partitions.rb +52 -0
- data/lib/karafka/web/ui/lib/paginations/paginators/sets.rb +85 -0
- data/lib/karafka/web/ui/lib/paginations/watermark_offsets_based.rb +75 -0
- data/lib/karafka/web/ui/lib/ttl_cache.rb +82 -0
- data/lib/karafka/web/ui/models/cluster_info.rb +59 -0
- data/lib/karafka/web/ui/models/consumers_metrics.rb +46 -0
- data/lib/karafka/web/ui/models/{state.rb → consumers_state.rb} +6 -2
- data/lib/karafka/web/ui/models/health.rb +37 -7
- data/lib/karafka/web/ui/models/message.rb +123 -39
- data/lib/karafka/web/ui/models/metrics/aggregated.rb +196 -0
- data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb +50 -0
- data/lib/karafka/web/ui/models/metrics/charts/topics.rb +109 -0
- data/lib/karafka/web/ui/models/metrics/topics.rb +101 -0
- data/lib/karafka/web/ui/models/partition.rb +27 -0
- data/lib/karafka/web/ui/models/process.rb +12 -1
- data/lib/karafka/web/ui/models/status.rb +110 -22
- data/lib/karafka/web/ui/models/visibility_filter.rb +33 -0
- data/lib/karafka/web/ui/pro/app.rb +87 -19
- data/lib/karafka/web/ui/pro/controllers/cluster.rb +11 -0
- data/lib/karafka/web/ui/pro/controllers/consumers.rb +13 -7
- data/lib/karafka/web/ui/pro/controllers/dashboard.rb +54 -0
- data/lib/karafka/web/ui/pro/controllers/dlq.rb +1 -2
- data/lib/karafka/web/ui/pro/controllers/errors.rb +46 -10
- data/lib/karafka/web/ui/pro/controllers/explorer.rb +145 -15
- data/lib/karafka/web/ui/pro/controllers/health.rb +10 -2
- data/lib/karafka/web/ui/pro/controllers/messages.rb +62 -0
- data/lib/karafka/web/ui/pro/controllers/routing.rb +44 -0
- data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb +7 -1
- data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb +6 -2
- data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +7 -5
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb +3 -3
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb +5 -4
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +13 -4
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +3 -2
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb +7 -0
- data/lib/karafka/web/ui/pro/views/consumers/details.erb +21 -0
- data/lib/karafka/web/ui/pro/views/consumers/index.erb +4 -2
- data/lib/karafka/web/ui/pro/views/dashboard/_ranges_selector.erb +39 -0
- data/lib/karafka/web/ui/pro/views/dashboard/index.erb +82 -0
- data/lib/karafka/web/ui/pro/views/dlq/_topic.erb +1 -1
- data/lib/karafka/web/ui/pro/views/errors/_breadcrumbs.erb +8 -6
- data/lib/karafka/web/ui/pro/views/errors/_error.erb +2 -2
- data/lib/karafka/web/ui/pro/views/errors/_partition_option.erb +1 -1
- data/lib/karafka/web/ui/pro/views/errors/_table.erb +21 -0
- data/lib/karafka/web/ui/pro/views/errors/_title_with_select.erb +31 -0
- data/lib/karafka/web/ui/pro/views/errors/index.erb +9 -56
- data/lib/karafka/web/ui/pro/views/errors/partition.erb +17 -0
- data/lib/karafka/web/ui/pro/views/errors/show.erb +1 -1
- data/lib/karafka/web/ui/pro/views/explorer/_breadcrumbs.erb +6 -4
- data/lib/karafka/web/ui/pro/views/explorer/_filtered.erb +16 -0
- data/lib/karafka/web/ui/pro/views/explorer/_message.erb +14 -4
- data/lib/karafka/web/ui/pro/views/explorer/_no_topics.erb +7 -0
- data/lib/karafka/web/ui/pro/views/explorer/_partition_option.erb +3 -3
- data/lib/karafka/web/ui/pro/views/explorer/_topic.erb +1 -1
- data/lib/karafka/web/ui/pro/views/explorer/index.erb +12 -8
- data/lib/karafka/web/ui/pro/views/explorer/messages/_headers.erb +15 -0
- data/lib/karafka/web/ui/pro/views/explorer/messages/_key.erb +12 -0
- data/lib/karafka/web/ui/pro/views/explorer/partition/_details.erb +35 -0
- data/lib/karafka/web/ui/pro/views/explorer/partition/_messages.erb +1 -0
- data/lib/karafka/web/ui/pro/views/explorer/partition.erb +6 -4
- data/lib/karafka/web/ui/pro/views/explorer/show.erb +48 -5
- data/lib/karafka/web/ui/pro/views/explorer/topic/_details.erb +23 -0
- data/lib/karafka/web/ui/pro/views/explorer/topic/_empty.erb +3 -0
- data/lib/karafka/web/ui/pro/views/explorer/topic/_limited.erb +4 -0
- data/lib/karafka/web/ui/pro/views/explorer/topic.erb +51 -0
- data/lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb +16 -0
- data/lib/karafka/web/ui/pro/views/health/_no_data.erb +9 -0
- data/lib/karafka/web/ui/pro/views/health/_partition.erb +17 -15
- data/lib/karafka/web/ui/pro/views/health/_partition_offset.erb +40 -0
- data/lib/karafka/web/ui/pro/views/health/_tabs.erb +27 -0
- data/lib/karafka/web/ui/pro/views/health/offsets.erb +71 -0
- data/lib/karafka/web/ui/pro/views/health/overview.erb +68 -0
- data/lib/karafka/web/ui/pro/views/jobs/_job.erb +6 -3
- data/lib/karafka/web/ui/pro/views/jobs/index.erb +4 -1
- data/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb +37 -0
- data/lib/karafka/web/ui/pro/views/routing/_detail.erb +25 -0
- data/lib/karafka/web/ui/pro/views/routing/_topic.erb +23 -0
- data/lib/karafka/web/ui/pro/views/routing/index.erb +10 -0
- data/lib/karafka/web/ui/pro/views/routing/show.erb +26 -0
- data/lib/karafka/web/ui/pro/views/shared/_navigation.erb +7 -10
- data/lib/karafka/web/ui/public/images/logo-gray.svg +28 -0
- data/lib/karafka/web/ui/public/javascripts/application.js +30 -0
- data/lib/karafka/web/ui/public/javascripts/chart.min.js +14 -0
- data/lib/karafka/web/ui/public/javascripts/charts.js +330 -0
- data/lib/karafka/web/ui/public/javascripts/datepicker.js +6 -0
- data/lib/karafka/web/ui/public/javascripts/live_poll.js +48 -12
- data/lib/karafka/web/ui/public/javascripts/offset_datetime.js +74 -0
- data/lib/karafka/web/ui/public/javascripts/tabs.js +59 -0
- data/lib/karafka/web/ui/public/stylesheets/application.css +11 -0
- data/lib/karafka/web/ui/public/stylesheets/datepicker.min.css +12 -0
- data/lib/karafka/web/ui/views/cluster/_no_partitions.erb +3 -0
- data/lib/karafka/web/ui/views/cluster/_partition.erb +20 -22
- data/lib/karafka/web/ui/views/cluster/index.erb +6 -1
- data/lib/karafka/web/ui/views/consumers/_consumer.erb +1 -1
- data/lib/karafka/web/ui/views/consumers/_counters.erb +6 -4
- data/lib/karafka/web/ui/views/consumers/_summary.erb +3 -3
- data/lib/karafka/web/ui/views/consumers/index.erb +3 -1
- data/lib/karafka/web/ui/views/dashboard/_feature_pro.erb +3 -0
- data/lib/karafka/web/ui/views/dashboard/_not_enough_data.erb +15 -0
- data/lib/karafka/web/ui/views/dashboard/_ranges_selector.erb +23 -0
- data/lib/karafka/web/ui/views/dashboard/index.erb +95 -0
- data/lib/karafka/web/ui/views/errors/_detail.erb +12 -0
- data/lib/karafka/web/ui/views/errors/_error.erb +2 -2
- data/lib/karafka/web/ui/views/errors/show.erb +1 -1
- data/lib/karafka/web/ui/views/jobs/index.erb +3 -1
- data/lib/karafka/web/ui/views/layout.erb +10 -3
- data/lib/karafka/web/ui/views/routing/_consumer_group.erb +8 -6
- data/lib/karafka/web/ui/views/routing/_detail.erb +2 -2
- data/lib/karafka/web/ui/views/routing/_topic.erb +1 -1
- data/lib/karafka/web/ui/views/routing/show.erb +1 -1
- data/lib/karafka/web/ui/views/shared/_brand.erb +2 -2
- data/lib/karafka/web/ui/views/shared/_chart.erb +14 -0
- data/lib/karafka/web/ui/views/shared/_content.erb +2 -2
- data/lib/karafka/web/ui/views/shared/_feature_pro.erb +1 -1
- data/lib/karafka/web/ui/views/shared/_flashes.erb +9 -0
- data/lib/karafka/web/ui/views/shared/_footer.erb +22 -0
- data/lib/karafka/web/ui/views/shared/_header.erb +15 -9
- data/lib/karafka/web/ui/views/shared/_live_poll.erb +7 -0
- data/lib/karafka/web/ui/views/shared/_navigation.erb +5 -8
- data/lib/karafka/web/ui/views/shared/_no_paginated_data.erb +9 -0
- data/lib/karafka/web/ui/views/shared/_pagination.erb +17 -13
- data/lib/karafka/web/ui/views/shared/_tab_nav.erb +7 -0
- data/lib/karafka/web/ui/views/shared/exceptions/not_found.erb +34 -32
- data/lib/karafka/web/ui/views/shared/exceptions/pro_only.erb +45 -43
- data/lib/karafka/web/ui/views/status/failures/_consumers_reports_schema_state.erb +15 -0
- data/lib/karafka/web/ui/views/status/failures/_enabled.erb +8 -0
- data/lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb +11 -0
- data/lib/karafka/web/ui/views/status/failures/{_initial_state.erb → _initial_consumers_state.erb} +3 -3
- data/lib/karafka/web/ui/views/status/failures/_partitions.erb +14 -6
- data/lib/karafka/web/ui/views/status/info/_components.erb +21 -1
- data/lib/karafka/web/ui/views/status/show.erb +62 -5
- data/lib/karafka/web/ui/views/status/successes/_enabled.erb +1 -0
- data/lib/karafka/web/ui/views/status/warnings/_replication.erb +19 -0
- data/lib/karafka/web/version.rb +1 -1
- data/lib/karafka/web.rb +11 -0
- data.tar.gz.sig +0 -0
- metadata +124 -39
- metadata.gz.sig +0 -0
- data/lib/karafka/web/processing/consumers/aggregator.rb +0 -130
- data/lib/karafka/web/tracking/contracts/base.rb +0 -34
- data/lib/karafka/web/ui/lib/paginate_array.rb +0 -38
- data/lib/karafka/web/ui/pro/views/explorer/_encryption_enabled.erb +0 -18
- data/lib/karafka/web/ui/pro/views/explorer/partition/_watermark_offsets.erb +0 -10
- data/lib/karafka/web/ui/pro/views/health/index.erb +0 -60
- /data/lib/karafka/web/ui/pro/views/explorer/{_detail.erb → messages/_detail.erb} +0 -0
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Lib
|
7
|
+
module Paginations
|
8
|
+
# Namespace for commands that build paginated resources based on the provided page
|
9
|
+
module Paginators
|
10
|
+
# A simple wrapper for paginating array related data structures
|
11
|
+
# We call this with plural (same with `Sets`) to avoid confusion with Ruby classes
|
12
|
+
class Arrays < Base
|
13
|
+
class << self
|
14
|
+
# @param array [Array] array we want to paginate
|
15
|
+
# @param current_page [Integer] page we want to be on
|
16
|
+
# @return [Array<Array, Boolean>] Array with two elements: first is the array with
|
17
|
+
# data of the given page and second is a boolean flag with info if the elements we got
|
18
|
+
# are from the last page
|
19
|
+
def call(array, current_page)
|
20
|
+
slices = array.each_slice(per_page).to_a
|
21
|
+
current_data = slices[current_page - 1] || []
|
22
|
+
last_page = !(slices.count >= current_page - 1 && current_data.size >= per_page)
|
23
|
+
|
24
|
+
[current_data, last_page]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Lib
|
7
|
+
module Paginations
|
8
|
+
module Paginators
|
9
|
+
# Base paginator
|
10
|
+
class Base
|
11
|
+
class << self
|
12
|
+
# @return [Integer] number of elements per page
|
13
|
+
def per_page
|
14
|
+
::Karafka::Web.config.ui.per_page
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Lib
|
7
|
+
module Paginations
|
8
|
+
module Paginators
|
9
|
+
# Paginator for selecting proper range of partitions for each page
|
10
|
+
# For topics with a lot of partitions we cannot get all the data efficiently, that
|
11
|
+
# is why we limit number of partitions per page and reduce the operations
|
12
|
+
# that way. This allows us to effectively display more while not having to fetch
|
13
|
+
# more partitions then the number of messages per page.
|
14
|
+
# In cases like this we distribute partitions evenly part of partitions on each of
|
15
|
+
# the pages. This may become unreliable for partitions that are not evenly
|
16
|
+
# distributed but this allows us to display data for as many partitions as we want
|
17
|
+
# without overloading the system
|
18
|
+
class Partitions < Base
|
19
|
+
class << self
|
20
|
+
# Computers the partitions slice, materialized page and the limitations status
|
21
|
+
# for a given page
|
22
|
+
# @param partitions_count [Integer] number of partitions for a given topic
|
23
|
+
# @param current_page [Integer] current page
|
24
|
+
# @return [Array<Array<Integer>, Integer, Boolean>] list of partitions that should
|
25
|
+
# be active on a given page, materialized page for them and info if we had to
|
26
|
+
# limit the partitions number on a given page
|
27
|
+
def call(partitions_count, current_page)
|
28
|
+
# How many "chunks" of partitions we will have
|
29
|
+
slices_count = (partitions_count / per_page.to_f).ceil
|
30
|
+
# How many partitions in a single slice should we have
|
31
|
+
in_slice = (partitions_count / slices_count.to_f).ceil
|
32
|
+
# Which "chunked" page do we want to get
|
33
|
+
materialized_page = (current_page / slices_count.to_f).ceil
|
34
|
+
# Which slice is the one we are operating on
|
35
|
+
active_slice_index = (current_page - 1) % slices_count
|
36
|
+
# All available slices so we can pick one that is active
|
37
|
+
partitions_slices = (0...partitions_count).each_slice(in_slice).to_a
|
38
|
+
# Select active partitions only
|
39
|
+
active_partitions = partitions_slices[active_slice_index]
|
40
|
+
# Are we limiting ourselves because of partition count
|
41
|
+
limited = slices_count > 1
|
42
|
+
|
43
|
+
[active_partitions, materialized_page, limited]
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Lib
|
7
|
+
module Paginations
|
8
|
+
module Paginators
|
9
|
+
# Paginator that allows us to take several lists/sets and iterate over them in a
|
10
|
+
# round-robin fashion.
|
11
|
+
#
|
12
|
+
# It does not have to iterate over all the elements from each set for higher pages
|
13
|
+
# making it much more effective than the naive implementation.
|
14
|
+
class Sets < Base
|
15
|
+
class << self
|
16
|
+
# @param counts [Array<Integer>] sets elements counts
|
17
|
+
# @param current_page [Integer] page number
|
18
|
+
# @return [Hash<Integer, Range>] hash with integer keys indicating the count
|
19
|
+
# location and the range needed to be taken of elements (counting backwards) for
|
20
|
+
# each partition
|
21
|
+
def call(counts, current_page)
|
22
|
+
return {} if current_page < 1
|
23
|
+
|
24
|
+
lists = counts.dup.map.with_index { |el, i| [i, el] }
|
25
|
+
|
26
|
+
curr_item_index = 0
|
27
|
+
curr_list_index = 0
|
28
|
+
items_to_skip_count = per_page * (current_page - 1)
|
29
|
+
|
30
|
+
loop do
|
31
|
+
lists_count = lists.length
|
32
|
+
return {} if lists_count.zero?
|
33
|
+
|
34
|
+
shortest_list_count = lists.map(&:last).min
|
35
|
+
mover = (shortest_list_count - curr_item_index)
|
36
|
+
items_we_are_considering_count = lists_count * mover
|
37
|
+
|
38
|
+
if items_we_are_considering_count >= items_to_skip_count
|
39
|
+
curr_item_index += items_to_skip_count / lists_count
|
40
|
+
curr_list_index = items_to_skip_count % lists_count
|
41
|
+
break
|
42
|
+
else
|
43
|
+
curr_item_index = shortest_list_count
|
44
|
+
lists.delete_if { |x| x.last == shortest_list_count }
|
45
|
+
items_to_skip_count -= items_we_are_considering_count
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
page_items = []
|
50
|
+
largest_list_count = lists.map(&:last).max
|
51
|
+
|
52
|
+
while page_items.length < per_page && curr_item_index < largest_list_count
|
53
|
+
curr_list = lists[curr_list_index]
|
54
|
+
|
55
|
+
if curr_item_index < curr_list.last
|
56
|
+
page_items << [curr_list.first, curr_item_index]
|
57
|
+
end
|
58
|
+
|
59
|
+
curr_list_index += 1
|
60
|
+
if curr_list_index == lists.length
|
61
|
+
curr_list_index = 0
|
62
|
+
curr_item_index += 1
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
hashed = Hash.new { |h, k| h[k] = [] }
|
67
|
+
|
68
|
+
page_items.each do |el|
|
69
|
+
hashed[el.first] << el.last
|
70
|
+
end
|
71
|
+
|
72
|
+
hashed.each do |key, value|
|
73
|
+
hashed[key] = (value.first..value.last)
|
74
|
+
end
|
75
|
+
|
76
|
+
hashed
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Lib
|
7
|
+
module Paginations
|
8
|
+
# Watermark offsets single message pagination engine
|
9
|
+
#
|
10
|
+
# It is used to provide pagination for single message displays (explorer, errors)
|
11
|
+
class WatermarkOffsetsBased < Base
|
12
|
+
# @param current_offset [Integer] current message offset
|
13
|
+
# @param low_watermark_offset [Integer]
|
14
|
+
# @param high_watermark_offset [Integer]
|
15
|
+
def initialize(
|
16
|
+
current_offset,
|
17
|
+
low_watermark_offset,
|
18
|
+
high_watermark_offset
|
19
|
+
)
|
20
|
+
@low_watermark_offset = low_watermark_offset
|
21
|
+
@high_watermark_offset = high_watermark_offset
|
22
|
+
@previous_offset = current_offset + 1
|
23
|
+
@current_offset = current_offset
|
24
|
+
@next_offset = current_offset - 1
|
25
|
+
super()
|
26
|
+
end
|
27
|
+
|
28
|
+
# @return [Boolean] show pagination only when there are other things to present
|
29
|
+
def paginate?
|
30
|
+
return true if @current_offset > @low_watermark_offset
|
31
|
+
return true if @current_offset < @high_watermark_offset - 1
|
32
|
+
|
33
|
+
false
|
34
|
+
end
|
35
|
+
|
36
|
+
# @return [Boolean] provide link to the first (newest)
|
37
|
+
def first_offset?
|
38
|
+
@current_offset < @high_watermark_offset - 1
|
39
|
+
end
|
40
|
+
|
41
|
+
# @return [Integer] highest available offset
|
42
|
+
def first_offset
|
43
|
+
@high_watermark_offset - 1
|
44
|
+
end
|
45
|
+
|
46
|
+
# @return [Boolean]
|
47
|
+
def previous_offset?
|
48
|
+
@current_offset < @high_watermark_offset - 1
|
49
|
+
end
|
50
|
+
|
51
|
+
# @return [Boolean] We always show current offset
|
52
|
+
def current_offset?
|
53
|
+
true
|
54
|
+
end
|
55
|
+
|
56
|
+
# @return [String] shows as current page pagination the offset
|
57
|
+
def current_label
|
58
|
+
@current_offset.to_s
|
59
|
+
end
|
60
|
+
|
61
|
+
# @return [Boolean] if not lowest, show
|
62
|
+
def next_offset?
|
63
|
+
@current_offset > @low_watermark_offset
|
64
|
+
end
|
65
|
+
|
66
|
+
# @return [String] params offset key
|
67
|
+
def offset_key
|
68
|
+
'offset'
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
# Non info related extra components used in the UI
|
7
|
+
module Lib
|
8
|
+
# Ttl Cache for caching things in-memory
|
9
|
+
# @note It **is** thread-safe
|
10
|
+
class TtlCache
|
11
|
+
include ::Karafka::Core::Helpers::Time
|
12
|
+
|
13
|
+
# @param ttl [Integer] time in ms how long should this cache keep data
|
14
|
+
def initialize(ttl)
|
15
|
+
@ttl = ttl
|
16
|
+
@times = {}
|
17
|
+
@values = {}
|
18
|
+
@mutex = Mutex.new
|
19
|
+
end
|
20
|
+
|
21
|
+
# Reads data from the cache
|
22
|
+
#
|
23
|
+
# @param key [String, Symbol] key for the cache read
|
24
|
+
# @return [Object] anything that was cached
|
25
|
+
def read(key)
|
26
|
+
@mutex.synchronize do
|
27
|
+
evict
|
28
|
+
@values[key]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# Writes to the cache
|
33
|
+
#
|
34
|
+
# @param key [String, Symbol] key for the cache
|
35
|
+
# @param value [Object] value we want to cache
|
36
|
+
# @return [Object] value we have written
|
37
|
+
def write(key, value)
|
38
|
+
@mutex.synchronize do
|
39
|
+
@times[key] = monotonic_now + @ttl
|
40
|
+
@values[key] = value
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# Reads from the cache and if value not present, will run the block and store its result
|
45
|
+
# in the cache
|
46
|
+
#
|
47
|
+
# @param key [String, Symbol] key for the cache read
|
48
|
+
# @return [Object] anything that was cached or yielded
|
49
|
+
def fetch(key)
|
50
|
+
@mutex.synchronize do
|
51
|
+
evict
|
52
|
+
|
53
|
+
return @values[key] if @values.key?(key)
|
54
|
+
|
55
|
+
@values[key] = yield
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Clears the whole cache
|
60
|
+
def clear
|
61
|
+
@mutex.synchronize do
|
62
|
+
@times.clear
|
63
|
+
@values.clear
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
# Removes expired elements from the cache
|
70
|
+
def evict
|
71
|
+
@times.each do |key, time|
|
72
|
+
next if time >= monotonic_now
|
73
|
+
|
74
|
+
@times.delete(key)
|
75
|
+
@values.delete(key)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Models
|
7
|
+
# Wraps around the `Karafka::Admin#cluster_info` with caching and some additional aliases
|
8
|
+
# so we can reference relevant information easily
|
9
|
+
class ClusterInfo
|
10
|
+
class << self
|
11
|
+
# Gets us all the cluster metadata info
|
12
|
+
#
|
13
|
+
# @param cached [Boolean] should we use cached data (true by default)
|
14
|
+
# @return [Rdkafka::Metadata] cluster metadata info
|
15
|
+
def fetch(cached: true)
|
16
|
+
cache = ::Karafka::Web.config.ui.cache
|
17
|
+
|
18
|
+
cluster_info = cache.read(:cluster_info)
|
19
|
+
|
20
|
+
if cluster_info.nil? || !cached
|
21
|
+
cluster_info = cache.write(:cluster_info, Karafka::Admin.cluster_info)
|
22
|
+
end
|
23
|
+
|
24
|
+
cluster_info
|
25
|
+
end
|
26
|
+
|
27
|
+
# Returns us all the info about available topics from the cluster
|
28
|
+
#
|
29
|
+
# @param cached [Boolean] should we use cached data (true by default)
|
30
|
+
# @return [Array<Ui::Models::Topic>] topics details
|
31
|
+
def topics(cached: true)
|
32
|
+
fetch(cached: cached)
|
33
|
+
.topics
|
34
|
+
.map { |topic| Topic.new(topic) }
|
35
|
+
end
|
36
|
+
|
37
|
+
# Fetches us details about particular topic
|
38
|
+
#
|
39
|
+
# @param topic_name [String] name of the topic we are looking for
|
40
|
+
# @param cached [Boolean] should we use cached data (true by default)
|
41
|
+
# @return [Ui::Models::Topic] topic details
|
42
|
+
def topic(topic_name, cached: true)
|
43
|
+
topics(cached: cached)
|
44
|
+
.find { |topic_data| topic_data.topic_name == topic_name }
|
45
|
+
.tap { |topic| topic || raise(Web::Errors::Ui::NotFoundError, topic_name) }
|
46
|
+
end
|
47
|
+
|
48
|
+
# @param topic_name [String] name of the topic we are looking for
|
49
|
+
# @param cached [Boolean] should we use cached data (true by default)
|
50
|
+
# @return [Integer] number of partitions in a given topic
|
51
|
+
def partitions_count(topic_name, cached: true)
|
52
|
+
topic(topic_name, cached: cached).partition_count
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Models
|
7
|
+
# Model representing the current consumers metrics most recent state
|
8
|
+
class ConsumersMetrics < Lib::HashProxy
|
9
|
+
class << self
|
10
|
+
# @return [State, false] current consumers metrics or false if not found
|
11
|
+
def current
|
12
|
+
state = fetch
|
13
|
+
|
14
|
+
return false unless state
|
15
|
+
|
16
|
+
# Do not return the state in case web-ui is not enabled because we need our
|
17
|
+
# internal deserializer for it to operate. False will force user to go to the
|
18
|
+
# status page
|
19
|
+
return false unless Models::Status.new.enabled.success?
|
20
|
+
|
21
|
+
state = state.payload
|
22
|
+
new(state)
|
23
|
+
end
|
24
|
+
|
25
|
+
# @return [State] current consumers metrics
|
26
|
+
# @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised when there is no metrics.
|
27
|
+
def current!
|
28
|
+
current || raise(::Karafka::Web::Errors::Ui::NotFoundError)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
# @return [::Karafka::Messages::Message, nil] most recent state or nil if none
|
34
|
+
def fetch
|
35
|
+
::Karafka::Admin.read_topic(
|
36
|
+
Karafka::Web.config.topics.consumers.metrics,
|
37
|
+
0,
|
38
|
+
1
|
39
|
+
).last
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -7,7 +7,7 @@ module Karafka
|
|
7
7
|
# Represents the current consumer processes aggregated state
|
8
8
|
# This state is the core of Karafka reporting. It holds the most important aggregated data
|
9
9
|
# as well as pointers to states of particular consumers and their details.
|
10
|
-
class
|
10
|
+
class ConsumersState < Lib::HashProxy
|
11
11
|
extend ::Karafka::Core::Helpers::Time
|
12
12
|
|
13
13
|
class << self
|
@@ -18,6 +18,10 @@ module Karafka
|
|
18
18
|
state = fetch
|
19
19
|
|
20
20
|
return false unless state
|
21
|
+
# Do not return the state in case web-ui is not enabled because we need our
|
22
|
+
# internal deserializer for it to operate. False will force user to go to the
|
23
|
+
# status page
|
24
|
+
return false unless Models::Status.new.enabled.success?
|
21
25
|
|
22
26
|
state = state.payload
|
23
27
|
evict_expired_processes(state)
|
@@ -28,7 +32,7 @@ module Karafka
|
|
28
32
|
|
29
33
|
# @return [State] current aggregated state
|
30
34
|
# @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised when there is no current
|
31
|
-
# state.
|
35
|
+
# state.
|
32
36
|
def current!
|
33
37
|
current || raise(::Karafka::Web::Errors::Ui::NotFoundError)
|
34
38
|
end
|
@@ -12,21 +12,51 @@ module Karafka
|
|
12
12
|
def current(state)
|
13
13
|
stats = {}
|
14
14
|
|
15
|
+
fetch_topics_data(state, stats)
|
16
|
+
fetch_rebalance_ages(state, stats)
|
17
|
+
|
18
|
+
stats
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
# Aggregates data on a per topic basis (in the context of a consumer group)
|
24
|
+
# @param state [Hash]
|
25
|
+
# @param stats [Hash] hash where we will store all the aggregated data
|
26
|
+
def fetch_topics_data(state, stats)
|
15
27
|
iterate_partitions(state) do |process, consumer_group, topic, partition|
|
16
28
|
cg_name = consumer_group.id
|
17
29
|
t_name = topic.name
|
18
30
|
pt_id = partition.id
|
19
31
|
|
20
|
-
stats[cg_name] ||= {}
|
21
|
-
stats[cg_name][t_name] ||= {}
|
22
|
-
stats[cg_name][t_name][pt_id] = partition
|
23
|
-
stats[cg_name][t_name][pt_id][:process] = process
|
32
|
+
stats[cg_name] ||= { topics: {} }
|
33
|
+
stats[cg_name][:topics][t_name] ||= {}
|
34
|
+
stats[cg_name][:topics][t_name][pt_id] = partition
|
35
|
+
stats[cg_name][:topics][t_name][pt_id][:process] = process
|
24
36
|
end
|
25
|
-
|
26
|
-
stats
|
27
37
|
end
|
28
38
|
|
29
|
-
|
39
|
+
# Aggregates rebalances ages data
|
40
|
+
# @param state [Hash]
|
41
|
+
# @param stats [Hash] hash where we will store all the aggregated data
|
42
|
+
def fetch_rebalance_ages(state, stats)
|
43
|
+
iterate_partitions(state) do |process, consumer_group|
|
44
|
+
cg_name = consumer_group.id
|
45
|
+
dispatched_at = process.dispatched_at
|
46
|
+
|
47
|
+
ages = consumer_group[:subscription_groups].values.map do |sub_group_details|
|
48
|
+
rebalance_age_ms = sub_group_details[:state][:rebalance_age] || 0
|
49
|
+
dispatched_at - rebalance_age_ms / 1_000
|
50
|
+
end
|
51
|
+
|
52
|
+
stats[cg_name][:rebalance_ages] ||= []
|
53
|
+
stats[cg_name][:rebalance_ages] += ages
|
54
|
+
end
|
55
|
+
|
56
|
+
stats.each_value do |details|
|
57
|
+
details[:rebalanced_at] = details[:rebalance_ages].max
|
58
|
+
end
|
59
|
+
end
|
30
60
|
|
31
61
|
# Iterates over all partitions, yielding with extra expanded details
|
32
62
|
#
|