karafka-web 0.11.3 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/Gemfile +0 -2
- data/Gemfile.lock +45 -32
- data/bin/integrations +44 -0
- data/bin/rspecs +6 -2
- data/docker-compose.yml +1 -1
- data/karafka-web.gemspec +1 -1
- data/lib/karafka/web/app.rb +2 -3
- data/lib/karafka/web/cli/help.rb +1 -1
- data/lib/karafka/web/config.rb +8 -0
- data/lib/karafka/web/contracts/base.rb +2 -4
- data/lib/karafka/web/contracts/config.rb +5 -5
- data/lib/karafka/web/management/actions/enable.rb +14 -1
- data/lib/karafka/web/management/migrations/consumers_reports/1761645571_rename_process_name_to_id.rb +38 -0
- data/lib/karafka/web/management/migrator.rb +3 -2
- data/lib/karafka/web/pro/commanding/commands/base.rb +1 -1
- data/lib/karafka/web/pro/commanding/contracts/config.rb +2 -4
- data/lib/karafka/web/pro/commanding/handlers/partitions/tracker.rb +2 -3
- data/lib/karafka/web/pro/ui/controllers/scheduled_messages/schedules_controller.rb +1 -2
- data/lib/karafka/web/pro/ui/controllers/topics/distributions_controller.rb +1 -3
- data/lib/karafka/web/pro/ui/lib/branding/contracts/config.rb +2 -4
- data/lib/karafka/web/pro/ui/lib/policies/contracts/config.rb +2 -4
- data/lib/karafka/web/pro/ui/lib/search/contracts/config.rb +3 -5
- data/lib/karafka/web/pro/ui/lib/search/contracts/form.rb +3 -5
- data/lib/karafka/web/pro/ui/lib/search/runner.rb +14 -1
- data/lib/karafka/web/pro/ui/routes/errors.rb +3 -3
- data/lib/karafka/web/pro/ui/routes/explorer.rb +3 -3
- data/lib/karafka/web/pro/ui/views/health/_no_partition_data.erb +9 -0
- data/lib/karafka/web/pro/ui/views/health/_partitions_with_fallback.erb +41 -0
- data/lib/karafka/web/pro/ui/views/health/changes.erb +12 -13
- data/lib/karafka/web/pro/ui/views/health/lags.erb +12 -13
- data/lib/karafka/web/pro/ui/views/health/offsets.erb +12 -13
- data/lib/karafka/web/pro/ui/views/health/overview.erb +15 -16
- data/lib/karafka/web/processing/consumer.rb +8 -3
- data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +1 -1
- data/lib/karafka/web/processing/consumers/aggregators/state.rb +5 -5
- data/lib/karafka/web/processing/consumers/contracts/state.rb +1 -1
- data/lib/karafka/web/processing/consumers/reports_migrator.rb +49 -0
- data/lib/karafka/web/processing/time_series_tracker.rb +1 -1
- data/lib/karafka/web/tracking/consumers/contracts/report.rb +1 -1
- data/lib/karafka/web/tracking/consumers/contracts/topic.rb +1 -0
- data/lib/karafka/web/tracking/consumers/listeners/errors.rb +2 -1
- data/lib/karafka/web/tracking/consumers/listeners/processing.rb +46 -0
- data/lib/karafka/web/tracking/consumers/listeners/statistics.rb +1 -0
- data/lib/karafka/web/tracking/consumers/sampler/enrichers/base.rb +20 -0
- data/lib/karafka/web/tracking/consumers/sampler/enrichers/consumer_groups.rb +116 -0
- data/lib/karafka/web/tracking/consumers/sampler/metrics/base.rb +20 -0
- data/lib/karafka/web/tracking/consumers/sampler/metrics/container.rb +113 -0
- data/lib/karafka/web/tracking/consumers/sampler/metrics/jobs.rb +60 -0
- data/lib/karafka/web/tracking/consumers/sampler/metrics/network.rb +48 -0
- data/lib/karafka/web/tracking/consumers/sampler/metrics/os.rb +206 -0
- data/lib/karafka/web/tracking/consumers/sampler/metrics/server.rb +33 -0
- data/lib/karafka/web/tracking/consumers/sampler.rb +34 -215
- data/lib/karafka/web/tracking/contracts/error.rb +1 -0
- data/lib/karafka/web/tracking/helpers/ttls/hash.rb +2 -3
- data/lib/karafka/web/tracking/helpers/ttls/stats.rb +1 -2
- data/lib/karafka/web/tracking/producers/listeners/errors.rb +2 -1
- data/lib/karafka/web/tracking/ui/errors.rb +76 -0
- data/lib/karafka/web/ui/base.rb +19 -9
- data/lib/karafka/web/ui/controllers/requests/execution_wrapper.rb +2 -4
- data/lib/karafka/web/ui/controllers/requests/params.rb +1 -1
- data/lib/karafka/web/ui/helpers/application_helper.rb +1 -1
- data/lib/karafka/web/ui/helpers/paths_helper.rb +6 -9
- data/lib/karafka/web/ui/lib/sorter.rb +1 -1
- data/lib/karafka/web/ui/models/health.rb +14 -9
- data/lib/karafka/web/ui/models/jobs.rb +4 -6
- data/lib/karafka/web/ui/models/message.rb +7 -8
- data/lib/karafka/web/ui/models/metrics/aggregated.rb +4 -4
- data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb +1 -2
- data/lib/karafka/web/ui/models/metrics/charts/topics.rb +2 -2
- data/lib/karafka/web/ui/models/metrics/topics.rb +3 -4
- data/lib/karafka/web/ui/models/recurring_tasks/schedule.rb +1 -1
- data/lib/karafka/web/ui/public/javascripts/application.min.js.gz +0 -0
- data/lib/karafka/web/ui/public/stylesheets/application.min.css +3 -0
- data/lib/karafka/web/ui/public/stylesheets/application.min.css.br +0 -0
- data/lib/karafka/web/ui/public/stylesheets/application.min.css.gz +0 -0
- data/lib/karafka/web/ui/routes/errors.rb +3 -3
- data/lib/karafka/web/ui/views/shared/exceptions/unhandled_error.erb +42 -0
- data/lib/karafka/web/version.rb +1 -1
- data/lib/karafka/web.rb +2 -3
- data/package-lock.json +180 -236
- data/package.json +3 -3
- data/renovate.json +13 -0
- metadata +18 -3
|
@@ -146,9 +146,9 @@ module Karafka
|
|
|
146
146
|
# @param page [Integer] which page we want to get
|
|
147
147
|
def topic_page(topic_id, partitions_ids, page)
|
|
148
148
|
# This is the bottleneck, for each partition we make one request :(
|
|
149
|
-
offsets = partitions_ids.
|
|
149
|
+
offsets = partitions_ids.to_h do |partition_id|
|
|
150
150
|
[partition_id, Models::WatermarkOffsets.find(topic_id, partition_id)]
|
|
151
|
-
end
|
|
151
|
+
end
|
|
152
152
|
|
|
153
153
|
# Count number of elements we have in each partition
|
|
154
154
|
# This assumes linear presence until low. If not, gaps will be filled like we fill
|
|
@@ -158,7 +158,7 @@ module Karafka
|
|
|
158
158
|
# Establish initial offsets for the iterator (where to start) per partition
|
|
159
159
|
# We do not use the negative lookup iterator because we already can compute starting
|
|
160
160
|
# offsets. This saves a lot of calls to Kafka
|
|
161
|
-
ranges = Sets.call(counts, page).
|
|
161
|
+
ranges = Sets.call(counts, page).to_h do |partition_position, partition_range|
|
|
162
162
|
partition_id = partitions_ids.to_a[partition_position]
|
|
163
163
|
watermarks = offsets[partition_id]
|
|
164
164
|
|
|
@@ -169,7 +169,7 @@ module Karafka
|
|
|
169
169
|
|
|
170
170
|
# This range represents offsets we want to fetch
|
|
171
171
|
[partition_id, lowest..highest]
|
|
172
|
-
end
|
|
172
|
+
end
|
|
173
173
|
|
|
174
174
|
# We start on our topic from the lowest offset for each expected partition
|
|
175
175
|
iterator = Karafka::Pro::Iterator.new(
|
|
@@ -208,18 +208,17 @@ module Karafka
|
|
|
208
208
|
end
|
|
209
209
|
|
|
210
210
|
[
|
|
211
|
-
aggregated.values.
|
|
211
|
+
aggregated.values.sum([]) { |partition| partition.values.reverse },
|
|
212
212
|
!Sets.call(counts, page + 1).empty?
|
|
213
213
|
]
|
|
214
214
|
end
|
|
215
215
|
|
|
216
216
|
private
|
|
217
217
|
|
|
218
|
-
# @param args [Object] anything required by the admin `#read_topic`
|
|
219
218
|
# @return [Array<Karafka::Messages::Message>, false] topic partition messages or false
|
|
220
219
|
# in case we hit a non-existing offset
|
|
221
|
-
def read_topic(*
|
|
222
|
-
Lib::Admin.read_topic(*
|
|
220
|
+
def read_topic(*)
|
|
221
|
+
Lib::Admin.read_topic(*)
|
|
223
222
|
rescue Rdkafka::RdkafkaError => e
|
|
224
223
|
return false if e.code == :auto_offset_reset
|
|
225
224
|
|
|
@@ -103,10 +103,10 @@ module Karafka
|
|
|
103
103
|
base = sample.last.dup
|
|
104
104
|
|
|
105
105
|
DELTA_KEYS.each do |key|
|
|
106
|
-
base[key] = previous.last[key] + (sample.last[key] - previous.last[key]) / 2
|
|
106
|
+
base[key] = previous.last[key] + ((sample.last[key] - previous.last[key]) / 2)
|
|
107
107
|
end
|
|
108
108
|
|
|
109
|
-
filled << [previous.first + (sample.first - previous.first) / 2, base]
|
|
109
|
+
filled << [previous.first + ((sample.first - previous.first) / 2), base]
|
|
110
110
|
end
|
|
111
111
|
|
|
112
112
|
filled << sample
|
|
@@ -182,12 +182,12 @@ module Karafka
|
|
|
182
182
|
# @param current [Hash]
|
|
183
183
|
# @return [Hash] delta computed values
|
|
184
184
|
def compute_deltas(previous, current)
|
|
185
|
-
DELTA_KEYS.
|
|
185
|
+
DELTA_KEYS.to_h do |delta_key|
|
|
186
186
|
[
|
|
187
187
|
delta_key,
|
|
188
188
|
current.fetch(delta_key) - previous.fetch(delta_key)
|
|
189
189
|
]
|
|
190
|
-
end
|
|
190
|
+
end
|
|
191
191
|
end
|
|
192
192
|
end
|
|
193
193
|
end
|
|
@@ -36,11 +36,11 @@ module Karafka
|
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
# Extract the lag stored only from all the data
|
|
39
|
-
per_topic = @data.to_h.
|
|
39
|
+
per_topic = @data.to_h.to_h do |topic, metrics|
|
|
40
40
|
extracted = metrics.map { |metric| [metric.first, metric.last[:lag_hybrid]] }
|
|
41
41
|
|
|
42
42
|
[topic, extracted]
|
|
43
|
-
end
|
|
43
|
+
end
|
|
44
44
|
|
|
45
45
|
# We name it with a space because someone may have a topic called "total" and we
|
|
46
46
|
# want to avoid collisions
|
|
@@ -43,9 +43,9 @@ module Karafka
|
|
|
43
43
|
|
|
44
44
|
# Always align the order of topics in hash based on their name so it is
|
|
45
45
|
# independent from the reported order
|
|
46
|
-
extracted[range] = range_extracted.keys.sort.
|
|
46
|
+
extracted[range] = range_extracted.keys.sort.to_h do |key|
|
|
47
47
|
[key, range_extracted[key]]
|
|
48
|
-
end
|
|
48
|
+
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
extracted
|
|
@@ -67,8 +67,7 @@ module Karafka
|
|
|
67
67
|
.select { |val| val.is_a?(Hash) }
|
|
68
68
|
.flat_map(&:keys)
|
|
69
69
|
.uniq
|
|
70
|
-
.
|
|
71
|
-
.to_h
|
|
70
|
+
.to_h { |key| [key, nil] }
|
|
72
71
|
.freeze
|
|
73
72
|
|
|
74
73
|
# Normalize data in between topics reportings
|
|
@@ -50,7 +50,7 @@ module Karafka
|
|
|
50
50
|
new(candidate.payload)
|
|
51
51
|
rescue Rdkafka::RdkafkaError => e
|
|
52
52
|
# If any of "topic missing" is raised, we return false but other errors we re-raise
|
|
53
|
-
raise(e) unless EXPECTED_RDKAFKA_ERRORS.
|
|
53
|
+
raise(e) unless EXPECTED_RDKAFKA_ERRORS.include?(e.code)
|
|
54
54
|
|
|
55
55
|
false
|
|
56
56
|
end
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -11,10 +11,10 @@ module Karafka
|
|
|
11
11
|
controller = build(Controllers::ErrorsController)
|
|
12
12
|
|
|
13
13
|
r.get Integer do |offset|
|
|
14
|
-
if params.current_offset
|
|
15
|
-
r.redirect root_path('errors', params.current_offset)
|
|
16
|
-
else
|
|
14
|
+
if params.current_offset == -1
|
|
17
15
|
controller.show(offset)
|
|
16
|
+
else
|
|
17
|
+
r.redirect root_path('errors', params.current_offset)
|
|
18
18
|
end
|
|
19
19
|
end
|
|
20
20
|
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
<div class="hidden">
|
|
2
|
+
<%== partial 'shared/controls' %>
|
|
3
|
+
</div>
|
|
4
|
+
|
|
5
|
+
<main class="flex items-center justify-center min-h-screen">
|
|
6
|
+
<div id="content" class="max-w-2xl w-full mx-auto">
|
|
7
|
+
<div class="flex items-center justify-center vh-100">
|
|
8
|
+
<div class="text-center">
|
|
9
|
+
<h1 class="text-8xl font-bold">500</h1>
|
|
10
|
+
|
|
11
|
+
<p class="text-3xl mt-5">
|
|
12
|
+
<span class="text-red-500">Oops!</span> Internal Server Error.
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
<div class="mt-5">
|
|
16
|
+
<p class="mb-10 text-lg font-light">
|
|
17
|
+
Something went wrong while processing your request.
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
<p class="text-lg font-light mb-5">This error has been reported and logged. Please try the following:</p>
|
|
21
|
+
|
|
22
|
+
<ul class="list-disc text-left text-lg mb-10 pl-5 font-light">
|
|
23
|
+
<li>Visit the <a href="<%= root_path('status') %>" class="text-blue-500 underline">Status</a> page to check if there are any system issues</li>
|
|
24
|
+
<li>Check the <a href="<%= root_path('errors') %>" class="text-blue-500 underline">Errors</a> page for details about this error</li>
|
|
25
|
+
<li>Verify that all required topics exist and are accessible</li>
|
|
26
|
+
<li>Ensure you have a working connection with your Kafka cluster</li>
|
|
27
|
+
<li>Try refreshing the page or navigating back to retry your action</li>
|
|
28
|
+
</ul>
|
|
29
|
+
|
|
30
|
+
<p class="mb-10 text-lg font-light">
|
|
31
|
+
If the problem persists, please check your application logs for more details.
|
|
32
|
+
</p>
|
|
33
|
+
|
|
34
|
+
<p>
|
|
35
|
+
<a href="<%= root_path %>" class="btn btn-primary mr-2 rounded-md py-1">Go Home</a>
|
|
36
|
+
<a href="<%= root_path('status') %>" class="btn btn-success text-white rounded-md">Status page</a>
|
|
37
|
+
</p>
|
|
38
|
+
</div>
|
|
39
|
+
</div>
|
|
40
|
+
</div>
|
|
41
|
+
</div>
|
|
42
|
+
</main>
|
data/lib/karafka/web/version.rb
CHANGED
data/lib/karafka/web.rb
CHANGED
|
@@ -26,8 +26,7 @@ module Karafka
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
# Sets up the whole configuration
|
|
29
|
-
|
|
30
|
-
def setup(&block)
|
|
29
|
+
def setup(&)
|
|
31
30
|
# You should never reconfigure Web UI after it has been enabled
|
|
32
31
|
raise Errors::LateSetupError, 'Always call #setup before #enable!' if config.enabled
|
|
33
32
|
|
|
@@ -38,7 +37,7 @@ module Karafka
|
|
|
38
37
|
Pro::Loader.pre_setup_all(config)
|
|
39
38
|
end
|
|
40
39
|
|
|
41
|
-
Config.configure(&
|
|
40
|
+
Config.configure(&)
|
|
42
41
|
|
|
43
42
|
Pro::Loader.post_setup_all(config) if Karafka.pro?
|
|
44
43
|
|