karafka-web 0.8.1 → 0.8.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +19 -19
- data/docker-compose.yml +1 -1
- data/lib/karafka/web/config.rb +2 -0
- data/lib/karafka/web/installer.rb +4 -1
- data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb +1 -1
- data/lib/karafka/web/tracking/consumers/listeners/booting.rb +34 -0
- data/lib/karafka/web/tracking/consumers/sampler.rb +8 -2
- data/lib/karafka/web/tracking/producers/listeners/booting.rb +24 -0
- data/lib/karafka/web/tracking/scheduler.rb +0 -5
- data/lib/karafka/web/ui/helpers/application_helper.rb +2 -0
- data/lib/karafka/web/ui/models/process.rb +9 -0
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +7 -1
- data/lib/karafka/web/ui/pro/views/consumers/subscriptions.erb +11 -11
- data/lib/karafka/web/ui/pro/views/health/changes.erb +2 -2
- data/lib/karafka/web/ui/pro/views/health/lags.erb +2 -2
- data/lib/karafka/web/ui/pro/views/health/offsets.erb +2 -2
- data/lib/karafka/web/ui/pro/views/health/overview.erb +2 -2
- data/lib/karafka/web/version.rb +1 -1
- data/lib/karafka/web.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +4 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 457da6c2e847225be5d27f3a30fcf8f99a4a132f8109204315ac042e24d1bd35
|
4
|
+
data.tar.gz: c277f44c30a34ee2748ac4e9b56833814b522da87f6cc1841d21aca618af58be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9ff1df82abd7f4f9f7002ae8a9e3e4e3fef2a71fbac820ff700d52c1926a5a81bf8e141ac6b928693d6276d8c16ef496efc11cada7574da217f4fb31efe31b5b
|
7
|
+
data.tar.gz: 3f39b8155794c25a39464d71c35f33ee82e6b59f92b55301f20ff69f1c58bc2cdf32af976720c548c2262893e25e0f8c8d31b8c033a248feb929bd088bc46d6c
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# Karafka Web changelog
|
2
2
|
|
3
|
+
## 0.8.2 (2024-02-16)
|
4
|
+
- [Enhancement] Defer scheduler background thread creation until needed allowing for forks.
|
5
|
+
- [Enhancement] Tag forks with fork indication + ppid reference when operating in swarm.
|
6
|
+
- [Fix] Fix issue where Health tabs would not be visible when no data reported.
|
7
|
+
- [Fix] Stopped processes subscriptions lacks indicator of no groups.
|
8
|
+
- [Fix] Terminated process state not supported in the web ui.
|
9
|
+
- [Fix] Rebalance reason can be empty on a SG when no network.
|
10
|
+
|
3
11
|
## 0.8.1 (2024-02-01)
|
4
12
|
- [Enhancement] Introduce "Lags" health view.
|
5
13
|
- [Enhancement] Remove "Stored Lag" and "Committed Offset" from Health Overview due to Lags Tab.
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka-web (0.8.
|
4
|
+
karafka-web (0.8.2)
|
5
5
|
erubi (~> 1.4)
|
6
6
|
karafka (>= 2.3.0, < 2.4.0)
|
7
7
|
karafka-core (>= 2.3.0, < 2.4.0)
|
@@ -26,7 +26,7 @@ GEM
|
|
26
26
|
byebug (11.1.3)
|
27
27
|
concurrent-ruby (1.2.3)
|
28
28
|
connection_pool (2.4.1)
|
29
|
-
diff-lcs (1.5.
|
29
|
+
diff-lcs (1.5.1)
|
30
30
|
docile (1.4.0)
|
31
31
|
drb (2.2.0)
|
32
32
|
ruby2_keywords
|
@@ -42,14 +42,14 @@ GEM
|
|
42
42
|
zeitwerk (~> 2.3)
|
43
43
|
karafka-core (2.3.0)
|
44
44
|
karafka-rdkafka (>= 0.14.8, < 0.15.0)
|
45
|
-
karafka-rdkafka (0.14.
|
45
|
+
karafka-rdkafka (0.14.9)
|
46
46
|
ffi (~> 1.15)
|
47
47
|
mini_portile2 (~> 2.6)
|
48
48
|
rake (> 12)
|
49
49
|
mini_portile2 (2.8.5)
|
50
|
-
minitest (5.
|
50
|
+
minitest (5.22.0)
|
51
51
|
mutex_m (0.2.0)
|
52
|
-
rack (3.0.
|
52
|
+
rack (3.0.9)
|
53
53
|
rack-test (2.1.0)
|
54
54
|
rack (>= 1.3)
|
55
55
|
rackup (0.2.3)
|
@@ -58,19 +58,19 @@ GEM
|
|
58
58
|
rake (13.1.0)
|
59
59
|
roda (3.76.0)
|
60
60
|
rack
|
61
|
-
rspec (3.
|
62
|
-
rspec-core (~> 3.
|
63
|
-
rspec-expectations (~> 3.
|
64
|
-
rspec-mocks (~> 3.
|
65
|
-
rspec-core (3.
|
66
|
-
rspec-support (~> 3.
|
67
|
-
rspec-expectations (3.
|
61
|
+
rspec (3.13.0)
|
62
|
+
rspec-core (~> 3.13.0)
|
63
|
+
rspec-expectations (~> 3.13.0)
|
64
|
+
rspec-mocks (~> 3.13.0)
|
65
|
+
rspec-core (3.13.0)
|
66
|
+
rspec-support (~> 3.13.0)
|
67
|
+
rspec-expectations (3.13.0)
|
68
68
|
diff-lcs (>= 1.2.0, < 2.0)
|
69
|
-
rspec-support (~> 3.
|
70
|
-
rspec-mocks (3.
|
69
|
+
rspec-support (~> 3.13.0)
|
70
|
+
rspec-mocks (3.13.0)
|
71
71
|
diff-lcs (>= 1.2.0, < 2.0)
|
72
|
-
rspec-support (~> 3.
|
73
|
-
rspec-support (3.
|
72
|
+
rspec-support (~> 3.13.0)
|
73
|
+
rspec-support (3.13.0)
|
74
74
|
ruby2_keywords (0.0.5)
|
75
75
|
simplecov (0.22.0)
|
76
76
|
docile (~> 1.1)
|
@@ -81,11 +81,11 @@ GEM
|
|
81
81
|
tilt (2.3.0)
|
82
82
|
tzinfo (2.0.6)
|
83
83
|
concurrent-ruby (~> 1.0)
|
84
|
-
waterdrop (2.6.
|
84
|
+
waterdrop (2.6.14)
|
85
85
|
karafka-core (>= 2.2.3, < 3.0.0)
|
86
86
|
zeitwerk (~> 2.3)
|
87
87
|
webrick (1.8.1)
|
88
|
-
zeitwerk (2.6.
|
88
|
+
zeitwerk (2.6.13)
|
89
89
|
|
90
90
|
PLATFORMS
|
91
91
|
ruby
|
@@ -101,4 +101,4 @@ DEPENDENCIES
|
|
101
101
|
simplecov
|
102
102
|
|
103
103
|
BUNDLED WITH
|
104
|
-
2.5.
|
104
|
+
2.5.6
|
data/docker-compose.yml
CHANGED
data/lib/karafka/web/config.rb
CHANGED
@@ -69,6 +69,7 @@ module Karafka
|
|
69
69
|
|
70
70
|
# Listeners needed for the Web UI to track consumer related changes
|
71
71
|
setting :listeners, default: [
|
72
|
+
Tracking::Consumers::Listeners::Booting.new,
|
72
73
|
Tracking::Consumers::Listeners::Status.new,
|
73
74
|
Tracking::Consumers::Listeners::Errors.new,
|
74
75
|
Tracking::Consumers::Listeners::Connections.new,
|
@@ -88,6 +89,7 @@ module Karafka
|
|
88
89
|
|
89
90
|
# Listeners needed for the Web UI to track producers related stuff
|
90
91
|
setting :listeners, default: [
|
92
|
+
Tracking::Producers::Listeners::Booting.new,
|
91
93
|
Tracking::Producers::Listeners::Errors.new
|
92
94
|
]
|
93
95
|
end
|
@@ -12,6 +12,7 @@ module Karafka
|
|
12
12
|
#
|
13
13
|
# @param replication_factor [Integer] replication factor we want to use (1 by default)
|
14
14
|
def install(replication_factor: 1)
|
15
|
+
enable!
|
15
16
|
puts
|
16
17
|
puts 'Installing Karafka Web UI...'
|
17
18
|
puts
|
@@ -21,7 +22,6 @@ module Karafka
|
|
21
22
|
puts
|
22
23
|
Management::Actions::CreateTopics.new.call(replication_factor)
|
23
24
|
wait_for_topics
|
24
|
-
enable!
|
25
25
|
Management::Actions::CreateInitialStates.new.call
|
26
26
|
puts
|
27
27
|
puts 'Running data migrations...'
|
@@ -36,6 +36,7 @@ module Karafka
|
|
36
36
|
#
|
37
37
|
# @param replication_factor [Integer] replication factor we want to use (1 by default)
|
38
38
|
def migrate(replication_factor: 1)
|
39
|
+
enable!
|
39
40
|
puts
|
40
41
|
puts 'Creating necessary topics and populating state data...'
|
41
42
|
puts
|
@@ -53,6 +54,7 @@ module Karafka
|
|
53
54
|
# Removes all the Karafka topics and creates them again with the same replication factor
|
54
55
|
# @param replication_factor [Integer] replication factor we want to use (1 by default)
|
55
56
|
def reset(replication_factor: 1)
|
57
|
+
enable!
|
56
58
|
puts
|
57
59
|
puts 'Resetting Karafka Web UI...'
|
58
60
|
puts
|
@@ -71,6 +73,7 @@ module Karafka
|
|
71
73
|
|
72
74
|
# Removes all the Karafka Web topics and cleans after itself.
|
73
75
|
def uninstall
|
76
|
+
enable!
|
74
77
|
puts
|
75
78
|
puts 'Uninstalling Karafka Web UI...'
|
76
79
|
puts
|
@@ -19,7 +19,7 @@ module Karafka
|
|
19
19
|
required(:stateage) { |val| val.is_a?(Integer) && val >= 0 }
|
20
20
|
required(:rebalance_age) { |val| val.is_a?(Integer) && val >= 0 }
|
21
21
|
required(:rebalance_cnt) { |val| val.is_a?(Integer) && val >= 0 }
|
22
|
-
required(:rebalance_reason) { |val| val.is_a?(String)
|
22
|
+
required(:rebalance_reason) { |val| val.is_a?(String) }
|
23
23
|
required(:poll_age) { |val| val.is_a?(Numeric) && val >= 0 }
|
24
24
|
end
|
25
25
|
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Tracking
|
6
|
+
module Consumers
|
7
|
+
module Listeners
|
8
|
+
# Listener needed to start schedulers and other things that we need to collect and report
|
9
|
+
# data. We do not want to start this when code is loaded because it may not be fork
|
10
|
+
# compatible that way
|
11
|
+
class Booting < Base
|
12
|
+
# Starts (if needed) the Web UI tracking scheduler thread that periodically pings
|
13
|
+
# reporters to report needed data (when it is time).
|
14
|
+
#
|
15
|
+
# @param _event [Karafka::Core::Monitoring::Event]
|
16
|
+
def on_app_running(_event)
|
17
|
+
::Karafka::Web.config.tracking.scheduler.async_call
|
18
|
+
end
|
19
|
+
|
20
|
+
# Updates the web producer after fork if needed and adds ppid to nodes
|
21
|
+
# @param _event [Karafka::Core::Monitoring::Event]
|
22
|
+
def on_swarm_node_after_fork(_event)
|
23
|
+
::Karafka::Process.tags.add(:node_ppid, "ppid:#{::Process.ppid}")
|
24
|
+
|
25
|
+
return if Karafka::Web.config.producer == Karafka::App.config.producer
|
26
|
+
|
27
|
+
Web.config.producer = Karafka::App.config.producer
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -46,7 +46,6 @@ module Karafka
|
|
46
46
|
end
|
47
47
|
@subscription_groups = {}
|
48
48
|
@errors = []
|
49
|
-
@started_at = float_now
|
50
49
|
@pauses = {}
|
51
50
|
@jobs = {}
|
52
51
|
@shell = MemoizedShell.new
|
@@ -71,7 +70,7 @@ module Karafka
|
|
71
70
|
dispatched_at: float_now,
|
72
71
|
|
73
72
|
process: {
|
74
|
-
started_at:
|
73
|
+
started_at: started_at,
|
75
74
|
name: process_name,
|
76
75
|
status: ::Karafka::App.config.internal.status.to_s,
|
77
76
|
listeners: listeners,
|
@@ -129,6 +128,13 @@ module Karafka
|
|
129
128
|
|
130
129
|
private
|
131
130
|
|
131
|
+
# @return [Float] time of start of this process
|
132
|
+
# @note We memoize it on first run as forks should have their creation time matching the
|
133
|
+
# fork time.
|
134
|
+
def started_at
|
135
|
+
@started_at ||= float_now
|
136
|
+
end
|
137
|
+
|
132
138
|
# @return [Numeric] % utilization of all the threads. 100% means all the threads are
|
133
139
|
# utilized all the time within the given time window. 0% means, nothing is happening
|
134
140
|
# most if not all the time.
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Tracking
|
6
|
+
module Producers
|
7
|
+
module Listeners
|
8
|
+
# Listener needed to start schedulers and other things that we need to collect and report
|
9
|
+
# data. We do not want to start this when code is loaded because it may not be fork
|
10
|
+
# compatible that way
|
11
|
+
class Booting < Base
|
12
|
+
# Starts (if needed) the Web UI tracking scheduler thread that periodically pings
|
13
|
+
# reporters to report needed data (when it is time).
|
14
|
+
#
|
15
|
+
# @param _event [Karafka::Core::Monitoring::Event]
|
16
|
+
def on_producer_connected(_event)
|
17
|
+
::Karafka::Web.config.tracking.scheduler.async_call
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -41,11 +41,13 @@ module Karafka
|
|
41
41
|
def status_bg(status)
|
42
42
|
case status
|
43
43
|
when 'initialized' then 'bg-success'
|
44
|
+
when 'supervising' then 'bg-success'
|
44
45
|
when 'running' then 'bg-success'
|
45
46
|
when 'quieting' then 'bg-warning'
|
46
47
|
when 'quiet' then 'bg-warning text-dark'
|
47
48
|
when 'stopping' then 'bg-warning text-dark'
|
48
49
|
when 'stopped' then 'bg-danger'
|
50
|
+
when 'terminated' then 'bg-danger'
|
49
51
|
else
|
50
52
|
raise ::Karafka::Errors::UnsupportedCaseError, status
|
51
53
|
end
|
@@ -52,6 +52,15 @@ module Karafka
|
|
52
52
|
.sum
|
53
53
|
end
|
54
54
|
|
55
|
+
# @return [Boolean] true if there are any active subscriptions, otherwise false.
|
56
|
+
def subscribed?
|
57
|
+
return false if consumer_groups.empty?
|
58
|
+
|
59
|
+
consumer_groups.any? do |cg|
|
60
|
+
!cg.subscription_groups.empty?
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
55
64
|
# @return [Integer] number of partitions to which we are currently subscribed
|
56
65
|
def subscribed_partitions_count
|
57
66
|
consumer_groups
|
@@ -76,7 +76,13 @@
|
|
76
76
|
|
77
77
|
<span class="text-end mt-3">
|
78
78
|
<small>
|
79
|
-
Last rebalance reason:
|
79
|
+
Last rebalance reason:
|
80
|
+
|
81
|
+
<% if subscription_group.rebalance_reason.empty? %>
|
82
|
+
Unknown
|
83
|
+
<% else %>
|
84
|
+
<%= subscription_group.rebalance_reason %>
|
85
|
+
<% end %>
|
80
86
|
</small>
|
81
87
|
</span>
|
82
88
|
</div>
|
@@ -8,16 +8,16 @@
|
|
8
8
|
|
9
9
|
<%== partial 'consumers/consumer/tabs' %>
|
10
10
|
|
11
|
-
<% if @process.
|
11
|
+
<% if @process.subscribed? %>
|
12
|
+
<div class="container">
|
13
|
+
<%==
|
14
|
+
render_each(
|
15
|
+
@process.consumer_groups,
|
16
|
+
'consumers/consumer/_consumer_group',
|
17
|
+
local: :consumer_group
|
18
|
+
)
|
19
|
+
%>
|
20
|
+
</div>
|
21
|
+
<% else %>
|
12
22
|
<%== partial 'consumers/consumer/no_subscriptions' %>
|
13
23
|
<% end %>
|
14
|
-
|
15
|
-
<div class="container">
|
16
|
-
<%==
|
17
|
-
render_each(
|
18
|
-
@process.consumer_groups,
|
19
|
-
'consumers/consumer/_consumer_group',
|
20
|
-
local: :consumer_group
|
21
|
-
)
|
22
|
-
%>
|
23
|
-
</div>
|
@@ -1,9 +1,9 @@
|
|
1
1
|
<%== view_title('Consumers groups changes details') %>
|
2
2
|
|
3
|
+
<%== partial 'health/tabs' %>
|
4
|
+
|
3
5
|
<% if @stats.empty? %>
|
4
6
|
<%== partial 'health/no_data' %>
|
5
|
-
<% else %>
|
6
|
-
<%== partial 'health/tabs' %>
|
7
7
|
<% end %>
|
8
8
|
|
9
9
|
<% @stats.each_with_index do |(cg_name, details), index| %>
|
@@ -1,9 +1,9 @@
|
|
1
1
|
<%== view_title('Consumers groups lags details') %>
|
2
2
|
|
3
|
+
<%== partial 'health/tabs' %>
|
4
|
+
|
3
5
|
<% if @stats.empty? %>
|
4
6
|
<%== partial 'health/no_data' %>
|
5
|
-
<% else %>
|
6
|
-
<%== partial 'health/tabs' %>
|
7
7
|
<% end %>
|
8
8
|
|
9
9
|
<% @stats.each_with_index do |(cg_name, details), index| %>
|
@@ -1,9 +1,9 @@
|
|
1
1
|
<%== view_title('Consumers groups offsets details') %>
|
2
2
|
|
3
|
+
<%== partial 'health/tabs' %>
|
4
|
+
|
3
5
|
<% if @stats.empty? %>
|
4
6
|
<%== partial 'health/no_data' %>
|
5
|
-
<% else %>
|
6
|
-
<%== partial 'health/tabs' %>
|
7
7
|
<% end %>
|
8
8
|
|
9
9
|
<% @stats.each_with_index do |(cg_name, details), index| %>
|
@@ -1,9 +1,9 @@
|
|
1
1
|
<%== view_title('Consumers groups overview') %>
|
2
2
|
|
3
|
+
<%== partial 'health/tabs' %>
|
4
|
+
|
3
5
|
<% if @stats.empty? %>
|
4
6
|
<%== partial 'health/no_data' %>
|
5
|
-
<% else %>
|
6
|
-
<%== partial 'health/tabs' %>
|
7
7
|
<% end %>
|
8
8
|
|
9
9
|
<% @stats.each_with_index do |(cg_name, details), index| %>
|
data/lib/karafka/web/version.rb
CHANGED
data/lib/karafka/web.rb
CHANGED
@@ -15,8 +15,9 @@ module Karafka
|
|
15
15
|
class << self
|
16
16
|
# @return [WaterDrop::Producer, nil] waterdrop messages producer or nil if not yet fully
|
17
17
|
# initialized. It may not be fully initialized until the configuration is done
|
18
|
+
# @note Do NOT memoize producer as it may be updated after forking
|
18
19
|
def producer
|
19
|
-
|
20
|
+
Web.config.producer
|
20
21
|
end
|
21
22
|
|
22
23
|
# @return [String] root path of this gem
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-web
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2024-02-
|
38
|
+
date: 2024-02-16 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: erubi
|
@@ -231,6 +231,7 @@ files:
|
|
231
231
|
- lib/karafka/web/tracking/consumers/contracts/subscription_group.rb
|
232
232
|
- lib/karafka/web/tracking/consumers/contracts/topic.rb
|
233
233
|
- lib/karafka/web/tracking/consumers/listeners/base.rb
|
234
|
+
- lib/karafka/web/tracking/consumers/listeners/booting.rb
|
234
235
|
- lib/karafka/web/tracking/consumers/listeners/connections.rb
|
235
236
|
- lib/karafka/web/tracking/consumers/listeners/errors.rb
|
236
237
|
- lib/karafka/web/tracking/consumers/listeners/pausing.rb
|
@@ -248,6 +249,7 @@ files:
|
|
248
249
|
- lib/karafka/web/tracking/helpers/ttls/windows.rb
|
249
250
|
- lib/karafka/web/tracking/memoized_shell.rb
|
250
251
|
- lib/karafka/web/tracking/producers/listeners/base.rb
|
252
|
+
- lib/karafka/web/tracking/producers/listeners/booting.rb
|
251
253
|
- lib/karafka/web/tracking/producers/listeners/errors.rb
|
252
254
|
- lib/karafka/web/tracking/producers/reporter.rb
|
253
255
|
- lib/karafka/web/tracking/producers/sampler.rb
|
metadata.gz.sig
CHANGED
Binary file
|