karafka-web 0.7.7 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +7 -7
- data/karafka-web.gemspec +1 -1
- data/lib/karafka/web/config.rb +36 -2
- data/lib/karafka/web/processing/consumers/metrics.rb +4 -1
- data/lib/karafka/web/processing/consumers/state.rb +4 -1
- data/lib/karafka/web/tracking/consumers/reporter.rb +21 -30
- data/lib/karafka/web/tracking/producers/reporter.rb +6 -4
- data/lib/karafka/web/tracking/producers/sampler.rb +3 -0
- data/lib/karafka/web/tracking/reporter.rb +25 -0
- data/lib/karafka/web/tracking/scheduler.rb +46 -0
- data/lib/karafka/web/ui/lib/admin.rb +56 -0
- data/lib/karafka/web/ui/models/cluster_info.rb +2 -2
- data/lib/karafka/web/ui/models/consumers_metrics.rb +4 -2
- data/lib/karafka/web/ui/models/consumers_state.rb +4 -2
- data/lib/karafka/web/ui/models/counters.rb +1 -1
- data/lib/karafka/web/ui/models/message.rb +2 -2
- data/lib/karafka/web/ui/models/processes.rb +1 -1
- data/lib/karafka/web/ui/models/watermark_offsets.rb +1 -1
- data/lib/karafka/web/ui/pro/controllers/explorer.rb +16 -5
- data/lib/karafka/web/ui/pro/views/errors/_error.erb +1 -1
- data/lib/karafka/web/ui/pro/views/explorer/_message.erb +1 -1
- data/lib/karafka/web/ui/public/javascripts/live_poll.js +1 -1
- data/lib/karafka/web/ui/views/errors/_error.erb +1 -1
- data/lib/karafka/web/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +7 -5
- metadata.gz.sig +0 -0
- data/lib/karafka/web/tracking/producers/listeners/reporter.rb +0 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2d46244304483184c3be59a0686497a9f948e9b666dccf00961b560e60db4655
|
4
|
+
data.tar.gz: 329686ce548063787fe06290d67650793ebef6c014f43b753780ad120671a8b1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5a18a3c23f63382c6c41dc5ac7f82f893bc14a55e7ec68af9cf16da0f47a45aee77ca9626973570cb9d6abe9355e0712f117424e00619b7b0198748c3bfbeb55
|
7
|
+
data.tar.gz: f639bd90ca21c09fcd798fc289079cafeb9c752d261ecf88df9762e347df738bf260c658f0f15b3b08c3708daf091217c0159d85ac975380463a65a45d876d0b
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# Karafka Web changelog
|
2
2
|
|
3
|
+
## 0.7.8 (2023-10-24)
|
4
|
+
- [Enhancement] Support transactional producer usage with Web UI.
|
5
|
+
- [Fix] Fix a bug where critical errors (like `IRB::Abort`) would not abort the ongoing transaction.
|
6
|
+
- [Fix] Prevent a scenario where an ongoing transactional producer would have stats emitted and an error that could not have been dispatched because of the transaction, creating a dead-lock.
|
7
|
+
- [Fix] Make sure that the `recent` displays the most recent non-compacted, non-system message.
|
8
|
+
- [Fix] Improve the `recent` message display to compensate for aborted transactions.
|
9
|
+
- [Fix] Fix `ReferenceError: response is not defined` that occurs when Web UI returns refresh non 200.
|
10
|
+
|
3
11
|
## 0.7.7 (2023-10-20)
|
4
12
|
- [Fix] Remove `thor` as a CLI engine due to breaking changes.
|
5
13
|
|
data/Gemfile.lock
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka-web (0.7.
|
4
|
+
karafka-web (0.7.8)
|
5
5
|
erubi (~> 1.4)
|
6
|
-
karafka (>= 2.2.
|
6
|
+
karafka (>= 2.2.9, < 3.0.0)
|
7
7
|
karafka-core (>= 2.2.2, < 3.0.0)
|
8
8
|
roda (~> 3.68, >= 3.69)
|
9
9
|
tilt (~> 2.0)
|
@@ -36,9 +36,9 @@ GEM
|
|
36
36
|
ffi (1.16.3)
|
37
37
|
i18n (1.14.1)
|
38
38
|
concurrent-ruby (~> 1.0)
|
39
|
-
karafka (2.2.
|
39
|
+
karafka (2.2.9)
|
40
40
|
karafka-core (>= 2.2.2, < 2.3.0)
|
41
|
-
waterdrop (>= 2.6.
|
41
|
+
waterdrop (>= 2.6.10, < 3.0.0)
|
42
42
|
zeitwerk (~> 2.3)
|
43
43
|
karafka-core (2.2.3)
|
44
44
|
concurrent-ruby (>= 1.1)
|
@@ -47,7 +47,7 @@ GEM
|
|
47
47
|
ffi (~> 1.15)
|
48
48
|
mini_portile2 (~> 2.6)
|
49
49
|
rake (> 12)
|
50
|
-
mini_portile2 (2.8.
|
50
|
+
mini_portile2 (2.8.5)
|
51
51
|
minitest (5.20.0)
|
52
52
|
mutex_m (0.1.2)
|
53
53
|
rack (3.0.8)
|
@@ -82,8 +82,8 @@ GEM
|
|
82
82
|
tilt (2.3.0)
|
83
83
|
tzinfo (2.0.6)
|
84
84
|
concurrent-ruby (~> 1.0)
|
85
|
-
waterdrop (2.6.
|
86
|
-
karafka-core (>= 2.
|
85
|
+
waterdrop (2.6.10)
|
86
|
+
karafka-core (>= 2.2.3, < 3.0.0)
|
87
87
|
zeitwerk (~> 2.3)
|
88
88
|
webrick (1.8.1)
|
89
89
|
zeitwerk (2.6.12)
|
data/karafka-web.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.licenses = %w[LGPL-3.0 Commercial]
|
18
18
|
|
19
19
|
spec.add_dependency 'erubi', '~> 1.4'
|
20
|
-
spec.add_dependency 'karafka', '>= 2.2.
|
20
|
+
spec.add_dependency 'karafka', '>= 2.2.9', '< 3.0.0'
|
21
21
|
spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
|
22
22
|
spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
|
23
23
|
spec.add_dependency 'tilt', '~> 2.0'
|
data/lib/karafka/web/config.rb
CHANGED
@@ -36,6 +36,10 @@ module Karafka
|
|
36
36
|
# 5 seconds should be enough
|
37
37
|
setting :interval, default: 5_000
|
38
38
|
|
39
|
+
# Main Web UI reporting scheduler that runs a background thread and reports periodically
|
40
|
+
# from the consumer reporter and producer reporter
|
41
|
+
setting :scheduler, default: Tracking::Scheduler.new
|
42
|
+
|
39
43
|
setting :consumers do
|
40
44
|
# Reports the metrics collected in the sampler
|
41
45
|
setting :reporter, default: Tracking::Consumers::Reporter.new
|
@@ -58,8 +62,7 @@ module Karafka
|
|
58
62
|
setting :sampler, default: Tracking::Producers::Sampler.new
|
59
63
|
|
60
64
|
setting :listeners, default: [
|
61
|
-
Tracking::Producers::Listeners::Errors.new
|
62
|
-
Tracking::Producers::Listeners::Reporter.new
|
65
|
+
Tracking::Producers::Listeners::Errors.new
|
63
66
|
]
|
64
67
|
end
|
65
68
|
end
|
@@ -114,6 +117,37 @@ module Karafka
|
|
114
117
|
# In some cases you may want to limit what is being displayed due to the type of data you
|
115
118
|
# are dealing with
|
116
119
|
setting :visibility_filter, default: Ui::Models::VisibilityFilter.new
|
120
|
+
|
121
|
+
# Specific kafka settings that are tuned to operate within the Web UI interface.
|
122
|
+
#
|
123
|
+
# Please do not change them unless you know what you are doing as their misconfiguration
|
124
|
+
# may cause Web UI to misbehave
|
125
|
+
#
|
126
|
+
# The settings are inherited as follows:
|
127
|
+
# 1. root routing level `kafka` settings
|
128
|
+
# 2. admin `kafka` settings
|
129
|
+
# 3. web ui `kafka` settings from here
|
130
|
+
#
|
131
|
+
# Those settings impact ONLY Web UI interface and do not affect other scopes. This is done
|
132
|
+
# on purpose as we want to improve responsiveness of the interface by tuning some of the
|
133
|
+
# settings and this is not that relevant for processing itself.
|
134
|
+
#
|
135
|
+
# option [Hash] extra changes to the default admin kafka settings
|
136
|
+
setting :kafka, default: {
|
137
|
+
# optimizes the responsiveness of the Web UI in three scenarios:
|
138
|
+
# - topics to which writes happen only in transactions so EOF is yield faster
|
139
|
+
# - heavily compacted topics
|
140
|
+
# - Web UI topics read operations when using transactional producer
|
141
|
+
#
|
142
|
+
# This can be configured to be higher if you do not use transactional WaterDrop producer.
|
143
|
+
# This value is used when last message (first from the high watermark offset) is the
|
144
|
+
# transaction commit message. In cases like this the EOF gets propagated after this time
|
145
|
+
# so we have to wait. Default 500ms means, that for some views, where we take our data
|
146
|
+
# that might have been committed via transactional producer, we would wait for 1 second
|
147
|
+
# to get needed data. If you are experiencing timeouts or other issues with the Web IU
|
148
|
+
# interface, you can increase this.
|
149
|
+
'fetch.wait.max.ms': 100
|
150
|
+
}
|
117
151
|
end
|
118
152
|
end
|
119
153
|
end
|
@@ -14,7 +14,10 @@ module Karafka
|
|
14
14
|
metrics_message = ::Karafka::Admin.read_topic(
|
15
15
|
Karafka::Web.config.topics.consumers.metrics,
|
16
16
|
0,
|
17
|
-
|
17
|
+
# We need to take more in case there would be transactions running.
|
18
|
+
# In theory we could take two but this compensates for any involuntary
|
19
|
+
# revocations and cases where two producers would write to the same state
|
20
|
+
5
|
18
21
|
).last
|
19
22
|
|
20
23
|
return metrics_message.payload if metrics_message
|
@@ -14,7 +14,10 @@ module Karafka
|
|
14
14
|
state_message = ::Karafka::Admin.read_topic(
|
15
15
|
Karafka::Web.config.topics.consumers.states,
|
16
16
|
0,
|
17
|
-
|
17
|
+
# We need to take more in case there would be transactions running.
|
18
|
+
# In theory we could take two but this compensates for any involuntary
|
19
|
+
# revocations and cases where two producers would write to the same state
|
20
|
+
5
|
18
21
|
).last
|
19
22
|
|
20
23
|
return state_message.payload if state_message
|
@@ -5,10 +5,7 @@ module Karafka
|
|
5
5
|
module Tracking
|
6
6
|
module Consumers
|
7
7
|
# Reports the collected data about the process and sends it, so we can use it in the UI
|
8
|
-
class Reporter
|
9
|
-
include ::Karafka::Core::Helpers::Time
|
10
|
-
include ::Karafka::Helpers::Async
|
11
|
-
|
8
|
+
class Reporter < Tracking::Reporter
|
12
9
|
# Minimum number of messages to produce to produce them in sync mode
|
13
10
|
# This acts as a small back-off not to overload the system in case we would have
|
14
11
|
# extremely big number of errors happening
|
@@ -21,12 +18,31 @@ module Karafka
|
|
21
18
|
MUTEX = Mutex.new
|
22
19
|
|
23
20
|
def initialize
|
21
|
+
super
|
24
22
|
# Move back so first report is dispatched fast to indicate, that the process is alive
|
25
23
|
@tracked_at = monotonic_now - 10_000
|
26
24
|
@report_contract = Consumers::Contracts::Report.new
|
27
25
|
@error_contract = Tracking::Contracts::Error.new
|
28
26
|
end
|
29
27
|
|
28
|
+
# We never report in initializing phase because things are not yet fully configured
|
29
|
+
# We never report in the initialized because server is not yet ready until Karafka is
|
30
|
+
# fully running and some of the things like listeners are not yet available
|
31
|
+
#
|
32
|
+
# This method will also be `false` in case we are not running in `karafka server` or
|
33
|
+
# in embedding, because in those cases Karafka does not go beyond the `initialized` phase
|
34
|
+
#
|
35
|
+
# @return [Boolean] are we able to report consumer state
|
36
|
+
def active?
|
37
|
+
# If we do not have a producer that we could use to report or it was closed, we cannot
|
38
|
+
# and should not report
|
39
|
+
return false unless super
|
40
|
+
return false if ::Karafka::App.initializing?
|
41
|
+
return false if ::Karafka::App.initialized?
|
42
|
+
|
43
|
+
true
|
44
|
+
end
|
45
|
+
|
30
46
|
# Dispatches the current state from sampler to appropriate topics
|
31
47
|
#
|
32
48
|
# @param forced [Boolean] should we report bypassing the time frequency or should we
|
@@ -41,11 +57,6 @@ module Karafka
|
|
41
57
|
sampler.sample
|
42
58
|
|
43
59
|
MUTEX.synchronize do
|
44
|
-
# Start background thread only when needed
|
45
|
-
# This prevents us from starting it too early or for non-consumer processes where
|
46
|
-
# Karafka is being included
|
47
|
-
async_call unless @running
|
48
|
-
|
49
60
|
return unless report?(forced)
|
50
61
|
|
51
62
|
@tracked_at = monotonic_now
|
@@ -97,31 +108,11 @@ module Karafka
|
|
97
108
|
|
98
109
|
private
|
99
110
|
|
100
|
-
# Reports the process state once in a while
|
101
|
-
def call
|
102
|
-
@running = true
|
103
|
-
|
104
|
-
# We won't track more often anyhow but want to try frequently not to miss a window
|
105
|
-
# We need to convert the sleep interval into seconds for sleep
|
106
|
-
sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
|
107
|
-
|
108
|
-
loop do
|
109
|
-
report
|
110
|
-
|
111
|
-
sleep(sleep_time)
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
111
|
# @param forced [Boolean] is this report forced. Forced means that as long as we can
|
116
112
|
# flush we will flush
|
117
113
|
# @return [Boolean] Should we report or is it not yet time to do so
|
118
114
|
def report?(forced)
|
119
|
-
|
120
|
-
return false if ::Karafka::App.initializing?
|
121
|
-
# We never report in the initialized because server is not yet ready until Karafka is
|
122
|
-
# fully running and some of the things like listeners are not yet available
|
123
|
-
return false if ::Karafka::App.initialized?
|
124
|
-
|
115
|
+
return false unless active?
|
125
116
|
return true if forced
|
126
117
|
|
127
118
|
(monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
|
@@ -9,9 +9,7 @@ module Karafka
|
|
9
9
|
# @note Producer reported does not have to operate with the `forced` dispatch mainly
|
10
10
|
# because there is no expectation on immediate status updates for producers and their
|
11
11
|
# dispatch flow is always periodic based.
|
12
|
-
class Reporter
|
13
|
-
include ::Karafka::Core::Helpers::Time
|
14
|
-
|
12
|
+
class Reporter < Tracking::Reporter
|
15
13
|
# Minimum number of messages to produce to produce them in sync mode
|
16
14
|
# This acts as a small back-off not to overload the system in case we would have
|
17
15
|
# extremely big number of errors happening
|
@@ -24,6 +22,7 @@ module Karafka
|
|
24
22
|
MUTEX = Mutex.new
|
25
23
|
|
26
24
|
def initialize
|
25
|
+
super
|
27
26
|
# If there are any errors right after we started sampling, dispatch them immediately
|
28
27
|
@tracked_at = monotonic_now - 10_000
|
29
28
|
@error_contract = Tracking::Contracts::Error.new
|
@@ -62,7 +61,7 @@ module Karafka
|
|
62
61
|
|
63
62
|
# @return [Boolean] Should we report or is it not yet time to do so
|
64
63
|
def report?
|
65
|
-
return false unless
|
64
|
+
return false unless active?
|
66
65
|
|
67
66
|
(monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
|
68
67
|
end
|
@@ -93,6 +92,9 @@ module Karafka
|
|
93
92
|
# and we can just safely ignore this
|
94
93
|
rescue WaterDrop::Errors::ProducerClosedError
|
95
94
|
nil
|
95
|
+
rescue StandardError => e
|
96
|
+
p '------------------------------------------------'
|
97
|
+
p e
|
96
98
|
end
|
97
99
|
end
|
98
100
|
end
|
@@ -26,6 +26,9 @@ module Karafka
|
|
26
26
|
# We cannot report and track the same time, that is why we use mutex here. To make sure
|
27
27
|
# that samples aggregations and counting does not interact with reporter flushing.
|
28
28
|
def track
|
29
|
+
# Prevents deadlocks when something producer related fails in the Web UI reporter
|
30
|
+
return yield(self) if Reporter::MUTEX.owned?
|
31
|
+
|
29
32
|
Reporter::MUTEX.synchronize do
|
30
33
|
yield(self)
|
31
34
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Tracking
|
6
|
+
# Base reporter from which all the reports should inherit
|
7
|
+
class Reporter
|
8
|
+
include ::Karafka::Core::Helpers::Time
|
9
|
+
|
10
|
+
# Can this reporter report. Since some reporters may report only in part of the processes
|
11
|
+
# where Karafka is used (like `karafka server`) each may implement more complex rules.
|
12
|
+
#
|
13
|
+
# The basic is not to report unless we have a producer and this producer is active
|
14
|
+
#
|
15
|
+
# @return [Boolean]
|
16
|
+
def active?
|
17
|
+
return false unless ::Karafka::App.producer
|
18
|
+
return false unless ::Karafka::App.producer.status.active?
|
19
|
+
|
20
|
+
true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Tracking
|
6
|
+
# Triggers reporters to report in an async mode in a separate thread
|
7
|
+
# We report this way to prevent any potential dead-locks in cases we would be emitting
|
8
|
+
# statistics during transactions.
|
9
|
+
#
|
10
|
+
# We should never use the notifications thread for sensitive IO bound operations.
|
11
|
+
class Scheduler
|
12
|
+
include ::Karafka::Helpers::Async
|
13
|
+
|
14
|
+
# Creates the scheduler and runs its internal reporting
|
15
|
+
def initialize
|
16
|
+
async_call
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
# Reports the process state once in a while
|
22
|
+
def call
|
23
|
+
# We won't track more often anyhow but want to try frequently not to miss a window
|
24
|
+
# We need to convert the sleep interval into seconds for sleep
|
25
|
+
sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
|
26
|
+
|
27
|
+
loop do
|
28
|
+
# Not every reporter may be active at a given stage or in a context of a given process
|
29
|
+
# We select only those that decided that they are active.
|
30
|
+
reporters.select(&:active?).each(&:report)
|
31
|
+
|
32
|
+
sleep(sleep_time)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# @return [Array] consumers and producers reporters
|
37
|
+
def reporters
|
38
|
+
@reporters ||= [
|
39
|
+
::Karafka::Web.config.tracking.consumers.reporter,
|
40
|
+
::Karafka::Web.config.tracking.producers.reporter
|
41
|
+
].freeze
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
module Ui
|
6
|
+
module Lib
|
7
|
+
# Wrapper around Karafka Admin that alters its behaviours or injects Web UI interface
|
8
|
+
# specific settings that optimize the responsiveness of the UI when operating on topics
|
9
|
+
#
|
10
|
+
# @note Not all commands need those optimizations, hence we alter only those that need
|
11
|
+
# that and we only expose those admin commands that are used in the Web-UI interface
|
12
|
+
# component.
|
13
|
+
#
|
14
|
+
# @note We expose here only admin methods used in the Web UI interface. Processing uses the
|
15
|
+
# `Karafka::Admin` with the defaults
|
16
|
+
class Admin
|
17
|
+
class << self
|
18
|
+
extend Forwardable
|
19
|
+
|
20
|
+
def_delegators ::Karafka::Admin, :read_watermark_offsets, :cluster_info
|
21
|
+
|
22
|
+
# Allows us to read messages from the topic
|
23
|
+
#
|
24
|
+
# @param name [String, Symbol] topic name
|
25
|
+
# @param partition [Integer] partition
|
26
|
+
# @param count [Integer] how many messages we want to get at most
|
27
|
+
# @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
|
28
|
+
# (default) we will start from the latest offset. If time is provided, the appropriate
|
29
|
+
# offset will be resolved. If negative beyond -1 is provided, we move backwards more.
|
30
|
+
# @param settings [Hash] kafka extra settings (optional)
|
31
|
+
#
|
32
|
+
# @return [Array<Karafka::Messages::Message>] array with messages
|
33
|
+
def read_topic(name, partition, count, start_offset = -1, settings = {})
|
34
|
+
::Karafka::Admin.read_topic(
|
35
|
+
name,
|
36
|
+
partition,
|
37
|
+
count,
|
38
|
+
start_offset,
|
39
|
+
# Merge our Web UI specific settings
|
40
|
+
config.merge(settings)
|
41
|
+
)
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
# @return [Hash] kafka config for Web UI interface.
|
47
|
+
# @note It does **not** affect tracking or processing
|
48
|
+
def config
|
49
|
+
::Karafka::Web.config.ui.kafka
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -4,7 +4,7 @@ module Karafka
|
|
4
4
|
module Web
|
5
5
|
module Ui
|
6
6
|
module Models
|
7
|
-
# Wraps around the `
|
7
|
+
# Wraps around the `Lib::Admin#cluster_info` with caching and some additional aliases
|
8
8
|
# so we can reference relevant information easily
|
9
9
|
class ClusterInfo
|
10
10
|
class << self
|
@@ -18,7 +18,7 @@ module Karafka
|
|
18
18
|
cluster_info = cache.read(:cluster_info)
|
19
19
|
|
20
20
|
if cluster_info.nil? || !cached
|
21
|
-
cluster_info = cache.write(:cluster_info,
|
21
|
+
cluster_info = cache.write(:cluster_info, Lib::Admin.cluster_info)
|
22
22
|
end
|
23
23
|
|
24
24
|
cluster_info
|
@@ -32,10 +32,12 @@ module Karafka
|
|
32
32
|
|
33
33
|
# @return [::Karafka::Messages::Message, nil] most recent state or nil if none
|
34
34
|
def fetch
|
35
|
-
::
|
35
|
+
Lib::Admin.read_topic(
|
36
36
|
Karafka::Web.config.topics.consumers.metrics,
|
37
37
|
0,
|
38
|
-
|
38
|
+
# We need to take last two and not the last because in case of a transactional
|
39
|
+
# producer, the last one will match the transaction commit message
|
40
|
+
2
|
39
41
|
).last
|
40
42
|
end
|
41
43
|
end
|
@@ -41,10 +41,12 @@ module Karafka
|
|
41
41
|
|
42
42
|
# @return [::Karafka::Messages::Message, nil] most recent state or nil if none
|
43
43
|
def fetch
|
44
|
-
::
|
44
|
+
Lib::Admin.read_topic(
|
45
45
|
Karafka::Web.config.topics.consumers.states,
|
46
46
|
0,
|
47
|
-
|
47
|
+
# We need to take last two and not the last because in case of a transactional
|
48
|
+
# producer, the last one will match the transaction commit message
|
49
|
+
2
|
48
50
|
).last
|
49
51
|
end
|
50
52
|
|
@@ -17,7 +17,7 @@ module Karafka
|
|
17
17
|
# @param offset [Integer]
|
18
18
|
# @raise [::Karafka::Web::Errors::Ui::NotFoundError] when not found
|
19
19
|
def find(topic_id, partition_id, offset)
|
20
|
-
message =
|
20
|
+
message = Lib::Admin.read_topic(
|
21
21
|
topic_id,
|
22
22
|
partition_id,
|
23
23
|
1,
|
@@ -195,7 +195,7 @@ module Karafka
|
|
195
195
|
# @return [Array<Karafka::Messages::Message>, false] topic partition messages or false
|
196
196
|
# in case we hit a non-existing offset
|
197
197
|
def read_topic(*args)
|
198
|
-
::
|
198
|
+
Lib::Admin.read_topic(*args)
|
199
199
|
rescue Rdkafka::RdkafkaError => e
|
200
200
|
return false if e.code == :auto_offset_reset
|
201
201
|
|
@@ -44,7 +44,7 @@ module Karafka
|
|
44
44
|
.map { |process| process[:offset] }
|
45
45
|
.sort
|
46
46
|
|
47
|
-
::
|
47
|
+
Lib::Admin.read_topic(
|
48
48
|
::Karafka::Web.config.topics.consumers.reports,
|
49
49
|
0,
|
50
50
|
# We set 10k here because we start from the latest offset of the reports, hence
|
@@ -13,7 +13,7 @@ module Karafka
|
|
13
13
|
# @param partition_id [Integer]
|
14
14
|
# @return [WatermarkOffsets]
|
15
15
|
def find(topic_id, partition_id)
|
16
|
-
offsets = ::
|
16
|
+
offsets = Lib::Admin.read_watermark_offsets(topic_id, partition_id)
|
17
17
|
|
18
18
|
new(
|
19
19
|
low: offsets.first,
|
@@ -132,11 +132,22 @@ module Karafka
|
|
132
132
|
active_partitions, = Paginators::Partitions.call(partitions_count, 1)
|
133
133
|
end
|
134
134
|
|
135
|
-
|
136
|
-
messages, = Models::Message.topic_page(topic_id, active_partitions, 1)
|
135
|
+
recent = nil
|
137
136
|
|
138
|
-
#
|
139
|
-
|
137
|
+
# This selects first pages with most recent messages and moves to next if first
|
138
|
+
# contains only compacted data, etc.
|
139
|
+
#
|
140
|
+
# We do it until we find a message we could refer to (if doable) within first
|
141
|
+
# ten pages
|
142
|
+
10.times do |page|
|
143
|
+
messages, = Models::Message.topic_page(topic_id, active_partitions, page + 1)
|
144
|
+
|
145
|
+
# Selects newest out of all partitions
|
146
|
+
# Reject compacted messages and transaction-related once
|
147
|
+
recent = messages.reject { |message| message.is_a?(Array) }.max_by(&:timestamp)
|
148
|
+
|
149
|
+
break if recent
|
150
|
+
end
|
140
151
|
|
141
152
|
recent || raise(::Karafka::Web::Errors::Ui::NotFoundError)
|
142
153
|
|
@@ -184,7 +195,7 @@ module Karafka
|
|
184
195
|
# @param partition_id [Integer]
|
185
196
|
# @param time [Time] time of the message
|
186
197
|
def closest(topic_id, partition_id, time)
|
187
|
-
target = ::
|
198
|
+
target = Lib::Admin.read_topic(topic_id, partition_id, 1, time).first
|
188
199
|
|
189
200
|
partition_path = "explorer/#{topic_id}/#{partition_id}"
|
190
201
|
partition_path += "?offset=#{target.offset}" if target
|
@@ -1,7 +1,7 @@
|
|
1
1
|
<% if error_msg.is_a?(Array) %>
|
2
2
|
<tr>
|
3
3
|
<td colspan="5" class="text-center text-muted">
|
4
|
-
This error
|
4
|
+
This offset does not contain error data. The message may have been compacted or is a system entry.
|
5
5
|
</td>
|
6
6
|
</tr>
|
7
7
|
<% else %>
|
@@ -7,7 +7,7 @@
|
|
7
7
|
<%= message[1] %>
|
8
8
|
</td>
|
9
9
|
<td colspan="3" class="text-center text-muted">
|
10
|
-
This message
|
10
|
+
This offset does not contain any data. The message may have been compacted or is a system entry.
|
11
11
|
</td>
|
12
12
|
</tr>
|
13
13
|
<% else %>
|
@@ -1,7 +1,7 @@
|
|
1
1
|
<% if error_msg.is_a?(Array) %>
|
2
2
|
<tr>
|
3
3
|
<td colspan="5" class="text-center text-muted">
|
4
|
-
This error
|
4
|
+
This offset does not contain error data. The message may have been compacted or is a system entry.
|
5
5
|
</td>
|
6
6
|
</tr>
|
7
7
|
<% else %>
|
data/lib/karafka/web/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-web
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-10-
|
38
|
+
date: 2023-10-24 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: erubi
|
@@ -57,7 +57,7 @@ dependencies:
|
|
57
57
|
requirements:
|
58
58
|
- - ">="
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: 2.2.
|
60
|
+
version: 2.2.9
|
61
61
|
- - "<"
|
62
62
|
- !ruby/object:Gem::Version
|
63
63
|
version: 3.0.0
|
@@ -67,7 +67,7 @@ dependencies:
|
|
67
67
|
requirements:
|
68
68
|
- - ">="
|
69
69
|
- !ruby/object:Gem::Version
|
70
|
-
version: 2.2.
|
70
|
+
version: 2.2.9
|
71
71
|
- - "<"
|
72
72
|
- !ruby/object:Gem::Version
|
73
73
|
version: 3.0.0
|
@@ -225,10 +225,11 @@ files:
|
|
225
225
|
- lib/karafka/web/tracking/memoized_shell.rb
|
226
226
|
- lib/karafka/web/tracking/producers/listeners/base.rb
|
227
227
|
- lib/karafka/web/tracking/producers/listeners/errors.rb
|
228
|
-
- lib/karafka/web/tracking/producers/listeners/reporter.rb
|
229
228
|
- lib/karafka/web/tracking/producers/reporter.rb
|
230
229
|
- lib/karafka/web/tracking/producers/sampler.rb
|
230
|
+
- lib/karafka/web/tracking/reporter.rb
|
231
231
|
- lib/karafka/web/tracking/sampler.rb
|
232
|
+
- lib/karafka/web/tracking/scheduler.rb
|
232
233
|
- lib/karafka/web/tracking/ttl_array.rb
|
233
234
|
- lib/karafka/web/tracking/ttl_hash.rb
|
234
235
|
- lib/karafka/web/ui/app.rb
|
@@ -247,6 +248,7 @@ files:
|
|
247
248
|
- lib/karafka/web/ui/controllers/status.rb
|
248
249
|
- lib/karafka/web/ui/helpers/application_helper.rb
|
249
250
|
- lib/karafka/web/ui/helpers/paths_helper.rb
|
251
|
+
- lib/karafka/web/ui/lib/admin.rb
|
250
252
|
- lib/karafka/web/ui/lib/hash_proxy.rb
|
251
253
|
- lib/karafka/web/ui/lib/paginations/base.rb
|
252
254
|
- lib/karafka/web/ui/lib/paginations/offset_based.rb
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,21 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Web
|
5
|
-
module Tracking
|
6
|
-
module Producers
|
7
|
-
module Listeners
|
8
|
-
# Special listener that we use to report data about producers states
|
9
|
-
# We don't have to have a separate thread for reporting, because producers have their
|
10
|
-
# own internal threads for changes polling and we can utilize this thread
|
11
|
-
class Reporter < Base
|
12
|
-
# @param _event [Karafka::Core::Monitoring::Event]
|
13
|
-
def on_statistics_emitted(_event)
|
14
|
-
reporter.report
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|