karafka-web 0.7.4 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +10 -4
- data/CHANGELOG.md +13 -0
- data/Gemfile.lock +3 -3
- data/bin/wait_for_kafka +24 -0
- data/docker-compose.yml +17 -16
- data/karafka-web.gemspec +1 -1
- data/lib/karafka/web/errors.rb +10 -1
- data/lib/karafka/web/installer.rb +27 -2
- data/lib/karafka/web/management/create_topics.rb +52 -46
- data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +56 -46
- data/lib/karafka/web/processing/consumers/metrics.rb +4 -0
- data/lib/karafka/web/processing/consumers/state.rb +4 -0
- data/lib/karafka/web/processing/time_series_tracker.rb +4 -1
- data/lib/karafka/web/ui/app.rb +1 -1
- data/lib/karafka/web/ui/base.rb +1 -1
- data/lib/karafka/web/ui/helpers/application_helper.rb +3 -2
- data/lib/karafka/web/ui/models/health.rb +5 -1
- data/lib/karafka/web/ui/pro/app.rb +1 -1
- data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +24 -8
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +0 -3
- data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +40 -34
- data/lib/karafka/web/ui/pro/views/routing/_detail.erb +11 -24
- data/lib/karafka/web/ui/public/javascripts/bootstrap.min.js +0 -1
- data/lib/karafka/web/ui/public/javascripts/chart.min.js +0 -1
- data/lib/karafka/web/ui/public/javascripts/timeago.min.js +5 -0
- data/lib/karafka/web/ui/public/stylesheets/bootstrap.min.css +0 -1
- data/lib/karafka/web/ui/views/consumers/_counters.erb +21 -7
- data/lib/karafka/web/ui/views/routing/_detail.erb +11 -24
- data/lib/karafka/web/ui/views/shared/_header.erb +1 -1
- data/lib/karafka/web/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +5 -5
- metadata.gz.sig +0 -0
- data/lib/karafka/web/ui/public/stylesheets/bootstrap.min.css.map +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 32657edc369df2240e6788c6972d0c83073263ad3e2cc416ff4d24bde1677f18
|
|
4
|
+
data.tar.gz: d013c25e74a87d1820f60912805888c42c8432a24e1ed33674276f8647328585
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 148e9fbe63c0029b1181aece3569f2822e108240b94c78e960b7e24a2921553dc64cd5c3d59f95fe555df2265bfd216fac97ba8906dbae2cb8c8611a66963400
|
|
7
|
+
data.tar.gz: c015c0c8cc8d15351c2620e3a4a87ee10a986277bf1f3e50fc0c7d2e9d66f839d596bf94273b067eac7fb2f1f231453ac91119ec125cea90fb8bd6eff8943333
|
checksums.yaml.gz.sig
CHANGED
|
Binary file
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -20,6 +20,7 @@ jobs:
|
|
|
20
20
|
fail-fast: false
|
|
21
21
|
matrix:
|
|
22
22
|
ruby:
|
|
23
|
+
- '3.3.0-preview2'
|
|
23
24
|
- '3.2'
|
|
24
25
|
- '3.1'
|
|
25
26
|
- '3.0'
|
|
@@ -28,18 +29,19 @@ jobs:
|
|
|
28
29
|
- ruby: '3.2'
|
|
29
30
|
coverage: 'true'
|
|
30
31
|
steps:
|
|
31
|
-
- uses: actions/checkout@
|
|
32
|
+
- uses: actions/checkout@v4
|
|
32
33
|
- name: Install package dependencies
|
|
33
34
|
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
|
34
35
|
|
|
35
36
|
- name: Start Kafka with docker-compose
|
|
36
37
|
run: |
|
|
37
|
-
docker-compose up -d
|
|
38
|
+
docker-compose up -d || (sleep 5 && docker-compose up -d)
|
|
38
39
|
|
|
39
40
|
- name: Set up Ruby
|
|
40
41
|
uses: ruby/setup-ruby@v1
|
|
41
42
|
with:
|
|
42
43
|
ruby-version: ${{matrix.ruby}}
|
|
44
|
+
bundler-cache: true
|
|
43
45
|
|
|
44
46
|
- name: Install latest bundler
|
|
45
47
|
run: |
|
|
@@ -51,6 +53,10 @@ jobs:
|
|
|
51
53
|
bundle config set without development
|
|
52
54
|
bundle install --jobs 4 --retry 3
|
|
53
55
|
|
|
56
|
+
- name: Wait for Kafka
|
|
57
|
+
run: |
|
|
58
|
+
bundle exec bin/wait_for_kafka
|
|
59
|
+
|
|
54
60
|
- name: Run all tests
|
|
55
61
|
env:
|
|
56
62
|
GITHUB_COVERAGE: ${{matrix.coverage}}
|
|
@@ -62,7 +68,7 @@ jobs:
|
|
|
62
68
|
strategy:
|
|
63
69
|
fail-fast: false
|
|
64
70
|
steps:
|
|
65
|
-
- uses: actions/checkout@
|
|
71
|
+
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
|
|
66
72
|
with:
|
|
67
73
|
fetch-depth: 0
|
|
68
74
|
|
|
@@ -83,7 +89,7 @@ jobs:
|
|
|
83
89
|
strategy:
|
|
84
90
|
fail-fast: false
|
|
85
91
|
steps:
|
|
86
|
-
- uses: actions/checkout@
|
|
92
|
+
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
|
|
87
93
|
with:
|
|
88
94
|
fetch-depth: 0
|
|
89
95
|
- name: Run Coditsu
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Karafka Web changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.6 (2023-10-10)
|
|
4
|
+
- [Fix] Fix nested SASL/SAML data visible in the routing details (#173)
|
|
5
|
+
|
|
6
|
+
## 0.7.5 (2023-09-29)
|
|
7
|
+
- [Enhancement] Update order of topics creation for the setup of Web to support zero-downtime setup of Web in running Karafka projects.
|
|
8
|
+
- [Enhancement] Add space delimiter to counters numbers to make them look better.
|
|
9
|
+
- [Improvement] Normalize per-process job tables and health tables structure (topic name on top).
|
|
10
|
+
- [Fix] Fix a case where charts aggregated data would not include all topics.
|
|
11
|
+
- [Fix] Make sure, that most recent per partition data for Health is never overwritten by an old state from a previous partition owner.
|
|
12
|
+
- [Fix] Cache assets for 1 year instead of 7 days.
|
|
13
|
+
- [Fix] Remove source maps pointing to non-existing locations.
|
|
14
|
+
- [Maintenance] Include license and copyrights notice for `timeago.js` that was missing in the JS min file.
|
|
15
|
+
|
|
3
16
|
## 0.7.4 (2023-09-19)
|
|
4
17
|
- [Improvement] Skip aggregations on older schemas during upgrades. This only skips process-reports (that are going to be rolled) on the 5s window in case of an upgrade that should not be a rolling one anyhow. This simplifies the operations and minimizes the risk on breaking upgrades.
|
|
5
18
|
- [Fix] Fix not working `ps` for macOS.
|
data/Gemfile.lock
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
karafka-web (0.7.
|
|
4
|
+
karafka-web (0.7.6)
|
|
5
5
|
erubi (~> 1.4)
|
|
6
|
-
karafka (>= 2.2.
|
|
6
|
+
karafka (>= 2.2.6, < 3.0.0)
|
|
7
7
|
karafka-core (>= 2.2.2, < 3.0.0)
|
|
8
8
|
roda (~> 3.68, >= 3.69)
|
|
9
9
|
tilt (~> 2.0)
|
|
@@ -26,7 +26,7 @@ GEM
|
|
|
26
26
|
ffi (1.15.5)
|
|
27
27
|
i18n (1.14.1)
|
|
28
28
|
concurrent-ruby (~> 1.0)
|
|
29
|
-
karafka (2.2.
|
|
29
|
+
karafka (2.2.6)
|
|
30
30
|
karafka-core (>= 2.2.2, < 2.3.0)
|
|
31
31
|
thor (>= 0.20)
|
|
32
32
|
waterdrop (>= 2.6.6, < 3.0.0)
|
data/bin/wait_for_kafka
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
# Waits for Kafka to be ready
|
|
4
|
+
# Useful in CI where Kafka needs to be fully started before we run any tests
|
|
5
|
+
|
|
6
|
+
require 'karafka'
|
|
7
|
+
|
|
8
|
+
Karafka::App.setup do |config|
|
|
9
|
+
config.kafka[:'bootstrap.servers'] = '127.0.0.1:9092'
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
60.times do
|
|
13
|
+
begin
|
|
14
|
+
# Stop if we can connect to the cluster and get info
|
|
15
|
+
exit if Karafka::Admin.cluster_info
|
|
16
|
+
rescue Rdkafka::RdkafkaError
|
|
17
|
+
puts "Kafka not available, retrying..."
|
|
18
|
+
sleep(1)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
puts 'Kafka not available!'
|
|
23
|
+
|
|
24
|
+
exit 1
|
data/docker-compose.yml
CHANGED
|
@@ -1,22 +1,23 @@
|
|
|
1
1
|
version: '2'
|
|
2
|
-
services:
|
|
3
|
-
zookeeper:
|
|
4
|
-
container_name: karafka_web_21_zookeeper
|
|
5
|
-
image: wurstmeister/zookeeper
|
|
6
|
-
restart: on-failure
|
|
7
|
-
ports:
|
|
8
|
-
- '2181:2181'
|
|
9
2
|
|
|
3
|
+
services:
|
|
10
4
|
kafka:
|
|
11
|
-
container_name:
|
|
12
|
-
image:
|
|
5
|
+
container_name: kafka
|
|
6
|
+
image: confluentinc/cp-kafka:7.5.1
|
|
7
|
+
|
|
13
8
|
ports:
|
|
14
|
-
-
|
|
9
|
+
- 9092:9092
|
|
10
|
+
|
|
15
11
|
environment:
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
CLUSTER_ID: kafka-docker-cluster-1
|
|
13
|
+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
|
14
|
+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
15
|
+
KAFKA_PROCESS_ROLES: broker,controller
|
|
16
|
+
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
|
|
17
|
+
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
|
|
18
|
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
|
|
19
|
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
|
|
20
|
+
KAFKA_BROKER_ID: 1
|
|
21
|
+
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
|
|
22
|
+
ALLOW_PLAINTEXT_LISTENER: 'yes'
|
|
19
23
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
|
20
|
-
volumes:
|
|
21
|
-
- /var/run/docker.sock:/var/run/docker.sock
|
|
22
|
-
restart: on-failure
|
data/karafka-web.gemspec
CHANGED
|
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
|
17
17
|
spec.licenses = %w[LGPL-3.0 Commercial]
|
|
18
18
|
|
|
19
19
|
spec.add_dependency 'erubi', '~> 1.4'
|
|
20
|
-
spec.add_dependency 'karafka', '>= 2.2.
|
|
20
|
+
spec.add_dependency 'karafka', '>= 2.2.6', '< 3.0.0'
|
|
21
21
|
spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
|
|
22
22
|
spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
|
|
23
23
|
spec.add_dependency 'tilt', '~> 2.0'
|
data/lib/karafka/web/errors.rb
CHANGED
|
@@ -17,9 +17,18 @@ module Karafka
|
|
|
17
17
|
# If you see this error, it probably means, that you did not bootstrap Web-UI correctly
|
|
18
18
|
MissingConsumersStateError = Class.new(BaseError)
|
|
19
19
|
|
|
20
|
-
#
|
|
20
|
+
# Raised when we try to materialize the state but the consumers states topic does not
|
|
21
|
+
# exist and we do not have a way to get the initial state.
|
|
22
|
+
# It differs from the above because above indicates that the topic exists but that there
|
|
23
|
+
# is no initial state, while this indicates, that there is no consumers states topic.
|
|
24
|
+
MissingConsumersStatesTopicError = Class.new(BaseError)
|
|
25
|
+
|
|
26
|
+
# Similar to the above. It should be created during install / migration
|
|
21
27
|
MissingConsumersMetricsError = Class.new(BaseError)
|
|
22
28
|
|
|
29
|
+
# Similar to the one related to consumers states
|
|
30
|
+
MissingConsumersMetricsTopicError = Class.new(BaseError)
|
|
31
|
+
|
|
23
32
|
# This error occurs when consumer running older version of the web-ui tries to materialize
|
|
24
33
|
# states from newer versions. Karafka Web-UI provides only backwards compatibility, so
|
|
25
34
|
# you need to have an up-to-date consumer materializing reported states.
|
|
@@ -18,7 +18,7 @@ module Karafka
|
|
|
18
18
|
puts 'Creating necessary topics and populating state data...'
|
|
19
19
|
puts
|
|
20
20
|
Management::CreateTopics.new.call(replication_factor)
|
|
21
|
-
|
|
21
|
+
wait_for_topics
|
|
22
22
|
Management::CreateInitialStates.new.call
|
|
23
23
|
puts
|
|
24
24
|
Management::ExtendBootFile.new.call
|
|
@@ -36,6 +36,7 @@ module Karafka
|
|
|
36
36
|
puts 'Creating necessary topics and populating state data...'
|
|
37
37
|
puts
|
|
38
38
|
Management::CreateTopics.new.call(replication_factor)
|
|
39
|
+
wait_for_topics
|
|
39
40
|
Management::CreateInitialStates.new.call
|
|
40
41
|
puts
|
|
41
42
|
puts("Migration #{green('completed')}. Have fun!")
|
|
@@ -51,7 +52,7 @@ module Karafka
|
|
|
51
52
|
Management::DeleteTopics.new.call
|
|
52
53
|
puts
|
|
53
54
|
Management::CreateTopics.new.call(replication_factor)
|
|
54
|
-
|
|
55
|
+
wait_for_topics
|
|
55
56
|
Management::CreateInitialStates.new.call
|
|
56
57
|
puts
|
|
57
58
|
puts("Resetting #{green('completed')}. Have fun!")
|
|
@@ -74,6 +75,30 @@ module Karafka
|
|
|
74
75
|
def enable!
|
|
75
76
|
Management::Enable.new.call
|
|
76
77
|
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# Waits with a message, that we are waiting on topics
|
|
82
|
+
# This is not doing much, just waiting as there are some cases that it takes a bit of time
|
|
83
|
+
# for Kafka to actually propagate new topics knowledge across the cluster. We give it that
|
|
84
|
+
# bit of time just in case.
|
|
85
|
+
def wait_for_topics
|
|
86
|
+
puts
|
|
87
|
+
print 'Waiting for the topics to synchronize in the cluster'
|
|
88
|
+
wait(5)
|
|
89
|
+
puts
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Waits for given number of seconds and prints `.` every second.
|
|
93
|
+
# @param time_in_seconds [Integer] time of wait
|
|
94
|
+
def wait(time_in_seconds)
|
|
95
|
+
time_in_seconds.times do
|
|
96
|
+
sleep(1)
|
|
97
|
+
print '.'
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
print "\n"
|
|
101
|
+
end
|
|
77
102
|
end
|
|
78
103
|
end
|
|
79
104
|
end
|
|
@@ -9,54 +9,36 @@ module Karafka
|
|
|
9
9
|
# Runs the creation process
|
|
10
10
|
#
|
|
11
11
|
# @param replication_factor [Integer] replication factor for Web-UI topics
|
|
12
|
+
#
|
|
13
|
+
# @note The order of creation of those topics is important. In order to support the
|
|
14
|
+
# zero-downtime bootstrap, we use the presence of the states topic and its initial state
|
|
15
|
+
# existence as an indicator that the setup went as expected. It the consumers states
|
|
16
|
+
# topic exists and contains needed data, it means all went as expected and that
|
|
17
|
+
# topics created before it also exist (as no error).
|
|
12
18
|
def call(replication_factor)
|
|
13
19
|
consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
|
|
14
20
|
consumers_metrics_topic = ::Karafka::Web.config.topics.consumers.metrics
|
|
15
21
|
consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
|
|
16
22
|
errors_topic = ::Karafka::Web.config.topics.errors
|
|
17
23
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
exists(consumers_states_topic)
|
|
21
|
-
else
|
|
22
|
-
creating(consumers_states_topic)
|
|
23
|
-
# This topic needs to have one partition
|
|
24
|
-
::Karafka::Admin.create_topic(
|
|
25
|
-
consumers_states_topic,
|
|
26
|
-
1,
|
|
27
|
-
replication_factor,
|
|
28
|
-
# We care only about the most recent state, previous are irrelevant. So we can easily
|
|
29
|
-
# compact after one minute. We do not use this beyond the most recent collective
|
|
30
|
-
# state, hence it all can easily go away. We also limit the segment size to at most
|
|
31
|
-
# 100MB not to use more space ever.
|
|
32
|
-
{
|
|
33
|
-
'cleanup.policy': 'compact',
|
|
34
|
-
'retention.ms': 60 * 60 * 1_000,
|
|
35
|
-
'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
|
|
36
|
-
'segment.bytes': 104_857_600 # 100MB
|
|
37
|
-
}
|
|
38
|
-
)
|
|
39
|
-
created(consumers_states_topic)
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
if existing_topics_names.include?(consumers_metrics_topic)
|
|
43
|
-
exists(consumers_metrics_topic)
|
|
24
|
+
if existing_topics_names.include?(errors_topic)
|
|
25
|
+
exists(errors_topic)
|
|
44
26
|
else
|
|
45
|
-
creating(
|
|
46
|
-
#
|
|
47
|
-
#
|
|
27
|
+
creating(errors_topic)
|
|
28
|
+
# All the errors will be dispatched here
|
|
29
|
+
# This topic can have multiple partitions but we go with one by default. A single Ruby
|
|
30
|
+
# process should not crash that often and if there is an expectation of a higher volume
|
|
31
|
+
# of errors, this can be changed by the end user
|
|
48
32
|
::Karafka::Admin.create_topic(
|
|
49
|
-
|
|
33
|
+
errors_topic,
|
|
50
34
|
1,
|
|
51
35
|
replication_factor,
|
|
36
|
+
# Remove really old errors (older than 3 months just to preserve space)
|
|
52
37
|
{
|
|
53
|
-
'
|
|
54
|
-
'retention.ms': 60 * 60 * 1_000, # 1h
|
|
55
|
-
'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
|
|
56
|
-
'segment.bytes': 104_857_600 # 100MB
|
|
38
|
+
'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
|
|
57
39
|
}
|
|
58
40
|
)
|
|
59
|
-
created(
|
|
41
|
+
created(errors_topic)
|
|
60
42
|
end
|
|
61
43
|
|
|
62
44
|
if existing_topics_names.include?(consumers_reports_topic)
|
|
@@ -81,24 +63,48 @@ module Karafka
|
|
|
81
63
|
created(consumers_reports_topic)
|
|
82
64
|
end
|
|
83
65
|
|
|
84
|
-
if existing_topics_names.include?(
|
|
85
|
-
exists(
|
|
66
|
+
if existing_topics_names.include?(consumers_metrics_topic)
|
|
67
|
+
exists(consumers_metrics_topic)
|
|
86
68
|
else
|
|
87
|
-
creating(
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
# process should not crash that often and if there is an expectation of a higher volume
|
|
91
|
-
# of errors, this can be changed by the end user
|
|
69
|
+
creating(consumers_metrics_topic)
|
|
70
|
+
# This topic needs to have one partition
|
|
71
|
+
# Same as states - only most recent is relevant as it is a materialized state
|
|
92
72
|
::Karafka::Admin.create_topic(
|
|
93
|
-
|
|
73
|
+
consumers_metrics_topic,
|
|
94
74
|
1,
|
|
95
75
|
replication_factor,
|
|
96
|
-
# Remove really old errors (older than 3 months just to preserve space)
|
|
97
76
|
{
|
|
98
|
-
'
|
|
77
|
+
'cleanup.policy': 'compact',
|
|
78
|
+
'retention.ms': 60 * 60 * 1_000, # 1h
|
|
79
|
+
'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
|
|
80
|
+
'segment.bytes': 104_857_600 # 100MB
|
|
99
81
|
}
|
|
100
82
|
)
|
|
101
|
-
created(
|
|
83
|
+
created(consumers_metrics_topic)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Create only if needed
|
|
87
|
+
if existing_topics_names.include?(consumers_states_topic)
|
|
88
|
+
exists(consumers_states_topic)
|
|
89
|
+
else
|
|
90
|
+
creating(consumers_states_topic)
|
|
91
|
+
# This topic needs to have one partition
|
|
92
|
+
::Karafka::Admin.create_topic(
|
|
93
|
+
consumers_states_topic,
|
|
94
|
+
1,
|
|
95
|
+
replication_factor,
|
|
96
|
+
# We care only about the most recent state, previous are irrelevant. So we can easily
|
|
97
|
+
# compact after one minute. We do not use this beyond the most recent collective
|
|
98
|
+
# state, hence it all can easily go away. We also limit the segment size to at most
|
|
99
|
+
# 100MB not to use more space ever.
|
|
100
|
+
{
|
|
101
|
+
'cleanup.policy': 'compact',
|
|
102
|
+
'retention.ms': 60 * 60 * 1_000,
|
|
103
|
+
'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
|
|
104
|
+
'segment.bytes': 104_857_600 # 100MB
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
created(consumers_states_topic)
|
|
102
108
|
end
|
|
103
109
|
end
|
|
104
110
|
|
|
@@ -86,65 +86,75 @@ module Karafka
|
|
|
86
86
|
|
|
87
87
|
# Materializes the current state of consumers group data
|
|
88
88
|
#
|
|
89
|
-
# At the moment we report only topics lags but the format we are using supports
|
|
90
|
-
# extending this information in the future if it would be needed.
|
|
91
|
-
#
|
|
92
89
|
# @return [Hash] hash with nested consumers and their topics details structure
|
|
93
90
|
# @note We do **not** report on a per partition basis because it would significantly
|
|
94
91
|
# increase needed storage.
|
|
95
92
|
def materialize_consumers_groups_current_state
|
|
96
93
|
cgs = {}
|
|
97
94
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
95
|
+
iterate_partitions_data do |group_name, topic_name, partitions_data|
|
|
96
|
+
lags = partitions_data
|
|
97
|
+
.map { |p_details| p_details.fetch(:lag, -1) }
|
|
98
|
+
.reject(&:negative?)
|
|
99
|
+
|
|
100
|
+
lags_stored = partitions_data
|
|
101
|
+
.map { |p_details| p_details.fetch(:lag_stored, -1) }
|
|
102
|
+
.reject(&:negative?)
|
|
103
103
|
|
|
104
|
-
|
|
105
|
-
.map { |p_details| p_details.fetch(:
|
|
104
|
+
offsets_hi = partitions_data
|
|
105
|
+
.map { |p_details| p_details.fetch(:hi_offset, -1) }
|
|
106
106
|
.reject(&:negative?)
|
|
107
107
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
108
|
+
# Last stable offsets freeze durations - we pick the max freeze to indicate
|
|
109
|
+
# the longest open transaction that potentially may be hanging
|
|
110
|
+
ls_offsets_fd = partitions_data
|
|
111
|
+
.map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
|
|
112
|
+
.reject(&:negative?)
|
|
113
|
+
|
|
114
|
+
cgs[group_name] ||= {}
|
|
115
|
+
cgs[group_name][topic_name] = {
|
|
116
|
+
lag_stored: lags_stored.sum,
|
|
117
|
+
lag: lags.sum,
|
|
118
|
+
pace: offsets_hi.sum,
|
|
119
|
+
# Take max last stable offset duration without any change. This can
|
|
120
|
+
# indicate a hanging transaction, because the offset will not move forward
|
|
121
|
+
# and will stay with a growing freeze duration when stuck
|
|
122
|
+
ls_offset_fd: ls_offsets_fd.max || 0
|
|
123
|
+
}
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
cgs
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Converts our reports data into an iterator per partition
|
|
130
|
+
# Compensates for a case where same partition data would be available for a short
|
|
131
|
+
# period of time in multiple processes reports due to rebalances.
|
|
132
|
+
def iterate_partitions_data
|
|
133
|
+
cgs_topics = Hash.new { |h, v| h[v] = Hash.new { |h2, v2| h2[v2] = {} } }
|
|
134
|
+
|
|
135
|
+
# We need to sort them in case we have same reports containing data about same
|
|
136
|
+
# topics partitions. Mostly during shutdowns and rebalances
|
|
137
|
+
@active_reports
|
|
138
|
+
.values
|
|
139
|
+
.sort_by { |report| report.fetch(:dispatched_at) }
|
|
140
|
+
.map { |details| details.fetch(:consumer_groups) }
|
|
141
|
+
.each do |consumer_groups|
|
|
142
|
+
consumer_groups.each do |group_name, group_details|
|
|
143
|
+
group_details.fetch(:subscription_groups).each_value do |sg_details|
|
|
144
|
+
sg_details.fetch(:topics).each do |topic_name, topic_details|
|
|
145
|
+
topic_details.fetch(:partitions).each do |partition_id, partition_data|
|
|
146
|
+
cgs_topics[group_name][topic_name][partition_id] = partition_data
|
|
147
|
+
end
|
|
148
|
+
end
|
|
142
149
|
end
|
|
143
150
|
end
|
|
144
151
|
end
|
|
145
|
-
end
|
|
146
152
|
|
|
147
|
-
|
|
153
|
+
cgs_topics.each do |group_name, topics_data|
|
|
154
|
+
topics_data.each do |topic_name, partitions_data|
|
|
155
|
+
yield(group_name, topic_name, partitions_data.values)
|
|
156
|
+
end
|
|
157
|
+
end
|
|
148
158
|
end
|
|
149
159
|
end
|
|
150
160
|
end
|
|
@@ -20,6 +20,10 @@ module Karafka
|
|
|
20
20
|
return metrics_message.payload if metrics_message
|
|
21
21
|
|
|
22
22
|
raise(::Karafka::Web::Errors::Processing::MissingConsumersMetricsError)
|
|
23
|
+
rescue Rdkafka::RdkafkaError => e
|
|
24
|
+
raise(e) unless e.code == :unknown_partition
|
|
25
|
+
|
|
26
|
+
raise(::Karafka::Web::Errors::Processing::MissingConsumersMetricsTopicError)
|
|
23
27
|
end
|
|
24
28
|
end
|
|
25
29
|
end
|
|
@@ -20,6 +20,10 @@ module Karafka
|
|
|
20
20
|
return state_message.payload if state_message
|
|
21
21
|
|
|
22
22
|
raise(::Karafka::Web::Errors::Processing::MissingConsumersStateError)
|
|
23
|
+
rescue Rdkafka::RdkafkaError => e
|
|
24
|
+
raise(e) unless e.code == :unknown_partition
|
|
25
|
+
|
|
26
|
+
raise(::Karafka::Web::Errors::Processing::MissingConsumersStatesTopicError)
|
|
23
27
|
end
|
|
24
28
|
end
|
|
25
29
|
end
|
|
@@ -116,9 +116,12 @@ module Karafka
|
|
|
116
116
|
# available
|
|
117
117
|
times << values.last unless values.empty?
|
|
118
118
|
|
|
119
|
+
# Keep the most recent state out of many that would come from the same time moment
|
|
120
|
+
# Squash in case there would be two events from the same time
|
|
121
|
+
times.reverse!
|
|
119
122
|
times.uniq!(&:first)
|
|
123
|
+
times.reverse!
|
|
120
124
|
|
|
121
|
-
# Squash in case there would be two events from the same time
|
|
122
125
|
times.sort_by!(&:first)
|
|
123
126
|
|
|
124
127
|
@historicals[range_name] = times.last(limit)
|
data/lib/karafka/web/ui/app.rb
CHANGED
data/lib/karafka/web/ui/base.rb
CHANGED
|
@@ -16,7 +16,7 @@ module Karafka
|
|
|
16
16
|
root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public'),
|
|
17
17
|
# Cache all static files for the end user for as long as possible
|
|
18
18
|
# We can do it because we ship per version assets so they invalidate with gem bumps
|
|
19
|
-
headers: { 'Cache-Control' => 'max-age=
|
|
19
|
+
headers: { 'Cache-Control' => 'max-age=31536000, immutable' }
|
|
20
20
|
)
|
|
21
21
|
plugin :render_each
|
|
22
22
|
plugin :partials
|
|
@@ -101,12 +101,13 @@ module Karafka
|
|
|
101
101
|
|
|
102
102
|
# Converts number to a more friendly delimiter based version
|
|
103
103
|
# @param number [Numeric]
|
|
104
|
+
# @param delimiter [String] delimiter (comma by default)
|
|
104
105
|
# @return [String] number with delimiter
|
|
105
|
-
def number_with_delimiter(number)
|
|
106
|
+
def number_with_delimiter(number, delimiter = ',')
|
|
106
107
|
return '' unless number
|
|
107
108
|
|
|
108
109
|
parts = number.to_s.to_str.split('.')
|
|
109
|
-
parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/,
|
|
110
|
+
parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
|
|
110
111
|
parts.join('.')
|
|
111
112
|
end
|
|
112
113
|
|
|
@@ -62,7 +62,11 @@ module Karafka
|
|
|
62
62
|
#
|
|
63
63
|
# @param state [State]
|
|
64
64
|
def iterate_partitions(state)
|
|
65
|
-
processes
|
|
65
|
+
# By default processes are sort by name and this is not what we want here
|
|
66
|
+
# We want to make sure that the newest data is processed the last, so we get
|
|
67
|
+
# the most accurate state in case of deployments and shutdowns, etc without the
|
|
68
|
+
# expired processes partitions data overwriting the newly created processes
|
|
69
|
+
processes = Processes.active(state).sort_by!(&:dispatched_at)
|
|
66
70
|
|
|
67
71
|
processes.each do |process|
|
|
68
72
|
process.consumer_groups.each do |consumer_group|
|