karafka-web 0.7.4 → 0.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +10 -4
  4. data/CHANGELOG.md +13 -0
  5. data/Gemfile.lock +3 -3
  6. data/bin/wait_for_kafka +24 -0
  7. data/docker-compose.yml +17 -16
  8. data/karafka-web.gemspec +1 -1
  9. data/lib/karafka/web/errors.rb +10 -1
  10. data/lib/karafka/web/installer.rb +27 -2
  11. data/lib/karafka/web/management/create_topics.rb +52 -46
  12. data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +56 -46
  13. data/lib/karafka/web/processing/consumers/metrics.rb +4 -0
  14. data/lib/karafka/web/processing/consumers/state.rb +4 -0
  15. data/lib/karafka/web/processing/time_series_tracker.rb +4 -1
  16. data/lib/karafka/web/ui/app.rb +1 -1
  17. data/lib/karafka/web/ui/base.rb +1 -1
  18. data/lib/karafka/web/ui/helpers/application_helper.rb +3 -2
  19. data/lib/karafka/web/ui/models/health.rb +5 -1
  20. data/lib/karafka/web/ui/pro/app.rb +1 -1
  21. data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +24 -8
  22. data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +0 -3
  23. data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +40 -34
  24. data/lib/karafka/web/ui/pro/views/routing/_detail.erb +11 -24
  25. data/lib/karafka/web/ui/public/javascripts/bootstrap.min.js +0 -1
  26. data/lib/karafka/web/ui/public/javascripts/chart.min.js +0 -1
  27. data/lib/karafka/web/ui/public/javascripts/timeago.min.js +5 -0
  28. data/lib/karafka/web/ui/public/stylesheets/bootstrap.min.css +0 -1
  29. data/lib/karafka/web/ui/views/consumers/_counters.erb +21 -7
  30. data/lib/karafka/web/ui/views/routing/_detail.erb +11 -24
  31. data/lib/karafka/web/ui/views/shared/_header.erb +1 -1
  32. data/lib/karafka/web/version.rb +1 -1
  33. data.tar.gz.sig +0 -0
  34. metadata +5 -5
  35. metadata.gz.sig +0 -0
  36. data/lib/karafka/web/ui/public/stylesheets/bootstrap.min.css.map +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7a31ebcde158ca18cc353d3fefad976b7dfb8d9fe3b3114ea8f43b533229434a
4
- data.tar.gz: f9870e5268ebebb38c838dc8a220030beaebfe30d4d53cd63537c5c2f442e2d4
3
+ metadata.gz: 32657edc369df2240e6788c6972d0c83073263ad3e2cc416ff4d24bde1677f18
4
+ data.tar.gz: d013c25e74a87d1820f60912805888c42c8432a24e1ed33674276f8647328585
5
5
  SHA512:
6
- metadata.gz: '02152094838c0be1606f49b4963e7d0791f1800bc7b7fbf71240866ed84e73bc264322415b48272faf46c715d70568fc054fc7183406d182d3c95eeb664019db'
7
- data.tar.gz: 787a16d7f91f7ccb907a3cbea0599337bd7525b1f2982f023b639227df1b52511afcc763422c71d0e2bf34216d3f3a4fe6794ae8824abeea9302b84071de8ca5
6
+ metadata.gz: 148e9fbe63c0029b1181aece3569f2822e108240b94c78e960b7e24a2921553dc64cd5c3d59f95fe555df2265bfd216fac97ba8906dbae2cb8c8611a66963400
7
+ data.tar.gz: c015c0c8cc8d15351c2620e3a4a87ee10a986277bf1f3e50fc0c7d2e9d66f839d596bf94273b067eac7fb2f1f231453ac91119ec125cea90fb8bd6eff8943333
checksums.yaml.gz.sig CHANGED
Binary file
@@ -20,6 +20,7 @@ jobs:
20
20
  fail-fast: false
21
21
  matrix:
22
22
  ruby:
23
+ - '3.3.0-preview2'
23
24
  - '3.2'
24
25
  - '3.1'
25
26
  - '3.0'
@@ -28,18 +29,19 @@ jobs:
28
29
  - ruby: '3.2'
29
30
  coverage: 'true'
30
31
  steps:
31
- - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
32
+ - uses: actions/checkout@v4
32
33
  - name: Install package dependencies
33
34
  run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
34
35
 
35
36
  - name: Start Kafka with docker-compose
36
37
  run: |
37
- docker-compose up -d
38
+ docker-compose up -d || (sleep 5 && docker-compose up -d)
38
39
 
39
40
  - name: Set up Ruby
40
41
  uses: ruby/setup-ruby@v1
41
42
  with:
42
43
  ruby-version: ${{matrix.ruby}}
44
+ bundler-cache: true
43
45
 
44
46
  - name: Install latest bundler
45
47
  run: |
@@ -51,6 +53,10 @@ jobs:
51
53
  bundle config set without development
52
54
  bundle install --jobs 4 --retry 3
53
55
 
56
+ - name: Wait for Kafka
57
+ run: |
58
+ bundle exec bin/wait_for_kafka
59
+
54
60
  - name: Run all tests
55
61
  env:
56
62
  GITHUB_COVERAGE: ${{matrix.coverage}}
@@ -62,7 +68,7 @@ jobs:
62
68
  strategy:
63
69
  fail-fast: false
64
70
  steps:
65
- - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
71
+ - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
66
72
  with:
67
73
  fetch-depth: 0
68
74
 
@@ -83,7 +89,7 @@ jobs:
83
89
  strategy:
84
90
  fail-fast: false
85
91
  steps:
86
- - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
92
+ - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
87
93
  with:
88
94
  fetch-depth: 0
89
95
  - name: Run Coditsu
data/CHANGELOG.md CHANGED
@@ -1,5 +1,18 @@
1
1
  # Karafka Web changelog
2
2
 
3
+ ## 0.7.6 (2023-10-10)
4
+ - [Fix] Fix nested SASL/SAML data visible in the routing details (#173)
5
+
6
+ ## 0.7.5 (2023-09-29)
7
+ - [Enhancement] Update order of topics creation for the setup of Web to support zero-downtime setup of Web in running Karafka projects.
8
+ - [Enhancement] Add space delimiter to counters numbers to make them look better.
9
+ - [Improvement] Normalize per-process job tables and health tables structure (topic name on top).
10
+ - [Fix] Fix a case where charts aggregated data would not include all topics.
11
+ - [Fix] Make sure, that most recent per partition data for Health is never overwritten by an old state from a previous partition owner.
12
+ - [Fix] Cache assets for 1 year instead of 7 days.
13
+ - [Fix] Remove source maps pointing to non-existing locations.
14
+ - [Maintenance] Include license and copyrights notice for `timeago.js` that was missing in the JS min file.
15
+
3
16
  ## 0.7.4 (2023-09-19)
4
17
  - [Improvement] Skip aggregations on older schemas during upgrades. This only skips process-reports (that are going to be rolled) on the 5s window in case of an upgrade that should not be a rolling one anyhow. This simplifies the operations and minimizes the risk on breaking upgrades.
5
18
  - [Fix] Fix not working `ps` for macOS.
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka-web (0.7.4)
4
+ karafka-web (0.7.6)
5
5
  erubi (~> 1.4)
6
- karafka (>= 2.2.3, < 3.0.0)
6
+ karafka (>= 2.2.6, < 3.0.0)
7
7
  karafka-core (>= 2.2.2, < 3.0.0)
8
8
  roda (~> 3.68, >= 3.69)
9
9
  tilt (~> 2.0)
@@ -26,7 +26,7 @@ GEM
26
26
  ffi (1.15.5)
27
27
  i18n (1.14.1)
28
28
  concurrent-ruby (~> 1.0)
29
- karafka (2.2.3)
29
+ karafka (2.2.6)
30
30
  karafka-core (>= 2.2.2, < 2.3.0)
31
31
  thor (>= 0.20)
32
32
  waterdrop (>= 2.6.6, < 3.0.0)
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Waits for Kafka to be ready
4
+ # Useful in CI where Kafka needs to be fully started before we run any tests
5
+
6
+ require 'karafka'
7
+
8
+ Karafka::App.setup do |config|
9
+ config.kafka[:'bootstrap.servers'] = '127.0.0.1:9092'
10
+ end
11
+
12
+ 60.times do
13
+ begin
14
+ # Stop if we can connect to the cluster and get info
15
+ exit if Karafka::Admin.cluster_info
16
+ rescue Rdkafka::RdkafkaError
17
+ puts "Kafka not available, retrying..."
18
+ sleep(1)
19
+ end
20
+ end
21
+
22
+ puts 'Kafka not available!'
23
+
24
+ exit 1
data/docker-compose.yml CHANGED
@@ -1,22 +1,23 @@
1
1
  version: '2'
2
- services:
3
- zookeeper:
4
- container_name: karafka_web_21_zookeeper
5
- image: wurstmeister/zookeeper
6
- restart: on-failure
7
- ports:
8
- - '2181:2181'
9
2
 
3
+ services:
10
4
  kafka:
11
- container_name: karafka_web_21_kafka
12
- image: wurstmeister/kafka
5
+ container_name: kafka
6
+ image: confluentinc/cp-kafka:7.5.1
7
+
13
8
  ports:
14
- - '9092:9092'
9
+ - 9092:9092
10
+
15
11
  environment:
16
- KAFKA_ADVERTISED_HOST_NAME: localhost
17
- KAFKA_ADVERTISED_PORT: 9092
18
- KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
12
+ CLUSTER_ID: kafka-docker-cluster-1
13
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
14
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
15
+ KAFKA_PROCESS_ROLES: broker,controller
16
+ KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
17
+ KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
18
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
19
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
20
+ KAFKA_BROKER_ID: 1
21
+ KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
22
+ ALLOW_PLAINTEXT_LISTENER: 'yes'
19
23
  KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
20
- volumes:
21
- - /var/run/docker.sock:/var/run/docker.sock
22
- restart: on-failure
data/karafka-web.gemspec CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
17
17
  spec.licenses = %w[LGPL-3.0 Commercial]
18
18
 
19
19
  spec.add_dependency 'erubi', '~> 1.4'
20
- spec.add_dependency 'karafka', '>= 2.2.3', '< 3.0.0'
20
+ spec.add_dependency 'karafka', '>= 2.2.6', '< 3.0.0'
21
21
  spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
22
22
  spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
23
23
  spec.add_dependency 'tilt', '~> 2.0'
@@ -17,9 +17,18 @@ module Karafka
17
17
  # If you see this error, it probably means, that you did not bootstrap Web-UI correctly
18
18
  MissingConsumersStateError = Class.new(BaseError)
19
19
 
20
- # Similar to the above. It should be created during install
20
+ # Raised when we try to materialize the state but the consumers states topic does not
21
+ # exist and we do not have a way to get the initial state.
22
+ # It differs from the above because above indicates that the topic exists but that there
23
+ # is no initial state, while this indicates, that there is no consumers states topic.
24
+ MissingConsumersStatesTopicError = Class.new(BaseError)
25
+
26
+ # Similar to the above. It should be created during install / migration
21
27
  MissingConsumersMetricsError = Class.new(BaseError)
22
28
 
29
+ # Similar to the one related to consumers states
30
+ MissingConsumersMetricsTopicError = Class.new(BaseError)
31
+
23
32
  # This error occurs when consumer running older version of the web-ui tries to materialize
24
33
  # states from newer versions. Karafka Web-UI provides only backwards compatibility, so
25
34
  # you need to have an up-to-date consumer materializing reported states.
@@ -18,7 +18,7 @@ module Karafka
18
18
  puts 'Creating necessary topics and populating state data...'
19
19
  puts
20
20
  Management::CreateTopics.new.call(replication_factor)
21
- puts
21
+ wait_for_topics
22
22
  Management::CreateInitialStates.new.call
23
23
  puts
24
24
  Management::ExtendBootFile.new.call
@@ -36,6 +36,7 @@ module Karafka
36
36
  puts 'Creating necessary topics and populating state data...'
37
37
  puts
38
38
  Management::CreateTopics.new.call(replication_factor)
39
+ wait_for_topics
39
40
  Management::CreateInitialStates.new.call
40
41
  puts
41
42
  puts("Migration #{green('completed')}. Have fun!")
@@ -51,7 +52,7 @@ module Karafka
51
52
  Management::DeleteTopics.new.call
52
53
  puts
53
54
  Management::CreateTopics.new.call(replication_factor)
54
- puts
55
+ wait_for_topics
55
56
  Management::CreateInitialStates.new.call
56
57
  puts
57
58
  puts("Resetting #{green('completed')}. Have fun!")
@@ -74,6 +75,30 @@ module Karafka
74
75
  def enable!
75
76
  Management::Enable.new.call
76
77
  end
78
+
79
+ private
80
+
81
+ # Waits with a message, that we are waiting on topics
82
+ # This is not doing much, just waiting as there are some cases that it takes a bit of time
83
+ # for Kafka to actually propagate new topics knowledge across the cluster. We give it that
84
+ # bit of time just in case.
85
+ def wait_for_topics
86
+ puts
87
+ print 'Waiting for the topics to synchronize in the cluster'
88
+ wait(5)
89
+ puts
90
+ end
91
+
92
+ # Waits for given number of seconds and prints `.` every second.
93
+ # @param time_in_seconds [Integer] time of wait
94
+ def wait(time_in_seconds)
95
+ time_in_seconds.times do
96
+ sleep(1)
97
+ print '.'
98
+ end
99
+
100
+ print "\n"
101
+ end
77
102
  end
78
103
  end
79
104
  end
@@ -9,54 +9,36 @@ module Karafka
9
9
  # Runs the creation process
10
10
  #
11
11
  # @param replication_factor [Integer] replication factor for Web-UI topics
12
+ #
13
+ # @note The order of creation of those topics is important. In order to support the
14
+ # zero-downtime bootstrap, we use the presence of the states topic and its initial state
15
+ # existence as an indicator that the setup went as expected. It the consumers states
16
+ # topic exists and contains needed data, it means all went as expected and that
17
+ # topics created before it also exist (as no error).
12
18
  def call(replication_factor)
13
19
  consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
14
20
  consumers_metrics_topic = ::Karafka::Web.config.topics.consumers.metrics
15
21
  consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
16
22
  errors_topic = ::Karafka::Web.config.topics.errors
17
23
 
18
- # Create only if needed
19
- if existing_topics_names.include?(consumers_states_topic)
20
- exists(consumers_states_topic)
21
- else
22
- creating(consumers_states_topic)
23
- # This topic needs to have one partition
24
- ::Karafka::Admin.create_topic(
25
- consumers_states_topic,
26
- 1,
27
- replication_factor,
28
- # We care only about the most recent state, previous are irrelevant. So we can easily
29
- # compact after one minute. We do not use this beyond the most recent collective
30
- # state, hence it all can easily go away. We also limit the segment size to at most
31
- # 100MB not to use more space ever.
32
- {
33
- 'cleanup.policy': 'compact',
34
- 'retention.ms': 60 * 60 * 1_000,
35
- 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
36
- 'segment.bytes': 104_857_600 # 100MB
37
- }
38
- )
39
- created(consumers_states_topic)
40
- end
41
-
42
- if existing_topics_names.include?(consumers_metrics_topic)
43
- exists(consumers_metrics_topic)
24
+ if existing_topics_names.include?(errors_topic)
25
+ exists(errors_topic)
44
26
  else
45
- creating(consumers_metrics_topic)
46
- # This topic needs to have one partition
47
- # Same as states - only most recent is relevant as it is a materialized state
27
+ creating(errors_topic)
28
+ # All the errors will be dispatched here
29
+ # This topic can have multiple partitions but we go with one by default. A single Ruby
30
+ # process should not crash that often and if there is an expectation of a higher volume
31
+ # of errors, this can be changed by the end user
48
32
  ::Karafka::Admin.create_topic(
49
- consumers_metrics_topic,
33
+ errors_topic,
50
34
  1,
51
35
  replication_factor,
36
+ # Remove really old errors (older than 3 months just to preserve space)
52
37
  {
53
- 'cleanup.policy': 'compact',
54
- 'retention.ms': 60 * 60 * 1_000, # 1h
55
- 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
56
- 'segment.bytes': 104_857_600 # 100MB
38
+ 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
57
39
  }
58
40
  )
59
- created(consumers_metrics_topic)
41
+ created(errors_topic)
60
42
  end
61
43
 
62
44
  if existing_topics_names.include?(consumers_reports_topic)
@@ -81,24 +63,48 @@ module Karafka
81
63
  created(consumers_reports_topic)
82
64
  end
83
65
 
84
- if existing_topics_names.include?(errors_topic)
85
- exists(errors_topic)
66
+ if existing_topics_names.include?(consumers_metrics_topic)
67
+ exists(consumers_metrics_topic)
86
68
  else
87
- creating(errors_topic)
88
- # All the errors will be dispatched here
89
- # This topic can have multiple partitions but we go with one by default. A single Ruby
90
- # process should not crash that often and if there is an expectation of a higher volume
91
- # of errors, this can be changed by the end user
69
+ creating(consumers_metrics_topic)
70
+ # This topic needs to have one partition
71
+ # Same as states - only most recent is relevant as it is a materialized state
92
72
  ::Karafka::Admin.create_topic(
93
- errors_topic,
73
+ consumers_metrics_topic,
94
74
  1,
95
75
  replication_factor,
96
- # Remove really old errors (older than 3 months just to preserve space)
97
76
  {
98
- 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
77
+ 'cleanup.policy': 'compact',
78
+ 'retention.ms': 60 * 60 * 1_000, # 1h
79
+ 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
80
+ 'segment.bytes': 104_857_600 # 100MB
99
81
  }
100
82
  )
101
- created(errors_topic)
83
+ created(consumers_metrics_topic)
84
+ end
85
+
86
+ # Create only if needed
87
+ if existing_topics_names.include?(consumers_states_topic)
88
+ exists(consumers_states_topic)
89
+ else
90
+ creating(consumers_states_topic)
91
+ # This topic needs to have one partition
92
+ ::Karafka::Admin.create_topic(
93
+ consumers_states_topic,
94
+ 1,
95
+ replication_factor,
96
+ # We care only about the most recent state, previous are irrelevant. So we can easily
97
+ # compact after one minute. We do not use this beyond the most recent collective
98
+ # state, hence it all can easily go away. We also limit the segment size to at most
99
+ # 100MB not to use more space ever.
100
+ {
101
+ 'cleanup.policy': 'compact',
102
+ 'retention.ms': 60 * 60 * 1_000,
103
+ 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
104
+ 'segment.bytes': 104_857_600 # 100MB
105
+ }
106
+ )
107
+ created(consumers_states_topic)
102
108
  end
103
109
  end
104
110
 
@@ -86,65 +86,75 @@ module Karafka
86
86
 
87
87
  # Materializes the current state of consumers group data
88
88
  #
89
- # At the moment we report only topics lags but the format we are using supports
90
- # extending this information in the future if it would be needed.
91
- #
92
89
  # @return [Hash] hash with nested consumers and their topics details structure
93
90
  # @note We do **not** report on a per partition basis because it would significantly
94
91
  # increase needed storage.
95
92
  def materialize_consumers_groups_current_state
96
93
  cgs = {}
97
94
 
98
- @active_reports.each do |_, details|
99
- details.fetch(:consumer_groups).each do |group_name, group_details|
100
- group_details.fetch(:subscription_groups).each do |_sg_name, sg_details|
101
- sg_details.fetch(:topics).each do |topic_name, topic_details|
102
- partitions_data = topic_details.fetch(:partitions).values
95
+ iterate_partitions_data do |group_name, topic_name, partitions_data|
96
+ lags = partitions_data
97
+ .map { |p_details| p_details.fetch(:lag, -1) }
98
+ .reject(&:negative?)
99
+
100
+ lags_stored = partitions_data
101
+ .map { |p_details| p_details.fetch(:lag_stored, -1) }
102
+ .reject(&:negative?)
103
103
 
104
- lags = partitions_data
105
- .map { |p_details| p_details.fetch(:lag, -1) }
104
+ offsets_hi = partitions_data
105
+ .map { |p_details| p_details.fetch(:hi_offset, -1) }
106
106
  .reject(&:negative?)
107
107
 
108
- lags_stored = partitions_data
109
- .map { |p_details| p_details.fetch(:lag_stored, -1) }
110
- .reject(&:negative?)
111
-
112
- offsets_hi = partitions_data
113
- .map { |p_details| p_details.fetch(:hi_offset, -1) }
114
- .reject(&:negative?)
115
-
116
- # Last stable offsets freeze durations - we pick the max freeze to indicate
117
- # the longest open transaction that potentially may be hanging
118
- ls_offsets_fd = partitions_data
119
- .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
120
- .reject(&:negative?)
121
-
122
- # If there is no lag that would not be negative, it means we did not mark
123
- # any messages as consumed on this topic in any partitions, hence we cannot
124
- # compute lag easily
125
- # We do not want to initialize any data for this topic, when there is nothing
126
- # useful we could present
127
- #
128
- # In theory lag stored must mean that lag must exist but just to be sure we
129
- # check both here
130
- next if lags.empty? || lags_stored.empty?
131
-
132
- cgs[group_name] ||= {}
133
- cgs[group_name][topic_name] = {
134
- lag_stored: lags_stored.sum,
135
- lag: lags.sum,
136
- pace: offsets_hi.sum,
137
- # Take max last stable offset duration without any change. This can
138
- # indicate a hanging transaction, because the offset will not move forward
139
- # and will stay with a growing freeze duration when stuck
140
- ls_offset_fd: ls_offsets_fd.max
141
- }
108
+ # Last stable offsets freeze durations - we pick the max freeze to indicate
109
+ # the longest open transaction that potentially may be hanging
110
+ ls_offsets_fd = partitions_data
111
+ .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
112
+ .reject(&:negative?)
113
+
114
+ cgs[group_name] ||= {}
115
+ cgs[group_name][topic_name] = {
116
+ lag_stored: lags_stored.sum,
117
+ lag: lags.sum,
118
+ pace: offsets_hi.sum,
119
+ # Take max last stable offset duration without any change. This can
120
+ # indicate a hanging transaction, because the offset will not move forward
121
+ # and will stay with a growing freeze duration when stuck
122
+ ls_offset_fd: ls_offsets_fd.max || 0
123
+ }
124
+ end
125
+
126
+ cgs
127
+ end
128
+
129
+ # Converts our reports data into an iterator per partition
130
+ # Compensates for a case where same partition data would be available for a short
131
+ # period of time in multiple processes reports due to rebalances.
132
+ def iterate_partitions_data
133
+ cgs_topics = Hash.new { |h, v| h[v] = Hash.new { |h2, v2| h2[v2] = {} } }
134
+
135
+ # We need to sort them in case we have same reports containing data about same
136
+ # topics partitions. Mostly during shutdowns and rebalances
137
+ @active_reports
138
+ .values
139
+ .sort_by { |report| report.fetch(:dispatched_at) }
140
+ .map { |details| details.fetch(:consumer_groups) }
141
+ .each do |consumer_groups|
142
+ consumer_groups.each do |group_name, group_details|
143
+ group_details.fetch(:subscription_groups).each_value do |sg_details|
144
+ sg_details.fetch(:topics).each do |topic_name, topic_details|
145
+ topic_details.fetch(:partitions).each do |partition_id, partition_data|
146
+ cgs_topics[group_name][topic_name][partition_id] = partition_data
147
+ end
148
+ end
142
149
  end
143
150
  end
144
151
  end
145
- end
146
152
 
147
- cgs
153
+ cgs_topics.each do |group_name, topics_data|
154
+ topics_data.each do |topic_name, partitions_data|
155
+ yield(group_name, topic_name, partitions_data.values)
156
+ end
157
+ end
148
158
  end
149
159
  end
150
160
  end
@@ -20,6 +20,10 @@ module Karafka
20
20
  return metrics_message.payload if metrics_message
21
21
 
22
22
  raise(::Karafka::Web::Errors::Processing::MissingConsumersMetricsError)
23
+ rescue Rdkafka::RdkafkaError => e
24
+ raise(e) unless e.code == :unknown_partition
25
+
26
+ raise(::Karafka::Web::Errors::Processing::MissingConsumersMetricsTopicError)
23
27
  end
24
28
  end
25
29
  end
@@ -20,6 +20,10 @@ module Karafka
20
20
  return state_message.payload if state_message
21
21
 
22
22
  raise(::Karafka::Web::Errors::Processing::MissingConsumersStateError)
23
+ rescue Rdkafka::RdkafkaError => e
24
+ raise(e) unless e.code == :unknown_partition
25
+
26
+ raise(::Karafka::Web::Errors::Processing::MissingConsumersStatesTopicError)
23
27
  end
24
28
  end
25
29
  end
@@ -116,9 +116,12 @@ module Karafka
116
116
  # available
117
117
  times << values.last unless values.empty?
118
118
 
119
+ # Keep the most recent state out of many that would come from the same time moment
120
+ # Squash in case there would be two events from the same time
121
+ times.reverse!
119
122
  times.uniq!(&:first)
123
+ times.reverse!
120
124
 
121
- # Squash in case there would be two events from the same time
122
125
  times.sort_by!(&:first)
123
126
 
124
127
  @historicals[range_name] = times.last(limit)
@@ -16,7 +16,7 @@ module Karafka
16
16
 
17
17
  # Serve current version specific assets to prevent users from fetching old assets
18
18
  # after upgrade
19
- r.on(:assets, Karafka::Web::VERSION) do
19
+ r.on 'assets', Karafka::Web::VERSION do
20
20
  r.public
21
21
  end
22
22
 
@@ -16,7 +16,7 @@ module Karafka
16
16
  root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public'),
17
17
  # Cache all static files for the end user for as long as possible
18
18
  # We can do it because we ship per version assets so they invalidate with gem bumps
19
- headers: { 'Cache-Control' => 'max-age=604800' }
19
+ headers: { 'Cache-Control' => 'max-age=31536000, immutable' }
20
20
  )
21
21
  plugin :render_each
22
22
  plugin :partials
@@ -101,12 +101,13 @@ module Karafka
101
101
 
102
102
  # Converts number to a more friendly delimiter based version
103
103
  # @param number [Numeric]
104
+ # @param delimiter [String] delimiter (comma by default)
104
105
  # @return [String] number with delimiter
105
- def number_with_delimiter(number)
106
+ def number_with_delimiter(number, delimiter = ',')
106
107
  return '' unless number
107
108
 
108
109
  parts = number.to_s.to_str.split('.')
109
- parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, '\1,')
110
+ parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
110
111
  parts.join('.')
111
112
  end
112
113
 
@@ -62,7 +62,11 @@ module Karafka
62
62
  #
63
63
  # @param state [State]
64
64
  def iterate_partitions(state)
65
- processes = Processes.active(state)
65
+ # By default processes are sort by name and this is not what we want here
66
+ # We want to make sure that the newest data is processed the last, so we get
67
+ # the most accurate state in case of deployments and shutdowns, etc without the
68
+ # expired processes partitions data overwriting the newly created processes
69
+ processes = Processes.active(state).sort_by!(&:dispatched_at)
66
70
 
67
71
  processes.each do |process|
68
72
  process.consumer_groups.each do |consumer_group|
@@ -36,7 +36,7 @@ module Karafka
36
36
 
37
37
  # Serve current version specific assets to prevent users from fetching old assets
38
38
  # after upgrade
39
- r.on(:assets, Karafka::Web::VERSION) do
39
+ r.on 'assets', Karafka::Web::VERSION do
40
40
  r.public
41
41
  end
42
42