karafka-web 0.7.4 → 0.7.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +10 -4
  4. data/CHANGELOG.md +10 -0
  5. data/Gemfile.lock +3 -3
  6. data/bin/wait_for_kafka +24 -0
  7. data/docker-compose.yml +17 -16
  8. data/karafka-web.gemspec +1 -1
  9. data/lib/karafka/web/errors.rb +10 -1
  10. data/lib/karafka/web/installer.rb +27 -2
  11. data/lib/karafka/web/management/create_topics.rb +52 -46
  12. data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +56 -46
  13. data/lib/karafka/web/processing/consumers/metrics.rb +4 -0
  14. data/lib/karafka/web/processing/consumers/state.rb +4 -0
  15. data/lib/karafka/web/processing/time_series_tracker.rb +4 -1
  16. data/lib/karafka/web/ui/app.rb +1 -1
  17. data/lib/karafka/web/ui/base.rb +1 -1
  18. data/lib/karafka/web/ui/helpers/application_helper.rb +3 -2
  19. data/lib/karafka/web/ui/models/health.rb +5 -1
  20. data/lib/karafka/web/ui/pro/app.rb +1 -1
  21. data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +24 -8
  22. data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +0 -3
  23. data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +40 -34
  24. data/lib/karafka/web/ui/public/javascripts/bootstrap.min.js +0 -1
  25. data/lib/karafka/web/ui/public/javascripts/chart.min.js +0 -1
  26. data/lib/karafka/web/ui/public/javascripts/timeago.min.js +5 -0
  27. data/lib/karafka/web/ui/public/stylesheets/bootstrap.min.css +0 -1
  28. data/lib/karafka/web/ui/views/consumers/_counters.erb +21 -7
  29. data/lib/karafka/web/ui/views/shared/_header.erb +1 -1
  30. data/lib/karafka/web/version.rb +1 -1
  31. data.tar.gz.sig +0 -0
  32. metadata +5 -5
  33. metadata.gz.sig +0 -0
  34. data/lib/karafka/web/ui/public/stylesheets/bootstrap.min.css.map +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7a31ebcde158ca18cc353d3fefad976b7dfb8d9fe3b3114ea8f43b533229434a
4
- data.tar.gz: f9870e5268ebebb38c838dc8a220030beaebfe30d4d53cd63537c5c2f442e2d4
3
+ metadata.gz: caf2303a2e877dd3f06b738974c4093f7a9cf1bdb24b1152183ba9173432b640
4
+ data.tar.gz: 80b56d4efec23d589adb731c18e5b7137718862edc246fb2c32ff0baaadb84a5
5
5
  SHA512:
6
- metadata.gz: '02152094838c0be1606f49b4963e7d0791f1800bc7b7fbf71240866ed84e73bc264322415b48272faf46c715d70568fc054fc7183406d182d3c95eeb664019db'
7
- data.tar.gz: 787a16d7f91f7ccb907a3cbea0599337bd7525b1f2982f023b639227df1b52511afcc763422c71d0e2bf34216d3f3a4fe6794ae8824abeea9302b84071de8ca5
6
+ metadata.gz: 0b28806d2442202259c3c7c4cf3e9d2193e47c2a6b2113f31a5906019f94f4fb2f342ef170721e63ff8ff0e037f1c574509b9be0a98f0c3b8b5b9ff289bd4fbc
7
+ data.tar.gz: 9c6b42c67d4f305517e42c7446278ba7f5a65f7ad9c4ac52657ebc3231fe92619e6d2b6a35b857bb03b0e29568cfeb596ff4153bcd023ee30e0e235e0917af99
checksums.yaml.gz.sig CHANGED
Binary file
@@ -20,6 +20,7 @@ jobs:
20
20
  fail-fast: false
21
21
  matrix:
22
22
  ruby:
23
+ - '3.3.0-preview2'
23
24
  - '3.2'
24
25
  - '3.1'
25
26
  - '3.0'
@@ -28,18 +29,19 @@ jobs:
28
29
  - ruby: '3.2'
29
30
  coverage: 'true'
30
31
  steps:
31
- - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
32
+ - uses: actions/checkout@v4
32
33
  - name: Install package dependencies
33
34
  run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
34
35
 
35
36
  - name: Start Kafka with docker-compose
36
37
  run: |
37
- docker-compose up -d
38
+ docker-compose up -d || (sleep 5 && docker-compose up -d)
38
39
 
39
40
  - name: Set up Ruby
40
41
  uses: ruby/setup-ruby@v1
41
42
  with:
42
43
  ruby-version: ${{matrix.ruby}}
44
+ bundler-cache: true
43
45
 
44
46
  - name: Install latest bundler
45
47
  run: |
@@ -51,6 +53,10 @@ jobs:
51
53
  bundle config set without development
52
54
  bundle install --jobs 4 --retry 3
53
55
 
56
+ - name: Wait for Kafka
57
+ run: |
58
+ bundle exec bin/wait_for_kafka
59
+
54
60
  - name: Run all tests
55
61
  env:
56
62
  GITHUB_COVERAGE: ${{matrix.coverage}}
@@ -62,7 +68,7 @@ jobs:
62
68
  strategy:
63
69
  fail-fast: false
64
70
  steps:
65
- - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
71
+ - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
66
72
  with:
67
73
  fetch-depth: 0
68
74
 
@@ -83,7 +89,7 @@ jobs:
83
89
  strategy:
84
90
  fail-fast: false
85
91
  steps:
86
- - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
92
+ - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
87
93
  with:
88
94
  fetch-depth: 0
89
95
  - name: Run Coditsu
data/CHANGELOG.md CHANGED
@@ -1,5 +1,15 @@
1
1
  # Karafka Web changelog
2
2
 
3
+ ## 0.7.5 (2023-09-29)
4
+ - [Enhancement] Update order of topics creation for the setup of Web to support zero-downtime setup of Web in running Karafka projects.
5
+ - [Enhancement] Add space delimiter to counters numbers to make them look better.
6
+ - [Improvement] Normalize per-process job tables and health tables structure (topic name on top).
7
+ - [Fix] Fix a case where charts aggregated data would not include all topics.
8
+ - [Fix] Make sure, that most recent per partition data for Health is never overwritten by an old state from a previous partition owner.
9
+ - [Fix] Cache assets for 1 year instead of 7 days.
10
+ - [Fix] Remove source maps pointing to non-existing locations.
11
+ - [Maintenance] Include license and copyrights notice for `timeago.js` that was missing in the JS min file.
12
+
3
13
  ## 0.7.4 (2023-09-19)
4
14
  - [Improvement] Skip aggregations on older schemas during upgrades. This only skips process-reports (that are going to be rolled) on the 5s window in case of an upgrade that should not be a rolling one anyhow. This simplifies the operations and minimizes the risk on breaking upgrades.
5
15
  - [Fix] Fix not working `ps` for macOS.
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka-web (0.7.4)
4
+ karafka-web (0.7.5)
5
5
  erubi (~> 1.4)
6
- karafka (>= 2.2.3, < 3.0.0)
6
+ karafka (>= 2.2.6, < 3.0.0)
7
7
  karafka-core (>= 2.2.2, < 3.0.0)
8
8
  roda (~> 3.68, >= 3.69)
9
9
  tilt (~> 2.0)
@@ -26,7 +26,7 @@ GEM
26
26
  ffi (1.15.5)
27
27
  i18n (1.14.1)
28
28
  concurrent-ruby (~> 1.0)
29
- karafka (2.2.3)
29
+ karafka (2.2.6)
30
30
  karafka-core (>= 2.2.2, < 2.3.0)
31
31
  thor (>= 0.20)
32
32
  waterdrop (>= 2.6.6, < 3.0.0)
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Waits for Kafka to be ready
4
+ # Useful in CI where Kafka needs to be fully started before we run any tests
5
+
6
+ require 'karafka'
7
+
8
+ Karafka::App.setup do |config|
9
+ config.kafka[:'bootstrap.servers'] = '127.0.0.1:9092'
10
+ end
11
+
12
+ 60.times do
13
+ begin
14
+ # Stop if we can connect to the cluster and get info
15
+ exit if Karafka::Admin.cluster_info
16
+ rescue Rdkafka::RdkafkaError
17
+ puts "Kafka not available, retrying..."
18
+ sleep(1)
19
+ end
20
+ end
21
+
22
+ puts 'Kafka not available!'
23
+
24
+ exit 1
data/docker-compose.yml CHANGED
@@ -1,22 +1,23 @@
1
1
  version: '2'
2
- services:
3
- zookeeper:
4
- container_name: karafka_web_21_zookeeper
5
- image: wurstmeister/zookeeper
6
- restart: on-failure
7
- ports:
8
- - '2181:2181'
9
2
 
3
+ services:
10
4
  kafka:
11
- container_name: karafka_web_21_kafka
12
- image: wurstmeister/kafka
5
+ container_name: kafka
6
+ image: confluentinc/cp-kafka:7.5.0
7
+
13
8
  ports:
14
- - '9092:9092'
9
+ - 9092:9092
10
+
15
11
  environment:
16
- KAFKA_ADVERTISED_HOST_NAME: localhost
17
- KAFKA_ADVERTISED_PORT: 9092
18
- KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
12
+ CLUSTER_ID: kafka-docker-cluster-1
13
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
14
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
15
+ KAFKA_PROCESS_ROLES: broker,controller
16
+ KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
17
+ KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
18
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
19
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
20
+ KAFKA_BROKER_ID: 1
21
+ KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
22
+ ALLOW_PLAINTEXT_LISTENER: 'yes'
19
23
  KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
20
- volumes:
21
- - /var/run/docker.sock:/var/run/docker.sock
22
- restart: on-failure
data/karafka-web.gemspec CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
17
17
  spec.licenses = %w[LGPL-3.0 Commercial]
18
18
 
19
19
  spec.add_dependency 'erubi', '~> 1.4'
20
- spec.add_dependency 'karafka', '>= 2.2.3', '< 3.0.0'
20
+ spec.add_dependency 'karafka', '>= 2.2.6', '< 3.0.0'
21
21
  spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
22
22
  spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
23
23
  spec.add_dependency 'tilt', '~> 2.0'
@@ -17,9 +17,18 @@ module Karafka
17
17
  # If you see this error, it probably means, that you did not bootstrap Web-UI correctly
18
18
  MissingConsumersStateError = Class.new(BaseError)
19
19
 
20
- # Similar to the above. It should be created during install
20
+ # Raised when we try to materialize the state but the consumers states topic does not
21
+ # exist and we do not have a way to get the initial state.
22
+ # It differs from the above because above indicates that the topic exists but that there
23
+ # is no initial state, while this indicates, that there is no consumers states topic.
24
+ MissingConsumersStatesTopicError = Class.new(BaseError)
25
+
26
+ # Similar to the above. It should be created during install / migration
21
27
  MissingConsumersMetricsError = Class.new(BaseError)
22
28
 
29
+ # Similar to the one related to consumers states
30
+ MissingConsumersMetricsTopicError = Class.new(BaseError)
31
+
23
32
  # This error occurs when consumer running older version of the web-ui tries to materialize
24
33
  # states from newer versions. Karafka Web-UI provides only backwards compatibility, so
25
34
  # you need to have an up-to-date consumer materializing reported states.
@@ -18,7 +18,7 @@ module Karafka
18
18
  puts 'Creating necessary topics and populating state data...'
19
19
  puts
20
20
  Management::CreateTopics.new.call(replication_factor)
21
- puts
21
+ wait_for_topics
22
22
  Management::CreateInitialStates.new.call
23
23
  puts
24
24
  Management::ExtendBootFile.new.call
@@ -36,6 +36,7 @@ module Karafka
36
36
  puts 'Creating necessary topics and populating state data...'
37
37
  puts
38
38
  Management::CreateTopics.new.call(replication_factor)
39
+ wait_for_topics
39
40
  Management::CreateInitialStates.new.call
40
41
  puts
41
42
  puts("Migration #{green('completed')}. Have fun!")
@@ -51,7 +52,7 @@ module Karafka
51
52
  Management::DeleteTopics.new.call
52
53
  puts
53
54
  Management::CreateTopics.new.call(replication_factor)
54
- puts
55
+ wait_for_topics
55
56
  Management::CreateInitialStates.new.call
56
57
  puts
57
58
  puts("Resetting #{green('completed')}. Have fun!")
@@ -74,6 +75,30 @@ module Karafka
74
75
  def enable!
75
76
  Management::Enable.new.call
76
77
  end
78
+
79
+ private
80
+
81
+ # Waits with a message, that we are waiting on topics
82
+ # This is not doing much, just waiting as there are some cases that it takes a bit of time
83
+ # for Kafka to actually propagate new topics knowledge across the cluster. We give it that
84
+ # bit of time just in case.
85
+ def wait_for_topics
86
+ puts
87
+ print 'Waiting for the topics to synchronize in the cluster'
88
+ wait(5)
89
+ puts
90
+ end
91
+
92
+ # Waits for given number of seconds and prints `.` every second.
93
+ # @param time_in_seconds [Integer] time of wait
94
+ def wait(time_in_seconds)
95
+ time_in_seconds.times do
96
+ sleep(1)
97
+ print '.'
98
+ end
99
+
100
+ print "\n"
101
+ end
77
102
  end
78
103
  end
79
104
  end
@@ -9,54 +9,36 @@ module Karafka
9
9
  # Runs the creation process
10
10
  #
11
11
  # @param replication_factor [Integer] replication factor for Web-UI topics
12
+ #
13
+ # @note The order of creation of those topics is important. In order to support the
14
+ # zero-downtime bootstrap, we use the presence of the states topic and its initial state
15
+ # existence as an indicator that the setup went as expected. It the consumers states
16
+ # topic exists and contains needed data, it means all went as expected and that
17
+ # topics created before it also exist (as no error).
12
18
  def call(replication_factor)
13
19
  consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
14
20
  consumers_metrics_topic = ::Karafka::Web.config.topics.consumers.metrics
15
21
  consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
16
22
  errors_topic = ::Karafka::Web.config.topics.errors
17
23
 
18
- # Create only if needed
19
- if existing_topics_names.include?(consumers_states_topic)
20
- exists(consumers_states_topic)
21
- else
22
- creating(consumers_states_topic)
23
- # This topic needs to have one partition
24
- ::Karafka::Admin.create_topic(
25
- consumers_states_topic,
26
- 1,
27
- replication_factor,
28
- # We care only about the most recent state, previous are irrelevant. So we can easily
29
- # compact after one minute. We do not use this beyond the most recent collective
30
- # state, hence it all can easily go away. We also limit the segment size to at most
31
- # 100MB not to use more space ever.
32
- {
33
- 'cleanup.policy': 'compact',
34
- 'retention.ms': 60 * 60 * 1_000,
35
- 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
36
- 'segment.bytes': 104_857_600 # 100MB
37
- }
38
- )
39
- created(consumers_states_topic)
40
- end
41
-
42
- if existing_topics_names.include?(consumers_metrics_topic)
43
- exists(consumers_metrics_topic)
24
+ if existing_topics_names.include?(errors_topic)
25
+ exists(errors_topic)
44
26
  else
45
- creating(consumers_metrics_topic)
46
- # This topic needs to have one partition
47
- # Same as states - only most recent is relevant as it is a materialized state
27
+ creating(errors_topic)
28
+ # All the errors will be dispatched here
29
+ # This topic can have multiple partitions but we go with one by default. A single Ruby
30
+ # process should not crash that often and if there is an expectation of a higher volume
31
+ # of errors, this can be changed by the end user
48
32
  ::Karafka::Admin.create_topic(
49
- consumers_metrics_topic,
33
+ errors_topic,
50
34
  1,
51
35
  replication_factor,
36
+ # Remove really old errors (older than 3 months just to preserve space)
52
37
  {
53
- 'cleanup.policy': 'compact',
54
- 'retention.ms': 60 * 60 * 1_000, # 1h
55
- 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
56
- 'segment.bytes': 104_857_600 # 100MB
38
+ 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
57
39
  }
58
40
  )
59
- created(consumers_metrics_topic)
41
+ created(errors_topic)
60
42
  end
61
43
 
62
44
  if existing_topics_names.include?(consumers_reports_topic)
@@ -81,24 +63,48 @@ module Karafka
81
63
  created(consumers_reports_topic)
82
64
  end
83
65
 
84
- if existing_topics_names.include?(errors_topic)
85
- exists(errors_topic)
66
+ if existing_topics_names.include?(consumers_metrics_topic)
67
+ exists(consumers_metrics_topic)
86
68
  else
87
- creating(errors_topic)
88
- # All the errors will be dispatched here
89
- # This topic can have multiple partitions but we go with one by default. A single Ruby
90
- # process should not crash that often and if there is an expectation of a higher volume
91
- # of errors, this can be changed by the end user
69
+ creating(consumers_metrics_topic)
70
+ # This topic needs to have one partition
71
+ # Same as states - only most recent is relevant as it is a materialized state
92
72
  ::Karafka::Admin.create_topic(
93
- errors_topic,
73
+ consumers_metrics_topic,
94
74
  1,
95
75
  replication_factor,
96
- # Remove really old errors (older than 3 months just to preserve space)
97
76
  {
98
- 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
77
+ 'cleanup.policy': 'compact',
78
+ 'retention.ms': 60 * 60 * 1_000, # 1h
79
+ 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
80
+ 'segment.bytes': 104_857_600 # 100MB
99
81
  }
100
82
  )
101
- created(errors_topic)
83
+ created(consumers_metrics_topic)
84
+ end
85
+
86
+ # Create only if needed
87
+ if existing_topics_names.include?(consumers_states_topic)
88
+ exists(consumers_states_topic)
89
+ else
90
+ creating(consumers_states_topic)
91
+ # This topic needs to have one partition
92
+ ::Karafka::Admin.create_topic(
93
+ consumers_states_topic,
94
+ 1,
95
+ replication_factor,
96
+ # We care only about the most recent state, previous are irrelevant. So we can easily
97
+ # compact after one minute. We do not use this beyond the most recent collective
98
+ # state, hence it all can easily go away. We also limit the segment size to at most
99
+ # 100MB not to use more space ever.
100
+ {
101
+ 'cleanup.policy': 'compact',
102
+ 'retention.ms': 60 * 60 * 1_000,
103
+ 'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
104
+ 'segment.bytes': 104_857_600 # 100MB
105
+ }
106
+ )
107
+ created(consumers_states_topic)
102
108
  end
103
109
  end
104
110
 
@@ -86,65 +86,75 @@ module Karafka
86
86
 
87
87
  # Materializes the current state of consumers group data
88
88
  #
89
- # At the moment we report only topics lags but the format we are using supports
90
- # extending this information in the future if it would be needed.
91
- #
92
89
  # @return [Hash] hash with nested consumers and their topics details structure
93
90
  # @note We do **not** report on a per partition basis because it would significantly
94
91
  # increase needed storage.
95
92
  def materialize_consumers_groups_current_state
96
93
  cgs = {}
97
94
 
98
- @active_reports.each do |_, details|
99
- details.fetch(:consumer_groups).each do |group_name, group_details|
100
- group_details.fetch(:subscription_groups).each do |_sg_name, sg_details|
101
- sg_details.fetch(:topics).each do |topic_name, topic_details|
102
- partitions_data = topic_details.fetch(:partitions).values
95
+ iterate_partitions_data do |group_name, topic_name, partitions_data|
96
+ lags = partitions_data
97
+ .map { |p_details| p_details.fetch(:lag, -1) }
98
+ .reject(&:negative?)
99
+
100
+ lags_stored = partitions_data
101
+ .map { |p_details| p_details.fetch(:lag_stored, -1) }
102
+ .reject(&:negative?)
103
103
 
104
- lags = partitions_data
105
- .map { |p_details| p_details.fetch(:lag, -1) }
104
+ offsets_hi = partitions_data
105
+ .map { |p_details| p_details.fetch(:hi_offset, -1) }
106
106
  .reject(&:negative?)
107
107
 
108
- lags_stored = partitions_data
109
- .map { |p_details| p_details.fetch(:lag_stored, -1) }
110
- .reject(&:negative?)
111
-
112
- offsets_hi = partitions_data
113
- .map { |p_details| p_details.fetch(:hi_offset, -1) }
114
- .reject(&:negative?)
115
-
116
- # Last stable offsets freeze durations - we pick the max freeze to indicate
117
- # the longest open transaction that potentially may be hanging
118
- ls_offsets_fd = partitions_data
119
- .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
120
- .reject(&:negative?)
121
-
122
- # If there is no lag that would not be negative, it means we did not mark
123
- # any messages as consumed on this topic in any partitions, hence we cannot
124
- # compute lag easily
125
- # We do not want to initialize any data for this topic, when there is nothing
126
- # useful we could present
127
- #
128
- # In theory lag stored must mean that lag must exist but just to be sure we
129
- # check both here
130
- next if lags.empty? || lags_stored.empty?
131
-
132
- cgs[group_name] ||= {}
133
- cgs[group_name][topic_name] = {
134
- lag_stored: lags_stored.sum,
135
- lag: lags.sum,
136
- pace: offsets_hi.sum,
137
- # Take max last stable offset duration without any change. This can
138
- # indicate a hanging transaction, because the offset will not move forward
139
- # and will stay with a growing freeze duration when stuck
140
- ls_offset_fd: ls_offsets_fd.max
141
- }
108
+ # Last stable offsets freeze durations - we pick the max freeze to indicate
109
+ # the longest open transaction that potentially may be hanging
110
+ ls_offsets_fd = partitions_data
111
+ .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
112
+ .reject(&:negative?)
113
+
114
+ cgs[group_name] ||= {}
115
+ cgs[group_name][topic_name] = {
116
+ lag_stored: lags_stored.sum,
117
+ lag: lags.sum,
118
+ pace: offsets_hi.sum,
119
+ # Take max last stable offset duration without any change. This can
120
+ # indicate a hanging transaction, because the offset will not move forward
121
+ # and will stay with a growing freeze duration when stuck
122
+ ls_offset_fd: ls_offsets_fd.max || 0
123
+ }
124
+ end
125
+
126
+ cgs
127
+ end
128
+
129
+ # Converts our reports data into an iterator per partition
130
+ # Compensates for a case where same partition data would be available for a short
131
+ # period of time in multiple processes reports due to rebalances.
132
+ def iterate_partitions_data
133
+ cgs_topics = Hash.new { |h, v| h[v] = Hash.new { |h2, v2| h2[v2] = {} } }
134
+
135
+ # We need to sort them in case we have same reports containing data about same
136
+ # topics partitions. Mostly during shutdowns and rebalances
137
+ @active_reports
138
+ .values
139
+ .sort_by { |report| report.fetch(:dispatched_at) }
140
+ .map { |details| details.fetch(:consumer_groups) }
141
+ .each do |consumer_groups|
142
+ consumer_groups.each do |group_name, group_details|
143
+ group_details.fetch(:subscription_groups).each_value do |sg_details|
144
+ sg_details.fetch(:topics).each do |topic_name, topic_details|
145
+ topic_details.fetch(:partitions).each do |partition_id, partition_data|
146
+ cgs_topics[group_name][topic_name][partition_id] = partition_data
147
+ end
148
+ end
142
149
  end
143
150
  end
144
151
  end
145
- end
146
152
 
147
- cgs
153
+ cgs_topics.each do |group_name, topics_data|
154
+ topics_data.each do |topic_name, partitions_data|
155
+ yield(group_name, topic_name, partitions_data.values)
156
+ end
157
+ end
148
158
  end
149
159
  end
150
160
  end
@@ -20,6 +20,10 @@ module Karafka
20
20
  return metrics_message.payload if metrics_message
21
21
 
22
22
  raise(::Karafka::Web::Errors::Processing::MissingConsumersMetricsError)
23
+ rescue Rdkafka::RdkafkaError => e
24
+ raise(e) unless e.code == :unknown_partition
25
+
26
+ raise(::Karafka::Web::Errors::Processing::MissingConsumersMetricsTopicError)
23
27
  end
24
28
  end
25
29
  end
@@ -20,6 +20,10 @@ module Karafka
20
20
  return state_message.payload if state_message
21
21
 
22
22
  raise(::Karafka::Web::Errors::Processing::MissingConsumersStateError)
23
+ rescue Rdkafka::RdkafkaError => e
24
+ raise(e) unless e.code == :unknown_partition
25
+
26
+ raise(::Karafka::Web::Errors::Processing::MissingConsumersStatesTopicError)
23
27
  end
24
28
  end
25
29
  end
@@ -116,9 +116,12 @@ module Karafka
116
116
  # available
117
117
  times << values.last unless values.empty?
118
118
 
119
+ # Keep the most recent state out of many that would come from the same time moment
120
+ # Squash in case there would be two events from the same time
121
+ times.reverse!
119
122
  times.uniq!(&:first)
123
+ times.reverse!
120
124
 
121
- # Squash in case there would be two events from the same time
122
125
  times.sort_by!(&:first)
123
126
 
124
127
  @historicals[range_name] = times.last(limit)
@@ -16,7 +16,7 @@ module Karafka
16
16
 
17
17
  # Serve current version specific assets to prevent users from fetching old assets
18
18
  # after upgrade
19
- r.on(:assets, Karafka::Web::VERSION) do
19
+ r.on 'assets', Karafka::Web::VERSION do
20
20
  r.public
21
21
  end
22
22
 
@@ -16,7 +16,7 @@ module Karafka
16
16
  root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public'),
17
17
  # Cache all static files for the end user for as long as possible
18
18
  # We can do it because we ship per version assets so they invalidate with gem bumps
19
- headers: { 'Cache-Control' => 'max-age=604800' }
19
+ headers: { 'Cache-Control' => 'max-age=31536000, immutable' }
20
20
  )
21
21
  plugin :render_each
22
22
  plugin :partials
@@ -101,12 +101,13 @@ module Karafka
101
101
 
102
102
  # Converts number to a more friendly delimiter based version
103
103
  # @param number [Numeric]
104
+ # @param delimiter [String] delimiter (comma by default)
104
105
  # @return [String] number with delimiter
105
- def number_with_delimiter(number)
106
+ def number_with_delimiter(number, delimiter = ',')
106
107
  return '' unless number
107
108
 
108
109
  parts = number.to_s.to_str.split('.')
109
- parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, '\1,')
110
+ parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
110
111
  parts.join('.')
111
112
  end
112
113
 
@@ -62,7 +62,11 @@ module Karafka
62
62
  #
63
63
  # @param state [State]
64
64
  def iterate_partitions(state)
65
- processes = Processes.active(state)
65
+ # By default processes are sort by name and this is not what we want here
66
+ # We want to make sure that the newest data is processed the last, so we get
67
+ # the most accurate state in case of deployments and shutdowns, etc without the
68
+ # expired processes partitions data overwriting the newly created processes
69
+ processes = Processes.active(state).sort_by!(&:dispatched_at)
66
70
 
67
71
  processes.each do |process|
68
72
  process.consumer_groups.each do |consumer_group|
@@ -36,7 +36,7 @@ module Karafka
36
36
 
37
37
  # Serve current version specific assets to prevent users from fetching old assets
38
38
  # after upgrade
39
- r.on(:assets, Karafka::Web::VERSION) do
39
+ r.on 'assets', Karafka::Web::VERSION do
40
40
  r.public
41
41
  end
42
42