karafka-web 0.11.3 → 0.11.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +17 -0
  3. data/Gemfile +1 -2
  4. data/Gemfile.lock +58 -41
  5. data/bin/integrations +44 -0
  6. data/bin/rspecs +6 -2
  7. data/docker-compose.yml +1 -1
  8. data/karafka-web.gemspec +2 -2
  9. data/lib/karafka/web/app.rb +2 -3
  10. data/lib/karafka/web/cli/help.rb +1 -1
  11. data/lib/karafka/web/config.rb +14 -0
  12. data/lib/karafka/web/contracts/base.rb +2 -4
  13. data/lib/karafka/web/contracts/config.rb +6 -5
  14. data/lib/karafka/web/inflector.rb +1 -1
  15. data/lib/karafka/web/management/actions/enable.rb +14 -1
  16. data/lib/karafka/web/management/migrations/consumers_reports/1761645571_rename_process_name_to_id.rb +38 -0
  17. data/lib/karafka/web/management/migrator.rb +3 -2
  18. data/lib/karafka/web/pro/commanding/commands/base.rb +1 -1
  19. data/lib/karafka/web/pro/commanding/contracts/config.rb +2 -4
  20. data/lib/karafka/web/pro/commanding/handlers/partitions/tracker.rb +2 -3
  21. data/lib/karafka/web/pro/ui/controllers/cluster_controller.rb +1 -1
  22. data/lib/karafka/web/pro/ui/controllers/scheduled_messages/schedules_controller.rb +1 -2
  23. data/lib/karafka/web/pro/ui/controllers/topics/distributions_controller.rb +1 -3
  24. data/lib/karafka/web/pro/ui/lib/branding/contracts/config.rb +2 -4
  25. data/lib/karafka/web/pro/ui/lib/policies/contracts/config.rb +2 -4
  26. data/lib/karafka/web/pro/ui/lib/search/contracts/config.rb +3 -5
  27. data/lib/karafka/web/pro/ui/lib/search/contracts/form.rb +3 -5
  28. data/lib/karafka/web/pro/ui/lib/search/runner.rb +14 -1
  29. data/lib/karafka/web/pro/ui/routes/errors.rb +3 -3
  30. data/lib/karafka/web/pro/ui/routes/explorer.rb +3 -3
  31. data/lib/karafka/web/pro/ui/views/health/_no_partition_data.erb +9 -0
  32. data/lib/karafka/web/pro/ui/views/health/_partitions_with_fallback.erb +41 -0
  33. data/lib/karafka/web/pro/ui/views/health/changes.erb +12 -13
  34. data/lib/karafka/web/pro/ui/views/health/lags.erb +12 -13
  35. data/lib/karafka/web/pro/ui/views/health/offsets.erb +12 -13
  36. data/lib/karafka/web/pro/ui/views/health/overview.erb +15 -16
  37. data/lib/karafka/web/processing/consumer.rb +8 -3
  38. data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +1 -1
  39. data/lib/karafka/web/processing/consumers/aggregators/state.rb +5 -5
  40. data/lib/karafka/web/processing/consumers/contracts/state.rb +1 -1
  41. data/lib/karafka/web/processing/consumers/reports_migrator.rb +49 -0
  42. data/lib/karafka/web/processing/time_series_tracker.rb +1 -1
  43. data/lib/karafka/web/tracking/consumers/contracts/report.rb +1 -1
  44. data/lib/karafka/web/tracking/consumers/contracts/topic.rb +1 -0
  45. data/lib/karafka/web/tracking/consumers/listeners/errors.rb +2 -1
  46. data/lib/karafka/web/tracking/consumers/listeners/processing.rb +46 -0
  47. data/lib/karafka/web/tracking/consumers/listeners/statistics.rb +1 -0
  48. data/lib/karafka/web/tracking/consumers/sampler/enrichers/base.rb +20 -0
  49. data/lib/karafka/web/tracking/consumers/sampler/enrichers/consumer_groups.rb +116 -0
  50. data/lib/karafka/web/tracking/consumers/sampler/metrics/base.rb +20 -0
  51. data/lib/karafka/web/tracking/consumers/sampler/metrics/container.rb +113 -0
  52. data/lib/karafka/web/tracking/consumers/sampler/metrics/jobs.rb +60 -0
  53. data/lib/karafka/web/tracking/consumers/sampler/metrics/network.rb +48 -0
  54. data/lib/karafka/web/tracking/consumers/sampler/metrics/os.rb +206 -0
  55. data/lib/karafka/web/tracking/consumers/sampler/metrics/server.rb +33 -0
  56. data/lib/karafka/web/tracking/consumers/sampler.rb +34 -215
  57. data/lib/karafka/web/tracking/contracts/error.rb +1 -0
  58. data/lib/karafka/web/tracking/helpers/ttls/hash.rb +2 -3
  59. data/lib/karafka/web/tracking/helpers/ttls/stats.rb +1 -2
  60. data/lib/karafka/web/tracking/producers/listeners/errors.rb +2 -1
  61. data/lib/karafka/web/tracking/ui/errors.rb +76 -0
  62. data/lib/karafka/web/ui/base.rb +26 -11
  63. data/lib/karafka/web/ui/controllers/requests/execution_wrapper.rb +2 -4
  64. data/lib/karafka/web/ui/controllers/requests/params.rb +1 -1
  65. data/lib/karafka/web/ui/helpers/application_helper.rb +1 -1
  66. data/lib/karafka/web/ui/helpers/paths_helper.rb +7 -10
  67. data/lib/karafka/web/ui/lib/cache.rb +1 -1
  68. data/lib/karafka/web/ui/lib/hash_proxy.rb +1 -1
  69. data/lib/karafka/web/ui/lib/paginations/paginators/sets.rb +1 -1
  70. data/lib/karafka/web/ui/lib/sorter.rb +1 -1
  71. data/lib/karafka/web/ui/models/broker.rb +1 -1
  72. data/lib/karafka/web/ui/models/health.rb +14 -9
  73. data/lib/karafka/web/ui/models/jobs.rb +4 -6
  74. data/lib/karafka/web/ui/models/message.rb +7 -8
  75. data/lib/karafka/web/ui/models/metrics/aggregated.rb +4 -4
  76. data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb +1 -2
  77. data/lib/karafka/web/ui/models/metrics/charts/topics.rb +2 -2
  78. data/lib/karafka/web/ui/models/metrics/topics.rb +3 -4
  79. data/lib/karafka/web/ui/models/recurring_tasks/schedule.rb +1 -1
  80. data/lib/karafka/web/ui/public/javascripts/application.min.js.gz +0 -0
  81. data/lib/karafka/web/ui/public/stylesheets/application.min.css +6 -0
  82. data/lib/karafka/web/ui/public/stylesheets/application.min.css.br +0 -0
  83. data/lib/karafka/web/ui/public/stylesheets/application.min.css.gz +0 -0
  84. data/lib/karafka/web/ui/routes/errors.rb +3 -3
  85. data/lib/karafka/web/ui/views/shared/exceptions/unhandled_error.erb +42 -0
  86. data/lib/karafka/web/version.rb +1 -1
  87. data/lib/karafka/web.rb +2 -3
  88. data/package-lock.json +180 -236
  89. data/package.json +3 -3
  90. data/renovate.json +13 -0
  91. metadata +20 -5
@@ -57,10 +57,15 @@ module Karafka
57
57
  pt_id = partition.id
58
58
 
59
59
  stats[cg_id] ||= { topics: {} }
60
- stats[cg_id][:topics][t_name] ||= {}
61
- stats[cg_id][:topics][t_name][pt_id] = partition
62
- stats[cg_id][:topics][t_name][pt_id][:process] = process
63
- stats[cg_id][:topics][t_name][pt_id][:subscription_group_id] = sg_id
60
+
61
+ stats[cg_id][:topics][t_name] ||= {
62
+ partitions: {},
63
+ partitions_count: topic.partitions_cnt
64
+ }
65
+
66
+ stats[cg_id][:topics][t_name][:partitions][pt_id] = partition
67
+ stats[cg_id][:topics][t_name][:partitions][pt_id][:process] = process
68
+ stats[cg_id][:topics][t_name][:partitions][pt_id][:subscription_group_id] = sg_id
64
69
  end
65
70
  end
66
71
 
@@ -74,7 +79,7 @@ module Karafka
74
79
 
75
80
  ages = consumer_group[:subscription_groups].values.map do |sub_group_details|
76
81
  rebalance_age_ms = sub_group_details[:state][:rebalance_age] || 0
77
- dispatched_at - rebalance_age_ms / 1_000
82
+ dispatched_at - (rebalance_age_ms / 1_000)
78
83
  end
79
84
 
80
85
  stats[cg_name][:rebalance_ages] ||= Set.new
@@ -119,15 +124,15 @@ module Karafka
119
124
  stats.each_value do |cg_data|
120
125
  topics = cg_data[:topics]
121
126
 
122
- topics.each do |topic_name, t_data|
123
- topics[topic_name] = Hash[t_data.sort_by { |key, _| key }]
127
+ topics.each_value do |t_data|
128
+ t_data[:partitions] = t_data[:partitions].sort_by { |key, _| key }.to_h
124
129
  end
125
130
 
126
- cg_data[:topics] = Hash[topics.sort_by { |key, _| key }]
131
+ cg_data[:topics] = topics.sort_by { |key, _| key }.to_h
127
132
  end
128
133
 
129
134
  # Ensure that all consumer groups are always in the same order
130
- Hash[stats.sort_by { |key, _| key }]
135
+ stats.sort_by { |key, _| key }.to_h
131
136
  end
132
137
  end
133
138
  end
@@ -30,16 +30,14 @@ module Karafka
30
30
  end
31
31
 
32
32
  # Creates a new Jobs object with selected jobs
33
- # @param block [Proc] select proc
34
33
  # @return [Jobs] selected jobs enclosed with the Jobs object
35
- def select(&block)
36
- self.class.new(super(&block))
34
+ def select(&)
35
+ self.class.new(super)
37
36
  end
38
37
 
39
38
  # Allows for iteration over jobs
40
- # @param block [Proc] block to call for each job
41
- def each(&block)
42
- @jobs_array.each(&block)
39
+ def each(&)
40
+ @jobs_array.each(&)
43
41
  end
44
42
  end
45
43
  end
@@ -146,9 +146,9 @@ module Karafka
146
146
  # @param page [Integer] which page we want to get
147
147
  def topic_page(topic_id, partitions_ids, page)
148
148
  # This is the bottleneck, for each partition we make one request :(
149
- offsets = partitions_ids.map do |partition_id|
149
+ offsets = partitions_ids.to_h do |partition_id|
150
150
  [partition_id, Models::WatermarkOffsets.find(topic_id, partition_id)]
151
- end.to_h
151
+ end
152
152
 
153
153
  # Count number of elements we have in each partition
154
154
  # This assumes linear presence until low. If not, gaps will be filled like we fill
@@ -158,7 +158,7 @@ module Karafka
158
158
  # Establish initial offsets for the iterator (where to start) per partition
159
159
  # We do not use the negative lookup iterator because we already can compute starting
160
160
  # offsets. This saves a lot of calls to Kafka
161
- ranges = Sets.call(counts, page).map do |partition_position, partition_range|
161
+ ranges = Sets.call(counts, page).to_h do |partition_position, partition_range|
162
162
  partition_id = partitions_ids.to_a[partition_position]
163
163
  watermarks = offsets[partition_id]
164
164
 
@@ -169,7 +169,7 @@ module Karafka
169
169
 
170
170
  # This range represents offsets we want to fetch
171
171
  [partition_id, lowest..highest]
172
- end.to_h
172
+ end
173
173
 
174
174
  # We start on our topic from the lowest offset for each expected partition
175
175
  iterator = Karafka::Pro::Iterator.new(
@@ -208,18 +208,17 @@ module Karafka
208
208
  end
209
209
 
210
210
  [
211
- aggregated.values.map(&:values).map(&:reverse).reduce(:+),
211
+ aggregated.values.sum([]) { |partition| partition.values.reverse },
212
212
  !Sets.call(counts, page + 1).empty?
213
213
  ]
214
214
  end
215
215
 
216
216
  private
217
217
 
218
- # @param args [Object] anything required by the admin `#read_topic`
219
218
  # @return [Array<Karafka::Messages::Message>, false] topic partition messages or false
220
219
  # in case we hit a non-existing offset
221
- def read_topic(*args)
222
- Lib::Admin.read_topic(*args)
220
+ def read_topic(*)
221
+ Lib::Admin.read_topic(*)
223
222
  rescue Rdkafka::RdkafkaError => e
224
223
  return false if e.code == :auto_offset_reset
225
224
 
@@ -103,10 +103,10 @@ module Karafka
103
103
  base = sample.last.dup
104
104
 
105
105
  DELTA_KEYS.each do |key|
106
- base[key] = previous.last[key] + (sample.last[key] - previous.last[key]) / 2
106
+ base[key] = previous.last[key] + ((sample.last[key] - previous.last[key]) / 2)
107
107
  end
108
108
 
109
- filled << [previous.first + (sample.first - previous.first) / 2, base]
109
+ filled << [previous.first + ((sample.first - previous.first) / 2), base]
110
110
  end
111
111
 
112
112
  filled << sample
@@ -182,12 +182,12 @@ module Karafka
182
182
  # @param current [Hash]
183
183
  # @return [Hash] delta computed values
184
184
  def compute_deltas(previous, current)
185
- DELTA_KEYS.map do |delta_key|
185
+ DELTA_KEYS.to_h do |delta_key|
186
186
  [
187
187
  delta_key,
188
188
  current.fetch(delta_key) - previous.fetch(delta_key)
189
189
  ]
190
- end.to_h
190
+ end
191
191
  end
192
192
  end
193
193
  end
@@ -38,8 +38,7 @@ module Karafka
38
38
  # @return [String] JSON with data about all the charts we were interested in
39
39
  def with(*args)
40
40
  args
41
- .map { |name| [name.to_sym, public_send(name)] }
42
- .to_h
41
+ .to_h { |name| [name.to_sym, public_send(name)] }
43
42
  .to_json
44
43
  end
45
44
 
@@ -36,11 +36,11 @@ module Karafka
36
36
  end
37
37
 
38
38
  # Extract the lag stored only from all the data
39
- per_topic = @data.to_h.map do |topic, metrics|
39
+ per_topic = @data.to_h.to_h do |topic, metrics|
40
40
  extracted = metrics.map { |metric| [metric.first, metric.last[:lag_hybrid]] }
41
41
 
42
42
  [topic, extracted]
43
- end.to_h
43
+ end
44
44
 
45
45
  # We name it with a space because someone may have a topic called "total" and we
46
46
  # want to avoid collisions
@@ -43,9 +43,9 @@ module Karafka
43
43
 
44
44
  # Always align the order of topics in hash based on their name so it is
45
45
  # independent from the reported order
46
- extracted[range] = range_extracted.keys.sort.map do |key|
46
+ extracted[range] = range_extracted.keys.sort.to_h do |key|
47
47
  [key, range_extracted[key]]
48
- end.to_h
48
+ end
49
49
  end
50
50
 
51
51
  extracted
@@ -67,8 +67,7 @@ module Karafka
67
67
  .select { |val| val.is_a?(Hash) }
68
68
  .flat_map(&:keys)
69
69
  .uniq
70
- .map { |key| [key, nil] }
71
- .to_h
70
+ .to_h { |key| [key, nil] }
72
71
  .freeze
73
72
 
74
73
  # Normalize data in between topics reportings
@@ -50,7 +50,7 @@ module Karafka
50
50
  new(candidate.payload)
51
51
  rescue Rdkafka::RdkafkaError => e
52
52
  # If any of "topic missing" is raised, we return false but other errors we re-raise
53
- raise(e) unless EXPECTED_RDKAFKA_ERRORS.any? { |code| e.code == code }
53
+ raise(e) unless EXPECTED_RDKAFKA_ERRORS.include?(e.code)
54
54
 
55
55
  false
56
56
  end
@@ -2431,6 +2431,9 @@
2431
2431
  --btn-shadow: 0 0 0 0 oklch(0% 0 0/0), 0 0 0 0 oklch(0% 0 0/0);
2432
2432
  isolation: isolate;
2433
2433
  }
2434
+ .isolate {
2435
+ isolation: isolate;
2436
+ }
2434
2437
  .stack {
2435
2438
  display: inline-grid;
2436
2439
  grid-template-columns: 3px 4px 1fr 4px 3px;
@@ -3861,6 +3864,9 @@
3861
3864
  .alert {
3862
3865
  border-radius: var(--radius-sm);
3863
3866
  }
3867
+ .rounded {
3868
+ border-radius: 0.25rem;
3869
+ }
3864
3870
  .rounded-box {
3865
3871
  border-radius: var(--radius-box);
3866
3872
  }
@@ -11,10 +11,10 @@ module Karafka
11
11
  controller = build(Controllers::ErrorsController)
12
12
 
13
13
  r.get Integer do |offset|
14
- if params.current_offset != -1
15
- r.redirect root_path('errors', params.current_offset)
16
- else
14
+ if params.current_offset == -1
17
15
  controller.show(offset)
16
+ else
17
+ r.redirect root_path('errors', params.current_offset)
18
18
  end
19
19
  end
20
20
 
@@ -0,0 +1,42 @@
1
+ <div class="hidden">
2
+ <%== partial 'shared/controls' %>
3
+ </div>
4
+
5
+ <main class="flex items-center justify-center min-h-screen">
6
+ <div id="content" class="max-w-2xl w-full mx-auto">
7
+ <div class="flex items-center justify-center vh-100">
8
+ <div class="text-center">
9
+ <h1 class="text-8xl font-bold">500</h1>
10
+
11
+ <p class="text-3xl mt-5">
12
+ <span class="text-red-500">Oops!</span> Internal Server Error.
13
+ </p>
14
+
15
+ <div class="mt-5">
16
+ <p class="mb-10 text-lg font-light">
17
+ Something went wrong while processing your request.
18
+ </p>
19
+
20
+ <p class="text-lg font-light mb-5">This error has been reported and logged. Please try the following:</p>
21
+
22
+ <ul class="list-disc text-left text-lg mb-10 pl-5 font-light">
23
+ <li>Visit the <a href="<%= root_path('status') %>" class="text-blue-500 underline">Status</a> page to check if there are any system issues</li>
24
+ <li>Check the <a href="<%= root_path('errors') %>" class="text-blue-500 underline">Errors</a> page for details about this error</li>
25
+ <li>Verify that all required topics exist and are accessible</li>
26
+ <li>Ensure you have a working connection with your Kafka cluster</li>
27
+ <li>Try refreshing the page or navigating back to retry your action</li>
28
+ </ul>
29
+
30
+ <p class="mb-10 text-lg font-light">
31
+ If the problem persists, please check your application logs for more details.
32
+ </p>
33
+
34
+ <p>
35
+ <a href="<%= root_path %>" class="btn btn-primary mr-2 rounded-md py-1">Go Home</a>
36
+ <a href="<%= root_path('status') %>" class="btn btn-success text-white rounded-md">Status page</a>
37
+ </p>
38
+ </div>
39
+ </div>
40
+ </div>
41
+ </div>
42
+ </main>
@@ -3,6 +3,6 @@
3
3
  module Karafka
4
4
  module Web
5
5
  # Current gem version
6
- VERSION = '0.11.3'
6
+ VERSION = '0.11.5'
7
7
  end
8
8
  end
data/lib/karafka/web.rb CHANGED
@@ -26,8 +26,7 @@ module Karafka
26
26
  end
27
27
 
28
28
  # Sets up the whole configuration
29
- # @param [Block] block configuration block
30
- def setup(&block)
29
+ def setup(&)
31
30
  # You should never reconfigure Web UI after it has been enabled
32
31
  raise Errors::LateSetupError, 'Always call #setup before #enable!' if config.enabled
33
32
 
@@ -38,7 +37,7 @@ module Karafka
38
37
  Pro::Loader.pre_setup_all(config)
39
38
  end
40
39
 
41
- Config.configure(&block)
40
+ Config.configure(&)
42
41
 
43
42
  Pro::Loader.post_setup_all(config) if Karafka.pro?
44
43