karafka-web 0.7.9 → 0.8.0.rc1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (163) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +21 -6
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +66 -0
  6. data/Gemfile.lock +22 -22
  7. data/docker-compose.yml +3 -1
  8. data/karafka-web.gemspec +2 -2
  9. data/lib/karafka/web/config.rb +16 -3
  10. data/lib/karafka/web/contracts/config.rb +7 -2
  11. data/lib/karafka/web/errors.rb +12 -0
  12. data/lib/karafka/web/inflector.rb +33 -0
  13. data/lib/karafka/web/installer.rb +20 -11
  14. data/lib/karafka/web/management/actions/base.rb +36 -0
  15. data/lib/karafka/web/management/actions/clean_boot_file.rb +33 -0
  16. data/lib/karafka/web/management/actions/create_initial_states.rb +77 -0
  17. data/lib/karafka/web/management/actions/create_topics.rb +139 -0
  18. data/lib/karafka/web/management/actions/delete_topics.rb +30 -0
  19. data/lib/karafka/web/management/actions/enable.rb +117 -0
  20. data/lib/karafka/web/management/actions/extend_boot_file.rb +39 -0
  21. data/lib/karafka/web/management/actions/migrate_states_data.rb +18 -0
  22. data/lib/karafka/web/management/migrations/0_base.rb +58 -0
  23. data/lib/karafka/web/management/migrations/0_set_initial_consumers_metrics.rb +36 -0
  24. data/lib/karafka/web/management/migrations/0_set_initial_consumers_state.rb +43 -0
  25. data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_metrics.rb +26 -0
  26. data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_state.rb +23 -0
  27. data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_metrics.rb +24 -0
  28. data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_state.rb +20 -0
  29. data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_metrics.rb +24 -0
  30. data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_state.rb +20 -0
  31. data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_metrics.rb +36 -0
  32. data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_states.rb +32 -0
  33. data/lib/karafka/web/management/migrator.rb +117 -0
  34. data/lib/karafka/web/processing/consumer.rb +39 -38
  35. data/lib/karafka/web/processing/consumers/aggregators/metrics.rb +15 -7
  36. data/lib/karafka/web/processing/consumers/aggregators/state.rb +8 -3
  37. data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb +5 -1
  38. data/lib/karafka/web/processing/publisher.rb +59 -0
  39. data/lib/karafka/web/tracking/consumers/contracts/job.rb +3 -2
  40. data/lib/karafka/web/tracking/consumers/contracts/partition.rb +1 -0
  41. data/lib/karafka/web/tracking/consumers/contracts/report.rb +6 -1
  42. data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb +10 -1
  43. data/lib/karafka/web/tracking/consumers/listeners/connections.rb +49 -0
  44. data/lib/karafka/web/tracking/consumers/listeners/pausing.rb +7 -4
  45. data/lib/karafka/web/tracking/consumers/listeners/processing.rb +78 -70
  46. data/lib/karafka/web/tracking/consumers/listeners/statistics.rb +40 -13
  47. data/lib/karafka/web/tracking/consumers/sampler.rb +82 -25
  48. data/lib/karafka/web/tracking/helpers/ttls/array.rb +72 -0
  49. data/lib/karafka/web/tracking/helpers/ttls/hash.rb +34 -0
  50. data/lib/karafka/web/tracking/helpers/ttls/stats.rb +49 -0
  51. data/lib/karafka/web/tracking/helpers/ttls/windows.rb +32 -0
  52. data/lib/karafka/web/tracking/reporter.rb +1 -0
  53. data/lib/karafka/web/ui/app.rb +22 -4
  54. data/lib/karafka/web/ui/base.rb +18 -2
  55. data/lib/karafka/web/ui/controllers/base.rb +34 -4
  56. data/lib/karafka/web/ui/controllers/become_pro.rb +1 -1
  57. data/lib/karafka/web/ui/controllers/cluster.rb +33 -9
  58. data/lib/karafka/web/ui/controllers/consumers.rb +8 -2
  59. data/lib/karafka/web/ui/controllers/dashboard.rb +2 -2
  60. data/lib/karafka/web/ui/controllers/errors.rb +2 -2
  61. data/lib/karafka/web/ui/controllers/jobs.rb +55 -5
  62. data/lib/karafka/web/ui/controllers/requests/params.rb +5 -0
  63. data/lib/karafka/web/ui/controllers/responses/deny.rb +15 -0
  64. data/lib/karafka/web/ui/controllers/responses/file.rb +23 -0
  65. data/lib/karafka/web/ui/controllers/responses/{data.rb → render.rb} +3 -3
  66. data/lib/karafka/web/ui/controllers/routing.rb +11 -2
  67. data/lib/karafka/web/ui/controllers/status.rb +1 -1
  68. data/lib/karafka/web/ui/helpers/application_helper.rb +70 -0
  69. data/lib/karafka/web/ui/lib/hash_proxy.rb +29 -14
  70. data/lib/karafka/web/ui/lib/sorter.rb +170 -0
  71. data/lib/karafka/web/ui/models/counters.rb +6 -0
  72. data/lib/karafka/web/ui/models/health.rb +23 -2
  73. data/lib/karafka/web/ui/models/jobs.rb +48 -0
  74. data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb +33 -0
  75. data/lib/karafka/web/ui/models/metrics/charts/topics.rb +1 -10
  76. data/lib/karafka/web/ui/models/process.rb +2 -1
  77. data/lib/karafka/web/ui/models/status.rb +23 -7
  78. data/lib/karafka/web/ui/models/topic.rb +3 -1
  79. data/lib/karafka/web/ui/models/visibility_filter.rb +16 -0
  80. data/lib/karafka/web/ui/pro/app.rb +44 -6
  81. data/lib/karafka/web/ui/pro/controllers/cluster.rb +1 -0
  82. data/lib/karafka/web/ui/pro/controllers/consumers.rb +52 -6
  83. data/lib/karafka/web/ui/pro/controllers/dashboard.rb +1 -1
  84. data/lib/karafka/web/ui/pro/controllers/dlq.rb +1 -1
  85. data/lib/karafka/web/ui/pro/controllers/errors.rb +3 -3
  86. data/lib/karafka/web/ui/pro/controllers/explorer.rb +8 -8
  87. data/lib/karafka/web/ui/pro/controllers/health.rb +34 -2
  88. data/lib/karafka/web/ui/pro/controllers/jobs.rb +11 -0
  89. data/lib/karafka/web/ui/pro/controllers/messages.rb +42 -0
  90. data/lib/karafka/web/ui/pro/controllers/routing.rb +11 -2
  91. data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb +8 -2
  92. data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb +14 -8
  93. data/lib/karafka/web/ui/pro/views/consumers/_counters.erb +8 -6
  94. data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb +4 -1
  95. data/lib/karafka/web/ui/pro/views/consumers/consumer/_no_jobs.erb +1 -1
  96. data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb +1 -3
  97. data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb +28 -11
  98. data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb +10 -3
  99. data/lib/karafka/web/ui/pro/views/consumers/index.erb +3 -3
  100. data/lib/karafka/web/ui/pro/views/consumers/pending_jobs.erb +43 -0
  101. data/lib/karafka/web/ui/pro/views/consumers/{jobs.erb → running_jobs.erb} +11 -10
  102. data/lib/karafka/web/ui/pro/views/dashboard/index.erb +7 -1
  103. data/lib/karafka/web/ui/pro/views/explorer/message/_message_actions.erb +18 -0
  104. data/lib/karafka/web/ui/pro/views/explorer/message/_metadata.erb +43 -0
  105. data/lib/karafka/web/ui/pro/views/explorer/message/_payload.erb +21 -0
  106. data/lib/karafka/web/ui/pro/views/explorer/message/_payload_actions.erb +19 -0
  107. data/lib/karafka/web/ui/pro/views/explorer/show.erb +9 -84
  108. data/lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb +8 -0
  109. data/lib/karafka/web/ui/pro/views/health/_partition.erb +1 -3
  110. data/lib/karafka/web/ui/pro/views/health/_partition_offset.erb +4 -4
  111. data/lib/karafka/web/ui/pro/views/health/_partition_times.erb +32 -0
  112. data/lib/karafka/web/ui/pro/views/health/_tabs.erb +9 -0
  113. data/lib/karafka/web/ui/pro/views/health/changes.erb +66 -0
  114. data/lib/karafka/web/ui/pro/views/health/offsets.erb +14 -14
  115. data/lib/karafka/web/ui/pro/views/health/overview.erb +11 -11
  116. data/lib/karafka/web/ui/pro/views/jobs/_job.erb +1 -1
  117. data/lib/karafka/web/ui/pro/views/jobs/_no_jobs.erb +1 -1
  118. data/lib/karafka/web/ui/pro/views/jobs/pending.erb +39 -0
  119. data/lib/karafka/web/ui/pro/views/jobs/running.erb +39 -0
  120. data/lib/karafka/web/ui/pro/views/routing/_consumer_group.erb +2 -2
  121. data/lib/karafka/web/ui/pro/views/routing/_topic.erb +9 -0
  122. data/lib/karafka/web/ui/pro/views/routing/show.erb +12 -0
  123. data/lib/karafka/web/ui/pro/views/shared/_navigation.erb +1 -1
  124. data/lib/karafka/web/ui/public/javascripts/application.js +10 -0
  125. data/lib/karafka/web/ui/public/stylesheets/application.css +4 -0
  126. data/lib/karafka/web/ui/views/cluster/_breadcrumbs.erb +16 -0
  127. data/lib/karafka/web/ui/views/cluster/_tabs.erb +27 -0
  128. data/lib/karafka/web/ui/views/cluster/brokers.erb +27 -0
  129. data/lib/karafka/web/ui/views/cluster/topics.erb +35 -0
  130. data/lib/karafka/web/ui/views/consumers/_counters.erb +8 -6
  131. data/lib/karafka/web/ui/views/consumers/_summary.erb +2 -2
  132. data/lib/karafka/web/ui/views/consumers/index.erb +3 -3
  133. data/lib/karafka/web/ui/views/dashboard/_ranges_selector.erb +23 -7
  134. data/lib/karafka/web/ui/views/dashboard/index.erb +19 -8
  135. data/lib/karafka/web/ui/views/errors/show.erb +2 -23
  136. data/lib/karafka/web/ui/views/jobs/_breadcrumbs.erb +17 -1
  137. data/lib/karafka/web/ui/views/jobs/_job.erb +1 -1
  138. data/lib/karafka/web/ui/views/jobs/_no_jobs.erb +1 -1
  139. data/lib/karafka/web/ui/views/jobs/_tabs.erb +27 -0
  140. data/lib/karafka/web/ui/views/jobs/{index.erb → pending.erb} +9 -7
  141. data/lib/karafka/web/ui/{pro/views/jobs/index.erb → views/jobs/running.erb} +9 -11
  142. data/lib/karafka/web/ui/views/routing/_consumer_group.erb +14 -12
  143. data/lib/karafka/web/ui/views/shared/_navigation.erb +1 -1
  144. data/lib/karafka/web/ui/views/shared/_pagination.erb +1 -1
  145. data/lib/karafka/web/ui/views/shared/exceptions/not_allowed.erb +37 -0
  146. data/lib/karafka/web/ui/views/status/show.erb +17 -2
  147. data/lib/karafka/web/ui/views/status/warnings/_routing_topics_presence.erb +15 -0
  148. data/lib/karafka/web/version.rb +1 -1
  149. data/lib/karafka/web.rb +6 -2
  150. data.tar.gz.sig +0 -0
  151. metadata +61 -26
  152. metadata.gz.sig +0 -0
  153. data/lib/karafka/web/management/base.rb +0 -34
  154. data/lib/karafka/web/management/clean_boot_file.rb +0 -31
  155. data/lib/karafka/web/management/create_initial_states.rb +0 -101
  156. data/lib/karafka/web/management/create_topics.rb +0 -133
  157. data/lib/karafka/web/management/delete_topics.rb +0 -28
  158. data/lib/karafka/web/management/enable.rb +0 -102
  159. data/lib/karafka/web/management/extend_boot_file.rb +0 -37
  160. data/lib/karafka/web/tracking/ttl_array.rb +0 -59
  161. data/lib/karafka/web/tracking/ttl_hash.rb +0 -16
  162. data/lib/karafka/web/ui/pro/views/dashboard/_ranges_selector.erb +0 -39
  163. data/lib/karafka/web/ui/views/cluster/index.erb +0 -74
@@ -1,102 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Web
5
- module Management
6
- # @note This runs on each process start that has `karafka.rb`. It needs to be executed
7
- # also in the context of other processes types and not only karafka server, because it
8
- # installs producers instrumentation and routing as well.
9
- class Enable < Base
10
- # Enables routing consumer group and subscribes Web-UI listeners
11
- def call
12
- extend_routing
13
- subscribe_to_monitor
14
- subscribe_to_close_web_producer
15
- end
16
-
17
- private
18
-
19
- # Enables all the needed routes
20
- def extend_routing
21
- ::Karafka::App.routes.draw do
22
- web_deserializer = ::Karafka::Web::Deserializer.new
23
-
24
- consumer_group ::Karafka::Web.config.processing.consumer_group do
25
- # Topic we listen on to materialize the states
26
- topic ::Karafka::Web.config.topics.consumers.reports do
27
- config(active: false)
28
- active ::Karafka::Web.config.processing.active
29
- # Since we materialize state in intervals, we can poll for half of this time without
30
- # impacting the reporting responsiveness
31
- max_wait_time ::Karafka::Web.config.processing.interval / 2
32
- max_messages 1_000
33
- consumer ::Karafka::Web::Processing::Consumer
34
- # This needs to be true in order not to reload the consumer in dev. This consumer
35
- # should not be affected by the end user development process
36
- consumer_persistence true
37
- deserializer web_deserializer
38
- manual_offset_management true
39
- # Start from the most recent data, do not materialize historical states
40
- # This prevents us from dealing with cases, where client id would be changed and
41
- # consumer group name would be renamed and we would start consuming all historical
42
- initial_offset 'latest'
43
- end
44
-
45
- # We define those three here without consumption, so Web understands how to deserialize
46
- # them when used / viewed
47
- topic ::Karafka::Web.config.topics.consumers.states do
48
- config(active: false)
49
- active false
50
- deserializer web_deserializer
51
- end
52
-
53
- topic ::Karafka::Web.config.topics.consumers.metrics do
54
- config(active: false)
55
- active false
56
- deserializer web_deserializer
57
- end
58
-
59
- topic ::Karafka::Web.config.topics.errors do
60
- config(active: false)
61
- active false
62
- deserializer web_deserializer
63
- end
64
- end
65
- end
66
- end
67
-
68
- # Subscribes with all needed listeners
69
- def subscribe_to_monitor
70
- # Installs all the consumer related listeners
71
- ::Karafka::Web.config.tracking.consumers.listeners.each do |listener|
72
- ::Karafka.monitor.subscribe(listener)
73
- end
74
-
75
- # Installs all the producer related listeners into Karafka default listener and
76
- # into Karafka::Web listener in case it would be different than the Karafka one
77
- ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
78
- ::Karafka.producer.monitor.subscribe(listener)
79
-
80
- # Do not instrument twice in case only one default producer is used
81
- next if ::Karafka.producer == ::Karafka::Web.producer
82
-
83
- ::Karafka::Web.producer.monitor.subscribe(listener)
84
- end
85
- end
86
-
87
- # In most cases we want to close the producer if possible.
88
- # While we cannot do it easily in user processes and we should rely on WaterDrop
89
- # finalization logic, we can do it in `karafka server` on terminate
90
- #
91
- # In other places, this producer anyhow should not be used.
92
- def subscribe_to_close_web_producer
93
- ::Karafka::App.monitor.subscribe('app.terminated') do
94
- # If Web producer is the same as `Karafka.producer` it will do nothing as you can
95
- # call `#close` multiple times without side effects
96
- ::Karafka::Web.producer.close
97
- end
98
- end
99
- end
100
- end
101
- end
102
- end
@@ -1,37 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Web
5
- module Management
6
- # Extends the boot file with Web components
7
- class ExtendBootFile < Base
8
- # Code that is needed in the `karafka.rb` to connect Web UI to Karafka
9
- ENABLER_CODE = 'Karafka::Web.enable!'
10
-
11
- # Template with initial Web UI configuration
12
- # Session secret needs to be set per user and per env
13
- SETUP_TEMPLATE = <<~CONFIG.freeze
14
- Karafka::Web.setup do |config|
15
- # You may want to set it per ENV. This value was randomly generated.
16
- config.ui.sessions.secret = '#{SecureRandom.hex(32)}'
17
- end
18
-
19
- #{ENABLER_CODE}
20
- CONFIG
21
-
22
- # Adds needed code
23
- def call
24
- if File.read(Karafka.boot_file).include?(ENABLER_CODE)
25
- puts "Web UI #{already} installed."
26
- else
27
- puts 'Updating the Karafka boot file...'
28
- File.open(Karafka.boot_file, 'a') do |f|
29
- f << "\n#{SETUP_TEMPLATE}\n"
30
- end
31
- puts "Karafka boot file #{successfully} updated."
32
- end
33
- end
34
- end
35
- end
36
- end
37
- end
@@ -1,59 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Web
5
- module Tracking
6
- # Array that allows us to store data points that expire over time automatically.
7
- class TtlArray
8
- include ::Karafka::Core::Helpers::Time
9
- include Enumerable
10
-
11
- # @param ttl [Integer] milliseconds ttl
12
- def initialize(ttl)
13
- @ttl = ttl
14
- @accu = []
15
- end
16
-
17
- # Iterates over only active elements
18
- def each
19
- clear
20
-
21
- @accu.each do |sample|
22
- yield sample[:value]
23
- end
24
- end
25
-
26
- # @param value [Object] adds value to the array
27
- # @return [Object] added element
28
- def <<(value)
29
- @accu << { value: value, added_at: monotonic_now }
30
-
31
- clear
32
-
33
- value
34
- end
35
-
36
- # @return [Boolean] is the array empty
37
- def empty?
38
- clear
39
- @accu.empty?
40
- end
41
-
42
- # @return [Array] pure array version with only active elements
43
- def to_a
44
- clear
45
- super
46
- end
47
-
48
- private
49
-
50
- # Evicts outdated samples
51
- def clear
52
- @accu.delete_if do |sample|
53
- monotonic_now - sample[:added_at] > @ttl
54
- end
55
- end
56
- end
57
- end
58
- end
59
- end
@@ -1,16 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Web
5
- module Tracking
6
- # Hash that accumulates data that has an expiration date (ttl)
7
- # Used to keep track of metrics in a window
8
- class TtlHash < Hash
9
- # @param ttl [Integer] milliseconds ttl
10
- def initialize(ttl)
11
- super() { |k, v| k[v] = TtlArray.new(ttl) }
12
- end
13
- end
14
- end
15
- end
16
- end
@@ -1,39 +0,0 @@
1
- <div class="container">
2
- <div class="row">
3
- <div class="col-sm-12 text-end">
4
- <div class="btn-group btn-group-sm" role="group" aria-label="Small button group">
5
- <%
6
- path = root_path('dashboard?range=seconds')
7
- active = params.current_range == :seconds ? 'active' : false
8
- %>
9
- <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
10
- 5 minutes
11
- </a>
12
-
13
- <%
14
- path = root_path('dashboard?range=minutes')
15
- active = params.current_range == :minutes ? 'active' : false
16
- %>
17
- <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
18
- 1 hour
19
- </a>
20
-
21
- <%
22
- path = root_path('dashboard?range=hours')
23
- active = params.current_range == :hours ? 'active' : false
24
- %>
25
- <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
26
- 24 hours
27
- </a>
28
-
29
- <%
30
- path = root_path('dashboard?range=days')
31
- active = params.current_range == :days ? 'active' : false
32
- %>
33
- <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
34
- 7 days
35
- </a>
36
- </div>
37
- </div>
38
- </div>
39
- </div>
@@ -1,74 +0,0 @@
1
- <%== view_title('Cluster informations') %>
2
-
3
- <div class="container mb-5">
4
- <div class="row mb-3">
5
- <div class="col-sm-12">
6
- <h4 class="mb-4">
7
- Brokers
8
- </h4>
9
- <hr/>
10
- </div>
11
- </div>
12
-
13
- <div class="row">
14
- <div class="col-lg-12">
15
- <table class="processes bg-white table table-hover table-bordered table-striped">
16
- <thead>
17
- <tr class="align-middle">
18
- <th>Broker id</th>
19
- <th>Name</th>
20
- <th>Port</th>
21
- </tr>
22
- </thead>
23
- <tbody>
24
- <%==
25
- each_partial(
26
- @cluster_info.brokers,
27
- 'cluster/broker'
28
- )
29
- %>
30
- </tbody>
31
- </table>
32
- </div>
33
- </div>
34
- </div>
35
-
36
- <div class="container mb-5">
37
- <div class="row mb-3">
38
- <div class="col-sm-12">
39
- <h4 class="mb-4">
40
- Topics and partitions
41
- </h4>
42
- <hr/>
43
- </div>
44
- </div>
45
- <div class="row">
46
- <div class="col-lg-12">
47
- <% if @partitions.empty? && params.current_page <= 1 %>
48
- <%== partial 'cluster/no_partitions' %>
49
- <% elsif @partitions.empty? %>
50
- <%== partial 'shared/no_paginated_data' %>
51
- <% else %>
52
- <table class="processes bg-white table table-hover table-bordered table-striped">
53
- <thead>
54
- <tr class="align-middle">
55
- <th>Topic name</th>
56
- <th>Partition id</th>
57
- <th>Leader</th>
58
- <th>Replica count</th>
59
- <th>In sync brokers</th>
60
- </tr>
61
- </thead>
62
- <tbody>
63
- <%==
64
- each_partial(
65
- @partitions,
66
- 'cluster/partition'
67
- )
68
- %>
69
- </tbody>
70
- </table>
71
- <% end %>
72
- </div>
73
- </div>
74
- </div>