karafka-web 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +8 -1
- data/Gemfile.lock +1 -1
- data/bin/karafka-web +1 -26
- data/lib/karafka/web/cli.rb +80 -0
- data/lib/karafka/web/installer.rb +65 -28
- data/lib/karafka/web/version.rb +1 -1
- data/lib/karafka/web.rb +0 -6
- data.tar.gz.sig +0 -0
- metadata +3 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3a70fc5cb35d0e6e60a348e3efe91113b54d0c62310b7c3af4e7584aa70cfb70
|
4
|
+
data.tar.gz: 43a96b7e7a4622721dbc6ba144852335aecbfe907049bc47ab563ea07f40eb54
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db0b153569ca93f1164a851216e0f224acc56e6a7533a35a86a44f6f2f8bffc1630e5ccf7c2e53a14455206401c45b59b1728eb266c59360be8f952794d3011b
|
7
|
+
data.tar.gz: 794c77edb27f7f9a397c3de891b146f9020653b84bad46e1f1b537a1d547ffc793211e7bd5ffb0f426c0b8fa9b6359230bb3fe3ee8fa5ec668cb0e1d39782d5a
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,8 +1,15 @@
|
|
1
1
|
# Karafka Web changelog
|
2
2
|
|
3
|
+
## 0.1.3 (2023-02-14)
|
4
|
+
- Skip topics creation if web topics already exist (do not raise error)
|
5
|
+
- Support ability to provide replication factor in the install command
|
6
|
+
- Provide ability to reset the state with a `reset` command. It will remove and re-create the topics.
|
7
|
+
- Provide ability to uninstall the web via the CLI `uninstall` command
|
8
|
+
- Remove the `Karafka::Web.bootstrap!` method as the install should happen via `bundle exec karafka-web install`
|
9
|
+
|
3
10
|
## 0.1.2 (2023-02-10)
|
4
11
|
- Provide more comprehensive info when lag stored and stored offset are not available.
|
5
|
-
- Setup rspec scaffold
|
12
|
+
- Setup rspec scaffold.
|
6
13
|
|
7
14
|
## 0.1.1 (2023-01-30)
|
8
15
|
- Rename `Karafka::Web.bootstrap_topics!` to `Karafka::Web.bootstrap!` and expand it with the zero state injection.
|
data/Gemfile.lock
CHANGED
data/bin/karafka-web
CHANGED
@@ -4,30 +4,5 @@ require 'karafka'
|
|
4
4
|
require 'karafka/web'
|
5
5
|
|
6
6
|
::Karafka::Cli::Base.load
|
7
|
-
include ::Karafka::Helpers::Colorize
|
8
7
|
|
9
|
-
|
10
|
-
when 'install'
|
11
|
-
puts
|
12
|
-
puts 'Installing Karafka Web UI...'
|
13
|
-
puts
|
14
|
-
puts 'Creating necessary topics and populating state data...'
|
15
|
-
Karafka::Web.bootstrap!
|
16
|
-
puts 'Updating the Karafka boot file...'
|
17
|
-
|
18
|
-
enabler = 'Karafka::Web.enable!'
|
19
|
-
|
20
|
-
if File.read(Karafka.boot_file).include?(enabler)
|
21
|
-
puts "Web UI #{green('already')} installed."
|
22
|
-
else
|
23
|
-
File.open(Karafka.boot_file, 'a') do |f|
|
24
|
-
f << "\n#{enabler}\n"
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
puts
|
29
|
-
puts("Installation #{green('completed')}. Have fun!")
|
30
|
-
puts
|
31
|
-
else
|
32
|
-
raise NotImplementedError, "#{ARGV[0]} is not supported"
|
33
|
-
end
|
8
|
+
::Karafka::Web::Cli.start
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Web
|
5
|
+
# Karafka itself depends on Thor, so we can use it
|
6
|
+
class Cli < Thor
|
7
|
+
include ::Karafka::Helpers::Colorize
|
8
|
+
|
9
|
+
# Code that is needed in the `karafka.rb` to connect Web UI to Karafka
|
10
|
+
ENABLER_CODE = 'Karafka::Web.enable!'
|
11
|
+
|
12
|
+
private_constant :ENABLER_CODE
|
13
|
+
|
14
|
+
package_name 'Karafka Web'
|
15
|
+
|
16
|
+
desc 'install', 'Installs the Web UI'
|
17
|
+
method_option(
|
18
|
+
:replication_factor,
|
19
|
+
desc: 'Replication factor for created topics',
|
20
|
+
default: 1,
|
21
|
+
type: :numeric
|
22
|
+
)
|
23
|
+
# Installs Karafka Web
|
24
|
+
def install
|
25
|
+
puts
|
26
|
+
puts 'Installing Karafka Web UI...'
|
27
|
+
puts
|
28
|
+
puts 'Creating necessary topics and populating state data...'
|
29
|
+
|
30
|
+
Karafka::Web::Installer.new.bootstrap!(replication_factor: options[:replication_factor])
|
31
|
+
|
32
|
+
puts 'Updating the Karafka boot file...'
|
33
|
+
|
34
|
+
if File.read(Karafka.boot_file).include?(ENABLER_CODE)
|
35
|
+
puts "Web UI #{green('already')} installed."
|
36
|
+
else
|
37
|
+
File.open(Karafka.boot_file, 'a') do |f|
|
38
|
+
f << "\n#{ENABLER_CODE}\n"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
puts
|
43
|
+
puts("Installation #{green('completed')}. Have fun!")
|
44
|
+
puts
|
45
|
+
end
|
46
|
+
|
47
|
+
desc 'reset', 'Resets the Web UI by removing all the Web topics and creating them again'
|
48
|
+
# Resets Karafka Web
|
49
|
+
def reset
|
50
|
+
puts
|
51
|
+
puts 'Resetting Karafka Web UI...'
|
52
|
+
Karafka::Web::Installer.new.reset!
|
53
|
+
puts
|
54
|
+
puts("Resetting #{green('completed')}. Have fun!")
|
55
|
+
puts
|
56
|
+
end
|
57
|
+
|
58
|
+
desc 'uninstall', 'Removes all the Web UI topics and the enabled code'
|
59
|
+
# Uninstalls Karafka Web
|
60
|
+
def uninstall
|
61
|
+
puts
|
62
|
+
puts 'Uninstalling Karafka Web UI...'
|
63
|
+
Karafka::Web::Installer.new.uninstall!
|
64
|
+
|
65
|
+
puts 'Updating the Karafka boot file...'
|
66
|
+
|
67
|
+
karafka_rb = File.readlines(Karafka.boot_file)
|
68
|
+
if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
|
69
|
+
karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
|
70
|
+
|
71
|
+
File.write(Karafka.boot_file, karafka_rb.join)
|
72
|
+
end
|
73
|
+
|
74
|
+
puts
|
75
|
+
puts("Uninstalling #{green('completed')}. Goodbye!")
|
76
|
+
puts
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -13,6 +13,30 @@ module Karafka
|
|
13
13
|
bootstrap_state!
|
14
14
|
end
|
15
15
|
|
16
|
+
# Removes all the Karafka topics and creates them again with the same replication factor
|
17
|
+
def reset!
|
18
|
+
states_topic = ::Karafka::Web.config.topics.consumers.states
|
19
|
+
replication_factor = ::Karafka::Admin
|
20
|
+
.cluster_info
|
21
|
+
.topics
|
22
|
+
.find { |topic| topic[:topic_name] == states_topic }
|
23
|
+
.fetch(:partitions)
|
24
|
+
.first
|
25
|
+
.fetch(:replica_count)
|
26
|
+
|
27
|
+
uninstall!
|
28
|
+
bootstrap!(replication_factor: replication_factor)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Removes all the Karafka Web topics
|
32
|
+
def uninstall!
|
33
|
+
[
|
34
|
+
::Karafka::Web.config.topics.consumers.states,
|
35
|
+
::Karafka::Web.config.topics.consumers.reports,
|
36
|
+
::Karafka::Web.config.topics.errors
|
37
|
+
].each { |topic_name| ::Karafka::Admin.delete_topic(topic_name) }
|
38
|
+
end
|
39
|
+
|
16
40
|
# Adds the extra needed consumer group, topics and routes for Web UI to be able to operate
|
17
41
|
def enable!
|
18
42
|
::Karafka::App.routes.draw do
|
@@ -61,36 +85,49 @@ module Karafka
|
|
61
85
|
#
|
62
86
|
# @param replication_factor [Integer]
|
63
87
|
def bootstrap_topics!(replication_factor = 1)
|
64
|
-
|
65
|
-
::Karafka::Admin.create_topic(
|
66
|
-
::Karafka::Web.config.topics.consumers.states,
|
67
|
-
1,
|
68
|
-
replication_factor,
|
69
|
-
# We care only about the most recent state, previous are irrelevant
|
70
|
-
{ 'cleanup.policy': 'compact' }
|
71
|
-
)
|
88
|
+
existing_topics = ::Karafka::Admin.cluster_info.topics.map { |topic| topic[:topic_name] }
|
72
89
|
|
73
|
-
|
74
|
-
::Karafka::
|
75
|
-
|
76
|
-
1,
|
77
|
-
replication_factor,
|
78
|
-
# We do not need to to store this data for longer than 7 days as this data is only used
|
79
|
-
# to materialize the end states
|
80
|
-
# On the other hand we do not want to have it really short-living because in case of a
|
81
|
-
# consumer crash, we may want to use this info to catch up and backfill the state
|
82
|
-
{ 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
|
83
|
-
)
|
90
|
+
consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
|
91
|
+
consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
|
92
|
+
errors_topic = ::Karafka::Web.config.topics.errors
|
84
93
|
|
85
|
-
#
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
+
# Create only if needed
|
95
|
+
unless existing_topics.include?(consumers_states_topic)
|
96
|
+
# This topic needs to have one partition
|
97
|
+
::Karafka::Admin.create_topic(
|
98
|
+
consumers_states_topic,
|
99
|
+
1,
|
100
|
+
replication_factor,
|
101
|
+
# We care only about the most recent state, previous are irrelevant
|
102
|
+
{ 'cleanup.policy': 'compact' }
|
103
|
+
)
|
104
|
+
end
|
105
|
+
|
106
|
+
unless existing_topics.include?(consumers_reports_topic)
|
107
|
+
# This topic needs to have one partition
|
108
|
+
::Karafka::Admin.create_topic(
|
109
|
+
consumers_reports_topic,
|
110
|
+
1,
|
111
|
+
replication_factor,
|
112
|
+
# We do not need to to store this data for longer than 7 days as this data is only used
|
113
|
+
# to materialize the end states
|
114
|
+
# On the other hand we do not want to have it really short-living because in case of a
|
115
|
+
# consumer crash, we may want to use this info to catch up and backfill the state
|
116
|
+
{ 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
|
117
|
+
)
|
118
|
+
end
|
119
|
+
|
120
|
+
unless existing_topics.include?(errors_topic)
|
121
|
+
# All the errors will be dispatched here
|
122
|
+
# This topic can have multiple partitions but we go with one by default. A single Ruby
|
123
|
+
# process should not crash that often and if there is an expectation of a higher volume
|
124
|
+
# of errors, this can be changed by the end user
|
125
|
+
::Karafka::Admin.create_topic(
|
126
|
+
errors_topic,
|
127
|
+
1,
|
128
|
+
replication_factor
|
129
|
+
)
|
130
|
+
end
|
94
131
|
|
95
132
|
bootstrap_state!
|
96
133
|
end
|
data/lib/karafka/web/version.rb
CHANGED
data/lib/karafka/web.rb
CHANGED
@@ -27,12 +27,6 @@ module Karafka
|
|
27
27
|
Config.config
|
28
28
|
end
|
29
29
|
|
30
|
-
# Creates all the needed topics for the admin UI to work and populates initial (empty) set
|
31
|
-
# of data, so the UI will work even when no Karafka servers are started
|
32
|
-
def bootstrap!
|
33
|
-
Installer.new.bootstrap!
|
34
|
-
end
|
35
|
-
|
36
30
|
# Activates all the needed routing and sets up listener, etc
|
37
31
|
# This needs to run **after** the optional configuration of the web component
|
38
32
|
def enable!
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-web
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-02-
|
38
|
+
date: 2023-02-14 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: erubi
|
@@ -163,6 +163,7 @@ files:
|
|
163
163
|
- karafka-web.gemspec
|
164
164
|
- lib/karafka/web.rb
|
165
165
|
- lib/karafka/web/app.rb
|
166
|
+
- lib/karafka/web/cli.rb
|
166
167
|
- lib/karafka/web/config.rb
|
167
168
|
- lib/karafka/web/deserializer.rb
|
168
169
|
- lib/karafka/web/errors.rb
|
metadata.gz.sig
CHANGED
Binary file
|