kuber_kit 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -1
- data/README.md +16 -3
- data/TODO.md +1 -3
- data/example/configurations/review.rb +2 -1
- data/example/images/app_sources/Dockerfile +1 -1
- data/example/images/ruby_app/image.rb +4 -4
- data/example/infrastructure/build_servers.rb +8 -0
- data/kuber_kit.gemspec +1 -0
- data/lib/kuber_kit.rb +36 -19
- data/lib/kuber_kit/actions/configuration_loader.rb +3 -0
- data/lib/kuber_kit/actions/env_file_reader.rb +3 -0
- data/lib/kuber_kit/actions/image_compiler.rb +14 -10
- data/lib/kuber_kit/actions/kubectl_applier.rb +4 -1
- data/lib/kuber_kit/actions/service_deployer.rb +4 -0
- data/lib/kuber_kit/actions/service_reader.rb +4 -0
- data/lib/kuber_kit/actions/template_reader.rb +3 -0
- data/lib/kuber_kit/cli.rb +12 -6
- data/lib/kuber_kit/configs.rb +24 -22
- data/lib/kuber_kit/container.rb +17 -13
- data/lib/kuber_kit/core/artifacts/artifact_store.rb +9 -28
- data/lib/kuber_kit/core/build_servers/abstract_build_server.rb +21 -0
- data/lib/kuber_kit/core/build_servers/build_server.rb +24 -0
- data/lib/kuber_kit/core/build_servers/build_server_store.rb +18 -0
- data/lib/kuber_kit/core/configuration.rb +7 -3
- data/lib/kuber_kit/core/configuration_definition.rb +10 -0
- data/lib/kuber_kit/core/configuration_factory.rb +10 -1
- data/lib/kuber_kit/core/configuration_store.rb +14 -24
- data/lib/kuber_kit/core/env_files/env_file_store.rb +8 -23
- data/lib/kuber_kit/core/image_store.rb +8 -18
- data/lib/kuber_kit/core/registries/registry_store.rb +8 -23
- data/lib/kuber_kit/core/service_store.rb +13 -23
- data/lib/kuber_kit/core/store.rb +48 -0
- data/lib/kuber_kit/core/templates/template_store.rb +9 -28
- data/lib/kuber_kit/image_compiler/build_server_pool.rb +30 -0
- data/lib/kuber_kit/image_compiler/build_server_pool_factory.rb +13 -0
- data/lib/kuber_kit/image_compiler/image_build_dir_creator.rb +13 -7
- data/lib/kuber_kit/image_compiler/image_dependency_resolver.rb +25 -5
- data/lib/kuber_kit/preprocessing/file_preprocessor.rb +5 -4
- data/lib/kuber_kit/shell/abstract_shell.rb +4 -0
- data/lib/kuber_kit/shell/{bash_commands.rb → commands/bash_commands.rb} +1 -1
- data/lib/kuber_kit/shell/{docker_commands.rb → commands/docker_commands.rb} +1 -1
- data/lib/kuber_kit/shell/{git_commands.rb → commands/git_commands.rb} +1 -1
- data/lib/kuber_kit/shell/{kubectl_commands.rb → commands/kubectl_commands.rb} +1 -1
- data/lib/kuber_kit/shell/{rsync_commands.rb → commands/rsync_commands.rb} +9 -3
- data/lib/kuber_kit/shell/local_shell.rb +24 -5
- data/lib/kuber_kit/shell/ssh_session.rb +60 -0
- data/lib/kuber_kit/shell/ssh_shell.rb +77 -0
- data/lib/kuber_kit/tools/file_presence_checker.rb +6 -2
- data/lib/kuber_kit/version.rb +1 -1
- metadata +30 -9
- data/lib/kuber_kit/preprocessing/dir_preprocessor.rb +0 -19
- data/lib/kuber_kit/tools/files_sync.rb +0 -10
@@ -0,0 +1,48 @@
|
|
1
|
+
class KuberKit::Core::Store
|
2
|
+
NotFoundError = Class.new(KuberKit::NotFoundError)
|
3
|
+
AlreadyAddedError = Class.new(KuberKit::Error)
|
4
|
+
|
5
|
+
attr_reader :object_class_name
|
6
|
+
|
7
|
+
def initialize(object_class_name)
|
8
|
+
@object_class_name = object_class_name
|
9
|
+
end
|
10
|
+
|
11
|
+
def add(item_name, item)
|
12
|
+
unless item.is_a?(object_class_name)
|
13
|
+
raise ArgumentError.new("#{self.object_class_name}: should be an instance of #{object_class_name}, got: #{item.inspect}")
|
14
|
+
end
|
15
|
+
|
16
|
+
unless items[item_name].nil?
|
17
|
+
raise AlreadyAddedError, "#{self.object_class_name}: item with name #{item_name} was already added"
|
18
|
+
end
|
19
|
+
|
20
|
+
items[item_name] = item
|
21
|
+
end
|
22
|
+
|
23
|
+
def get(item_name)
|
24
|
+
item = items[item_name]
|
25
|
+
|
26
|
+
if item.nil?
|
27
|
+
raise NotFoundError, "#{self.object_class_name}: item '#{item_name}' not found"
|
28
|
+
end
|
29
|
+
|
30
|
+
item
|
31
|
+
end
|
32
|
+
|
33
|
+
def items
|
34
|
+
@items ||= {}
|
35
|
+
end
|
36
|
+
|
37
|
+
def reset!
|
38
|
+
@items = {}
|
39
|
+
end
|
40
|
+
|
41
|
+
def size
|
42
|
+
items.count
|
43
|
+
end
|
44
|
+
|
45
|
+
def exists?(name)
|
46
|
+
!items[name].nil?
|
47
|
+
end
|
48
|
+
end
|
@@ -1,19 +1,6 @@
|
|
1
1
|
class KuberKit::Core::Templates::TemplateStore
|
2
|
-
NotFoundError = Class.new(KuberKit::NotFoundError)
|
3
|
-
AlreadyAddedError = Class.new(KuberKit::Error)
|
4
|
-
|
5
2
|
def add(template)
|
6
|
-
|
7
|
-
|
8
|
-
if !template.is_a?(KuberKit::Core::Templates::AbstractTemplate)
|
9
|
-
raise ArgumentError.new("should be an instance of KuberKit::Core::Templates::AbstractTemplate, got: #{template.inspect}")
|
10
|
-
end
|
11
|
-
|
12
|
-
unless @@templates[template.name].nil?
|
13
|
-
raise AlreadyAddedError, "template #{template.name} was already added"
|
14
|
-
end
|
15
|
-
|
16
|
-
@@templates[template.name] = template
|
3
|
+
store.add(template.name, template)
|
17
4
|
end
|
18
5
|
|
19
6
|
def get(template_name)
|
@@ -24,14 +11,7 @@ class KuberKit::Core::Templates::TemplateStore
|
|
24
11
|
end
|
25
12
|
|
26
13
|
def get_global(template_name)
|
27
|
-
|
28
|
-
template = @@templates[template_name]
|
29
|
-
|
30
|
-
if template.nil?
|
31
|
-
raise NotFoundError, "template '#{template_name}' not found"
|
32
|
-
end
|
33
|
-
|
34
|
-
template
|
14
|
+
store.get(template_name)
|
35
15
|
end
|
36
16
|
|
37
17
|
def get_from_configuration(template_name)
|
@@ -40,14 +20,15 @@ class KuberKit::Core::Templates::TemplateStore
|
|
40
20
|
end
|
41
21
|
|
42
22
|
def reset!
|
43
|
-
|
44
|
-
end
|
45
|
-
|
46
|
-
def all_definitions
|
47
|
-
@@templates ||= {}
|
23
|
+
store.reset!
|
48
24
|
end
|
49
25
|
|
50
26
|
def exists?(template_name)
|
51
|
-
|
27
|
+
store.exists?(template_name)
|
52
28
|
end
|
29
|
+
|
30
|
+
private
|
31
|
+
def store
|
32
|
+
@@store ||= KuberKit::Core::Store.new(KuberKit::Core::Templates::AbstractTemplate)
|
33
|
+
end
|
53
34
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class KuberKit::ImageCompiler::BuildServerPool
|
2
|
+
attr_reader :ssh_shells, :local_shell
|
3
|
+
|
4
|
+
def initialize(local_shell:, build_servers:, ssh_shell_class:)
|
5
|
+
@local_shell = local_shell
|
6
|
+
@ssh_shell_class = ssh_shell_class
|
7
|
+
@ssh_shells = connect_to_ssh_shells(build_servers)
|
8
|
+
end
|
9
|
+
|
10
|
+
def get_shell
|
11
|
+
if @ssh_shells.any?
|
12
|
+
@ssh_shells.sample
|
13
|
+
else
|
14
|
+
@local_shell
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def disconnect_all
|
19
|
+
@ssh_shells.each(&:disconnect)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
def connect_to_ssh_shells(build_servers)
|
24
|
+
build_servers.map do |bs|
|
25
|
+
shell = @ssh_shell_class.new
|
26
|
+
shell.connect(host: bs.host, user: bs.user, port: bs.port)
|
27
|
+
shell
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
class KuberKit::ImageCompiler::BuildServerPoolFactory
|
2
|
+
include KuberKit::Import[
|
3
|
+
"shell.local_shell",
|
4
|
+
]
|
5
|
+
|
6
|
+
def create(ssh_shell_class: KuberKit::Shell::SshShell)
|
7
|
+
KuberKit::ImageCompiler::BuildServerPool.new(
|
8
|
+
local_shell: local_shell,
|
9
|
+
build_servers: KuberKit.current_configuration.build_servers,
|
10
|
+
ssh_shell_class: ssh_shell_class,
|
11
|
+
)
|
12
|
+
end
|
13
|
+
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
class KuberKit::ImageCompiler::ImageBuildDirCreator
|
2
2
|
include KuberKit::Import[
|
3
|
-
"preprocessing.dir_preprocessor",
|
4
3
|
"preprocessing.file_preprocessor",
|
5
4
|
"shell.bash_commands",
|
5
|
+
"shell.local_shell",
|
6
6
|
"configs"
|
7
7
|
]
|
8
8
|
|
@@ -14,16 +14,22 @@ class KuberKit::ImageCompiler::ImageBuildDirCreator
|
|
14
14
|
bash_commands.mkdir_p(shell, build_dir)
|
15
15
|
|
16
16
|
if image.build_context_dir
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
)
|
17
|
+
# Sync build context and then preprocess
|
18
|
+
shell.sync(image.build_context_dir, build_dir)
|
19
|
+
|
20
|
+
shell.recursive_list_files(build_dir).each do |file_path|
|
21
|
+
file_preprocessor.compile(
|
22
|
+
shell, file_path,
|
23
|
+
context_helper: context_helper
|
24
|
+
)
|
25
|
+
end
|
21
26
|
end
|
22
27
|
|
28
|
+
# Sync dockerfile and then preprocess
|
23
29
|
target_dockerfile = File.join(build_dir, configs.image_dockerfile_name)
|
30
|
+
shell.sync(image.dockerfile_path, target_dockerfile)
|
24
31
|
file_preprocessor.compile(
|
25
|
-
shell,
|
26
|
-
destination_path: target_dockerfile,
|
32
|
+
shell, target_dockerfile,
|
27
33
|
context_helper: context_helper
|
28
34
|
)
|
29
35
|
|
@@ -3,20 +3,40 @@ class KuberKit::ImageCompiler::ImageDependencyResolver
|
|
3
3
|
DependencyNotFoundError = Class.new(KuberKit::NotFoundError)
|
4
4
|
|
5
5
|
include KuberKit::Import[
|
6
|
-
"core.image_store"
|
6
|
+
"core.image_store",
|
7
|
+
"configs"
|
7
8
|
]
|
9
|
+
|
10
|
+
Contract Or[Symbol, ArrayOf[Symbol]], Proc => Any
|
11
|
+
def each_with_deps(image_names, &block)
|
12
|
+
compile_limit = configs.compile_simultaneous_limit
|
13
|
+
|
14
|
+
resolved_dependencies = []
|
15
|
+
next_dependencies = get_next(image_names, limit: compile_limit)
|
16
|
+
|
17
|
+
while (next_dependencies - resolved_dependencies).any?
|
18
|
+
block.call(next_dependencies)
|
19
|
+
resolved_dependencies += next_dependencies
|
20
|
+
next_dependencies = get_next(image_names, resolved: resolved_dependencies, limit: compile_limit)
|
21
|
+
end
|
22
|
+
|
23
|
+
block.call(image_names - resolved_dependencies)
|
24
|
+
end
|
8
25
|
|
9
|
-
Contract
|
10
|
-
resolved: Optional[ArrayOf[Symbol]]
|
26
|
+
Contract Or[Symbol, ArrayOf[Symbol]], KeywordArgs[
|
27
|
+
resolved: Optional[ArrayOf[Symbol]],
|
28
|
+
limit: Optional[Maybe[Num]]
|
11
29
|
] => Any
|
12
|
-
def get_next(image_names, resolved: [])
|
30
|
+
def get_next(image_names, resolved: [], limit: nil)
|
13
31
|
deps = Array(image_names).map { |i| get_recursive_deps(i) }.flatten.uniq
|
14
32
|
|
15
33
|
ready_to_resolve = deps.select do |dep_name|
|
16
34
|
unresolved_deps = get_deps(dep_name) - resolved
|
17
35
|
unresolved_deps.empty?
|
18
36
|
end
|
19
|
-
ready_to_resolve - resolved
|
37
|
+
unresolved_deps = ready_to_resolve - resolved
|
38
|
+
unresolved_deps = unresolved_deps.take(limit) if limit
|
39
|
+
unresolved_deps
|
20
40
|
end
|
21
41
|
|
22
42
|
def get_recursive_deps(image_name, dependency_tree: [])
|
@@ -1,6 +1,7 @@
|
|
1
1
|
class KuberKit::Preprocessing::FilePreprocessor
|
2
2
|
include KuberKit::Import[
|
3
|
-
"preprocessing.text_preprocessor"
|
3
|
+
"preprocessing.text_preprocessor",
|
4
|
+
"shell.bash_commands"
|
4
5
|
]
|
5
6
|
|
6
7
|
PreprocessingError = Class.new(KuberKit::Error)
|
@@ -8,7 +9,7 @@ class KuberKit::Preprocessing::FilePreprocessor
|
|
8
9
|
def compile(shell, source_path, destination_path: nil, context_helper: nil)
|
9
10
|
destination_path ||= source_path
|
10
11
|
|
11
|
-
prepare_destination_dir(destination_path)
|
12
|
+
prepare_destination_dir(shell, destination_path)
|
12
13
|
|
13
14
|
template = shell.read(source_path)
|
14
15
|
content = text_preprocessor.compile(template, context_helper: context_helper)
|
@@ -27,7 +28,7 @@ class KuberKit::Preprocessing::FilePreprocessor
|
|
27
28
|
raise PreprocessingError, "Error while processing template #{source_path}.\r\n#{message}"
|
28
29
|
end
|
29
30
|
|
30
|
-
def prepare_destination_dir(destination_path)
|
31
|
-
|
31
|
+
def prepare_destination_dir(shell, destination_path)
|
32
|
+
bash_commands.mkdir_p(shell, File.dirname(destination_path))
|
32
33
|
end
|
33
34
|
end
|
@@ -17,4 +17,8 @@ class KuberKit::Shell::AbstractShell
|
|
17
17
|
def recursive_list_files(path, name: nil)
|
18
18
|
raise KuberKit::NotImplementedError, "must be implemented"
|
19
19
|
end
|
20
|
+
|
21
|
+
def sync(local_path, remote_path, exclude: nil)
|
22
|
+
raise KuberKit::NotImplementedError, "must be implemented"
|
23
|
+
end
|
20
24
|
end
|
@@ -1,11 +1,17 @@
|
|
1
|
-
class KuberKit::Shell::RsyncCommands
|
2
|
-
def rsync(shell, source_path, target_path, exclude: nil)
|
1
|
+
class KuberKit::Shell::Commands::RsyncCommands
|
2
|
+
def rsync(shell, source_path, target_path, target_host: nil, exclude: nil)
|
3
3
|
# Add a trailing slash to directory to have behavior similar to CP command
|
4
4
|
if path_is_directory?(source_path) && !source_path.end_with?("/")
|
5
5
|
source_path = "#{source_path}/"
|
6
6
|
end
|
7
7
|
|
8
|
-
|
8
|
+
if target_host
|
9
|
+
destination = "#{target_host}:#{target_path}"
|
10
|
+
else
|
11
|
+
destination = target_path
|
12
|
+
end
|
13
|
+
|
14
|
+
args = [source_path, destination]
|
9
15
|
if exclude
|
10
16
|
args << "--exclude=#{exclude}"
|
11
17
|
end
|
@@ -3,21 +3,24 @@ require 'fileutils'
|
|
3
3
|
class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
|
4
4
|
include KuberKit::Import[
|
5
5
|
"tools.logger",
|
6
|
-
"shell.command_counter"
|
6
|
+
"shell.command_counter",
|
7
|
+
"shell.rsync_commands",
|
7
8
|
]
|
8
9
|
|
9
|
-
def exec!(command)
|
10
|
+
def exec!(command, log_command: true)
|
10
11
|
command_number = command_counter.get_number.to_s.rjust(2, "0")
|
11
12
|
|
12
|
-
|
13
|
+
if log_command
|
14
|
+
logger.info("Execute: [#{command_number}]: #{command.to_s.cyan}")
|
15
|
+
end
|
13
16
|
|
14
17
|
result = nil
|
15
18
|
IO.popen(command, err: [:child, :out]) do |io|
|
16
19
|
result = io.read.chomp.strip
|
17
20
|
end
|
18
21
|
|
19
|
-
if result && result != ""
|
20
|
-
logger.info("Finished
|
22
|
+
if result && result != "" && log_command
|
23
|
+
logger.info("Finished [#{command_number}] with result: \n#{result.grey}")
|
21
24
|
end
|
22
25
|
|
23
26
|
if $?.exitstatus != 0
|
@@ -27,6 +30,10 @@ class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
|
|
27
30
|
result
|
28
31
|
end
|
29
32
|
|
33
|
+
def sync(local_path, remote_path, exclude: nil)
|
34
|
+
rsync_commands.rsync(self, local_path, remote_path, exclude: exclude)
|
35
|
+
end
|
36
|
+
|
30
37
|
def read(file_path)
|
31
38
|
File.read(file_path)
|
32
39
|
end
|
@@ -41,6 +48,18 @@ class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
|
|
41
48
|
true
|
42
49
|
end
|
43
50
|
|
51
|
+
def delete(file_path)
|
52
|
+
exec!("rm #{file_path}")
|
53
|
+
end
|
54
|
+
|
55
|
+
def file_exists?(file_path)
|
56
|
+
exec!("test -f #{file_path} && echo 'true' || echo 'false'", log_command: false) == 'true'
|
57
|
+
end
|
58
|
+
|
59
|
+
def dir_exists?(dir_path)
|
60
|
+
exec!("test -d #{dir_path} && echo 'true' || echo 'false'", log_command: false) == 'true'
|
61
|
+
end
|
62
|
+
|
44
63
|
def recursive_list_files(path, name: nil)
|
45
64
|
command = %Q{find -L #{path} -type f}
|
46
65
|
command += " -name #{name}" if name
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'net/ssh'
|
2
|
+
|
3
|
+
class KuberKit::Shell::SshSession
|
4
|
+
SshSessionError = Class.new(KuberKit::Error)
|
5
|
+
|
6
|
+
attr_reader :session, :host, :user, :port
|
7
|
+
|
8
|
+
def initialize(host:, user:, port:)
|
9
|
+
@host = host
|
10
|
+
@user = user
|
11
|
+
@port = port
|
12
|
+
@session = Net::SSH.start(host, user, {port: port})
|
13
|
+
end
|
14
|
+
|
15
|
+
def connected?
|
16
|
+
!!@session
|
17
|
+
end
|
18
|
+
|
19
|
+
def disconnect
|
20
|
+
return unless connected?
|
21
|
+
@session.close
|
22
|
+
@session = nil
|
23
|
+
end
|
24
|
+
|
25
|
+
def exec!(command)
|
26
|
+
stdout_data = ''
|
27
|
+
stderr_data = ''
|
28
|
+
exit_code = nil
|
29
|
+
channel = session.open_channel do |ch|
|
30
|
+
ch.exec(command) do |ch, success|
|
31
|
+
if !success
|
32
|
+
raise SshSessionError, "Shell command failed: #{command}\r\n#{stdout_data}\r\n#{stderr_data}"
|
33
|
+
end
|
34
|
+
|
35
|
+
channel.on_data do |ch,data|
|
36
|
+
stdout_data += data
|
37
|
+
end
|
38
|
+
|
39
|
+
channel.on_extended_data do |ch,type,data|
|
40
|
+
stderr_data += data
|
41
|
+
end
|
42
|
+
|
43
|
+
channel.on_request('exit-status') do |ch,data|
|
44
|
+
exit_code = data.read_long
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
channel.wait
|
50
|
+
session.loop
|
51
|
+
|
52
|
+
stdout_data = stdout_data.chomp.strip
|
53
|
+
|
54
|
+
if exit_code != 0
|
55
|
+
raise SshSessionError, "Shell command failed: #{command}\r\n#{stdout_data}\r\n#{stderr_data}"
|
56
|
+
end
|
57
|
+
|
58
|
+
stdout_data
|
59
|
+
end
|
60
|
+
end
|