kuber_kit 0.1.8 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -1
  3. data/README.md +16 -3
  4. data/TODO.md +1 -3
  5. data/example/configurations/review.rb +2 -1
  6. data/example/images/app_sources/Dockerfile +1 -1
  7. data/example/images/ruby_app/image.rb +4 -4
  8. data/example/infrastructure/build_servers.rb +8 -0
  9. data/kuber_kit.gemspec +1 -0
  10. data/lib/kuber_kit.rb +36 -19
  11. data/lib/kuber_kit/actions/configuration_loader.rb +3 -0
  12. data/lib/kuber_kit/actions/env_file_reader.rb +3 -0
  13. data/lib/kuber_kit/actions/image_compiler.rb +14 -10
  14. data/lib/kuber_kit/actions/kubectl_applier.rb +4 -1
  15. data/lib/kuber_kit/actions/service_deployer.rb +4 -0
  16. data/lib/kuber_kit/actions/service_reader.rb +4 -0
  17. data/lib/kuber_kit/actions/template_reader.rb +3 -0
  18. data/lib/kuber_kit/cli.rb +12 -6
  19. data/lib/kuber_kit/configs.rb +24 -22
  20. data/lib/kuber_kit/container.rb +17 -13
  21. data/lib/kuber_kit/core/artifacts/artifact_store.rb +9 -28
  22. data/lib/kuber_kit/core/build_servers/abstract_build_server.rb +21 -0
  23. data/lib/kuber_kit/core/build_servers/build_server.rb +24 -0
  24. data/lib/kuber_kit/core/build_servers/build_server_store.rb +18 -0
  25. data/lib/kuber_kit/core/configuration.rb +7 -3
  26. data/lib/kuber_kit/core/configuration_definition.rb +10 -0
  27. data/lib/kuber_kit/core/configuration_factory.rb +10 -1
  28. data/lib/kuber_kit/core/configuration_store.rb +14 -24
  29. data/lib/kuber_kit/core/env_files/env_file_store.rb +8 -23
  30. data/lib/kuber_kit/core/image_store.rb +8 -18
  31. data/lib/kuber_kit/core/registries/registry_store.rb +8 -23
  32. data/lib/kuber_kit/core/service_store.rb +13 -23
  33. data/lib/kuber_kit/core/store.rb +48 -0
  34. data/lib/kuber_kit/core/templates/template_store.rb +9 -28
  35. data/lib/kuber_kit/image_compiler/build_server_pool.rb +30 -0
  36. data/lib/kuber_kit/image_compiler/build_server_pool_factory.rb +13 -0
  37. data/lib/kuber_kit/image_compiler/image_build_dir_creator.rb +13 -7
  38. data/lib/kuber_kit/image_compiler/image_dependency_resolver.rb +25 -5
  39. data/lib/kuber_kit/preprocessing/file_preprocessor.rb +5 -4
  40. data/lib/kuber_kit/shell/abstract_shell.rb +4 -0
  41. data/lib/kuber_kit/shell/{bash_commands.rb → commands/bash_commands.rb} +1 -1
  42. data/lib/kuber_kit/shell/{docker_commands.rb → commands/docker_commands.rb} +1 -1
  43. data/lib/kuber_kit/shell/{git_commands.rb → commands/git_commands.rb} +1 -1
  44. data/lib/kuber_kit/shell/{kubectl_commands.rb → commands/kubectl_commands.rb} +1 -1
  45. data/lib/kuber_kit/shell/{rsync_commands.rb → commands/rsync_commands.rb} +9 -3
  46. data/lib/kuber_kit/shell/local_shell.rb +24 -5
  47. data/lib/kuber_kit/shell/ssh_session.rb +60 -0
  48. data/lib/kuber_kit/shell/ssh_shell.rb +77 -0
  49. data/lib/kuber_kit/tools/file_presence_checker.rb +6 -2
  50. data/lib/kuber_kit/version.rb +1 -1
  51. metadata +30 -9
  52. data/lib/kuber_kit/preprocessing/dir_preprocessor.rb +0 -19
  53. data/lib/kuber_kit/tools/files_sync.rb +0 -10
@@ -0,0 +1,48 @@
1
+ class KuberKit::Core::Store
2
+ NotFoundError = Class.new(KuberKit::NotFoundError)
3
+ AlreadyAddedError = Class.new(KuberKit::Error)
4
+
5
+ attr_reader :object_class_name
6
+
7
+ def initialize(object_class_name)
8
+ @object_class_name = object_class_name
9
+ end
10
+
11
+ def add(item_name, item)
12
+ unless item.is_a?(object_class_name)
13
+ raise ArgumentError.new("#{self.object_class_name}: should be an instance of #{object_class_name}, got: #{item.inspect}")
14
+ end
15
+
16
+ unless items[item_name].nil?
17
+ raise AlreadyAddedError, "#{self.object_class_name}: item with name #{item_name} was already added"
18
+ end
19
+
20
+ items[item_name] = item
21
+ end
22
+
23
+ def get(item_name)
24
+ item = items[item_name]
25
+
26
+ if item.nil?
27
+ raise NotFoundError, "#{self.object_class_name}: item '#{item_name}' not found"
28
+ end
29
+
30
+ item
31
+ end
32
+
33
+ def items
34
+ @items ||= {}
35
+ end
36
+
37
+ def reset!
38
+ @items = {}
39
+ end
40
+
41
+ def size
42
+ items.count
43
+ end
44
+
45
+ def exists?(name)
46
+ !items[name].nil?
47
+ end
48
+ end
@@ -1,19 +1,6 @@
1
1
  class KuberKit::Core::Templates::TemplateStore
2
- NotFoundError = Class.new(KuberKit::NotFoundError)
3
- AlreadyAddedError = Class.new(KuberKit::Error)
4
-
5
2
  def add(template)
6
- @@templates ||= {}
7
-
8
- if !template.is_a?(KuberKit::Core::Templates::AbstractTemplate)
9
- raise ArgumentError.new("should be an instance of KuberKit::Core::Templates::AbstractTemplate, got: #{template.inspect}")
10
- end
11
-
12
- unless @@templates[template.name].nil?
13
- raise AlreadyAddedError, "template #{template.name} was already added"
14
- end
15
-
16
- @@templates[template.name] = template
3
+ store.add(template.name, template)
17
4
  end
18
5
 
19
6
  def get(template_name)
@@ -24,14 +11,7 @@ class KuberKit::Core::Templates::TemplateStore
24
11
  end
25
12
 
26
13
  def get_global(template_name)
27
- @@templates ||= {}
28
- template = @@templates[template_name]
29
-
30
- if template.nil?
31
- raise NotFoundError, "template '#{template_name}' not found"
32
- end
33
-
34
- template
14
+ store.get(template_name)
35
15
  end
36
16
 
37
17
  def get_from_configuration(template_name)
@@ -40,14 +20,15 @@ class KuberKit::Core::Templates::TemplateStore
40
20
  end
41
21
 
42
22
  def reset!
43
- @@templates = {}
44
- end
45
-
46
- def all_definitions
47
- @@templates ||= {}
23
+ store.reset!
48
24
  end
49
25
 
50
26
  def exists?(template_name)
51
- !all_definitions[template_name].nil?
27
+ store.exists?(template_name)
52
28
  end
29
+
30
+ private
31
+ def store
32
+ @@store ||= KuberKit::Core::Store.new(KuberKit::Core::Templates::AbstractTemplate)
33
+ end
53
34
  end
@@ -0,0 +1,30 @@
1
+ class KuberKit::ImageCompiler::BuildServerPool
2
+ attr_reader :ssh_shells, :local_shell
3
+
4
+ def initialize(local_shell:, build_servers:, ssh_shell_class:)
5
+ @local_shell = local_shell
6
+ @ssh_shell_class = ssh_shell_class
7
+ @ssh_shells = connect_to_ssh_shells(build_servers)
8
+ end
9
+
10
+ def get_shell
11
+ if @ssh_shells.any?
12
+ @ssh_shells.sample
13
+ else
14
+ @local_shell
15
+ end
16
+ end
17
+
18
+ def disconnect_all
19
+ @ssh_shells.each(&:disconnect)
20
+ end
21
+
22
+ private
23
+ def connect_to_ssh_shells(build_servers)
24
+ build_servers.map do |bs|
25
+ shell = @ssh_shell_class.new
26
+ shell.connect(host: bs.host, user: bs.user, port: bs.port)
27
+ shell
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,13 @@
1
+ class KuberKit::ImageCompiler::BuildServerPoolFactory
2
+ include KuberKit::Import[
3
+ "shell.local_shell",
4
+ ]
5
+
6
+ def create(ssh_shell_class: KuberKit::Shell::SshShell)
7
+ KuberKit::ImageCompiler::BuildServerPool.new(
8
+ local_shell: local_shell,
9
+ build_servers: KuberKit.current_configuration.build_servers,
10
+ ssh_shell_class: ssh_shell_class,
11
+ )
12
+ end
13
+ end
@@ -1,8 +1,8 @@
1
1
  class KuberKit::ImageCompiler::ImageBuildDirCreator
2
2
  include KuberKit::Import[
3
- "preprocessing.dir_preprocessor",
4
3
  "preprocessing.file_preprocessor",
5
4
  "shell.bash_commands",
5
+ "shell.local_shell",
6
6
  "configs"
7
7
  ]
8
8
 
@@ -14,16 +14,22 @@ class KuberKit::ImageCompiler::ImageBuildDirCreator
14
14
  bash_commands.mkdir_p(shell, build_dir)
15
15
 
16
16
  if image.build_context_dir
17
- dir_preprocessor.compile(
18
- shell, image.build_context_dir, build_dir,
19
- context_helper: context_helper
20
- )
17
+ # Sync build context and then preprocess
18
+ shell.sync(image.build_context_dir, build_dir)
19
+
20
+ shell.recursive_list_files(build_dir).each do |file_path|
21
+ file_preprocessor.compile(
22
+ shell, file_path,
23
+ context_helper: context_helper
24
+ )
25
+ end
21
26
  end
22
27
 
28
+ # Sync dockerfile and then preprocess
23
29
  target_dockerfile = File.join(build_dir, configs.image_dockerfile_name)
30
+ shell.sync(image.dockerfile_path, target_dockerfile)
24
31
  file_preprocessor.compile(
25
- shell, image.dockerfile_path,
26
- destination_path: target_dockerfile,
32
+ shell, target_dockerfile,
27
33
  context_helper: context_helper
28
34
  )
29
35
 
@@ -3,20 +3,40 @@ class KuberKit::ImageCompiler::ImageDependencyResolver
3
3
  DependencyNotFoundError = Class.new(KuberKit::NotFoundError)
4
4
 
5
5
  include KuberKit::Import[
6
- "core.image_store"
6
+ "core.image_store",
7
+ "configs"
7
8
  ]
9
+
10
+ Contract Or[Symbol, ArrayOf[Symbol]], Proc => Any
11
+ def each_with_deps(image_names, &block)
12
+ compile_limit = configs.compile_simultaneous_limit
13
+
14
+ resolved_dependencies = []
15
+ next_dependencies = get_next(image_names, limit: compile_limit)
16
+
17
+ while (next_dependencies - resolved_dependencies).any?
18
+ block.call(next_dependencies)
19
+ resolved_dependencies += next_dependencies
20
+ next_dependencies = get_next(image_names, resolved: resolved_dependencies, limit: compile_limit)
21
+ end
22
+
23
+ block.call(image_names - resolved_dependencies)
24
+ end
8
25
 
9
- Contract Any, KeywordArgs[
10
- resolved: Optional[ArrayOf[Symbol]]
26
+ Contract Or[Symbol, ArrayOf[Symbol]], KeywordArgs[
27
+ resolved: Optional[ArrayOf[Symbol]],
28
+ limit: Optional[Maybe[Num]]
11
29
  ] => Any
12
- def get_next(image_names, resolved: [])
30
+ def get_next(image_names, resolved: [], limit: nil)
13
31
  deps = Array(image_names).map { |i| get_recursive_deps(i) }.flatten.uniq
14
32
 
15
33
  ready_to_resolve = deps.select do |dep_name|
16
34
  unresolved_deps = get_deps(dep_name) - resolved
17
35
  unresolved_deps.empty?
18
36
  end
19
- ready_to_resolve - resolved
37
+ unresolved_deps = ready_to_resolve - resolved
38
+ unresolved_deps = unresolved_deps.take(limit) if limit
39
+ unresolved_deps
20
40
  end
21
41
 
22
42
  def get_recursive_deps(image_name, dependency_tree: [])
@@ -1,6 +1,7 @@
1
1
  class KuberKit::Preprocessing::FilePreprocessor
2
2
  include KuberKit::Import[
3
- "preprocessing.text_preprocessor"
3
+ "preprocessing.text_preprocessor",
4
+ "shell.bash_commands"
4
5
  ]
5
6
 
6
7
  PreprocessingError = Class.new(KuberKit::Error)
@@ -8,7 +9,7 @@ class KuberKit::Preprocessing::FilePreprocessor
8
9
  def compile(shell, source_path, destination_path: nil, context_helper: nil)
9
10
  destination_path ||= source_path
10
11
 
11
- prepare_destination_dir(destination_path)
12
+ prepare_destination_dir(shell, destination_path)
12
13
 
13
14
  template = shell.read(source_path)
14
15
  content = text_preprocessor.compile(template, context_helper: context_helper)
@@ -27,7 +28,7 @@ class KuberKit::Preprocessing::FilePreprocessor
27
28
  raise PreprocessingError, "Error while processing template #{source_path}.\r\n#{message}"
28
29
  end
29
30
 
30
- def prepare_destination_dir(destination_path)
31
- FileUtils.mkdir_p(File.dirname(destination_path))
31
+ def prepare_destination_dir(shell, destination_path)
32
+ bash_commands.mkdir_p(shell, File.dirname(destination_path))
32
33
  end
33
34
  end
@@ -17,4 +17,8 @@ class KuberKit::Shell::AbstractShell
17
17
  def recursive_list_files(path, name: nil)
18
18
  raise KuberKit::NotImplementedError, "must be implemented"
19
19
  end
20
+
21
+ def sync(local_path, remote_path, exclude: nil)
22
+ raise KuberKit::NotImplementedError, "must be implemented"
23
+ end
20
24
  end
@@ -1,4 +1,4 @@
1
- class KuberKit::Shell::BashCommands
1
+ class KuberKit::Shell::Commands::BashCommands
2
2
  def rm(shell, path)
3
3
  shell.exec!(%Q{rm "#{path}"})
4
4
  end
@@ -1,4 +1,4 @@
1
- class KuberKit::Shell::DockerCommands
1
+ class KuberKit::Shell::Commands::DockerCommands
2
2
  def build(shell, build_dir, args = [])
3
3
  default_args = ["--rm=true"]
4
4
  args_list = (default_args + args).join(" ")
@@ -1,4 +1,4 @@
1
- class KuberKit::Shell::GitCommands
1
+ class KuberKit::Shell::Commands::GitCommands
2
2
  def get_remote_url(shell, git_repo_path, remote_name: "origin")
3
3
  shell.exec! [
4
4
  "cd #{git_repo_path}",
@@ -1,7 +1,7 @@
1
1
  require 'json'
2
2
  require 'shellwords'
3
3
 
4
- class KuberKit::Shell::KubectlCommands
4
+ class KuberKit::Shell::Commands::KubectlCommands
5
5
  def apply_file(shell, file_path, kubeconfig_path: nil)
6
6
  command_parts = []
7
7
  if kubeconfig_path
@@ -1,11 +1,17 @@
1
- class KuberKit::Shell::RsyncCommands
2
- def rsync(shell, source_path, target_path, exclude: nil)
1
+ class KuberKit::Shell::Commands::RsyncCommands
2
+ def rsync(shell, source_path, target_path, target_host: nil, exclude: nil)
3
3
  # Add a trailing slash to directory to have behavior similar to CP command
4
4
  if path_is_directory?(source_path) && !source_path.end_with?("/")
5
5
  source_path = "#{source_path}/"
6
6
  end
7
7
 
8
- args = [source_path, target_path]
8
+ if target_host
9
+ destination = "#{target_host}:#{target_path}"
10
+ else
11
+ destination = target_path
12
+ end
13
+
14
+ args = [source_path, destination]
9
15
  if exclude
10
16
  args << "--exclude=#{exclude}"
11
17
  end
@@ -3,21 +3,24 @@ require 'fileutils'
3
3
  class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
4
4
  include KuberKit::Import[
5
5
  "tools.logger",
6
- "shell.command_counter"
6
+ "shell.command_counter",
7
+ "shell.rsync_commands",
7
8
  ]
8
9
 
9
- def exec!(command)
10
+ def exec!(command, log_command: true)
10
11
  command_number = command_counter.get_number.to_s.rjust(2, "0")
11
12
 
12
- logger.info("Executed command [#{command_number}]: #{command.to_s.cyan}")
13
+ if log_command
14
+ logger.info("Execute: [#{command_number}]: #{command.to_s.cyan}")
15
+ end
13
16
 
14
17
  result = nil
15
18
  IO.popen(command, err: [:child, :out]) do |io|
16
19
  result = io.read.chomp.strip
17
20
  end
18
21
 
19
- if result && result != ""
20
- logger.info("Finished command [#{command_number}] with result: \n#{result.grey}")
22
+ if result && result != "" && log_command
23
+ logger.info("Finished [#{command_number}] with result: \n#{result.grey}")
21
24
  end
22
25
 
23
26
  if $?.exitstatus != 0
@@ -27,6 +30,10 @@ class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
27
30
  result
28
31
  end
29
32
 
33
+ def sync(local_path, remote_path, exclude: nil)
34
+ rsync_commands.rsync(self, local_path, remote_path, exclude: exclude)
35
+ end
36
+
30
37
  def read(file_path)
31
38
  File.read(file_path)
32
39
  end
@@ -41,6 +48,18 @@ class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
41
48
  true
42
49
  end
43
50
 
51
+ def delete(file_path)
52
+ exec!("rm #{file_path}")
53
+ end
54
+
55
+ def file_exists?(file_path)
56
+ exec!("test -f #{file_path} && echo 'true' || echo 'false'", log_command: false) == 'true'
57
+ end
58
+
59
+ def dir_exists?(dir_path)
60
+ exec!("test -d #{dir_path} && echo 'true' || echo 'false'", log_command: false) == 'true'
61
+ end
62
+
44
63
  def recursive_list_files(path, name: nil)
45
64
  command = %Q{find -L #{path} -type f}
46
65
  command += " -name #{name}" if name
@@ -0,0 +1,60 @@
1
+ require 'net/ssh'
2
+
3
+ class KuberKit::Shell::SshSession
4
+ SshSessionError = Class.new(KuberKit::Error)
5
+
6
+ attr_reader :session, :host, :user, :port
7
+
8
+ def initialize(host:, user:, port:)
9
+ @host = host
10
+ @user = user
11
+ @port = port
12
+ @session = Net::SSH.start(host, user, {port: port})
13
+ end
14
+
15
+ def connected?
16
+ !!@session
17
+ end
18
+
19
+ def disconnect
20
+ return unless connected?
21
+ @session.close
22
+ @session = nil
23
+ end
24
+
25
+ def exec!(command)
26
+ stdout_data = ''
27
+ stderr_data = ''
28
+ exit_code = nil
29
+ channel = session.open_channel do |ch|
30
+ ch.exec(command) do |ch, success|
31
+ if !success
32
+ raise SshSessionError, "Shell command failed: #{command}\r\n#{stdout_data}\r\n#{stderr_data}"
33
+ end
34
+
35
+ channel.on_data do |ch,data|
36
+ stdout_data += data
37
+ end
38
+
39
+ channel.on_extended_data do |ch,type,data|
40
+ stderr_data += data
41
+ end
42
+
43
+ channel.on_request('exit-status') do |ch,data|
44
+ exit_code = data.read_long
45
+ end
46
+ end
47
+ end
48
+
49
+ channel.wait
50
+ session.loop
51
+
52
+ stdout_data = stdout_data.chomp.strip
53
+
54
+ if exit_code != 0
55
+ raise SshSessionError, "Shell command failed: #{command}\r\n#{stdout_data}\r\n#{stderr_data}"
56
+ end
57
+
58
+ stdout_data
59
+ end
60
+ end