kuber_kit 0.1.6 → 0.2.1

data/lib/kuber_kit/image_compiler/build_server_pool.rb

@@ -0,0 +1,30 @@
+class KuberKit::ImageCompiler::BuildServerPool
+  attr_reader :ssh_shells, :local_shell
+  
+  def initialize(local_shell:, build_servers:, ssh_shell_class:)
+    @local_shell     = local_shell
+    @ssh_shell_class = ssh_shell_class
+    @ssh_shells      = connect_to_ssh_shells(build_servers)
+  end
+
+  def get_shell
+    if @ssh_shells.any?
+      @ssh_shells.sample
+    else
+      @local_shell
+    end
+  end
+
+  def disconnect_all
+    @ssh_shells.each(&:disconnect)
+  end
+
+  private
+    def connect_to_ssh_shells(build_servers)
+      build_servers.map do |bs|
+        shell = @ssh_shell_class.new
+        shell.connect(host: bs.host, user: bs.user, port: bs.port)
+        shell
+      end
+    end
+end

data/lib/kuber_kit/image_compiler/build_server_pool_factory.rb

@@ -0,0 +1,13 @@
+class KuberKit::ImageCompiler::BuildServerPoolFactory
+  include KuberKit::Import[
+    "shell.local_shell",
+  ]
+
+  def create(ssh_shell_class: KuberKit::Shell::SshShell)
+    KuberKit::ImageCompiler::BuildServerPool.new(
+      local_shell:      local_shell,
+      build_servers:    KuberKit.current_configuration.build_servers,
+      ssh_shell_class:  ssh_shell_class,
+    )
+  end
+end

data/lib/kuber_kit/image_compiler/image_build_dir_creator.rb

@@ -1,8 +1,8 @@
 class KuberKit::ImageCompiler::ImageBuildDirCreator
   include KuberKit::Import[
-    "preprocessing.dir_preprocessor",
     "preprocessing.file_preprocessor",
     "shell.bash_commands",
+    "shell.local_shell",
     "configs"
   ]
 
@@ -14,16 +14,22 @@ class KuberKit::ImageCompiler::ImageBuildDirCreator
     bash_commands.mkdir_p(shell, build_dir)
 
     if image.build_context_dir
-      dir_preprocessor.compile(
-        shell, image.build_context_dir, build_dir,
-        context_helper: context_helper
-      )
+      # Sync build context and then preprocess
+      shell.sync(image.build_context_dir, build_dir)
+
+      shell.recursive_list_files(build_dir).each do |file_path|
+        file_preprocessor.compile(
+          shell, file_path, 
+          context_helper: context_helper
+        )
+      end
     end
 
+    # Sync dockerfile and then preprocess
     target_dockerfile = File.join(build_dir, configs.image_dockerfile_name)
+    shell.sync(image.dockerfile_path, target_dockerfile)
     file_preprocessor.compile(
-      shell, image.dockerfile_path, 
-      destination_path: target_dockerfile, 
+      shell, target_dockerfile,
       context_helper: context_helper
     )
 

data/lib/kuber_kit/image_compiler/image_dependency_resolver.rb

@@ -3,20 +3,40 @@ class KuberKit::ImageCompiler::ImageDependencyResolver
   DependencyNotFoundError = Class.new(KuberKit::NotFoundError)
 
   include KuberKit::Import[
-    "core.image_store"
+    "core.image_store",
+    "configs"
   ]
+
+  Contract Or[Symbol, ArrayOf[Symbol]], Proc => Any
+  def each_with_deps(image_names, &block)
+    compile_limit = configs.compile_simultaneous_limit
+
+    resolved_dependencies = []
+    next_dependencies = get_next(image_names, limit: compile_limit)
+
+    while (next_dependencies - resolved_dependencies).any?
+      block.call(next_dependencies)
+      resolved_dependencies += next_dependencies
+      next_dependencies = get_next(image_names, resolved: resolved_dependencies, limit: compile_limit)
+    end
+
+    block.call(image_names - resolved_dependencies)
+  end
   
-  Contract Any, KeywordArgs[
-    resolved: Optional[ArrayOf[Symbol]]
+  Contract Or[Symbol, ArrayOf[Symbol]], KeywordArgs[
+    resolved: Optional[ArrayOf[Symbol]],
+    limit:    Optional[Maybe[Num]]
   ] => Any
-  def get_next(image_names, resolved: [])
+  def get_next(image_names, resolved: [], limit: nil)
     deps = Array(image_names).map { |i| get_recursive_deps(i) }.flatten.uniq
 
     ready_to_resolve = deps.select do |dep_name|
       unresolved_deps = get_deps(dep_name) - resolved
       unresolved_deps.empty?
     end
-    ready_to_resolve - resolved
+    unresolved_deps = ready_to_resolve - resolved
+    unresolved_deps = unresolved_deps.take(limit) if limit
+    unresolved_deps
   end
 
   def get_recursive_deps(image_name, dependency_tree: [])

data/lib/kuber_kit/preprocessing/file_preprocessor.rb

@@ -1,6 +1,7 @@
 class KuberKit::Preprocessing::FilePreprocessor
   include KuberKit::Import[
-    "preprocessing.text_preprocessor"
+    "preprocessing.text_preprocessor",
+    "shell.bash_commands"
   ]
 
   PreprocessingError = Class.new(KuberKit::Error)
@@ -8,7 +9,7 @@ class KuberKit::Preprocessing::FilePreprocessor
   def compile(shell, source_path, destination_path: nil, context_helper: nil)
     destination_path ||= source_path
 
-    prepare_destination_dir(destination_path)
+    prepare_destination_dir(shell, destination_path)
 
     template = shell.read(source_path)
     content = text_preprocessor.compile(template, context_helper: context_helper)
@@ -27,7 +28,7 @@ class KuberKit::Preprocessing::FilePreprocessor
     raise PreprocessingError, "Error while processing template #{source_path}.\r\n#{message}"
   end
 
-  def prepare_destination_dir(destination_path)
-    FileUtils.mkdir_p(File.dirname(destination_path))
+  def prepare_destination_dir(shell, destination_path)
+    bash_commands.mkdir_p(shell, File.dirname(destination_path))
   end
 end

data/lib/kuber_kit/service_deployer/action_handler.rb

@@ -11,6 +11,6 @@ class KuberKit::ServiceDeployer::ActionHandler
 
     strategy_name = strategy_detector.call(service)
 
-    deployer.restart(shell, service, strategy_name)
+    deployer.deploy(shell, service, strategy_name)
   end
 end

data/lib/kuber_kit/service_deployer/strategies/kubernetes.rb

@@ -11,8 +11,15 @@ class KuberKit::ServiceDeployer::Strategies::Kubernetes < KuberKit::ServiceDeplo
     config_path    = "#{configs.service_config_dir}/#{service.name}.yml"
     shell.write(config_path, service_config)
 
-    kubeconfig_path = KuberKit.current_configuration.kubeconfig_path
-    kubectl_commands.apply_file(shell, config_path, kubeconfig_path: kubeconfig_path)
-    kubectl_commands.rolling_restart(shell, service.uri, kubeconfig_path: kubeconfig_path)
+    kubeconfig_path  = KuberKit.current_configuration.kubeconfig_path
+    deploy_namespace = KuberKit.current_configuration.deploy_namespace
+
+    kubectl_commands.apply_file(shell, config_path, kubeconfig_path: kubeconfig_path, namespace: deploy_namespace)
+
+    deployment_restart_enabled = service.attribute(:deployment_restart_enabled, default: true)
+    deployment_restart_name    = service.attribute(:deployment_restart_name, default: service.uri)
+    if deployment_restart_enabled
+      kubectl_commands.rolling_restart(shell, deployment_restart_name, kubeconfig_path: kubeconfig_path, namespace: deploy_namespace)
+    end
   end
 end

data/lib/kuber_kit/shell/abstract_shell.rb

@@ -17,4 +17,8 @@ class KuberKit::Shell::AbstractShell
   def recursive_list_files(path, name: nil)
     raise KuberKit::NotImplementedError, "must be implemented"
   end
+
+  def sync(local_path, remote_path, exclude: nil)
+    raise KuberKit::NotImplementedError, "must be implemented"
+  end
 end

data/lib/kuber_kit/shell/{bash_commands.rb → commands/bash_commands.rb}

@@ -1,4 +1,4 @@
-class KuberKit::Shell::BashCommands
+class KuberKit::Shell::Commands::BashCommands
   def rm(shell, path)
     shell.exec!(%Q{rm "#{path}"})
   end

data/lib/kuber_kit/shell/{docker_commands.rb → commands/docker_commands.rb}

@@ -1,4 +1,4 @@
-class KuberKit::Shell::DockerCommands
+class KuberKit::Shell::Commands::DockerCommands
   def build(shell, build_dir, args = [])
     default_args = ["--rm=true"]
     args_list = (default_args + args).join(" ")

data/lib/kuber_kit/shell/{git_commands.rb → commands/git_commands.rb}

@@ -1,4 +1,4 @@
-class KuberKit::Shell::GitCommands
+class KuberKit::Shell::Commands::GitCommands
   def get_remote_url(shell, git_repo_path, remote_name: "origin")
     shell.exec! [
       "cd #{git_repo_path}",

data/lib/kuber_kit/shell/commands/kubectl_commands.rb

@@ -0,0 +1,65 @@
+require 'json'
+require 'shellwords'
+
+class KuberKit::Shell::Commands::KubectlCommands
+  def kubectl_run(shell, command_list, kubeconfig_path: nil, namespace: nil, interactive: false)
+    command_parts = []
+    if kubeconfig_path
+      command_parts << "KUBECONFIG=#{kubeconfig_path}"
+    end
+
+    command_parts << "kubectl"
+
+    if namespace
+      command_parts << "-n #{namespace}"
+    end
+
+    command_parts += Array(command_list)
+
+    # TODO: investigate how to do it with shell.
+    if interactive
+      system(command_parts.join(" "))
+    else
+      shell.exec!(command_parts.join(" "))
+    end
+  end
+  
+  def apply_file(shell, file_path, kubeconfig_path: nil, namespace: nil)
+    kubectl_run(shell, "apply -f #{file_path}", kubeconfig_path: kubeconfig_path, namespace: namespace)
+  end
+
+  def exec(shell, pod_name, command, args: nil, kubeconfig_path: nil, interactive: false, namespace: nil)
+    command_parts = []
+    command_parts << "exec"
+
+    if args
+      command_parts << args
+    end
+
+    command_parts << pod_name
+    command_parts << "-- #{command}"
+    kubectl_run(shell, command_parts, kubeconfig_path: kubeconfig_path, interactive: interactive, namespace: namespace)
+  end
+
+  def rolling_restart(shell, deployment_name, kubeconfig_path: nil, namespace: namespace)
+    patch_deployment(shell, deployment_name, {
+      spec: {
+        template: {
+          metadata: {
+            labels: {
+              redeploy: "$(date +%s)"
+            }
+          }
+        }
+      }
+    }, kubeconfig_path: kubeconfig_path, namespace: namespace)
+  end
+
+  def patch_deployment(shell, deployment_name, specs, kubeconfig_path: nil, namespace: nil)
+    specs_json = JSON.dump(specs).gsub('"', '\"')
+
+    command = %Q{patch deployment #{deployment_name} -p "#{specs_json}"}
+
+    kubectl_run(shell, command, kubeconfig_path: kubeconfig_path, namespace: namespace)
+  end
+end

data/lib/kuber_kit/shell/commands/rsync_commands.rb

@@ -0,0 +1,32 @@
+class KuberKit::Shell::Commands::RsyncCommands
+  def rsync(shell, source_path, target_path, target_host: nil, exclude: nil, delete: true)
+    # Add a trailing slash to directory to have behavior similar to CP command
+    if path_is_directory?(source_path) && !source_path.end_with?("/")
+      source_path = "#{source_path}/"
+    end
+
+    if target_host
+      destination = "#{target_host}:#{target_path}"
+    else
+      destination = target_path
+    end
+
+    args = [source_path, destination]
+    if exclude
+      Array(exclude).each do |e|
+        args << "--exclude=#{e}"
+      end
+    end
+
+    if delete
+      args << "--delete"
+    end
+
+    shell.exec!(%Q{rsync -a #{args.join(' ')}})
+  end
+
+  private
+    def path_is_directory?(path)
+      File.directory?(path)
+    end
+end

data/lib/kuber_kit/shell/local_shell.rb

@@ -3,21 +3,24 @@ require 'fileutils'
 class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
   include KuberKit::Import[
     "tools.logger",
-    "shell.command_counter"
+    "shell.command_counter",
+    "shell.rsync_commands",
   ]
 
-  def exec!(command)
+  def exec!(command, log_command: true)
     command_number = command_counter.get_number.to_s.rjust(2, "0")
     
-    logger.info("Executed command [#{command_number}]: #{command.to_s.cyan}")
+    if log_command
+      logger.info("Execute: [#{command_number}]: #{command.to_s.cyan}")
+    end
 
     result = nil
     IO.popen(command, err: [:child, :out]) do |io|
       result = io.read.chomp.strip
     end
 
-    if result && result != ""
-      logger.info("Finished command [#{command_number}] with result: \n#{result.grey}")
+    if result && result != "" && log_command
+      logger.info("Finished [#{command_number}] with result: \n#{result.grey}")
     end
 
     if $?.exitstatus != 0
@@ -27,6 +30,10 @@ class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
     result
   end
 
+  def sync(local_path, remote_path, exclude: nil)
+    rsync_commands.rsync(self, local_path, remote_path, exclude: exclude)
+  end
+
   def read(file_path)
     File.read(file_path)
   end
@@ -41,6 +48,18 @@ class KuberKit::Shell::LocalShell < KuberKit::Shell::AbstractShell
     true
   end
 
+  def delete(file_path)
+    exec!("rm #{file_path}")
+  end
+
+  def file_exists?(file_path)
+    exec!("test -f #{file_path} && echo 'true' || echo 'false'", log_command: false) == 'true'
+  end
+
+  def dir_exists?(dir_path)
+    exec!("test -d #{dir_path} && echo 'true' || echo 'false'", log_command: false) == 'true'
+  end
+
   def recursive_list_files(path, name: nil)
     command = %Q{find -L #{path}  -type f}
     command += " -name #{name}" if name

data/lib/kuber_kit/shell/ssh_session.rb

@@ -0,0 +1,60 @@
+require 'net/ssh'
+
+class KuberKit::Shell::SshSession
+  SshSessionError = Class.new(KuberKit::Error)
+
+  attr_reader :session, :host, :user, :port
+
+  def initialize(host:, user:, port:)
+    @host = host
+    @user = user
+    @port = port
+    @session = Net::SSH.start(host, user, {port: port})
+  end
+
+  def connected?
+    !!@session
+  end
+  
+  def disconnect
+    return unless connected?
+    @session.close
+    @session = nil
+  end
+
+  def exec!(command)
+    stdout_data = ''
+    stderr_data = ''
+    exit_code = nil
+    channel = session.open_channel do |ch|
+      ch.exec(command) do |ch, success|
+        if !success
+          raise SshSessionError, "Shell command failed: #{command}\r\n#{stdout_data}\r\n#{stderr_data}"
+        end
+
+        channel.on_data do |ch,data|
+          stdout_data += data
+        end
+
+        channel.on_extended_data do |ch,type,data|
+          stderr_data += data
+        end
+
+        channel.on_request('exit-status') do |ch,data|
+          exit_code = data.read_long
+        end
+      end
+    end
+
+    channel.wait
+    session.loop
+
+    stdout_data = stdout_data.chomp.strip
+
+    if exit_code != 0
+      raise SshSessionError, "Shell command failed: #{command}\r\n#{stdout_data}\r\n#{stderr_data}"
+    end
+
+    stdout_data
+  end
+end

data/lib/kuber_kit/shell/ssh_shell.rb

@@ -0,0 +1,77 @@
+require 'tempfile'
+
+class KuberKit::Shell::SshShell < KuberKit::Shell::LocalShell
+  include KuberKit::Import[
+    "tools.logger",
+    "shell.command_counter",
+    "shell.rsync_commands",
+    "shell.local_shell"
+  ]
+
+  def connect(host:, user:, port:)
+    @ssh_session = KuberKit::Shell::SshSession.new(host: host, user: user, port: port)
+  end
+  
+  def connected?
+    @ssh_session && @ssh_session.connected?
+  end
+
+  def disconnect
+    @ssh_session.disconnect if @ssh_session
+  end
+
+  def exec!(command, log_command: true)
+    command_number = command_counter.get_number.to_s.rjust(2, "0")
+    
+    if log_command
+      logger.info("#{ssh_session.host.green} > Execute: [#{command_number}]: #{command.to_s.cyan}")
+    end
+
+    result = ssh_session.exec!(command)
+
+    if result && result != "" && log_command
+      logger.info("#{ssh_session.host.green} > Finished [#{command_number}] with result: \n#{result.grey}")
+    end
+
+    result
+  rescue KuberKit::Shell::SshSession::SshSessionError => e
+    raise ShellError.new(e.message)
+  end
+
+  def sync(local_path, remote_path, exclude: nil)
+    rsync_commands.rsync(
+      local_shell, local_path, remote_path, 
+      target_host: "#{ssh_session.user}@#{ssh_session.host}",
+      exclude:     exclude
+    )
+  end
+
+  def read(file_path)
+    exec!("cat #{file_path}")
+  end
+
+  def write(file_path, content)
+    Tempfile.create do |file| 
+      file << content
+      file.flush
+      sync(file.path, file_path)
+    end
+
+    logger.info("Created file #{file_path.to_s.cyan}\r\n#{content.grey}")
+
+    true
+  end
+
+  private
+    def ssh_session
+      unless connected?
+        raise ArgumentError, "ssh session is not created, please call #connect"
+      end
+
+      @ssh_session
+    end
+
+    def ensure_directory_exists(file_path)
+      exec!("mkdir -p #{file_path}")
+    end
+end