kuber_kit 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37500e0f3feb9e241251e04ae72ed0cc618d5dadf3e1c660a05658a0e5fefa15
-  data.tar.gz: 82d7c67f5f421fed5cbb94f14ad153dcb6f6f2f068817ffd518f6c0ea7cfe3f8
+  metadata.gz: b7fcc957995bcc680107ef41f95ef77d5ee20dda2c0063d31e21bf339ccdae2d
+  data.tar.gz: 80561e6e2562b9f195e6725c9dfcd785d36d741b4cf1779c7ccf1d071193c5ec
 SHA512:
-  metadata.gz: 63c544b4ee93bf01f7d36720effff85052643583b008d68327c10bcbceb5711bce976856da1e6c56c9f877720eb2735447574e534f9f97a966777156ee01bda3
-  data.tar.gz: d388233a8636e884ee033971bf8365f3b5e753dc43a22b5e03702dda764e0632003dc80448fd7a9f3378e98539e06bc25f8163dcd6c85fdd67afddba2f141c80
+  metadata.gz: bd95c6391e1880cbe6e92eb43fc13278da1cef2bcf47c0bd0e2aab66932c640bf8c709dae8df5b36961baf19d41acd6c846907f27cab237a7d37113642d6b8ee
+  data.tar.gz: 433d47c04f1a8f61e55e6dd664ab0818c0e687769cd29e9985acd74886abd2e5f4a2dacede60839c85797d71fa887e1196c61076837f6813c4fb55ac00fc25c2

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    kuber_kit (0.1.8)
+    kuber_kit (0.1.9)
       cli-ui
       contracts-lite
       dry-auto_inject
+      net-ssh
       thor
 
 GEM
@@ -29,6 +30,7 @@ GEM
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.3.0)
     method_source (1.0.0)
+    net-ssh (6.1.0)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)

data/README.md

@@ -1,4 +1,8 @@
-# Docker Kit
+# KuberKit
+
+[![Codeship Status for ArtStation/kuber_kit](https://app.codeship.com/projects/1286f0a6-3f90-4c1b-b426-721ed8a6571b/status?branch=master)](https://app.codeship.com/projects/417264)
+
+Solution for building & deploying applications on Kubernetes, written in Ruby.
 
 ## Installation
 
@@ -8,12 +12,21 @@ Add this line to your application's Gemfile:
 gem 'kuber_kit'
 ```
 
-## Launch compilation for example images
+## Development
+
+### Launch compilation
+
+```
+bin/kit compile ruby_app,ruby_app2 --path=./example
+```
+
+### Launch deployment
 
 ```
-bin/kit compile ruby_app,ruby_app2 --path=./example -C review
+bin/kit compile -s ruby_app,ruby_app2 --path=./example
 ```
 
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/TODO.md

@@ -1,6 +1,4 @@
-- limit amount of simultaneous image builds
 - list services and require confirmation before deployment
-- service factory should make sure that template exists
-- add build servers support
 - add build vars support (use images instead of containers)
+- template should be able to set default attributes
 - template should be able to depend on image?

data/example/configurations/review.rb

@@ -1,4 +1,5 @@
 KuberKit
   .define_configuration(:review)
   .use_registry(:review_default, as: :default)
-  .use_artifact(:kuber_kit_repo, as: :kuber_kit_repo)
+  .use_artifact(:kuber_kit_repo, as: :kuber_kit_repo)
+  .use_build_server(:remote_bs)

data/example/images/app_sources/Dockerfile

@@ -1,4 +1,4 @@
-FROM default/ruby
+FROM <%= image_url(:ruby) %>
 
 RUN mkdir -p /app
 WORKDIR /app

data/example/images/ruby_app/image.rb

@@ -3,13 +3,13 @@ KuberKit
   .registry(:default)
   .depends_on(:ruby, :app_sources)
   .before_build do |context_helper, build_dir|
-    # copy file local artifact
+    # copy file: local artifact
     source_path = context_helper.artifact_path(:kuber_kit_example_data, "test.txt")
     target_path = File.join(build_dir, "test.txt")
-    context_helper.shell.exec!("cp #{source_path} #{target_path}")
+    context_helper.shell.sync(source_path, target_path)
 
-    # copy file local artifact
+    # copy file: remote artifact
     source_path = context_helper.artifact_path(:kuber_kit_repo, "README.md")
     target_path = File.join(build_dir, "README.md")
-    context_helper.shell.exec!("cp #{source_path} #{target_path}")
+    context_helper.shell.sync(source_path, target_path)
   end

data/example/infrastructure/build_servers.rb

@@ -0,0 +1,8 @@
+ENV['SSH_TEST_HOST'] ||= "indocker.artstn.ninja"
+ENV['SSH_TEST_USER'] ||= "kuber_kit"
+
+KuberKit.add_build_server(
+  KuberKit::Core::BuildServers::BuildServer
+    .new(:remote_bs)
+    .setup(host: ENV['SSH_TEST_HOST'], user: ENV['SSH_TEST_USER'], port: 22)
+)

data/kuber_kit.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-auto_inject"
   spec.add_dependency "thor"
   spec.add_dependency "cli-ui"
+  spec.add_dependency "net-ssh"
 
   spec.add_development_dependency "bundler", "~> 1.17"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/kuber_kit.rb

@@ -31,18 +31,7 @@ module KuberKit
     autoload :ConfigurationFactory, 'core/configuration_factory'
     autoload :Configuration, 'core/configuration'
 
-    module ContextHelper
-      autoload :BaseHelper, 'core/context_helper/base_helper'
-      autoload :ImageHelper, 'core/context_helper/image_helper'
-      autoload :ServiceHelper, 'core/context_helper/service_helper'
-      autoload :ContextHelperFactory, 'core/context_helper/context_helper_factory'
-    end
-
-    module Registries
-      autoload :AbstractRegistry, 'core/registries/abstract_registry'
-      autoload :RegistryStore, 'core/registries/registry_store'
-      autoload :Registry, 'core/registries/registry'
-    end
+    autoload :Store, 'core/store'
 
     module Artifacts
       autoload :AbstractArtifact, 'core/artifacts/abstract_artifact'
@@ -51,12 +40,31 @@ module KuberKit
       autoload :Local, 'core/artifacts/local'
     end
 
+    module BuildServers
+      autoload :AbstractBuildServer, 'core/build_servers/abstract_build_server'
+      autoload :BuildServerStore, 'core/build_servers/build_server_store'
+      autoload :BuildServer, 'core/build_servers/build_server'
+    end
+
     module EnvFiles
       autoload :EnvFileStore, 'core/env_files/env_file_store'
       autoload :AbstractEnvFile, 'core/env_files/abstract_env_file'
       autoload :ArtifactFile, 'core/env_files/artifact_file'
     end
 
+    module ContextHelper
+      autoload :BaseHelper, 'core/context_helper/base_helper'
+      autoload :ImageHelper, 'core/context_helper/image_helper'
+      autoload :ServiceHelper, 'core/context_helper/service_helper'
+      autoload :ContextHelperFactory, 'core/context_helper/context_helper_factory'
+    end
+
+    module Registries
+      autoload :AbstractRegistry, 'core/registries/abstract_registry'
+      autoload :RegistryStore, 'core/registries/registry_store'
+      autoload :Registry, 'core/registries/registry'
+    end
+
     module Templates
       autoload :TemplateStore, 'core/templates/template_store'
       autoload :AbstractTemplate, 'core/templates/abstract_template'
@@ -67,22 +75,28 @@ module KuberKit
   module Tools
     autoload :FilePresenceChecker, 'tools/file_presence_checker'
     autoload :LoggerFactory, 'tools/logger_factory'
-    autoload :FilesSync, 'tools/files_sync'
   end
 
   module Shell
     autoload :AbstractShell, 'shell/abstract_shell'
     autoload :LocalShell, 'shell/local_shell'
+    autoload :SshShell, 'shell/ssh_shell'
+    autoload :SshSession, 'shell/ssh_session'
     autoload :CommandCounter, 'shell/command_counter'
-    autoload :BashCommands, 'shell/bash_commands'
-    autoload :DockerCommands, 'shell/docker_commands'
-    autoload :GitCommands, 'shell/git_commands'
-    autoload :RsyncCommands, 'shell/rsync_commands'
-    autoload :KubectlCommands, 'shell/kubectl_commands'
+
+    module Commands
+      autoload :BashCommands, 'shell/commands/bash_commands'
+      autoload :DockerCommands, 'shell/commands/docker_commands'
+      autoload :GitCommands, 'shell/commands/git_commands'
+      autoload :RsyncCommands, 'shell/commands/rsync_commands'
+      autoload :KubectlCommands, 'shell/commands/kubectl_commands'
+    end
   end
 
   module ImageCompiler
     autoload :ActionHandler, 'image_compiler/action_handler'
+    autoload :BuildServerPool, 'image_compiler/build_server_pool'
+    autoload :BuildServerPoolFactory, 'image_compiler/build_server_pool_factory'
     autoload :Compiler, 'image_compiler/compiler'
     autoload :ImageBuilder, 'image_compiler/image_builder'
     autoload :ImageBuildDirCreator, 'image_compiler/image_build_dir_creator'
@@ -93,7 +107,6 @@ module KuberKit
   module Preprocessing
     autoload :TextPreprocessor, 'preprocessing/text_preprocessor'
     autoload :FilePreprocessor, 'preprocessing/file_preprocessor'
-    autoload :DirPreprocessor, 'preprocessing/dir_preprocessor'
   end
 
   module ArtifactsSync
@@ -212,6 +225,10 @@ module KuberKit
       Container["core.template_store"].add(template)
     end
 
+    def add_build_server(build_server)
+      Container["core.build_server_store"].add(build_server)
+    end
+
     def build_helper(&proc)
       KuberKit::Core::ContextHelper::BaseHelper.class_exec(&proc)
     end

data/lib/kuber_kit/actions/configuration_loader.rb

@@ -50,8 +50,11 @@ class KuberKit::Actions::ConfigurationLoader
       files = service_store.load_definitions(services_path)
       task.update_title("Loaded #{files.count} service definitions")
     end
+    
+    true
   rescue KuberKit::Error => e
     ui.print_error("Error", e.message)
+    false
   end
   
   def load_configurations(configurations_path, configuration_name)

data/lib/kuber_kit/actions/env_file_reader.rb

@@ -11,7 +11,10 @@ class KuberKit::Actions::EnvFileReader
   def call(env_file_name, options)
     result = env_file_reader.call(local_shell, env_file_name)
     ui.print_info(env_file_name.to_s, JSON.pretty_generate(result))
+
+    true
   rescue KuberKit::Error => e
     ui.print_error("Error", e.message)
+    false
   end
 end

data/lib/kuber_kit/actions/image_compiler.rb

@@ -1,6 +1,7 @@
 class KuberKit::Actions::ImageCompiler
   include KuberKit::Import[
     "image_compiler.image_dependency_resolver",
+    "image_compiler.build_server_pool_factory",
     "shell.local_shell",
     "tools.logger",
     "ui",
@@ -10,29 +11,32 @@ class KuberKit::Actions::ImageCompiler
   Contract ArrayOf[Symbol], Hash => Any
   def call(image_names, options)
     build_id = generate_build_id
+    build_server_pool = build_server_pool_factory.create()
 
-    resolved_dependencies = []
-    dependencies = image_dependency_resolver.get_next(image_names)
-
-    while (dependencies - resolved_dependencies).any?
-      compile_simultaneously(dependencies, build_id)
-      resolved_dependencies += dependencies
-      dependencies = image_dependency_resolver.get_next(image_names, resolved: resolved_dependencies)
+    image_dependency_resolver.each_with_deps(image_names) do |dep_image_names|
+      result = compile_simultaneously(dep_image_names, build_id, build_server_pool)
+      abort unless result
     end
 
-    compile_simultaneously(image_names - resolved_dependencies, build_id)
+    build_server_pool.disconnect_all
+
+    true
   rescue KuberKit::Error => e
     ui.print_error("Error", e.message)
+
+    false
   end
 
   private
-    def compile_simultaneously(image_names, build_id)
+    def compile_simultaneously(image_names, build_id, build_server_pool)
       task_group = ui.create_task_group
       image_names.map do |image_name|
 
         logger.info("Started compiling: #{image_name.to_s.green}")
         task_group.add("Compiling #{image_name.to_s.yellow}") do |task|
-          image_compiler.call(local_shell, image_name, build_id)
+          shell = build_server_pool.get_shell
+          
+          image_compiler.call(shell, image_name, build_id)
 
           task.update_title("Compiled #{image_name.to_s.green}")
           logger.info("Finished compiling: #{image_name.to_s.green}")

data/lib/kuber_kit/actions/kubectl_applier.rb

@@ -12,8 +12,11 @@ class KuberKit::Actions::KubectlApplier
       kubectl_commands.apply_file(local_shell, file_path, kubeconfig_path: kubeconfig_path)
       task.update_title("Applied file: #{file_path}")
     end
-    nil
+
+    true
   rescue KuberKit::Error => e
     ui.print_error("Error", e.message)
+    
+    false
   end
 end

data/lib/kuber_kit/actions/service_deployer.rb

@@ -31,8 +31,12 @@ class KuberKit::Actions::ServiceDeployer
 
     compile_images(images_names)
     deploy_services(service_names)
+
+    true
   rescue KuberKit::Error => e
     ui.print_error("Error", e.message)
+
+    false
   end
 
   def deploy_services(service_names)

data/lib/kuber_kit/actions/service_reader.rb

@@ -10,7 +10,11 @@ class KuberKit::Actions::ServiceReader
     result = service_reader.call(local_shell, service_name)
 
     ui.print_info(service_name.to_s, result)
+
+    true
   rescue KuberKit::Error => e
     ui.print_error("Error", e.message)
+
+    false
   end
 end

data/lib/kuber_kit/actions/template_reader.rb

@@ -13,7 +13,10 @@ class KuberKit::Actions::TemplateReader
     result = reader.read(local_shell, template)
 
     ui.print_info(template_name.to_s, result)
+
+    true
   rescue KuberKit::Error => e
     ui.print_error("Error", e.message)
+    false
   end
 end

data/lib/kuber_kit/cli.rb

@@ -15,12 +15,18 @@ class KuberKit::CLI < Thor
     image_names = image_names_str.split(",").map(&:strip).map(&:to_sym)
 
     KuberKit::Container['actions.configuration_loader'].call(options)
-    KuberKit::Container['actions.image_compiler'].call(image_names, options)
-
-    logger = KuberKit::Container['tools.logger']
-    logger.info("---------------------------")
-    logger.info("Image compilation finished!")
-    logger.info("---------------------------")
+    result = KuberKit::Container['actions.image_compiler'].call(image_names, options)
+
+    if result
+      logger = KuberKit::Container['tools.logger']
+      logger.info("---------------------------")
+      logger.info("Image compilation finished!")
+      logger.info("---------------------------")
+    else
+      logger.info("-------------------------".red)
+      logger.info("Image compilation failed!".red)
+      logger.info("-------------------------".red)
+    end
   end
 
   desc "env ENV_FILE_NAME", "Return content of Env File ENV_FILE_NAME"

data/lib/kuber_kit/configs.rb

@@ -14,33 +14,35 @@ class KuberKit::Configs
     'tmp',
     'logs'
   ]
-  KUBER_KIT_DIRNAME      = "kuber_kit".freeze
-  IMAGES_DIRNAME         = "images".freeze
-  SERVICES_DIRNAME       = "services".freeze
-  INFRA_DIRNAME          = "infrastructure".freeze
-  CONFIGURATIONS_DIRNAME = "configurations".freeze
-  ARTIFACT_CLONE_DIR     = "/tmp/kuber_kit/artifacts"
-  SERVICE_CONFIG_DIR     = "/tmp/kuber_kit/services"
-  DEPLOY_STRATEGY        = :kubernetes
+  KUBER_KIT_DIRNAME           = "kuber_kit".freeze
+  IMAGES_DIRNAME              = "images".freeze
+  SERVICES_DIRNAME            = "services".freeze
+  INFRA_DIRNAME               = "infrastructure".freeze
+  CONFIGURATIONS_DIRNAME      = "configurations".freeze
+  ARTIFACT_CLONE_DIR          = "/tmp/kuber_kit/artifacts"
+  SERVICE_CONFIG_DIR          = "/tmp/kuber_kit/services"
+  DEPLOY_STRATEGY             = :kubernetes
+  COMPILE_SIMULTANEOUS_LIMIT  = 5
 
   attr_accessor :image_dockerfile_name, :image_build_context_dir, :image_tag,
                 :docker_ignore_list, :image_compile_dir, 
                 :kuber_kit_dirname, :images_dirname, :services_dirname, :infra_dirname, :configurations_dirname,
-                :artifact_clone_dir, :service_config_dir, :deploy_strategy
+                :artifact_clone_dir, :service_config_dir, :deploy_strategy, :compile_simultaneous_limit
 
   def initialize
-    @image_dockerfile_name   = IMAGE_DOCKERFILE_NAME
-    @image_build_context_dir = IMAGE_BUILD_CONTEXT_DIR
-    @image_tag               = IMAGE_TAG
-    @image_compile_dir       = IMAGE_COMPILE_DIR
-    @docker_ignore_list      = DOCKER_IGNORE_LIST
-    @kuber_kit_dirname       = KUBER_KIT_DIRNAME
-    @images_dirname          = IMAGES_DIRNAME
-    @services_dirname        = SERVICES_DIRNAME
-    @infra_dirname           = INFRA_DIRNAME
-    @configurations_dirname  = CONFIGURATIONS_DIRNAME
-    @artifact_clone_dir      = ARTIFACT_CLONE_DIR
-    @service_config_dir      = SERVICE_CONFIG_DIR
-    @deploy_strategy         = DEPLOY_STRATEGY
+    @image_dockerfile_name      = IMAGE_DOCKERFILE_NAME
+    @image_build_context_dir    = IMAGE_BUILD_CONTEXT_DIR
+    @image_tag                  = IMAGE_TAG
+    @image_compile_dir          = IMAGE_COMPILE_DIR
+    @docker_ignore_list         = DOCKER_IGNORE_LIST
+    @kuber_kit_dirname          = KUBER_KIT_DIRNAME
+    @images_dirname             = IMAGES_DIRNAME
+    @services_dirname           = SERVICES_DIRNAME
+    @infra_dirname              = INFRA_DIRNAME
+    @configurations_dirname     = CONFIGURATIONS_DIRNAME
+    @artifact_clone_dir         = ARTIFACT_CLONE_DIR
+    @service_config_dir         = SERVICE_CONFIG_DIR
+    @deploy_strategy            = DEPLOY_STRATEGY
+    @compile_simultaneous_limit = COMPILE_SIMULTANEOUS_LIMIT
   end
 end

data/lib/kuber_kit/container.rb

@@ -69,18 +69,22 @@ class KuberKit::Container
     KuberKit::Core::ConfigurationStore.new
   end
 
-  register "core.registry_store" do
-    KuberKit::Core::Registries::RegistryStore.new
-  end
-
   register "core.artifact_store" do
     KuberKit::Core::Artifacts::ArtifactStore.new
   end
 
+  register "core.build_server_store" do
+    KuberKit::Core::BuildServers::BuildServerStore.new
+  end
+
   register "core.env_file_store" do
     KuberKit::Core::EnvFiles::EnvFileStore.new
   end
 
+  register "core.registry_store" do
+    KuberKit::Core::Registries::RegistryStore.new
+  end
+
   register "core.template_store" do
     KuberKit::Core::Templates::TemplateStore.new
   end
@@ -102,23 +106,23 @@ class KuberKit::Container
   end
 
   register "shell.bash_commands" do
-    KuberKit::Shell::BashCommands.new
+    KuberKit::Shell::Commands::BashCommands.new
   end
 
   register "shell.docker_commands" do
-    KuberKit::Shell::DockerCommands.new
+    KuberKit::Shell::Commands::DockerCommands.new
   end
 
   register "shell.git_commands" do
-    KuberKit::Shell::GitCommands.new
+    KuberKit::Shell::Commands::GitCommands.new
   end
 
   register "shell.rsync_commands" do
-    KuberKit::Shell::RsyncCommands.new
+    KuberKit::Shell::Commands::RsyncCommands.new
   end
 
   register "shell.kubectl_commands" do
-    KuberKit::Shell::KubectlCommands.new
+    KuberKit::Shell::Commands::KubectlCommands.new
   end
 
   register "shell.local_shell" do
@@ -137,14 +141,14 @@ class KuberKit::Container
     KuberKit::Preprocessing::FilePreprocessor.new
   end
 
-  register "preprocessing.dir_preprocessor" do
-    KuberKit::Preprocessing::DirPreprocessor.new
-  end
-
   register "image_compiler.action_handler" do
     KuberKit::ImageCompiler::ActionHandler.new
   end
 
+  register "image_compiler.build_server_pool_factory" do
+    KuberKit::ImageCompiler::BuildServerPoolFactory.new
+  end
+
   register "image_compiler.compiler" do
     KuberKit::ImageCompiler::Compiler.new
   end