indocker 0.0.6 → 0.1.0

data/lib/indocker/artifacts/git.rb

@@ -0,0 +1,25 @@
+class Indocker::Artifacts::Git
+  attr_reader :name, :remote_name, :remote_url, :branch, :source_path, :target_path
+
+  def initialize(name:, remote_name:, remote_url:, branch:, source_path:, target_path:)
+    @name        = name
+    @remote_name = remote_name
+    @remote_url  = remote_url
+    @branch      = branch
+    @source_path = source_path
+    @target_path = target_path
+  end
+
+  def repository
+    @repository ||= Indocker::Repositories::Git.new(@name).setup(
+      remote_name: remote_name,
+      remote_url:  remote_url,
+      branch:      branch,
+      clone_path:  "/tmp/#{Indocker.configuration.name}/artifacts/git/#{project_name(remote_url)}/#{branch}"
+    )
+  end
+
+  def project_name(url)
+    url.split('/').last.gsub('.git', '')
+  end
+end

data/lib/indocker/build_context.rb

@@ -0,0 +1,87 @@
+require 'fileutils'
+
+class Indocker::BuildContext
+  attr_reader :session, :server, :configuration, :helper, :logger
+
+  def initialize(configuration:, build_server:, logger:)
+    @configuration = configuration
+    @logger = logger
+    @helper = Indocker::BuildContextHelper.new(@configuration, @build_server)
+    @server = build_server
+
+    if build_server
+      @session = Indocker::SshSession.new(
+        host: build_server.host,
+        user: build_server.user,
+        port: build_server.port,
+        logger: @logger
+      )
+    end
+
+    @compiled_images = Hash.new(false)
+  end
+
+  def exec!(command)
+    @session.exec!(command)
+  end
+
+  def close_session
+    @session.close if @session
+  end
+
+  def image_compiled?(image)
+    @compiled_images[image]
+  end
+
+  def set_busy(flag)
+    @busy = !!flag
+  end
+
+  def busy?
+    !!@busy
+  end
+
+  def set_compiled(image)
+    @compiled_images[image] = true
+  end
+
+  def build_image(image, build_dir, args: [])
+    image_name = image.image_name
+    registry = image.registry
+    tag        = image.tag
+
+    FileUtils.cd(build_dir) do
+      if @logger.debug?
+        @logger.debug("#{"Docker image content:".yellow}")
+
+        Dir[File.join(build_dir, '*')]
+          .map {|path|
+            file = path.gsub(build_dir, '')
+            @logger.debug("  .#{file}".yellow)
+          }
+          .join("\n")
+      end
+
+      if !@logger.debug?
+        args = args.push('-q')
+      end
+
+      build_args = args.join(' ')
+
+      res = Indocker::Docker.build(image.local_registry_url, build_args)
+
+      if res.exit_status != 0
+        @logger.error("image compilation :#{image.name} failed")
+        @logger.error(res.stdout)
+        exit 1
+      end
+
+      Indocker::Docker.tag(image.local_registry_url, image.registry_url)
+      Indocker::Docker.tag(image.local_registry_url, image.local_registry_url)
+
+      if !image.registry.is_local?
+        Indocker::Docker.push(image.registry_url)
+      end
+    end
+  end
+end

data/lib/indocker/build_context_helper.rb

@@ -0,0 +1,109 @@
+class Indocker::BuildContextHelper
+  attr_reader :configuration, :build_server
+
+  def initialize(configuration, build_server)
+    @configuration = configuration
+    @build_server = build_server
+    @clonned_repositories = Hash.new(false)
+  end
+
+  def image_url(image_sym)
+    path = Indocker.image_files.fetch(image_sym) do
+      Indocker.logger.error("image :#{image_sym} was not found in configuration :#{@configuration.name}")
+      exit 1
+    end
+
+    require path
+
+    image = @configuration.images.detect do |i|
+      i.name == image_sym
+    end
+
+    if image.nil?
+      raise ArgumentError.new("image :#{image_sym} was not found in configuration")
+    end
+
+    image.registry_url
+  end
+
+  def repository_path(name)
+    repo = repository(name)
+    repo.clone_path
+  end
+
+  def repository(name)
+    @configuration.repositories.fetch(name) do
+      raise ArgumentError.new("repository :#{name} is not defined in configuration")
+    end
+  end
+
+  def global_build_args
+    @global_build_args = Indocker::ContextArgs.new(nil, @configuration.global_build_args, nil)
+  end
+
+  def container_enabled?(container)
+    @configuration.enabled_containers.include?(container.name)
+  end
+
+  class Containers
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    def method_missing(name, *args)
+      if args.size > 0
+        raise ArgumentError.new("containers.#{name} does not accept arguments")
+      end
+
+      if Indocker.container_files.has_key?(name)
+        require Indocker.container_files.fetch(name)
+      end
+
+      container = @configuration.containers.detect do |container|
+        container.name == name
+      end
+
+      if !container
+        raise ArgumentError.new("container :#{name} was not found in configuration")
+      end
+
+      Container.new(@configuration, container)
+    end
+  end
+
+  class Container
+    def initialize(configuration, container)
+      @configuration = configuration
+      @container = container
+    end
+
+    def build_args
+      Indocker::ContextArgs.new(
+        nil,
+        Indocker::HashMerger.deep_merge(@container.build_args, @container.image.build_args),
+        nil,
+        @container
+      )
+    end
+
+    def hostname(number = nil)
+      Indocker::ContainerHelper.hostname(@configuration.name, @container, number)
+    end
+
+    def count
+      @container.get_start_option(:scale) || 1
+    end
+
+    def method_missing(name, *args)
+      @container.send(name, *args)
+    end
+  end
+
+  def containers
+    @containers ||= Containers.new(@configuration)
+  end
+
+  def get_binding
+    binding
+  end
+end

data/lib/indocker/build_context_pool.rb

@@ -0,0 +1,36 @@
+class Indocker::BuildContextPool
+  def initialize(configuration:, logger:)
+    @logger = logger
+    @configuration = configuration
+
+    @contexts = configuration.build_servers.map do |build_server|
+      Indocker::BuildContext.new(
+        logger: @logger,
+        configuration: configuration,
+        build_server: build_server,
+      )
+    end
+  end
+
+  def get
+    context = nil
+
+    loop do
+      context = @contexts.detect {|c| !c.busy?}
+      sleep(0.1)
+      break if context
+    end
+
+    context
+  end
+
+  def each(&proc)
+    @contexts.each(&proc)
+  end
+
+  def close_sessions
+    @contexts.each(&:close_session)
+  rescue => e
+    @logger.error("error during session close: #{e.inspect}")
+  end
+end

data/lib/indocker/build_server.rb

@@ -0,0 +1,2 @@
+class Indocker::BuildServer < Indocker::Server
+end

data/lib/indocker/colored_string.rb

@@ -0,0 +1,33 @@
+class String
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def yellow
+    colorize(33)
+  end
+
+  def blue
+    colorize(34)
+  end
+
+  def purple
+    colorize(35)
+  end
+
+  def cyan
+    colorize(36)
+  end
+
+  def grey
+    colorize(37)
+  end
+
+  def colorize(color_code)
+    "\e[#{color_code}m#{self}\e[0m"
+  end
+end

data/lib/indocker/concerns/inspectable.rb

@@ -0,0 +1,12 @@
+module Indocker::Concerns::Inspectable
+  def inspect
+    data = {}
+    data[:type] = self.class.to_s.split('::').last.downcase
+
+    instance_variables.each do |variable|
+      data[variable.to_s.gsub('@', '').to_sym] = instance_variable_get(variable)
+    end
+
+    data.inspect
+  end
+end

data/lib/indocker/configuration_deployer.rb

@@ -0,0 +1,525 @@
+require 'timeout'
+require 'benchmark'
+require 'tempfile'
+
+class Indocker::ConfigurationDeployer
+  REMOTE_OPERATION_TIMEOUT = 60
+
+  def initialize(logger)
+    Thread.abort_on_exception = true # abort all threads if exception occurs
+
+    @logger = logger
+    @global_logger = Logger.new(STDOUT)
+    @global_logger.formatter = logger.formatter
+
+    @progress = Indocker::DeploymentProgress.new(
+      Indocker.logger.level == Logger::DEBUG ? nil : Logger.new(STDOUT)
+    )
+  end
+
+  def run(configuration:, deploy_containers:, skip_tags:, deploy_tags:, skip_dependent:,
+          skip_containers:, servers:, skip_build:, force_restart:, skip_force_restart:, auto_confirm:, require_confirmation:)
+    build_context_pool = nil
+    deployer = nil
+
+    time = Benchmark.realtime do
+      if force_restart
+        @logger.warn("WARNING. All containers will be forced to restart.")
+      end
+
+      if skip_build
+        @logger.warn("WARNING. Images build step will be skipped")
+      end
+
+      preload_containers(configuration)
+
+      containers = find_deploy_containers(configuration, deploy_containers, deploy_tags, skip_dependent, skip_containers, servers, skip_tags, auto_confirm, require_confirmation)
+      containers = containers.uniq {|c| c.name}
+
+      clonner = Indocker::Repositories::Clonner.new(configuration, @logger)
+
+      @global_logger.info("Establishing ssh sessions to all servers...")
+
+      build_context_pool = Indocker::BuildContextPool.new(configuration: configuration, logger: @logger)
+      deployer = Indocker::ContainerDeployer.new(configuration: configuration, logger: @logger)
+
+      build_servers = configuration
+        .build_servers
+        .uniq { |s| s.host }
+
+      deploy_servers = containers
+        .map(&:servers)
+        .flatten
+        .uniq { |s| s.host }
+
+      servers = (deploy_servers + build_servers).uniq { |s| s.host }
+
+      @progress.setup(
+        binaries_servers: servers,
+        build_servers: build_servers,
+        deploy_servers: deploy_servers,
+        env_files: configuration.env_files.keys,
+        repositories: configuration.repositories.keys,
+        force_restart: force_restart,
+        skip_build: skip_build,
+        containers: containers,
+        artifact_servers: configuration.artifact_servers,
+      )
+
+      remote_operations = sync_indocker(servers)
+
+      wait_remote_operations(remote_operations)
+
+      remote_operations += sync_env_files(deploy_servers, configuration.env_files)
+      remote_operations += pull_repositories(clonner, build_servers, configuration.repositories)
+      remote_operations += sync_artifacts(clonner, configuration.artifact_servers)
+
+      wait_remote_operations(remote_operations)
+
+      update_crontab_redeploy_rules(configuration, build_servers.first)
+
+      containers.uniq.each do |container|
+        recursively_deploy_container(configuration, deployer, build_context_pool, container, containers, skip_build, force_restart, skip_force_restart)
+      end
+
+      Thread
+        .list
+        .each { |t| t.join if t != Thread.current }
+    end
+
+    @global_logger.info("Deployment finished".green)
+    @global_logger.info("Total time taken: #{time.round}s".green)
+  ensure
+    build_context_pool.close_sessions if build_context_pool
+    deployer.close_sessions if deployer
+  end
+
+  private
+
+  def wait_remote_operations(remote_operations)
+    remote_operations.each do |remote_operation|
+      begin
+        Timeout::timeout(REMOTE_OPERATION_TIMEOUT) do
+          remote_operation.thread.join
+        end
+      rescue Timeout::Error
+        @global_logger.error("Deployment aborted. Remote operation :#{remote_operation.operation} on server #{remote_operation.server.user}@#{remote_operation.server.host} did not finish during #{REMOTE_OPERATION_TIMEOUT} seconds.")
+        exit 1
+      end
+    end
+  end
+
+  def preload_containers(configuration)
+    configuration.enabled_containers.each do |container_name|
+      path = Indocker.container_files[container_name]
+      require path
+    end
+  end
+
+  def find_deploy_containers(configuration, deploy_containers, deploy_tags, skip_dependent, skip_containers, servers, skip_tags, auto_confirm, require_confirmation)
+    containers = []
+
+    deploy_tags.each do |tag|
+      containers += configuration.containers.select do |container|
+        container.tags.include?(tag)
+      end
+    end
+
+    skip_containers.each do |name|
+      container = configuration.containers.detect do |container|
+        container.name == name
+      end
+
+      if !container
+        @global_logger.error("Invalid --skip container :#{name} for configuration :#{configuration.name}")
+        @global_logger.info("Available containers:")
+
+        configuration.containers.sort_by(&:name).each do |container|
+          @global_logger.info("  - #{container.name}")
+        end
+
+        exit 1
+      end
+    end
+
+    deploy_containers.each do |name|
+      container = configuration.containers.detect do |container|
+        container.name == name
+      end
+
+      if container
+        containers.push(container)
+      else
+        @global_logger.error("container :#{name} was not found in configuration :#{configuration.name}")
+
+        exit 1
+      end
+    end
+
+    if deploy_tags.empty? && deploy_containers.empty?
+      containers = configuration.containers.select do |container|
+        configuration.enabled_containers.include?(container.name)
+      end
+    end
+
+    if !skip_dependent
+      containers = collect_dependent_containers(containers)
+    end
+
+    if !skip_dependent
+      containers = collect_soft_dependent_containers(containers, configuration)
+    end
+
+    extra_containers = containers.map(&:name) - configuration.enabled_containers
+
+    if !extra_containers.empty?
+      @global_logger.warn("configuration :#{configuration.name} does not include following containers: #{extra_containers.inspect}")
+      @global_logger.warn("they will be skipped during deployment")
+    end
+
+    containers = containers
+      .select { |container| configuration.container_enabled?(container) }
+      .select { |container| !skip_containers.include?(container.name) }
+      .select { |container|
+        (skip_tags & container.tags).empty?
+      }
+
+    if !servers.empty?
+      containers = containers.select {|c| !(c.servers.map(&:name) & servers).empty? }
+    end
+
+    if containers.empty?
+      @global_logger.error("at least one container should be specified for deployment")
+      exit 1
+    else
+      @global_logger.info("Following containers will be deployed:")
+
+      if servers.empty?
+        servers = containers.map(&:servers).flatten.uniq.map(&:name)
+      end
+
+      servers.each do |server_name|
+        @global_logger.info("")
+        @global_logger.info("List of containers for server #{server_name.to_s.yellow}")
+
+        containers.each do |container|
+          if container.servers.map(&:name).include?(server_name)
+            scale = container.get_start_option(:scale)
+            scale_msg = scale > 1 ? "  (#{container.get_start_option(:scale)})" : ''
+            @global_logger.info("  - #{container.name}#{scale_msg}")
+          end
+        end
+      end
+
+      if (require_confirmation || configuration.confirm_deployment) && !auto_confirm
+        @global_logger.info("\n")
+        @global_logger.info("Do you want to continue deployment? (y or n)")
+        result = gets.chomp
+
+        if result.downcase != 'y'
+          @global_logger.info("Deployment aborted")
+          exit 0
+        end
+      end
+
+      containers
+    end
+  end
+
+  def collect_dependent_containers(containers)
+    result = containers
+
+    result += containers.map do |container|
+      collect_dependent_containers(container.dependent_containers)
+    end.flatten
+
+    result.uniq
+  end
+
+  def collect_soft_dependent_containers(containers, configuration)
+    result = containers
+
+    result += containers.map do |container|
+      container.soft_dependent_containers.reject do |soft_dependent_container|
+        configuration.enabled_containers.include?(soft_dependent_container.name) &&
+        Indocker.launched?(soft_dependent_container.name)
+      end
+    end.flatten
+
+
+    result.uniq
+  end
+
+  def compile_image(configuration, image, build_context)
+    return if build_context.image_compiled?(image)
+
+    image.dependent_images.each do |dependent_image|
+      next if build_context.image_compiled?(dependent_image)
+      compile_image(configuration, dependent_image, build_context)
+    end
+
+    compiler = Indocker::Images::ImageCompiler.new
+
+    @logger.info("Image compilation started #{image.name.to_s.green}")
+
+    result = nil
+
+    time = Benchmark.realtime do
+      result = build_context
+        .session
+        .exec!(
+          "cd #{Indocker::IndockerHelper.indocker_dir} && ./bin/remote/compile -C #{Indocker.configuration_name} -i #{image.name} -s #{@logger.debug? ? '-d' : ''}"
+        )
+    end
+
+    Indocker::SshResultLogger
+      .new(@logger)
+      .log(result, "#{image.name.to_s.green} image compilation failed")
+
+    exit 1 if result.exit_code != 0
+
+    @logger.info("Image compilation completed #{image.name.to_s.green}. Time taken: #{time}")
+
+    build_context.set_compiled(image)
+  end
+
+  def recursively_deploy_container(configuration, deployer, build_context_pool, container, containers, skip_build, force_restart, skip_force_restart)
+    container.dependent_containers.each do |container|
+      recursively_deploy_container(configuration, deployer, build_context_pool, container, containers, skip_build, force_restart, skip_force_restart)
+    end
+
+    return if !containers.include?(container)
+
+    @progress.start_building_container(container)
+
+    if !skip_build
+      build_context = build_context_pool.get
+
+      build_context.set_busy(true)
+      compile_image(configuration, container.image, build_context)
+      build_context.set_busy(false)
+    end
+
+    @progress.finish_building_container(container)
+
+    deployer.deploy(container, force_restart, skip_force_restart, @progress)
+  end
+
+  class RemoteOperation
+    attr_reader :thread, :server, :operation, :message
+
+    def initialize(thread, server, operation, message = nil)
+      @thread = thread
+      @server = server
+      @operation = operation
+      @message = message
+    end
+  end
+
+  def pull_repositories(clonner, servers, repositories)
+    @logger.info("{timestamp}")
+    @logger.info("Clonning/pulling repositories")
+
+    remote_operations = []
+
+    servers.each do |server|
+      remote_operations += repositories.map do |alias_name, repository|
+        @progress.start_syncing_repository(server, alias_name)
+
+        thread = Thread.new do
+          session = Indocker::SshSession.new(
+            host: server.host,
+            user: server.user,
+            port: server.port,
+            logger: @logger
+          )
+
+          if repository.is_local?
+            @logger.info("Rsyncing repository :#{alias_name} from #{repository.root_path} to #{server.user}@#{server.host}:#{repository.clone_path}")
+
+            Indocker::Rsync.sync(
+              session,
+              File.join(repository.root_path, '.'),
+              repository.clone_path,
+              create_path: repository.clone_path,
+              raise_on_error: true
+            )
+          elsif repository.is_git?
+            @logger.info("Pulling repository #{alias_name.to_s.green} for #{server.user}@#{server.host}")
+            result = clonner.clone(session, repository)
+
+            if result.exit_code != 0
+              @logger.error("Repository :#{repository.name} was not clonned")
+              @logger.error(result.stderr_data)
+              exit 1
+            end
+          elsif repository.is_no_sync?
+            @logger.info("Skipping pull/sync operation for no_sync repository :#{alias_name} #{repository.clone_path}")
+          else
+            raise NotImplementedError.new("unsupported repository type: #{repository.inspect}")
+          end
+
+          @progress.finish_syncing_repository(server, alias_name)
+        end
+
+        RemoteOperation.new(thread, server, :repository_pull)
+      end
+    end
+
+    remote_operations
+  end
+
+  def sync_artifacts(clonner, artifact_servers)
+    @logger.info("{timestamp}")
+    @logger.info("Syncing git artifacts")
+
+    remote_operations = []
+
+    artifact_servers.each do |artifact, servers|
+      remote_operations += servers.map do |server|
+        @progress.start_syncing_artifact(server, artifact)
+
+        thread = Thread.new do
+          session = Indocker::SshSession.new(
+            host: server.host,
+            user: server.user,
+            port: server.port,
+            logger: @logger
+          )
+
+          @logger.info("Pulling git artifact  #{artifact.name.to_s.green} for #{server.user}@#{server.host}")
+          result = clonner.clone(session, artifact.repository)
+
+          if result.exit_code != 0
+            @logger.error("Artifact repository :#{artifact.repository.name} was not clonned")
+            @logger.error(result.stderr_data)
+            exit 1
+          end
+
+          source_path = File.join(artifact.repository.clone_path, artifact.source_path)
+          result = session.exec!("mkdir -p #{artifact.target_path}")
+          result = session.exec!("cp -r #{source_path} #{artifact.target_path}")
+
+          if !result.success?
+            @logger.error(result.stdout_data)
+            @logger.error(result.stderr_data)
+            exit 1
+          end
+
+          @progress.finish_syncing_artifact(server, artifact)
+        end
+
+        RemoteOperation.new(thread, server, :artifact_sync)
+      end
+    end
+
+    remote_operations
+  end
+
+  def update_crontab_redeploy_rules(configuration, server)
+    redeploy_containers = configuration.containers.select {|c| c.redeploy_schedule}.uniq
+    return if redeploy_containers.empty?
+
+    @logger.info("{timestamp}")
+
+    deploy_user       = "#{server.user}@#{server.host}"
+    crontab_filepath  = "~/#{server.user}/crontab"
+
+    crontab = Indocker::CrontabRedeployRulesBuilder
+      .new(
+        configuration:  configuration,
+        logger:         @logger,
+      )
+      .call(redeploy_containers)
+
+    tmp_crontab_file = Tempfile.new('crontab')
+    tmp_crontab_file.write(crontab)
+    tmp_crontab_file.close
+
+    @logger.info("Updating crontab file #{deploy_user}:#{crontab_filepath}")
+
+    Indocker::Shell.command("scp #{tmp_crontab_file.path} #{deploy_user}:#{crontab_filepath}", @logger)
+
+    tmp_crontab_file.unlink
+
+    Indocker::Shell.command("ssh #{deploy_user} 'crontab #{crontab_filepath}'", @logger)
+  end
+
+  def sync_indocker(servers)
+    @logger.info("{timestamp}")
+
+    servers.map do |server|
+      @progress.start_syncing_binaries(server)
+
+      thread = Thread.new do
+        session = Indocker::SshSession.new(
+          host: server.host,
+          user: server.user,
+          port: server.port,
+          logger: @logger
+        )
+
+        sync_path = Indocker::IndockerHelper.indocker_dir
+        @logger.info("Syncing indocker to #{server.user}@#{server.host}:#{sync_path}")
+
+        Indocker::Rsync.sync(
+          session,
+          File.join(Indocker.root_dir, '.'),
+          sync_path,
+          create_path: sync_path,
+          raise_on_error: true,
+        )
+
+        @progress.finish_syncing_binaries(server)
+      end
+
+      RemoteOperation.new(thread, server, :indocker_sync)
+    end
+  end
+
+  def sync_env_files(servers, env_files)
+    @logger.info("{timestamp}")
+
+    remote_operations = []
+
+    servers.map do |server|
+      remote_operations += env_files.map do |alias_name, env_file|
+        @progress.start_syncing_env_file(server, alias_name)
+
+        thread = Thread.new do
+          if env_file.is_a?(Indocker::EnvFiles::Local)
+            sync_path = File.join(Indocker.deploy_dir, 'env_files', File.basename(env_file.path))
+            @logger.info("Syncing env file :#{alias_name} from #{env_file.path} to #{server.user}@#{server.host}:#{sync_path}")
+
+            session = Indocker::SshSession.new(
+              host: server.host,
+              user: server.user,
+              port: server.port,
+              logger: @logger
+            )
+
+            session.exec!("mkdir -p #{Indocker::EnvFileHelper.folder}")
+
+            Indocker::Rsync.sync(
+              session,
+              env_file.path,
+              sync_path,
+              raise_on_error: true
+            )
+          elsif env_file.is_a?(Indocker::EnvFiles::Remote)
+            @logger.warn("Sync operation for remote env file :#{alias_name} is skipped")
+          else
+            @logger.error("unsupported env file type: #{env_file.inspect}")
+            raise "unsupported env file type: #{env_file.inspect}"
+          end
+
+          @progress.finish_syncing_env_file(server, alias_name)
+        end
+
+        RemoteOperation.new(thread, server, :env_file_sync)
+      end
+    end
+
+    remote_operations
+  end
+end