indocker 0.0.6 → 0.1.4

data/lib/indocker/deploy_context.rb

@@ -0,0 +1,126 @@
+require 'digest'
+require 'fileutils'
+
+class Indocker::DeployContext
+  attr_reader :server, :session
+
+  def initialize(logger:, configuration:, server:)
+    @logger = logger
+    @configuration = configuration
+    @server = server
+    @restart_policy = Indocker::Containers::RestartPolicy.new(configuration, logger)
+
+    if server
+      @session = Indocker::SshSession.new(
+        host: server.host,
+        user: server.user,
+        port: server.port,
+        logger: @logger
+      )
+    end
+  end
+
+  def exec!(command)
+    @session.exec!(command)
+  end
+
+  def close_session
+    @session.close if @session
+  end
+
+  def set_busy(flag)
+    @busy = !!flag
+  end
+
+  def busy?
+    !!@busy
+  end
+
+  def deploy(container, force_restart)
+    @logger.info("Deploying container: #{container.name.to_s.green}")
+    @logger.debug("Deploy dir: #{Indocker.deploy_dir}")
+
+    Indocker::Docker.pull(container.image.registry_url) if !container.image.registry.is_local?
+
+    container.networks.each do |network|
+      Indocker::Docker.create_network(
+        Indocker::NetworkHelper.name(@configuration.name, network)
+      )
+    end
+
+    container.volumes.each do |volume|
+      if volume.is_a?(Indocker::Volumes::External)
+        Indocker::Docker.create_volume(
+          Indocker::VolumeHelper.name(@configuration.name, volume)
+        )
+      end
+    end
+
+    container.get_start_option(:scale).times do |number|
+      arg_list = Indocker::DockerRunArgs
+        .get(container, @configuration, number)
+        .join(' ')
+
+      hostname = Indocker::ContainerHelper.hostname(@configuration.name, container, number)
+
+      # timestamp generation
+      run_cmd = Indocker::Docker.run_command(container.image.registry_url, arg_list, container.start_command, container.get_start_option(:service_args))
+
+      env_files = container.get_start_option(:env_files, default: [])
+        .map { |env_file|
+          env_file = @configuration.env_files.fetch(env_file)
+          File.read(Indocker::EnvFileHelper.path(env_file))
+        }
+        .join
+
+      image_id = Indocker::Docker.image_id(container.image.registry_url)
+      timestamp = Digest::MD5.hexdigest(run_cmd + image_id.to_s + env_files)
+
+      binary_path = File.join(File.expand_path(Indocker.deploy_dir), 'bin')
+      FileUtils.mkdir_p(binary_path)
+
+      binary_path = File.join(binary_path, hostname)
+
+      File.open(binary_path, 'w') { |f|
+        f.write("#!/bin/bash\n\n")
+        f.write(run_cmd)
+      }
+
+      FileUtils.chmod('+x', binary_path)
+
+      container_id = Indocker::Docker.container_id_by_name(hostname)
+
+      if !container_id || @restart_policy.restart?(container, timestamp) || force_restart
+        if container.before_start_proc
+          container.before_start_proc.call(container, number)
+        end
+
+        if container_id
+          Indocker::Docker.stop(hostname, skip_errors: true)
+        else
+          Indocker::Docker.rm(hostname, skip_errors: true)
+        end
+
+        Indocker::Docker.run(container.image.registry_url, arg_list, container.start_command, container.get_start_option(:service_args))
+
+        if container.after_start_proc
+          container.after_start_proc.call(container, number)
+        end
+
+        @restart_policy.update(container, timestamp)
+      else
+        @logger.info("Skipping restart for container #{container.name.to_s.green} as no changes were found")
+
+        if !container_id
+          @restart_policy.update(container, timestamp)
+        end
+      end
+
+      if container.after_deploy_proc
+        container.after_deploy_proc.call(container, number)
+      end
+    end
+
+    nil
+  end
+end

data/lib/indocker/deployment_checker.rb

@@ -0,0 +1,121 @@
+class Indocker::DeploymentChecker
+  def initialize(debug_logger, output_logger = nil)
+    @debug_logger = debug_logger
+
+    if output_logger
+      @logger = output_logger
+    else
+      @logger = Logger.new(STDOUT)
+      @logger.formatter = debug_logger.formatter
+    end
+  end
+
+  def run(configuration:, servers:, only_containers: [])
+    server_list = configuration.servers.map(&:name)
+
+    server_list = servers.map do |server_name|
+      server = configuration.servers.detect { |s| s.name == server_name }
+
+      if !server
+        @logger.error("Invalid server name specified :#{server_name} for configuration :#{configuration.name}")
+        exit 1
+      end
+
+      server
+    end
+
+    if server_list.empty?
+      server_list = configuration.servers
+    end
+
+    server_list = server_list.sort_by(&:name)
+
+    @logger.info("Following servers will be checked:")
+
+    server_list.each do |server|
+      @logger.info("  - #{server.name}")
+    end
+
+    names_by_server = {}
+
+    server_list.each do |server|
+      names_by_server[server.name] = []
+    end
+
+    containers_by_server = Indocker
+      .configuration
+      .containers
+      .select{ |container| only_containers.empty? || only_containers.include?(container.name) }
+      .map do |container|
+        (container.get_start_option(:scale) || 1).times do |number|
+          container.servers.each do |server|
+            names_by_server[server.name] << Indocker::ContainerHelper.hostname(configuration.name, container, number)
+          end
+        end
+      end
+
+    total_invalid_containers = []
+    total_missing_containers = []
+    
+    server_list.each do |server|
+      session = Indocker::SshSession.new(
+        host: server.host,
+        user: server.user,
+        port: server.port,
+        logger: @debug_logger
+      )
+
+      result = session.exec!('docker ps --filter="status=running" --format="{{.Names}}"')
+
+      if !result.success?
+        @logger.warn(result.stdout_data)
+        @logger.warn(result.stderr_data)
+        exit 1
+      end
+
+      names_list = result
+        .stdout_data
+        .strip
+        .split("\n")
+
+      invalid_containers = (names_list - names_by_server[server.name]).uniq
+
+      total_invalid_containers += invalid_containers
+
+      if !invalid_containers.empty?
+        @logger.warn("EXTRA containers for server #{server.name.to_s.yellow}:")
+
+        invalid_containers.each do |name|
+          @logger.warn("  - #{name}")
+        end
+      end
+
+      missing_containers = (names_by_server[server.name] - names_list).uniq
+      missing_containers = missing_containers.select do |name|
+        !(name.include?('migrations') || name.include?('gems_installer'))
+      end
+
+      total_missing_containers += missing_containers
+
+      if !missing_containers.empty?
+        @logger.warn("MISSING CONTAINERS for server #{server.name.to_s.yellow}:")
+
+        missing_containers.each do |name|
+          @logger.warn("  - #{name}")
+        end
+      end
+    end
+
+    { missing_containers: total_missing_containers, invalid_containers: total_invalid_containers }
+  end
+
+  def launched?(container_name, configuration:, servers:)
+    container = Indocker.containers.detect { |c| c.name == container_name.to_sym }
+    hostnames = (container.get_start_option(:scale) || 1).times.map do |number|
+      Indocker::ContainerHelper.hostname(configuration.name, container, number)
+    end
+
+    result = run(configuration: configuration, servers: servers, only_containers: [container.name])
+    result[:missing_containers].empty?
+  end
+end

data/lib/indocker/deployment_progress.rb

@@ -0,0 +1,495 @@
+require 'thread'
+
+class Indocker::DeploymentProgress
+  def initialize(logger)
+    @logger = logger
+
+    if logger
+      @logger.formatter = Proc.new do |severity, datetime, progname, msg|
+        "#{msg}\n"
+      end
+    end
+
+    @progress = Hash.new(:waiting)
+    @semaphore = Mutex.new
+
+    @synced_binaries = {
+      # Structure:
+      # server => {
+      #   start: time,
+      #   finish: time,
+      #   state: (:waiting|:in_progress|:finished)
+      # }
+    }
+
+    @synced_env_files = {
+      # Structure:
+      # env_file => {
+      #   server => {
+      #     start: time,
+      #     finish: time,
+      #     state: (:waiting|:in_progress|:finished)
+      #   }
+      # }
+    }
+
+    @synced_artifacts = {
+      # Structure:
+      # artifact => {
+      #   server => {
+      #     start: time,
+      #     finish: time,
+      #     state: (:waiting|:in_progress|:finished)
+      #   }
+      # }
+    }
+
+    @synced_repositories = {
+      # Structure:
+      # repository => {
+      #   server => {
+      #     start: time,
+      #     finish: time,
+      #     state: (:waiting|:in_progress|:finished)
+      #   }
+      # }
+    }
+
+    @deployed_containers = {
+      # Structure:
+      # container => {
+      #   server => {
+      #     build_start: time,
+      #     build_finish: time,
+      #     deploy_start: time,
+      #     deploy_finish: time,
+      #     state: (:waiting|:building|:waiting_deployment|:deploying|:finished)
+      #   }
+      # }
+    }
+  end
+
+  def setup(binaries_servers:, build_servers:, deploy_servers:, env_files:, artifact_servers:,
+            repositories:, force_restart:, skip_build:, containers:)
+    @force_restart = force_restart
+    @skip_build = skip_build
+
+    binaries_servers.each do |server|
+      @synced_binaries[server] = {
+        start: nil,
+        finish: nil,
+        state: :waiting
+      }
+    end
+
+    repositories.each do |repository|
+      @synced_repositories[repository] = {}
+
+      build_servers.each do |server|
+        @synced_repositories[repository][server] = {
+          start: nil,
+          finish: nil,
+          state: :waiting
+        }
+      end
+    end
+
+    env_files.each do |env_file|
+      @synced_env_files[env_file] = {}
+
+      deploy_servers.each do |server|
+        @synced_env_files[env_file][server] = {
+          start: nil,
+          finish: nil,
+          state: :waiting
+        }
+      end
+    end
+
+    artifact_servers.each do |artifact, servers|
+      @synced_artifacts[artifact] = {}
+
+      servers.each do |server|
+        @synced_artifacts[artifact][server] = {
+          start: nil,
+          finish: nil,
+          state: :waiting
+        }
+      end
+    end
+
+    @containers = containers
+
+    containers.each do |container|
+      @deployed_containers[container] = {}
+
+      container.servers.each do |server|
+        @deployed_containers[container][server] = {
+          build_start: nil,
+          build_finish: nil,
+          deploy_start: nil,
+          deploy_finish: nil,
+          state: :waiting
+        }
+      end
+    end
+
+    log
+  end
+
+  def start_syncing_binaries(server)
+    @semaphore.synchronize do
+      @synced_binaries[server][:start] = Time.now
+      @synced_binaries[server][:state] = :in_progress
+      log
+    end
+  end
+
+  def finish_syncing_binaries(server)
+    @semaphore.synchronize do
+      @synced_binaries[server][:finish] = Time.now
+      @synced_binaries[server][:state] = :finished
+      log
+    end
+  end
+
+  def start_syncing_env_file(server, env_file)
+    @semaphore.synchronize do
+      @synced_env_files[env_file][server][:start] = Time.now
+      log
+    end
+  end
+
+  def finish_syncing_env_file(server, env_file)
+    @semaphore.synchronize do
+      @synced_env_files[env_file][server][:finish] = Time.now
+      @synced_env_files[env_file][server][:state] = :finished
+      log
+    end
+  end
+
+  def start_syncing_artifact(server, artifact)
+    @semaphore.synchronize do
+      @synced_artifacts[artifact][server][:start] = Time.now
+      log
+    end
+  end
+
+  def finish_syncing_artifact(server, artifact)
+    @semaphore.synchronize do
+      @synced_artifacts[artifact][server][:finish] = Time.now
+      @synced_artifacts[artifact][server][:state] = :finished
+      log
+    end
+  end
+
+  def start_syncing_repository(server, repository)
+    @semaphore.synchronize do
+      @synced_repositories[repository][server][:start] = Time.now
+      log
+    end
+  end
+
+  def finish_syncing_repository(server, repository)
+    @semaphore.synchronize do
+      @synced_repositories[repository][server][:finish] = Time.now
+      @synced_repositories[repository][server][:state] = :finished
+      log
+    end
+  end
+
+  def start_building_container(container)
+    @semaphore.synchronize do
+      time = Time.now
+
+      @deployed_containers[container].each do |server, data|
+        next if data[:build_start]
+        data[:build_start] = time
+        data[:state] = :building
+      end
+
+      log
+    end
+  end
+
+  def finish_building_container(container)
+    @semaphore.synchronize do
+      time = Time.now
+
+      @deployed_containers[container].each do |server, data|
+        next if data[:build_finish]
+        data[:build_finish] = time
+        data[:state] = :waiting_deployment
+      end
+
+      log
+    end
+  end
+
+  def start_deploying_container(container, server)
+    @semaphore.synchronize do
+      @deployed_containers[container][server][:deploy_start] = Time.now
+      @deployed_containers[container][server][:state] = :deploying
+      log
+    end
+  end
+
+  def finish_deploying_container(container, server)
+    @semaphore.synchronize do
+      @deployed_containers[container][server][:deploy_finish] = Time.now
+      @deployed_containers[container][server][:state] = :finished
+      log
+    end
+  end
+
+  def log
+    return if !@logger
+    system("clear")
+
+    if @skip_build
+      @logger.info("Warning: Image build is skipped for all containers".purple)
+    end
+
+    if @force_restart
+      @logger.info("Warning: All containers will be force restarted".purple)
+    end
+
+    # BINARIES formatter
+    synced_binaries_servers = []
+    not_synced_binaries_servers = []
+
+    @synced_binaries.each do |server, data|
+      if data[:state] == :finished
+        synced_binaries_servers << server
+      else
+        not_synced_binaries_servers << server
+      end
+    end
+
+    if !synced_binaries_servers.empty?
+      @logger.info("Binaries synced:".green)
+
+      synced_binaries_servers.each do |server|
+        data = @synced_binaries[server]
+        total = ", total: #{(data[:finish] - data[:start]).round}s" if data[:finish]
+        @logger.info("  - #{server.name.to_s.yellow}#{total}")
+      end
+    end
+
+    if !not_synced_binaries_servers.empty?
+      @logger.info("Binaries syncing:".green)
+
+      not_synced_binaries_servers.each do |server|
+        data = @synced_binaries[server]
+        @logger.info("  - #{server.name.to_s.cyan} (#{data[:state]}...)")
+      end
+    end
+
+    # ENV FILES formatter
+    synced_env_files = []
+    not_synced_env_files = []
+
+    @synced_env_files.each do |env_file, server_info|
+      server_info.each do |server, data|
+        if data[:state] == :finished
+          if !synced_env_files.include?(env_file)
+            synced_env_files << env_file
+          end
+        else
+          if !not_synced_env_files.include?(env_file)
+            not_synced_env_files << env_file
+          end
+        end
+      end
+    end
+
+    if !synced_env_files.empty?
+      @logger.info("ENV files synced:".green)
+
+      synced_env_files.each do |env_file|
+        servers_data = @synced_env_files[env_file]
+
+        servers_data.each do |server, data|
+          next if data[:state] != :finished
+          total = ", total: #{(data[:finish] - data[:start]).round}s" if data[:finish]
+          @logger.info("  - #{env_file.to_s.yellow}, server: #{server.name}#{total}")
+        end
+      end
+    end
+
+    if !not_synced_env_files.empty?
+      @logger.info("ENV files syncing:".green)
+
+      not_synced_env_files.each do |env_file|
+        servers_data = @synced_env_files[env_file]
+
+        servers_data.each do |server, data|
+          next if data[:state] == :finished
+          @logger.info("  - #{env_file.to_s.cyan}, server: #{server.name} (#{data[:state]}...)")
+        end
+      end
+    end
+
+    # Artifacts formatter
+    synced_artifact = []
+    not_synced_artifact = []
+
+    @synced_artifacts.each do |artifact, server_info|
+      server_info.each do |server, data|
+        if data[:state] == :finished
+          if !synced_artifact.include?(artifact)
+            synced_artifact << artifact
+          end
+        else
+          if !not_synced_artifact.include?(artifact)
+            not_synced_artifact << artifact
+          end
+        end
+      end
+    end
+
+    if !synced_artifact.empty?
+      @logger.info("Artifacts synced:".green)
+
+      synced_artifact.each do |artifact|
+        servers_data = @synced_artifacts[artifact]
+
+        servers_data.each do |server, data|
+          next if data[:state] != :finished
+          total = ", total: #{(data[:finish] - data[:start]).round}s" if data[:finish]
+          @logger.info("  - #{artifact.name.to_s.yellow}, server: #{server.name}#{total}")
+        end
+      end
+    end
+
+    if !not_synced_artifact.empty?
+      @logger.info("Artifacts syncing:".green)
+
+      not_synced_artifact.each do |artifact|
+        servers_data = @synced_artifacts[artifact]
+
+        servers_data.each do |server, data|
+          next if data[:state] == :finished
+          @logger.info("  - #{artifact.name.to_s.cyan}, server: #{server.name} (#{data[:state]}...)")
+        end
+      end
+    end
+
+    # REPOSITORIES formatter
+    synced_repositories = []
+    not_synced_repositories = []
+
+    @synced_repositories.each do |repository, server_info|
+      server_info.each do |server, data|
+        if data[:state] == :finished
+          if !synced_repositories.include?(repository)
+            synced_repositories << repository
+          end
+        else
+          if !not_synced_repositories.include?(repository)
+            not_synced_repositories << repository
+          end
+        end
+      end
+    end
+
+    if !synced_repositories.empty?
+      @logger.info("Repositories synced:".green)
+
+      synced_repositories.each do |repository|
+        servers_data = @synced_repositories[repository]
+
+        servers_data.each do |server, data|
+          next if data[:state] != :finished
+          total = ", total: #{(data[:finish] - data[:start]).round}s" if data[:finish]
+          @logger.info("  - #{repository.to_s.yellow}, server: #{server.name}#{total}")
+        end
+      end
+    end
+
+    if !not_synced_repositories.empty?
+      @logger.info("Repositories syncing:".green)
+
+      not_synced_repositories.each do |repository|
+        servers_data = @synced_repositories[repository]
+
+        servers_data.each do |server, data|
+          next if data[:state] == :finished
+          @logger.info("  - #{repository.to_s.cyan}, server: #{server.name} (#{data[:state]}...)")
+        end
+      end
+    end
+
+    # Containers formatter
+    deployed_containers = []
+    not_deployed_containers = []
+
+    @deployed_containers.each do |container, server_info|
+      server_info.each do |server, data|
+        if data[:state] == :finished
+          if !deployed_containers.include?(container)
+            deployed_containers << container
+          end
+        else
+          if !not_deployed_containers.include?(container)
+            not_deployed_containers << container
+          end
+        end
+      end
+    end
+
+    if !deployed_containers.empty?
+      @logger.info("Deployed containers:".green)
+
+      deployed_containers = deployed_containers
+        .sort_by { |container|
+          @deployed_containers[container]
+            .map { |server, data| data[:deploy_finish] }
+            .reject(&:nil?)
+            .sort
+            .last
+            .to_i
+        }
+
+      deployed_containers.each do |container|
+        servers_data = @deployed_containers[container]
+
+        servers_data.each do |server, data|
+          next if data[:state] != :finished
+          total = ", total: #{(data[:deploy_finish] - data[:build_start]).round}s"
+          build = ", build: #{(data[:build_finish] - data[:build_start]).round}s"
+          deploy = ", deploy: #{(data[:deploy_finish] - data[:deploy_start]).round}s"
+          @logger.info("  - #{container.name.to_s.yellow}, server: #{server.name}#{build}#{deploy}#{total}")
+        end
+      end
+    end
+
+    if !not_deployed_containers.empty?
+      @logger.info("To be deployed containers:".green)
+
+      not_deployed_containers.each do |container|
+        servers_data = @deployed_containers[container]
+
+        servers_data.each do |server, data|
+          next if data[:state] == :finished
+
+          name = container.name.to_s
+
+          name = if data[:state] == :waiting
+            name
+          else
+            name.cyan
+          end
+
+          @logger.info("  - #{name}, server: #{server.name} (#{data[:state]}...)")
+        end
+      end
+    end
+  end
+
+  private
+
+  def format_time(time)
+    time.strftime("%H:%M:%S")
+  end
+end