indocker 0.5.4 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70bf1241f0783dede869906f9f359c2a2137778a3059acaea9ae2fc7db30bb38
-  data.tar.gz: f4d906fc4ae36b9bf4482c7e5802c03b4a8633c7f839a0ffb71bbfc1571a5e30
+  metadata.gz: 0eda49208a82e456326a838e76cab5152673a078e30668832c97c7c1b525a4fa
+  data.tar.gz: 5904dabfde8e6110821b104be6fd0809799a5deb3b35f991bed33627323bf1c9
 SHA512:
-  metadata.gz: d45a4d6ab954e10b76fbaf2b4ca098c1368fd6b6365756f24bb210757d5f221d0fcfcf5b11b378cc5e31de38d72967158c90d91aaa3c9df53334f8a2596f12f0
-  data.tar.gz: a0ae86f49931e812d34761ff20f73f704a010b742edb1968dae8e70fc7798edbd75cab519f34632d8f6840ab0b87f5454c70773c50627ac10fc5693e263819cb
+  metadata.gz: '020961f0cae0f634adf95f355d2e537bca061ec0380294f0a12ca75d0e72b54f19eb57d357dbe1f1dbef50cfd5d43896ed7f80107fe7ac822e6d3addd2645586'
+  data.tar.gz: f9ff2feb5c3288474635a3e7d31354c8065075068872865f51b6542e1c6933c0342e253260c58492cc7eb99e3ae09e1904692a9b1acd5a13751d467ca0edaf5d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    indocker (0.5.4)
+    indocker (0.6.1)
       net-ssh
 
 GEM

data/lib/indocker/artifacts/services/synchronizer.rb

@@ -13,14 +13,9 @@ class Indocker::Artifacts::Services::Synchronizer
       remote_operations += servers.map do |server|
         @progress.start_syncing_artifact(server, artifact)
 
-        thread = Thread.new do
+        Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :artifact_sync) do
           server.synchronize do
-            session = Indocker::SshSession.new(
-              host: server.host,
-              user: server.user,
-              port: server.port,
-              logger: @logger
-            )
+            session = Indocker::SshSession.for(server, logger: @logger)
 
             if artifact.is_git?
               @logger.info("Pulling git artifact  #{artifact.name.to_s.green} for #{server.user}@#{server.host}")
@@ -49,8 +44,6 @@ class Indocker::Artifacts::Services::Synchronizer
             @progress.finish_syncing_artifact(server, artifact)
           end
         end
-
-        Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :artifact_sync)
       end
     end
 

data/lib/indocker/deployment_checker.rb

@@ -58,12 +58,7 @@ class Indocker::DeploymentChecker
     total_missing_containers = []
     
     server_list.each do |server|
-      session = Indocker::SshSession.new(
-        host: server.host,
-        user: server.user,
-        port: server.port,
-        logger: @debug_logger
-      )
+      session = Indocker::SshSession.for(server, logger: @debug_logger)
 
       result = session.exec!('docker ps --filter="status=running" --format="{{.Names}}"')
 

data/lib/indocker/launchers/configuration_deployer.rb

@@ -105,8 +105,25 @@ class Indocker::Launchers::ConfigurationDeployer
 
   def wait_remote_operations(remote_operations)
     remote_operations.each do |remote_operation|
-      if remote_operation.thread.join(Indocker.remote_operation_timeout).nil?
-        @global_logger.error("Deployment aborted. Remote operation :#{remote_operation.operation} on server #{remote_operation.server.user}@#{remote_operation.server.host} did not finish during #{Indocker.remote_operation_timeout} seconds.")
+      server = remote_operation.server
+      target = "#{server.name} (#{server.user}@#{server.host})"
+
+      begin
+        if remote_operation.thread.join(Indocker.remote_operation_timeout).nil?
+          @global_logger.error("Deployment aborted. Remote operation :#{remote_operation.operation} on server #{target} did not finish during #{Indocker.remote_operation_timeout} seconds.")
+          exit 1
+        end
+      rescue StandardError => e
+        # Raised by the operation thread and re-raised here by #join, so the server
+        # that failed can be named instead of dumping a bare backtrace.
+        reason = e.message.to_s.empty? || e.message == e.class.name ? e.class.to_s : "#{e.class}: #{e.message}"
+
+        @global_logger.error("Deployment aborted. Remote operation :#{remote_operation.operation} on server #{target} failed: #{reason}")
+
+        if e.is_a?(Errno::ETIMEDOUT) || (defined?(Net::SSH::ConnectionTimeout) && e.is_a?(Net::SSH::ConnectionTimeout))
+          @global_logger.error("Server #{server.host} did not accept a connection within #{Indocker.ssh_connect_timeout} seconds. It is powered off, or its address does not accept TCP.")
+        end
+
         exit 1
       end
     end
@@ -380,13 +397,8 @@ class Indocker::Launchers::ConfigurationDeployer
       remote_operations += repositories.map do |alias_name, repository|
         @progress.start_syncing_repository(server, alias_name)
 
-        thread = Thread.new do
-          session = Indocker::SshSession.new(
-            host: server.host,
-            user: server.user,
-            port: server.port,
-            logger: @logger
-          )
+        Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :repository_pull) do
+          session = Indocker::SshSession.for(server, logger: @logger)
 
           if repository.is_local?
             @logger.info("Rsyncing repository :#{alias_name} from #{repository.root_path} to #{server.user}@#{server.host}:#{repository.clone_path}")
@@ -415,8 +427,6 @@ class Indocker::Launchers::ConfigurationDeployer
 
           @progress.finish_syncing_repository(server, alias_name)
         end
-
-        Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :repository_pull)
       end
     end
 
@@ -455,6 +465,7 @@ class Indocker::Launchers::ConfigurationDeployer
     @logger.info("Updating crontab file #{deploy_user}:#{crontab_filepath}")
 
     ssh_options = "-o BatchMode=yes -o ConnectTimeout=#{Indocker.ssh_connect_timeout}"
+    ssh_options = "#{ssh_options} -J #{server.jump_host}" if server.jump_host
 
     Indocker::Shell.command("scp #{ssh_options} #{tmp_crontab_file.path} #{deploy_user}:#{crontab_filepath}", @logger)
 
@@ -467,13 +478,8 @@ class Indocker::Launchers::ConfigurationDeployer
     servers.map do |server|
       @progress.start_syncing_binaries(server)
 
-      thread = Thread.new do
-        session = Indocker::SshSession.new(
-          host: server.host,
-          user: server.user,
-          port: server.port,
-          logger: @logger
-        )
+      Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :indocker_sync) do
+        session = Indocker::SshSession.for(server, logger: @logger)
 
         sync_path = Indocker::IndockerHelper.indocker_dir
         @logger.info("Syncing indocker to #{server.user}@#{server.host}:#{sync_path}")
@@ -488,8 +494,6 @@ class Indocker::Launchers::ConfigurationDeployer
 
         @progress.finish_syncing_binaries(server)
       end
-
-      Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :indocker_sync)
     end
   end
 
@@ -500,17 +504,12 @@ class Indocker::Launchers::ConfigurationDeployer
       remote_operations += env_files.map do |alias_name, env_file|
         @progress.start_syncing_env_file(server, alias_name)
 
-        thread = Thread.new do
+        Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :env_file_sync) do
           if env_file.is_a?(Indocker::EnvFiles::Local)
             sync_path = File.join(Indocker.deploy_dir, 'env_files', File.basename(env_file.path))
             @logger.info("Syncing env file :#{alias_name} from #{env_file.path} to #{server.user}@#{server.host}:#{sync_path}")
 
-            session = Indocker::SshSession.new(
-              host: server.host,
-              user: server.user,
-              port: server.port,
-              logger: @logger
-            )
+            session = Indocker::SshSession.for(server, logger: @logger)
 
             session.exec!("mkdir -p #{Indocker::EnvFileHelper.folder}")
 
@@ -529,8 +528,6 @@ class Indocker::Launchers::ConfigurationDeployer
 
           @progress.finish_syncing_env_file(server, alias_name)
         end
-
-        Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :env_file_sync)
       end
     end
 

data/lib/indocker/launchers/dto/remote_operation_dto.rb

@@ -1,10 +1,27 @@
 class Indocker::Launchers::DTO::RemoteOperationDTO
   attr_reader :thread, :server, :operation, :message
 
+  # Runs a remote operation in a thread that keeps its failure to itself, so that
+  # wait_remote_operations can re-raise it on join and report which server failed.
+  # Left to the global abort_on_exception/report_on_exception defaults, a failed ssh
+  # dumps a raw backtrace and tears the deploy down from inside the thread, naming
+  # no server at all. The flags are set inside the thread so that they are in place
+  # before the operation gets a chance to fail.
+  def self.spawn(server, operation, message = nil, &block)
+    thread = Thread.new do
+      Thread.current.abort_on_exception  = false
+      Thread.current.report_on_exception = false
+
+      block.call
+    end
+
+    new(thread, server, operation, message)
+  end
+
   def initialize(thread, server, operation, message = nil)
     @thread = thread
     @server = server
     @operation = operation
     @message = message
   end
-end
+end

data/lib/indocker/rsync.rb

@@ -41,7 +41,13 @@ class Indocker::Rsync
         "-o ConnectTimeout=#{Indocker.ssh_connect_timeout}",
         "-o ServerAliveInterval=10",
         "-o ServerAliveCountMax=3",
-      ].join(' ')
+      ]
+
+      # Tunnel through the jump host the server is defined with, so that a server
+      # reachable only over its internal address syncs like any other one.
+      ssh_command << "-J #{session.jump_host}" if session.jump_host
+
+      ssh_command = ssh_command.join(' ')
       command = "rsync --delete-after -a -e '#{ssh_command}' #{from} #{session.user}@#{session.host}:#{to}"
       Indocker.logger.debug("sync #{from} #{session.user}@#{session.host}:#{to}")
       Indocker::Shell.command(command, Indocker.logger, raise_on_error: raise_on_error)

data/lib/indocker/server.rb

@@ -1,13 +1,17 @@
 class Indocker::Server
   include Indocker::Concerns::Inspectable
 
-  attr_reader :name, :host, :user, :port
+  attr_reader :name, :host, :user, :port, :jump_host
 
-  def initialize(name:, host:, user:, port:)
+  # jump_host ("user@host") makes every connection to this server tunnel through
+  # another one. Servers whose public address is unreachable can still be deployed
+  # to over their internal address that way.
+  def initialize(name:, host:, user:, port:, jump_host: nil)
     @name = name
     @host = host
     @user = user
     @port = port
+    @jump_host = jump_host
   end
 
   def ==(value)

data/lib/indocker/server_pools/server_connection.rb

@@ -10,12 +10,7 @@ class Indocker::ServerPools::ServerConnection
   def create_session!
     return unless @server
     
-    @session = Indocker::SshSession.new(
-      host: @server.host,
-      user: @server.user,
-      port: @server.port,
-      logger: @logger
-    )
+    @session = Indocker::SshSession.for(@server, logger: @logger)
   end
 
   def exec!(command)

data/lib/indocker/ssh_session.rb

@@ -19,24 +19,47 @@ class Indocker::SshSession
     end
   end
 
-  attr_reader :host, :user, :port, :logger
+  attr_reader :host, :user, :port, :logger, :jump_host
+
+  # Opens a session to a server as it is defined in the configuration, including its
+  # jump host, if any. Every remote operation goes through here, so a server reached
+  # through a jump host is reached that way by binaries sync, container deploys and
+  # the deployment checker alike.
+  def self.for(server, logger:)
+    new(
+      host:      server.host,
+      user:      server.user,
+      port:      server.port,
+      jump_host: server.jump_host,
+      logger:    logger
+    )
+  end
 
-  def initialize(host:, user:, port:, logger:)
+  def initialize(host:, user:, port:, logger:, jump_host: nil)
     @host = host
     @user = user
     @port = port
+    @jump_host = jump_host
     @logger = logger
 
     if host != LOCALHOST
       require 'net/ssh'
-      @ssh = Net::SSH.start(@host, @user, {
+
+      options = {
         port:               @port,
         non_interactive:    true,
         timeout:            Indocker.ssh_connect_timeout,
         keepalive:          true,
         keepalive_interval: 10,
         keepalive_maxcount: 3,
-      })
+      }
+
+      if @jump_host
+        require 'net/ssh/proxy/command'
+        options[:proxy] = Net::SSH::Proxy::Command.new(jump_proxy_command)
+      end
+
+      @ssh = Net::SSH.start(@host, @user, options)
     end
   end
 
@@ -59,6 +82,32 @@ class Indocker::SshSession
   end
 
   private
+    # Net::SSH::Proxy::Jump builds this same `ssh -W` tunnel, but without BatchMode,
+    # connect timeout or -q: a jump host that asks for a password hangs the deploy, and
+    # every probe of a server whose sshd is still starting prints "channel 0: open failed"
+    # from the ssh client.
+    def jump_proxy_command
+      host = @jump_host
+      port = nil
+
+      if host =~ /\A(.+):(\d+)\z/
+        host = $1
+        port = $2
+      end
+
+      command = [
+        "ssh -q",
+        "-o BatchMode=yes",
+        "-o ConnectTimeout=#{Indocker.ssh_connect_timeout}",
+      ]
+
+      command << "-p #{port}" if port
+      command << "-W %h:%p"
+      command << host
+
+      command.join(' ')
+    end
+
     def exec_locally!(command)
       res = Indocker::Shell.command_with_result(command, @logger, skip_logging: false)
       ExecResult.new(res.stdout, '', res.exit_status, nil)

data/lib/indocker/version.rb

@@ -1,3 +1,3 @@
 module Indocker
-  VERSION = "0.5.4"
+  VERSION = "0.6.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indocker
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.1
 platform: ruby
 authors:
 - Ruslan Gatiyatov