indocker 0.5.4 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70bf1241f0783dede869906f9f359c2a2137778a3059acaea9ae2fc7db30bb38
-  data.tar.gz: f4d906fc4ae36b9bf4482c7e5802c03b4a8633c7f839a0ffb71bbfc1571a5e30
+  metadata.gz: 10915e6da5c1e8d2899b510a223d8761a0542880776b61e8078d38d23e261b28
+  data.tar.gz: dd047c45c09dd0f26b95ae668f00266c2f45c46ff1c4aecff14ef0155d9e5a59
 SHA512:
-  metadata.gz: d45a4d6ab954e10b76fbaf2b4ca098c1368fd6b6365756f24bb210757d5f221d0fcfcf5b11b378cc5e31de38d72967158c90d91aaa3c9df53334f8a2596f12f0
-  data.tar.gz: a0ae86f49931e812d34761ff20f73f704a010b742edb1968dae8e70fc7798edbd75cab519f34632d8f6840ab0b87f5454c70773c50627ac10fc5693e263819cb
+  metadata.gz: b879a49d9defa1cbe4dc56685e0467ff070c9d0310a6fd5531766c941e7b51028aa1ace5817dec1638dec944ccba8afbbceffa062095a658fdb248c615095253
+  data.tar.gz: 70814bb24a5a66da13c36a6d1350403686b6a5672f184aa691179cfb40850b74e66b059fc82f249fc052ebe942c2ed26326570ce041270d561f25ca7bb67021c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    indocker (0.5.4)
+    indocker (0.6.2)
       net-ssh
 
 GEM

data/lib/indocker/artifacts/services/synchronizer.rb

@@ -13,14 +13,9 @@ class Indocker::Artifacts::Services::Synchronizer
       remote_operations += servers.map do |server|
         @progress.start_syncing_artifact(server, artifact)
 
-        thread = Thread.new do
+        Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :artifact_sync) do
           server.synchronize do
-            session = Indocker::SshSession.new(
-              host: server.host,
-              user: server.user,
-              port: server.port,
-              logger: @logger
-            )
+            session = Indocker::SshSession.for(server, logger: @logger)
 
             if artifact.is_git?
               @logger.info("Pulling git artifact  #{artifact.name.to_s.green} for #{server.user}@#{server.host}")
@@ -49,8 +44,6 @@ class Indocker::Artifacts::Services::Synchronizer
             @progress.finish_syncing_artifact(server, artifact)
           end
         end
-
-        Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :artifact_sync)
       end
     end
 

data/lib/indocker/deployment_checker.rb

@@ -58,12 +58,7 @@ class Indocker::DeploymentChecker
     total_missing_containers = []
     
     server_list.each do |server|
-      session = Indocker::SshSession.new(
-        host: server.host,
-        user: server.user,
-        port: server.port,
-        logger: @debug_logger
-      )
+      session = Indocker::SshSession.for(server, logger: @debug_logger)
 
       result = session.exec!('docker ps --filter="status=running" --format="{{.Names}}"')
 

data/lib/indocker/launchers/configuration_deployer.rb

@@ -46,6 +46,9 @@ class Indocker::Launchers::ConfigurationDeployer
     deployer = Indocker::ContainerDeployer.new(configuration: configuration, logger: @logger)
 
     @global_logger.info("Establishing ssh sessions to all servers...")
+
+    warm_jump_hosts(deploy_servers + build_servers)
+
     build_server_pool.create_sessions!
 
     build_servers = configuration
@@ -103,10 +106,36 @@ class Indocker::Launchers::ConfigurationDeployer
 
   private
 
+  # Opens the shared connection to each jump host before anything runs in parallel, so the
+  # fan-out reuses it instead of racing to create one connection per operation.
+  def warm_jump_hosts(servers)
+    servers.map(&:jump_host).compact.uniq.each do |jump_host|
+      @logger.info("Establishing shared connection to jump host #{jump_host}")
+      Indocker::SshSession.warm_jump_host(jump_host, logger: @logger)
+    end
+  end
+
   def wait_remote_operations(remote_operations)
     remote_operations.each do |remote_operation|
-      if remote_operation.thread.join(Indocker.remote_operation_timeout).nil?
-        @global_logger.error("Deployment aborted. Remote operation :#{remote_operation.operation} on server #{remote_operation.server.user}@#{remote_operation.server.host} did not finish during #{Indocker.remote_operation_timeout} seconds.")
+      server = remote_operation.server
+      target = "#{server.name} (#{server.user}@#{server.host})"
+
+      begin
+        if remote_operation.thread.join(Indocker.remote_operation_timeout).nil?
+          @global_logger.error("Deployment aborted. Remote operation :#{remote_operation.operation} on server #{target} did not finish during #{Indocker.remote_operation_timeout} seconds.")
+          exit 1
+        end
+      rescue StandardError => e
+        # Raised by the operation thread and re-raised here by #join, so the server
+        # that failed can be named instead of dumping a bare backtrace.
+        reason = e.message.to_s.empty? || e.message == e.class.name ? e.class.to_s : "#{e.class}: #{e.message}"
+
+        @global_logger.error("Deployment aborted. Remote operation :#{remote_operation.operation} on server #{target} failed: #{reason}")
+
+        if e.is_a?(Errno::ETIMEDOUT) || (defined?(Net::SSH::ConnectionTimeout) && e.is_a?(Net::SSH::ConnectionTimeout))
+          @global_logger.error("Server #{server.host} did not accept a connection within #{Indocker.ssh_connect_timeout} seconds. It is powered off, or its address does not accept TCP.")
+        end
+
         exit 1
       end
     end
@@ -380,13 +409,8 @@ class Indocker::Launchers::ConfigurationDeployer
       remote_operations += repositories.map do |alias_name, repository|
         @progress.start_syncing_repository(server, alias_name)
 
-        thread = Thread.new do
-          session = Indocker::SshSession.new(
-            host: server.host,
-            user: server.user,
-            port: server.port,
-            logger: @logger
-          )
+        Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :repository_pull) do
+          session = Indocker::SshSession.for(server, logger: @logger)
 
           if repository.is_local?
             @logger.info("Rsyncing repository :#{alias_name} from #{repository.root_path} to #{server.user}@#{server.host}:#{repository.clone_path}")
@@ -415,8 +439,6 @@ class Indocker::Launchers::ConfigurationDeployer
 
           @progress.finish_syncing_repository(server, alias_name)
         end
-
-        Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :repository_pull)
       end
     end
 
@@ -455,6 +477,7 @@ class Indocker::Launchers::ConfigurationDeployer
     @logger.info("Updating crontab file #{deploy_user}:#{crontab_filepath}")
 
     ssh_options = "-o BatchMode=yes -o ConnectTimeout=#{Indocker.ssh_connect_timeout}"
+    ssh_options = "#{ssh_options} -J #{server.jump_host}" if server.jump_host
 
     Indocker::Shell.command("scp #{ssh_options} #{tmp_crontab_file.path} #{deploy_user}:#{crontab_filepath}", @logger)
 
@@ -467,13 +490,8 @@ class Indocker::Launchers::ConfigurationDeployer
     servers.map do |server|
       @progress.start_syncing_binaries(server)
 
-      thread = Thread.new do
-        session = Indocker::SshSession.new(
-          host: server.host,
-          user: server.user,
-          port: server.port,
-          logger: @logger
-        )
+      Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :indocker_sync) do
+        session = Indocker::SshSession.for(server, logger: @logger)
 
         sync_path = Indocker::IndockerHelper.indocker_dir
         @logger.info("Syncing indocker to #{server.user}@#{server.host}:#{sync_path}")
@@ -488,8 +506,6 @@ class Indocker::Launchers::ConfigurationDeployer
 
         @progress.finish_syncing_binaries(server)
       end
-
-      Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :indocker_sync)
     end
   end
 
@@ -500,17 +516,12 @@ class Indocker::Launchers::ConfigurationDeployer
       remote_operations += env_files.map do |alias_name, env_file|
         @progress.start_syncing_env_file(server, alias_name)
 
-        thread = Thread.new do
+        Indocker::Launchers::DTO::RemoteOperationDTO.spawn(server, :env_file_sync) do
           if env_file.is_a?(Indocker::EnvFiles::Local)
             sync_path = File.join(Indocker.deploy_dir, 'env_files', File.basename(env_file.path))
             @logger.info("Syncing env file :#{alias_name} from #{env_file.path} to #{server.user}@#{server.host}:#{sync_path}")
 
-            session = Indocker::SshSession.new(
-              host: server.host,
-              user: server.user,
-              port: server.port,
-              logger: @logger
-            )
+            session = Indocker::SshSession.for(server, logger: @logger)
 
             session.exec!("mkdir -p #{Indocker::EnvFileHelper.folder}")
 
@@ -529,8 +540,6 @@ class Indocker::Launchers::ConfigurationDeployer
 
           @progress.finish_syncing_env_file(server, alias_name)
         end
-
-        Indocker::Launchers::DTO::RemoteOperationDTO.new(thread, server, :env_file_sync)
       end
     end
 

data/lib/indocker/launchers/dto/remote_operation_dto.rb

@@ -1,10 +1,27 @@
 class Indocker::Launchers::DTO::RemoteOperationDTO
   attr_reader :thread, :server, :operation, :message
 
+  # Runs a remote operation in a thread that keeps its failure to itself, so that
+  # wait_remote_operations can re-raise it on join and report which server failed.
+  # Left to the global abort_on_exception/report_on_exception defaults, a failed ssh
+  # dumps a raw backtrace and tears the deploy down from inside the thread, naming
+  # no server at all. The flags are set inside the thread so that they are in place
+  # before the operation gets a chance to fail.
+  def self.spawn(server, operation, message = nil, &block)
+    thread = Thread.new do
+      Thread.current.abort_on_exception  = false
+      Thread.current.report_on_exception = false
+
+      block.call
+    end
+
+    new(thread, server, operation, message)
+  end
+
   def initialize(thread, server, operation, message = nil)
     @thread = thread
     @server = server
     @operation = operation
     @message = message
   end
-end
+end

data/lib/indocker/rsync.rb

@@ -41,7 +41,18 @@ class Indocker::Rsync
         "-o ConnectTimeout=#{Indocker.ssh_connect_timeout}",
         "-o ServerAliveInterval=10",
         "-o ServerAliveCountMax=3",
-      ].join(' ')
+      ]
+
+      # Tunnel through the jump host the server is defined with, so that a server
+      # reachable only over its internal address syncs like any other one. ProxyCommand
+      # rather than -J: it is the same tunnel every other remote operation uses, including
+      # the shared connection that keeps a parallel sync from exhausting the jump host's
+      # MaxStartups.
+      if session.jump_host
+        ssh_command << %Q{-o ProxyCommand="#{Indocker::SshSession.jump_proxy_command(session.jump_host)}"}
+      end
+
+      ssh_command = ssh_command.join(' ')
       command = "rsync --delete-after -a -e '#{ssh_command}' #{from} #{session.user}@#{session.host}:#{to}"
       Indocker.logger.debug("sync #{from} #{session.user}@#{session.host}:#{to}")
       Indocker::Shell.command(command, Indocker.logger, raise_on_error: raise_on_error)

data/lib/indocker/server.rb

@@ -1,13 +1,17 @@
 class Indocker::Server
   include Indocker::Concerns::Inspectable
 
-  attr_reader :name, :host, :user, :port
+  attr_reader :name, :host, :user, :port, :jump_host
 
-  def initialize(name:, host:, user:, port:)
+  # jump_host ("user@host") makes every connection to this server tunnel through
+  # another one. Servers whose public address is unreachable can still be deployed
+  # to over their internal address that way.
+  def initialize(name:, host:, user:, port:, jump_host: nil)
     @name = name
     @host = host
     @user = user
     @port = port
+    @jump_host = jump_host
   end
 
   def ==(value)

data/lib/indocker/server_pools/server_connection.rb

@@ -10,12 +10,7 @@ class Indocker::ServerPools::ServerConnection
   def create_session!
     return unless @server
     
-    @session = Indocker::SshSession.new(
-      host: @server.host,
-      user: @server.user,
-      port: @server.port,
-      logger: @logger
-    )
+    @session = Indocker::SshSession.for(@server, logger: @logger)
   end
 
   def exec!(command)

data/lib/indocker/ssh_session.rb

@@ -2,6 +2,7 @@ require_relative 'shell'
 
 class Indocker::SshSession
   LOCALHOST = 'localhost'.freeze
+  JUMP_CONTROL_PERSIST = '600'.freeze
 
   class ExecResult
     attr_reader :stdout_data, :stderr_data, :exit_code, :exit_signal
@@ -19,24 +20,94 @@ class Indocker::SshSession
     end
   end
 
-  attr_reader :host, :user, :port, :logger
+  attr_reader :host, :user, :port, :logger, :jump_host
+
+  # Builds the tunnel to a jump host, shared by Net::SSH sessions and by rsync.
+  #
+  # Net::SSH::Proxy::Jump would build a bare `ssh -W`: a jump host that asks for a password
+  # hangs the deploy, and probing a server whose sshd is still starting prints
+  # "channel 0: open failed" from the ssh client on every retry.
+  #
+  # Multiplexing matters even more. A deploy syncs and deploys to every server in parallel,
+  # and each of those opens its own tunnel; sshd drops everything above MaxStartups (10 by
+  # default), which surfaces as "connection closed by remote host" on random servers. Sharing
+  # one connection to the jump host keeps the whole fan-out down to a single tunnel.
+  def self.jump_proxy_command(jump_host)
+    require 'digest'
+
+    host = jump_host
+    port = nil
+
+    if host =~ /\A(.+):(\d+)\z/
+      host = $1
+      port = $2
+    end
+
+    control_path = "/tmp/indocker-jump-#{Digest::MD5.hexdigest(jump_host)[0, 10]}"
+
+    command = [
+      "ssh -q",
+      "-o BatchMode=yes",
+      "-o ConnectTimeout=#{Indocker.ssh_connect_timeout}",
+      "-o ControlMaster=auto",
+      "-o ControlPath=#{control_path}",
+      "-o ControlPersist=#{JUMP_CONTROL_PERSIST}",
+    ]
+
+    command << "-p #{port}" if port
+    command << "-W %h:%p"
+    command << host
 
-  def initialize(host:, user:, port:, logger:)
+    command.join(' ')
+  end
+
+  # Opens the shared connection to a jump host up front. Without it the first wave of
+  # parallel operations races to create the master connection and opens a burst of them.
+  def self.warm_jump_host(jump_host, logger:)
+    command = jump_proxy_command(jump_host).sub("-W %h:%p ", "")
+
+    Indocker::Shell.command("#{command} true", logger)
+  end
+
+  # Opens a session to a server as it is defined in the configuration, including its
+  # jump host, if any. Every remote operation goes through here, so a server reached
+  # through a jump host is reached that way by binaries sync, container deploys and
+  # the deployment checker alike.
+  def self.for(server, logger:)
+    new(
+      host:      server.host,
+      user:      server.user,
+      port:      server.port,
+      jump_host: server.jump_host,
+      logger:    logger
+    )
+  end
+
+  def initialize(host:, user:, port:, logger:, jump_host: nil)
     @host = host
     @user = user
     @port = port
+    @jump_host = jump_host
     @logger = logger
 
     if host != LOCALHOST
       require 'net/ssh'
-      @ssh = Net::SSH.start(@host, @user, {
+
+      options = {
         port:               @port,
         non_interactive:    true,
         timeout:            Indocker.ssh_connect_timeout,
         keepalive:          true,
         keepalive_interval: 10,
         keepalive_maxcount: 3,
-      })
+      }
+
+      if @jump_host
+        require 'net/ssh/proxy/command'
+        options[:proxy] = Net::SSH::Proxy::Command.new(self.class.jump_proxy_command(@jump_host))
+      end
+
+      @ssh = Net::SSH.start(@host, @user, options)
     end
   end
 

data/lib/indocker/version.rb

@@ -1,3 +1,3 @@
 module Indocker
-  VERSION = "0.5.4"
+  VERSION = "0.6.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indocker
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.2
 platform: ruby
 authors:
 - Ruslan Gatiyatov