dash 4.0.2 → 4.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2fca4cca966b5e857e1d537799367c1493701a1e18e48a691c53dddac6b2c36
-  data.tar.gz: 2627d3e285ddbee1da9b4bc0705c38fb3825345f554b3b7e8f85598810c0b354
+  metadata.gz: 762f586250ded9a4e4f60850a91b1358bd993bcb1d4c6e291dfec93dc6d516e2
+  data.tar.gz: d87d3161396bfdb38f9feb3d3946d0830bdd2b1dc2921e569a275e65f4285cb1
 SHA512:
-  metadata.gz: d40d1805b3f3b1ed9f0a79bf53a6d96709ea7c3693cc0d8319791e92b10c345aaa2743ef8fdbc568cf9b79de5b37659fb2fbbb75b7d4fac0f111322bc2f807c2
-  data.tar.gz: 2972113fc39cd3dcbdd6476d91634e29f757989ca4c84e9dbd1dac6443f6169b967a3f3c9b9c5d7ed9913d212e18d8239c1cd142981a7a14f90dea56769fca1b
+  metadata.gz: 86a1e05606121be7d85b83c2da117d8bcc2a2badbf080a5e5d0237b3175dd98ac9abba3d1afa154495d6be85923d8ef7d915e3bb877241b69a637ab47de00658
+  data.tar.gz: 9bca1f4085e3ad18ca778eba0dc3344b6592404839fdb4628a2a1bc05d81d869c4b4a837a18f66659f2c0b9fba307da4f14d877f0c088f709d72806a68bf2927

data/lib/dash/cli/base.rb

@@ -217,7 +217,7 @@ module Dash::Cli
 
             break
           rescue LockHeldError
-            release_server_lock_on(held)
+            roll_back_server_lock(held)
 
             unless details_shown
               # The holder can release between our failed mkdir and this read.
@@ -248,12 +248,28 @@ module Dash::Cli
 
             say "Waiting #{interval}s for the server lock (#{remaining}s remaining)...", :magenta
             sleep [ interval, remaining ].min
+          rescue StandardError
+            # Anything else - a dropped SSH connection on a host that idled
+            # through the wait, a full disk - leaves the hosts we did take
+            # locked. holding_server_lock? is still false, so with_server_lock's
+            # ensure never runs and nothing else would ever release them.
+            roll_back_server_lock(held)
+            raise
           end
         end
 
         DASH.holding_server_lock = true
       end
 
+      # Never raises: on the contention path a raise here would abandon the
+      # locks it was rolling back, and on the failure path it would replace the
+      # error the operator needs to see.
+      def roll_back_server_lock(held)
+        release_server_lock_on(held)
+      rescue StandardError => e
+        say "Error releasing the server lock on #{Array(held).join(", ")}: #{e.message}", :red
+      end
+
       def release_server_lock
         say "Releasing the server lock...", :magenta
         release_server_lock_on(server_lock_hosts)
@@ -264,12 +280,22 @@ module Dash::Cli
       # Only ever called with hosts this process actually locked, so a missing
       # directory means someone already cleaned up, not that we may delete
       # another deploy's lock.
+      #
+      # Every host is attempted even after one fails - stopping at the first
+      # error would strand the remaining locks with no one left to release them
+      # - and the first error is re-raised once the sweep is done.
       def release_server_lock_on(hosts)
+        error = nil
+
         Array(hosts).each do |host|
           execute_lock_release(lock: DASH.server_lock, hosts: host)
         rescue LockMissingError
           nil
+        rescue StandardError => e
+          error ||= e
         end
+
+        raise error if error
       end
 
       def server_lock_hosts

data/lib/dash/cli/lock.rb

@@ -1,8 +1,13 @@
 class Dash::Cli::Lock < Dash::Cli::Base
   desc "status", "Report lock status"
+  option :server, type: :boolean, default: false, desc: "Report the shared server lock instead of the deploy lock"
   def status
-    handle_missing_lock do
-      puts capture_lock_status
+    if options[:server]
+      report_server_lock_status
+    else
+      handle_missing_lock do
+        puts capture_lock_status
+      end
     end
   end
 
@@ -18,10 +23,16 @@ class Dash::Cli::Lock < Dash::Cli::Base
   end
 
   desc "release", "Release the deploy lock"
+  option :server, type: :boolean, default: false, desc: "Release the shared server lock instead of the deploy lock"
   def release
-    handle_missing_lock do
-      execute_lock_release
-      say "Released the deploy lock"
+    if options[:server]
+      release_server_lock_on(server_lock_hosts)
+      say "Released the server lock"
+    else
+      handle_missing_lock do
+        execute_lock_release
+        say "Released the deploy lock"
+      end
     end
   end
 
@@ -31,4 +42,23 @@ class Dash::Cli::Lock < Dash::Cli::Base
     rescue LockMissingError
       say "There is no deploy lock"
     end
+
+    # The server lock is taken per host rather than per destination, so report
+    # each one: an acquire that failed part-way leaves it held on a subset.
+    def report_server_lock_status
+      server_lock_hosts.each do |host|
+        if (status = server_lock_status_on(host))
+          say "Server lock on #{host}:"
+          puts status
+        else
+          say "There is no server lock on #{host}"
+        end
+      end
+    end
+
+    def server_lock_status_on(host)
+      capture_lock_status(lock: DASH.server_lock, hosts: host)
+    rescue LockMissingError
+      nil
+    end
 end

data/lib/dash/cli/proxy/loadbalancer_reboot.rb

@@ -19,6 +19,7 @@ class Dash::Cli::Proxy::LoadbalancerReboot
   def run
     execute *DASH.auditor.record("Rebooted loadbalancer"), verbosity: :debug
     execute *DASH.registry.login
+    ensure_network
 
     info "Stopping and removing #{DASH.loadbalancer.container_name} on #{host}, if running..."
     execute *DASH.loadbalancer.stop, raise_on_non_zero_exit: false
@@ -45,6 +46,16 @@ class Dash::Cli::Proxy::LoadbalancerReboot
   end
 
   private
+    # Before the old container is stopped: `docker run` against a missing
+    # network would otherwise fail with the edge already down. A dedicated
+    # loadbalancer host has no other path that creates the network
+    # (zoolutions/dash#140).
+    def ensure_network
+      execute *DASH.docker.create_network
+    rescue SSHKit::Command::Failed => e
+      raise unless e.message.include?("already exists")
+    end
+
     def wait_until_ready
       deadline = Time.now + READY_TIMEOUT
 

data/lib/dash/cli/proxy.rb

@@ -2,7 +2,7 @@ class Dash::Cli::Proxy < Dash::Cli::Base
   desc "boot", "Boot proxy on servers"
   def boot
     modify(lock: true, server_lock: true) do
-      on(DASH.hosts) do |host|
+      on(network_hosts) do |host|
         execute *DASH.docker.create_network
       rescue SSHKit::Command::Failed => e
         raise unless e.message.include?("already exists")
@@ -76,8 +76,10 @@ class Dash::Cli::Proxy < Dash::Cli::Base
           info "Starting loadbalancer on #{host}..."
           execute *DASH.registry.login
 
-          # Bring a pre-rename volume across before the container can be created
-          # against an empty one. A no-op once it has been.
+          # Bring a pre-rename host across before the container can be created
+          # against an empty volume or a network nothing else joined. A no-op
+          # once it has been.
+          execute *DASH.docker.connect_legacy_network_containers
           execute *DASH.loadbalancer.copy_legacy_config_volume
 
           # The load balancer terminates TLS and owns the cache, so its host
@@ -653,6 +655,15 @@ class Dash::Cli::Proxy < Dash::Cli::Base
   end
 
   private
+    # Every host that runs a container on the proxy network. A dedicated
+    # loadbalancer host is not in DASH.hosts, and nothing else creates the
+    # network there (zoolutions/dash#140).
+    def network_hosts
+      hosts = DASH.hosts
+      hosts |= [ DASH.config.proxy.effective_loadbalancer ] if DASH.config.proxy.load_balancing?
+      hosts
+    end
+
     # The host that owns TLS, and so the certificate store: the loadbalancer
     # host when load balancing (TLS terminates at the edge), else the primary
     # host - the same host `loadbalancer: true` would resolve to.

data/lib/dash/configuration/docs/sshkit.yml

@@ -20,6 +20,10 @@ sshkit:
   #
   # Kamal sets a long idle timeout of 900 seconds on connections to try to avoid
   # re-connection storms after an idle period, such as building an image or waiting for CI.
+  #
+  # A pooled connection that the network dropped while idle is evicted and the command
+  # retried once on a fresh connection, so a long wait elsewhere (a server-lock poll, a
+  # loadbalancer reboot) does not fail the next command on another host.
   pool_idle_timeout: 300
 
   # DNS retry settings

data/lib/dash/sshkit_with_ext.rb

@@ -170,6 +170,50 @@ class SSHKit::Backend::Netssh
       end
   end
   prepend LimitConcurrentStartsInstance
+
+  # A pooled session that sat idle while dash was busy elsewhere (a server-lock
+  # wait, a loadbalancer reboot) gets dropped by NATs and cloud networks without
+  # either end noticing: net-ssh only sends keepalives from inside its event
+  # loop, and the pool's liveness probe is a non-blocking `process(0)`, so the
+  # session looks fine until the first command on it dies. Evict that session
+  # and rerun the block once on a fresh connection.
+  #
+  # The block is rerun, so a drop *mid-command* runs the command twice. That
+  # is accepted: the errors here are the idle-drop ones, where nothing reached
+  # the server, and the alternative is the deploy failing on the next host.
+  module ReconnectOnStaleConnection
+    STALE_CONNECTION_ERRORS = [ Errno::ECONNRESET, Errno::EPIPE, Net::SSH::Disconnect, Net::SSH::Timeout ].freeze
+
+    private
+      def with_ssh
+        reconnected = false
+
+        begin
+          super do |ssh|
+            yield ssh
+          rescue *STALE_CONNECTION_ERRORS
+            evict_stale_session(ssh)
+            raise
+          end
+        rescue *STALE_CONNECTION_ERRORS => e
+          raise if reconnected
+
+          reconnected = true
+          SSHKit.config.output.warn("Reconnecting to #{host}: #{e.message}")
+          retry
+        end
+      end
+
+      # `close` waits for channel-close acknowledgements, which never come over
+      # a dead socket. `shutdown!` just closes the socket, and a closed session
+      # is what makes the pool drop it instead of caching it again.
+      def evict_stale_session(ssh)
+        ssh.shutdown!
+      rescue StandardError
+        nil
+      end
+  end
+  prepend ReconnectOnStaleConnection
 end
 
 class SSHKit::Runner::Parallel

data/lib/dash/version.rb

@@ -1,3 +1,3 @@
 module Dash
-  VERSION = "4.0.2"
+  VERSION = "4.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dash
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.0.4
 platform: ruby
 authors:
 - Mikael Henriksson