dash 4.0.2 → 4.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2fca4cca966b5e857e1d537799367c1493701a1e18e48a691c53dddac6b2c36
-  data.tar.gz: 2627d3e285ddbee1da9b4bc0705c38fb3825345f554b3b7e8f85598810c0b354
+  metadata.gz: 563633a7568cdc7c4567bbe4957af6e5212e37b3ae99ddda0e990552f949762f
+  data.tar.gz: dda127134f2d0ef7fc4e7636ce7132106842855f9084d082a5a3b65c41313b56
 SHA512:
-  metadata.gz: d40d1805b3f3b1ed9f0a79bf53a6d96709ea7c3693cc0d8319791e92b10c345aaa2743ef8fdbc568cf9b79de5b37659fb2fbbb75b7d4fac0f111322bc2f807c2
-  data.tar.gz: 2972113fc39cd3dcbdd6476d91634e29f757989ca4c84e9dbd1dac6443f6169b967a3f3c9b9c5d7ed9913d212e18d8239c1cd142981a7a14f90dea56769fca1b
+  metadata.gz: 8ebab61a7f7f4df70ecd48e5e95415198a01990e6ae6ce93dca062775f2d11b91d2a971bcf9b0b13b4cd3aa6940700c45f8fc73bc1c50be64fe9478e2e1d7ca8
+  data.tar.gz: 5cda006f0c95a79eea3198c9caa089d5e3b3d064860b6d031efe12b65513b2d4219ce04b62b5a9c66532bbda38681ba0ec8dd6cd51143b270588a804be647488

data/lib/dash/cli/base.rb

@@ -217,7 +217,7 @@ module Dash::Cli
 
             break
           rescue LockHeldError
-            release_server_lock_on(held)
+            roll_back_server_lock(held)
 
             unless details_shown
               # The holder can release between our failed mkdir and this read.
@@ -248,12 +248,28 @@ module Dash::Cli
 
             say "Waiting #{interval}s for the server lock (#{remaining}s remaining)...", :magenta
             sleep [ interval, remaining ].min
+          rescue StandardError
+            # Anything else - a dropped SSH connection on a host that idled
+            # through the wait, a full disk - leaves the hosts we did take
+            # locked. holding_server_lock? is still false, so with_server_lock's
+            # ensure never runs and nothing else would ever release them.
+            roll_back_server_lock(held)
+            raise
           end
         end
 
         DASH.holding_server_lock = true
       end
 
+      # Never raises: on the contention path a raise here would abandon the
+      # locks it was rolling back, and on the failure path it would replace the
+      # error the operator needs to see.
+      def roll_back_server_lock(held)
+        release_server_lock_on(held)
+      rescue StandardError => e
+        say "Error releasing the server lock on #{Array(held).join(", ")}: #{e.message}", :red
+      end
+
       def release_server_lock
         say "Releasing the server lock...", :magenta
         release_server_lock_on(server_lock_hosts)
@@ -264,12 +280,22 @@ module Dash::Cli
       # Only ever called with hosts this process actually locked, so a missing
       # directory means someone already cleaned up, not that we may delete
       # another deploy's lock.
+      #
+      # Every host is attempted even after one fails - stopping at the first
+      # error would strand the remaining locks with no one left to release them
+      # - and the first error is re-raised once the sweep is done.
       def release_server_lock_on(hosts)
+        error = nil
+
         Array(hosts).each do |host|
           execute_lock_release(lock: DASH.server_lock, hosts: host)
         rescue LockMissingError
           nil
+        rescue StandardError => e
+          error ||= e
         end
+
+        raise error if error
       end
 
       def server_lock_hosts

data/lib/dash/cli/lock.rb

@@ -1,8 +1,13 @@
 class Dash::Cli::Lock < Dash::Cli::Base
   desc "status", "Report lock status"
+  option :server, type: :boolean, default: false, desc: "Report the shared server lock instead of the deploy lock"
   def status
-    handle_missing_lock do
-      puts capture_lock_status
+    if options[:server]
+      report_server_lock_status
+    else
+      handle_missing_lock do
+        puts capture_lock_status
+      end
     end
   end
 
@@ -18,10 +23,16 @@ class Dash::Cli::Lock < Dash::Cli::Base
   end
 
   desc "release", "Release the deploy lock"
+  option :server, type: :boolean, default: false, desc: "Release the shared server lock instead of the deploy lock"
   def release
-    handle_missing_lock do
-      execute_lock_release
-      say "Released the deploy lock"
+    if options[:server]
+      release_server_lock_on(server_lock_hosts)
+      say "Released the server lock"
+    else
+      handle_missing_lock do
+        execute_lock_release
+        say "Released the deploy lock"
+      end
     end
   end
 
@@ -31,4 +42,23 @@ class Dash::Cli::Lock < Dash::Cli::Base
     rescue LockMissingError
       say "There is no deploy lock"
     end
+
+    # The server lock is taken per host rather than per destination, so report
+    # each one: an acquire that failed part-way leaves it held on a subset.
+    def report_server_lock_status
+      server_lock_hosts.each do |host|
+        if (status = server_lock_status_on(host))
+          say "Server lock on #{host}:"
+          puts status
+        else
+          say "There is no server lock on #{host}"
+        end
+      end
+    end
+
+    def server_lock_status_on(host)
+      capture_lock_status(lock: DASH.server_lock, hosts: host)
+    rescue LockMissingError
+      nil
+    end
 end

data/lib/dash/configuration/docs/sshkit.yml

@@ -20,6 +20,10 @@ sshkit:
   #
   # Kamal sets a long idle timeout of 900 seconds on connections to try to avoid
   # re-connection storms after an idle period, such as building an image or waiting for CI.
+  #
+  # A pooled connection that the network dropped while idle is evicted and the command
+  # retried once on a fresh connection, so a long wait elsewhere (a server-lock poll, a
+  # loadbalancer reboot) does not fail the next command on another host.
   pool_idle_timeout: 300
 
   # DNS retry settings

data/lib/dash/sshkit_with_ext.rb

@@ -170,6 +170,50 @@ class SSHKit::Backend::Netssh
       end
   end
   prepend LimitConcurrentStartsInstance
+
+  # A pooled session that sat idle while dash was busy elsewhere (a server-lock
+  # wait, a loadbalancer reboot) gets dropped by NATs and cloud networks without
+  # either end noticing: net-ssh only sends keepalives from inside its event
+  # loop, and the pool's liveness probe is a non-blocking `process(0)`, so the
+  # session looks fine until the first command on it dies. Evict that session
+  # and rerun the block once on a fresh connection.
+  #
+  # The block is rerun, so a drop *mid-command* runs the command twice. That
+  # is accepted: the errors here are the idle-drop ones, where nothing reached
+  # the server, and the alternative is the deploy failing on the next host.
+  module ReconnectOnStaleConnection
+    STALE_CONNECTION_ERRORS = [ Errno::ECONNRESET, Errno::EPIPE, Net::SSH::Disconnect, Net::SSH::Timeout ].freeze
+
+    private
+      def with_ssh
+        reconnected = false
+
+        begin
+          super do |ssh|
+            yield ssh
+          rescue *STALE_CONNECTION_ERRORS
+            evict_stale_session(ssh)
+            raise
+          end
+        rescue *STALE_CONNECTION_ERRORS => e
+          raise if reconnected
+
+          reconnected = true
+          SSHKit.config.output.warn("Reconnecting to #{host}: #{e.message}")
+          retry
+        end
+      end
+
+      # `close` waits for channel-close acknowledgements, which never come over
+      # a dead socket. `shutdown!` just closes the socket, and a closed session
+      # is what makes the pool drop it instead of caching it again.
+      def evict_stale_session(ssh)
+        ssh.shutdown!
+      rescue StandardError
+        nil
+      end
+  end
+  prepend ReconnectOnStaleConnection
 end
 
 class SSHKit::Runner::Parallel

data/lib/dash/version.rb

@@ -1,3 +1,3 @@
 module Dash
-  VERSION = "4.0.2"
+  VERSION = "4.0.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dash
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.0.3
 platform: ruby
 authors:
 - Mikael Henriksson