dash 4.0.1 → 4.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8651f7a91510814ccfa26283d3aef0f364caa5c99419a76ee6cdb15820680148
-  data.tar.gz: aef6a718f6f4d1427d6e20f73e3096eed8436426e32b6c6ea0a40f52e3588261
+  metadata.gz: 563633a7568cdc7c4567bbe4957af6e5212e37b3ae99ddda0e990552f949762f
+  data.tar.gz: dda127134f2d0ef7fc4e7636ce7132106842855f9084d082a5a3b65c41313b56
 SHA512:
-  metadata.gz: da7a30a00bf6f622aeeb13889670133285a202c5c4481a7b83cc8d980624db48a0c5d8007e0efaf723489d4ea20672999dc8791e5ff4240b6f4fa65ba54a7380
-  data.tar.gz: cdcf6391c305f5e84125687bb6f4174729a7c09819a08a5c8a26bd8de2fd5b10933606387f581187c94151574c2607d803689e3d97c2cffa00beab9e26375f09
+  metadata.gz: 8ebab61a7f7f4df70ecd48e5e95415198a01990e6ae6ce93dca062775f2d11b91d2a971bcf9b0b13b4cd3aa6940700c45f8fc73bc1c50be64fe9478e2e1d7ca8
+  data.tar.gz: 5cda006f0c95a79eea3198c9caa089d5e3b3d064860b6d031efe12b65513b2d4219ce04b62b5a9c66532bbda38681ba0ec8dd6cd51143b270588a804be647488

data/lib/dash/cli/base.rb

@@ -217,7 +217,7 @@ module Dash::Cli
 
             break
           rescue LockHeldError
-            release_server_lock_on(held)
+            roll_back_server_lock(held)
 
             unless details_shown
               # The holder can release between our failed mkdir and this read.
@@ -248,12 +248,28 @@ module Dash::Cli
 
             say "Waiting #{interval}s for the server lock (#{remaining}s remaining)...", :magenta
             sleep [ interval, remaining ].min
+          rescue StandardError
+            # Anything else - a dropped SSH connection on a host that idled
+            # through the wait, a full disk - leaves the hosts we did take
+            # locked. holding_server_lock? is still false, so with_server_lock's
+            # ensure never runs and nothing else would ever release them.
+            roll_back_server_lock(held)
+            raise
           end
         end
 
         DASH.holding_server_lock = true
       end
 
+      # Never raises: on the contention path a raise here would abandon the
+      # locks it was rolling back, and on the failure path it would replace the
+      # error the operator needs to see.
+      def roll_back_server_lock(held)
+        release_server_lock_on(held)
+      rescue StandardError => e
+        say "Error releasing the server lock on #{Array(held).join(", ")}: #{e.message}", :red
+      end
+
       def release_server_lock
         say "Releasing the server lock...", :magenta
         release_server_lock_on(server_lock_hosts)
@@ -264,12 +280,22 @@ module Dash::Cli
       # Only ever called with hosts this process actually locked, so a missing
       # directory means someone already cleaned up, not that we may delete
       # another deploy's lock.
+      #
+      # Every host is attempted even after one fails - stopping at the first
+      # error would strand the remaining locks with no one left to release them
+      # - and the first error is re-raised once the sweep is done.
       def release_server_lock_on(hosts)
+        error = nil
+
         Array(hosts).each do |host|
           execute_lock_release(lock: DASH.server_lock, hosts: host)
         rescue LockMissingError
           nil
+        rescue StandardError => e
+          error ||= e
         end
+
+        raise error if error
       end
 
       def server_lock_hosts

data/lib/dash/cli/lock.rb

@@ -1,8 +1,13 @@
 class Dash::Cli::Lock < Dash::Cli::Base
   desc "status", "Report lock status"
+  option :server, type: :boolean, default: false, desc: "Report the shared server lock instead of the deploy lock"
   def status
-    handle_missing_lock do
-      puts capture_lock_status
+    if options[:server]
+      report_server_lock_status
+    else
+      handle_missing_lock do
+        puts capture_lock_status
+      end
     end
   end
 
@@ -18,10 +23,16 @@ class Dash::Cli::Lock < Dash::Cli::Base
   end
 
   desc "release", "Release the deploy lock"
+  option :server, type: :boolean, default: false, desc: "Release the shared server lock instead of the deploy lock"
   def release
-    handle_missing_lock do
-      execute_lock_release
-      say "Released the deploy lock"
+    if options[:server]
+      release_server_lock_on(server_lock_hosts)
+      say "Released the server lock"
+    else
+      handle_missing_lock do
+        execute_lock_release
+        say "Released the deploy lock"
+      end
     end
   end
 
@@ -31,4 +42,23 @@ class Dash::Cli::Lock < Dash::Cli::Base
     rescue LockMissingError
       say "There is no deploy lock"
     end
+
+    # The server lock is taken per host rather than per destination, so report
+    # each one: an acquire that failed part-way leaves it held on a subset.
+    def report_server_lock_status
+      server_lock_hosts.each do |host|
+        if (status = server_lock_status_on(host))
+          say "Server lock on #{host}:"
+          puts status
+        else
+          say "There is no server lock on #{host}"
+        end
+      end
+    end
+
+    def server_lock_status_on(host)
+      capture_lock_status(lock: DASH.server_lock, hosts: host)
+    rescue LockMissingError
+      nil
+    end
 end

data/lib/dash/cli/proxy.rb

@@ -76,6 +76,10 @@ class Dash::Cli::Proxy < Dash::Cli::Base
           info "Starting loadbalancer on #{host}..."
           execute *DASH.registry.login
 
+          # Bring a pre-rename volume across before the container can be created
+          # against an empty one. A no-op once it has been.
+          execute *DASH.loadbalancer.copy_legacy_config_volume
+
           # The load balancer terminates TLS and owns the cache, so its host
           # needs the proxy secrets (acme credentials, cache store) just like
           # the proxy hosts do.

data/lib/dash/commands/base.rb

@@ -126,6 +126,34 @@ module Dash::Commands
         [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\\\''")}'" ]
       end
 
+      # Adopts a pre-rename docker volume: creates `volume` from `legacy` if, and
+      # only if, `volume` is absent and `legacy` is present. A host with neither
+      # exits 0. The first word must be a program, never `!` — see
+      # Dash::Commands::Proxy#copy_legacy_config_volume.
+      def copy_legacy_volume(legacy:, volume:, image:)
+        any \
+          volume_exists(volume),
+          negate(volume_exists(legacy)),
+          [ "(", *combine(docker(:volume, :create, volume), copy_between_volumes(legacy, volume, image: image)), ")" ]
+      end
+
+      def negate(command)
+        [ "!", *command ]
+      end
+
+      def volume_exists(name)
+        docker :volume, :inspect, name, ">", "/dev/null", "2>&1"
+      end
+
+      def copy_between_volumes(from, to, image:)
+        docker \
+          :run, "--rm", "--user", "root", "--entrypoint", "sh",
+          "--volume", "#{from}:/from",
+          "--volume", "#{to}:/to",
+          image,
+          "-c", "'cp -a /from/. /to/'"
+      end
+
       def docker(*args)
         args.compact.unshift :docker
       end

data/lib/dash/commands/loadbalancer.rb

@@ -37,6 +37,16 @@ class Dash::Commands::Loadbalancer < Dash::Commands::Base
     combine start, run, by: "||"
   end
 
+  # A dedicated load balancer host went through the 4.0 rename with nothing
+  # adopting its `kamal-loadbalancer-config` volume: the per-host proxies got
+  # Dash::Cli::Proxy::LegacyRename, the load balancer got a fresh empty volume
+  # and lost its routing table, dynamic domains and ACME cache. On a shared
+  # proxy host the volume is the proxy's own and LegacyRename already copied
+  # it, so this is a no-op there.
+  def copy_legacy_config_volume
+    copy_legacy_volume(legacy: legacy_config_volume_name, volume: config_volume_name, image: loadbalancer_config.run.image)
+  end
+
   def deploy(targets: [])
     docker :exec, container_name, "dash-proxy", "deploy", loadbalancer_config.config.service,
       *loadbalancer_config.deploy_command_args(targets: targets)
@@ -196,11 +206,15 @@ class Dash::Commands::Loadbalancer < Dash::Commands::Base
     # load balancer and a shared proxy host never fight over the same volume.
     # (The apps-config mount comes with run_args, via the proxy's run surface.)
     def config_volume
-      if on_proxy_host?
-        [ "--volume", "dash-proxy-config:/home/dash-proxy/.config/dash-proxy" ]
-      else
-        [ "--volume", "dash-loadbalancer-config:/home/dash-proxy/.config/dash-proxy" ]
-      end
+      [ "--volume", "#{config_volume_name}:/home/dash-proxy/.config/dash-proxy" ]
+    end
+
+    def config_volume_name
+      on_proxy_host? ? Dash::Configuration::Proxy::CONFIG_VOLUME : Dash::Configuration::Proxy::LOADBALANCER_CONFIG_VOLUME
+    end
+
+    def legacy_config_volume_name
+      on_proxy_host? ? Dash::Configuration::Proxy::LEGACY_CONFIG_VOLUME : Dash::Configuration::Proxy::LEGACY_LOADBALANCER_CONFIG_VOLUME
     end
 
     # The certificate store lives in whichever config volume this loadbalancer

data/lib/dash/commands/proxy.rb

@@ -51,21 +51,18 @@ class Dash::Commands::Proxy < Dash::Commands::Base
   # boot sequence, and its ubuntu base has sh and cp. `--user root` because the
   # image's own user cannot write the destination volume; `cp -a` preserves the
   # uid, which the rename leaves at 1001.
-  # The guard is negated and leads the chain, with `|| true` last, because
-  # shell `&&` and `||` share precedence and associate left: written as
-  # `exists || legacy_exists && create && copy` it would parse as
-  # `((exists || legacy_exists) && create) && copy` and re-copy the legacy
-  # volume over live state on every deploy. Leading with `! exists` makes the
-  # whole chain a single left-associative AND, which short-circuits correctly.
+  #
+  # Shape: `exists || ! legacy_exists || ( create && copy )`. The chain has to
+  # start with a real program: SSHKit prefixes the first word with
+  # `/usr/bin/env`, and `env !` is "No such file or directory" (exit 127), not
+  # shell negation. 4.0.0 led with `! docker volume inspect …`, so the whole
+  # chain failed silently into `|| true`, the proxy booted onto a volume docker
+  # created empty, and every host lost its routing table and ACME cache on the
+  # first deploy. The subshell groups create-and-copy because `&&` and `||`
+  # share precedence and associate left — without it a host that already has
+  # the new volume would still run the copy over live state.
   def copy_legacy_config_volume(volume: Dash::Configuration::Proxy::CONFIG_VOLUME, legacy: Dash::Configuration::Proxy::LEGACY_CONFIG_VOLUME)
-    any \
-      combine(
-        negate(volume_exists(volume)),
-        volume_exists(legacy),
-        docker(:volume, :create, volume),
-        copy_between_volumes(legacy, volume)
-      ),
-      [ :true ]
+    copy_legacy_volume(legacy: legacy, volume: volume, image: proxy_image)
   end
 
   # Stops and removes a pre-rename proxy container so the renamed one can claim
@@ -326,27 +323,10 @@ class Dash::Commands::Proxy < Dash::Commands::Base
       [ "--label", "#{CONFIG_DIGEST_LABEL}=#{digest}" ] if digest
     end
 
-    def negate(command)
-      [ "!", *command ]
-    end
-
-    def volume_exists(name)
-      docker :volume, :inspect, name, ">", "/dev/null", "2>&1"
-    end
-
     def container_exists(name)
       docker :container, :inspect, name, ">", "/dev/null", "2>&1"
     end
 
-    def copy_between_volumes(from, to)
-      docker \
-        :run, "--rm", "--user", "root", "--entrypoint", "sh",
-        "--volume", "#{from}:/from",
-        "--volume", "#{to}:/to",
-        proxy_image,
-        "-c", "'cp -a /from/. /to/'"
-    end
-
     # The image the volume copy borrows. The proxy this gem is pinned to is
     # already pulled by the time the copy runs, and `rake release` gates on
     # MINIMUM_VERSION being published, so this is always resolvable — unlike

data/lib/dash/configuration/docs/sshkit.yml

@@ -20,6 +20,10 @@ sshkit:
   #
   # Kamal sets a long idle timeout of 900 seconds on connections to try to avoid
   # re-connection storms after an idle period, such as building an image or waiting for CI.
+  #
+  # A pooled connection that the network dropped while idle is evicted and the command
+  # retried once on a fresh connection, so a long wait elsewhere (a server-lock poll, a
+  # loadbalancer reboot) does not fail the next command on another host.
   pool_idle_timeout: 300
 
   # DNS retry settings

data/lib/dash/sshkit_with_ext.rb

@@ -170,6 +170,50 @@ class SSHKit::Backend::Netssh
       end
   end
   prepend LimitConcurrentStartsInstance
+
+  # A pooled session that sat idle while dash was busy elsewhere (a server-lock
+  # wait, a loadbalancer reboot) gets dropped by NATs and cloud networks without
+  # either end noticing: net-ssh only sends keepalives from inside its event
+  # loop, and the pool's liveness probe is a non-blocking `process(0)`, so the
+  # session looks fine until the first command on it dies. Evict that session
+  # and rerun the block once on a fresh connection.
+  #
+  # The block is rerun, so a drop *mid-command* runs the command twice. That
+  # is accepted: the errors here are the idle-drop ones, where nothing reached
+  # the server, and the alternative is the deploy failing on the next host.
+  module ReconnectOnStaleConnection
+    STALE_CONNECTION_ERRORS = [ Errno::ECONNRESET, Errno::EPIPE, Net::SSH::Disconnect, Net::SSH::Timeout ].freeze
+
+    private
+      def with_ssh
+        reconnected = false
+
+        begin
+          super do |ssh|
+            yield ssh
+          rescue *STALE_CONNECTION_ERRORS
+            evict_stale_session(ssh)
+            raise
+          end
+        rescue *STALE_CONNECTION_ERRORS => e
+          raise if reconnected
+
+          reconnected = true
+          SSHKit.config.output.warn("Reconnecting to #{host}: #{e.message}")
+          retry
+        end
+      end
+
+      # `close` waits for channel-close acknowledgements, which never come over
+      # a dead socket. `shutdown!` just closes the socket, and a closed session
+      # is what makes the pool drop it instead of caching it again.
+      def evict_stale_session(ssh)
+        ssh.shutdown!
+      rescue StandardError
+        nil
+      end
+  end
+  prepend ReconnectOnStaleConnection
 end
 
 class SSHKit::Runner::Parallel

data/lib/dash/version.rb

@@ -1,3 +1,3 @@
 module Dash
-  VERSION = "4.0.1"
+  VERSION = "4.0.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dash
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.0.3
 platform: ruby
 authors:
 - Mikael Henriksson