dash 4.1.0 → 4.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32799cf5de0b149a2b36f88e0af2d671c82a5d05a3c54180f034d8574eb58bf8
-  data.tar.gz: 1e33d017d9ec6e3e3545ea0c1125d9d4b38a2e12557197b2c827e00e608b4aa0
+  metadata.gz: bb6c271cec9b9ab118649ffeaf503aa5ba31bd8d0f8057ce0a61adb93e130ef4
+  data.tar.gz: ce7f60b8e4a764e06ab868663f825134d8f73cac0e11f663793e6a12471ab490
 SHA512:
-  metadata.gz: bac661834cb626277175f2e0a9abbfb273975a0e70dd45779352798f547c5097ff836cfaed017316a207d59aa4d62b1b337112692db9fa7870aec2056bcd14e4
-  data.tar.gz: a8879e8f7b3250903eb854f23436b63892cdced0a303969039ebc809f29cce5231f3dda51faa2c5d3755b55edb8b543a007f96964272dc7afe1a1cf6c132ec6b
+  metadata.gz: 35204403e1680a42ce0b0726225d68c55119a74b387a9cd45189822499e5aa225c250ad82190944cbd4a5ff7aa976b3dcf02600e44b92a8a4ed165f86123c96b
+  data.tar.gz: cb520eb7f9028fad75e4537afcdcc1c7aea71aa6cb58c35a33b9c8d185781a4de9ad7a9f3d1f1f8daba8b73390b279bbd292ac1a82d531a303f34086c69c28d3

data/lib/dash/cli/app/boot.rb

@@ -1,4 +1,9 @@
 class Dash::Cli::App::Boot
+  # What `docker container ls --quiet` prints, and so what dash-proxy has always been
+  # handed as a target. `docker run --detach` prints the full 64-character id, so the
+  # target is its first twelve characters rather than a round trip of its own.
+  SHORT_CONTAINER_ID_LENGTH = 12
+
   attr_reader :host, :role, :version, :barrier, :sshkit, :cli
   delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit
   delegate :run_hook, to: :cli
@@ -58,8 +63,7 @@ class Dash::Cli::App::Boot
     end
 
     def capture_boot_state
-      output = capture_with_info(*app.boot_state(version), raise_on_non_zero_exit: false).to_s
-      clashing, _, running = output.partition(/^#{Regexp.escape(Dash::Commands::App::BOOT_STATE_SEPARATOR)}$/)
+      clashing, running = Dash::Commands::App.split_state(capture_with_info(*app.boot_state(version), raise_on_non_zero_exit: false))
 
       [ clashing.strip.presence, running.strip.presence ]
     end
@@ -70,9 +74,13 @@ class Dash::Cli::App::Boot
       execute *auditor.record_then("Booted app version #{version}", app.ensure_env_directory)
       upload! role.secrets_io(host), role.secrets_path, mode: "0600"
 
-      execute *app.run(hostname: hostname)
+      # `docker run --detach` prints the id of the container it just started, so the
+      # proxy target comes out of the run itself — asking docker for it again was a round
+      # trip spent re-reading something the host had already said.
+      container_id = capture_with_info(*app.run(hostname: hostname)).strip
+
       if running_proxy?
-        endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+        endpoint = container_id[0, SHORT_CONTAINER_ID_LENGTH]
         raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
 
         run_hook "pre-proxy-deploy", hosts: host.to_s, role: role.name
@@ -80,7 +88,7 @@ class Dash::Cli::App::Boot
         timing_healthy { execute *app.deploy(target: endpoint) }
         run_hook "post-proxy-deploy", hosts: host.to_s, role: role.name
       else
-        timing_healthy { Dash::Cli::Healthcheck::Poller.wait_for_healthy(role: role) { health_status } }
+        timing_healthy { Dash::Cli::Healthcheck::Poller.wait_for_healthy(role: role, &method(:readiness_status)) }
       end
     rescue => e
       error "Failed to boot #{role} on #{host}"
@@ -88,17 +96,21 @@ class Dash::Cli::App::Boot
       raise e
     end
 
-    # An exec probe is docker-invisible — the container declares no healthcheck, so
-    # `docker inspect` would only ever report its state. Poll the probe instead.
-    def health_status
-      role.healthcheck&.exec? ? exec_probe_status : capture_with_info(*app.status(version: version))
-    end
-
-    def exec_probe_status
-      execute *app.health_probe(version: version)
-      "healthy"
-    rescue SSHKit::Command::Failed
-      "exec probe exited non-zero"
+    # A role behind the proxy lets `dash-proxy deploy` block on the host until the
+    # container is healthy; a role without one now does the same, waiting in a shell loop
+    # on the host that streams its progress back rather than being polled from here once
+    # per attempt. The poller asks for the wait, and — only for an unchecked container it
+    # has just let through its readiness delay — for a plain confirming read.
+    #
+    # Neither capture suppresses a non-zero exit: a status that cannot be read is a broken
+    # command, and it has always failed the boot on the spot rather than being waited out.
+    def readiness_status(mode, seconds_left = nil)
+      if mode == :confirm
+        capture_with_info(*app.status(version: version))
+      else
+        capture_with_info *app.wait_for_ready(version: version, timeout: seconds_left),
+          interaction_handler: Dash::Cli::Healthcheck::ProgressReporter.new
+      end
     end
 
     # Every failed boot gets the container log, and the health probe history when the

data/lib/dash/cli/app.rb

@@ -186,8 +186,8 @@ class Dash::Cli::App < Dash::Cli::Base
     with_lock_if_stopping do
       on_roles(DASH.roles, hosts: DASH.app_hosts) do |host, role|
         app = DASH.app(role: role, host: host)
-        versions = capture_with_info(*app.list_versions, raise_on_non_zero_exit: false).split("\n")
-        versions -= [ capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip ]
+        listed, running = Dash::Commands::App.split_state(capture_with_info(*app.stale_state, raise_on_non_zero_exit: false))
+        versions = listed.strip.split("\n") - [ running.strip ]
 
         versions.each do |version|
           if stop

data/lib/dash/cli/build.rb

@@ -74,10 +74,8 @@ class Dash::Cli::Build < Dash::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    login_to_registry_remotely unless DASH.registry.local?
-
     forward_local_registry_port(DASH.hosts, **DASH.config.ssh.options) do
-      if (first_hosts = mirror_hosts).any?
+      if (first_hosts = login_and_mirror_hosts).any?
         #  Pull on a single host per mirror first to seed them
         say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
         pull_on_hosts(first_hosts)
@@ -225,19 +223,28 @@ class Dash::Cli::Build < Dash::Cli::Base
       end
     end
 
-    def mirror_hosts
-      if DASH.app_hosts.many?
-        mirror_hosts = Concurrent::Hash.new
-        on(DASH.app_hosts) do |host|
-          first_mirror = capture_with_info(*DASH.builder.first_mirror).strip.presence
-          mirror_hosts[first_mirror] ||= host.to_s if first_mirror
-        rescue SSHKit::Command::Failed => e
-          raise unless e.message =~ /error calling index: reflect: slice index out of range/
-        end
-        mirror_hosts.values
-      else
-        []
+    # The registry login and the mirror probe share one round trip per host. The probe only
+    # earns its keep where there is more than one app host to seed, so on a single host the
+    # login goes on its own; a local registry needs no login and the fold is the probe alone.
+    #
+    # A host with no mirror configured fails the `docker info` half with docker's own index
+    # error, which is what "no mirror" looks like. Anything else still raises - a rejected
+    # login short-circuits the `&&` and comes back with docker's `unauthorized`/`denied`,
+    # which this rescue does not match.
+    def login_and_mirror_hosts
+      unless DASH.app_hosts.many?
+        login_to_registry_remotely unless DASH.registry.local?
+        return []
+      end
+
+      mirror_hosts = Concurrent::Hash.new
+      on(DASH.app_hosts) do |host|
+        first_mirror = capture_with_info(*DASH.registry.login_then(DASH.builder.first_mirror)).strip.presence
+        mirror_hosts[first_mirror] ||= host.to_s if first_mirror
+      rescue SSHKit::Command::Failed => e
+        raise unless e.message =~ /error calling index: reflect: slice index out of range/
       end
+      mirror_hosts.values
     end
 
     # Audit, clean and pull share one round trip. validate_image keeps its own: folding it

data/lib/dash/cli/healthcheck/poller.rb

@@ -3,13 +3,19 @@ module Dash::Cli::Healthcheck::Poller
 
   NO_HEALTHCHECK = Dash::Commands::Base::NO_HEALTHCHECK
 
+  # The wait itself happens on the host now (Dash::Commands::App#wait_for_ready), which
+  # returns the moment the status is one this poller accepts and otherwise waits out the
+  # deadline it is given. So the block is called once for the wait - and once more only to
+  # confirm an unchecked container is still running after its readiness delay. Every
+  # decision below is the one the client-side poll made, in the same words; what shrank is
+  # the number of round trips it took to reach them.
   def wait_for_healthy(role:, &block)
     attempt = 1
     timeout_at = Time.now + DASH.config.deploy_timeout
     readiness_delay = role.readiness_delay
 
     begin
-      status = block.call
+      status = block.call(:wait, seconds_left(timeout_at))
 
       if unchecked?(status)
         ensure_no_healthcheck_drift(role, status)
@@ -20,7 +26,7 @@ module Dash::Cli::Healthcheck::Poller
           # Wait for the readiness delay and confirm it is still running
           if readiness_delay > 0
             sleep readiness_delay
-            status = block.call
+            status = block.call(:confirm)
             ensure_no_healthcheck_drift(role, status)
           end
         end
@@ -55,8 +61,14 @@ module Dash::Cli::Healthcheck::Poller
       status.to_s.delete_prefix("#{NO_HEALTHCHECK}:")
     end
 
+    # Shared with the host-side wait, which stops looking on exactly these - see
+    # Dash::Commands::Base::READY_STATUSES for why the two have to agree.
     def acceptable?(status)
-      status == "healthy" || (unchecked?(status) && docker_state(status) == "running")
+      Dash::Commands::Base::READY_STATUSES.include?(status)
+    end
+
+    def seconds_left(timeout_at)
+      [ (timeout_at - Time.now).ceil, 0 ].max
     end
 
     # The config asked docker to probe this container and docker is not probing it — the flags

data/lib/dash/cli/healthcheck/progress_reporter.rb

@@ -0,0 +1,39 @@
+# Turns the server-side readiness wait's progress lines back into the beacon the
+# client-side poll used to print. It is an SSHKit interaction handler, so it sees the
+# wait's stderr as it streams — the operator gets the same once-a-second feedback they
+# got when the laptop was the thing doing the polling, for one round trip instead of one
+# per attempt. The cadence is fixed at a second because the host loop's is.
+#
+# The stream is line-oriented but arrives in chunks (the SSH backend splits on packet
+# boundaries, not newlines), so data is buffered and only whole lines are reported. Only
+# stderr is buffered: the wait's stdout carries the final status, and stdout and stderr are
+# separate SSH streams whose chunks can interleave — folding both into one buffer would let
+# the status land in the middle of a half-arrived progress line and corrupt them both.
+# Anything on stderr that is not a progress line (docker's own complaints) is ignored.
+class Dash::Cli::Healthcheck::ProgressReporter
+  LINE = /\A#{Regexp.escape(Dash::Commands::Base::READINESS_PROGRESS_PREFIX)} (?<elapsed>\d+) (?<left>\d+)(?: |\z)/
+
+  def initialize
+    @buffer = +""
+    @mutex = Mutex.new
+  end
+
+  # SSHKit's interaction-handler contract.
+  def on_data(_command, stream_name, data, _channel = nil)
+    return unless stream_name == :stderr
+
+    @mutex.synchronize do
+      @buffer << data.to_s
+      while (newline = @buffer.index("\n"))
+        report @buffer.slice!(0..newline).chomp
+      end
+    end
+  end
+
+  private
+    def report(line)
+      match = LINE.match(line) or return
+
+      SSHKit.config.output.info "Container not ready yet, retrying in 1s (#{match[:elapsed]}s elapsed, #{match[:left]}s left)"
+    end
+end

data/lib/dash/cli/proxy/drift.rb

@@ -7,8 +7,23 @@ class Dash::Cli::Proxy::Drift
     @sshkit = sshkit
   end
 
+  # One `docker inspect` for everything a boot asks about the running proxy: whether it
+  # exists, which image tag it runs, and the digest it was booted with. Captured once per
+  # instance - `dash proxy boot` reads all three off it, and `dash doctor` only the first.
+  def state
+    @state ||= Dash::Commands::Proxy::State.parse(
+      capture_with_info(*proxy.inspect_state, raise_on_non_zero_exit: false)
+    )
+  end
+
   def container_exists?
-    capture_with_info(*proxy.container_id, raise_on_non_zero_exit: false).strip.present?
+    state.exists?
+  end
+
+  # The tag the running proxy was booted from, for the minimum-version gate. Nil when
+  # nothing is running - a host with no proxy has no version to be too old.
+  def version
+    state.version
   end
 
   # A proxy container has drifted when it was started with a different config
@@ -30,7 +45,7 @@ class Dash::Cli::Proxy::Drift
 
   private
     def current_digest
-      capture_with_info(*proxy.config_digest, raise_on_non_zero_exit: false).strip
+      state.digest.to_s
     end
 
     def proxy

data/lib/dash/cli/proxy/legacy_rename.rb

@@ -23,6 +23,13 @@
 # so a second deploy is a no-op. Nothing here removes the legacy network or
 # volume: an operator who wants them gone removes them by hand, and stage 3d
 # deletes this class outright.
+#
+# All three travel as one command (Dash::Commands::Proxy#prepare_boot), guarded on a
+# marker the host writes once it is verifiably past the rename - so a migrated host, and
+# a host installed fresh on 4.x that never had a kamal-proxy, run no docker command here
+# at all. The round trip itself is one the host already pays: the command carries the
+# apps-config `mkdir -p` too, which reads nothing the bridge writes. Stage 3d keeps the
+# mkdir and deletes the rest.
 class Dash::Cli::Proxy::LegacyRename
   attr_reader :host, :sshkit
   delegate :execute, to: :sshkit
@@ -33,26 +40,6 @@ class Dash::Cli::Proxy::LegacyRename
   end
 
   def run
-    bridge_network
-    adopt_config_volume
-    replace_legacy_container
+    execute *DASH.proxy(host).prepare_boot
   end
-
-  private
-    def bridge_network
-      execute *DASH.docker.connect_legacy_network_containers
-    end
-
-    def adopt_config_volume
-      execute *DASH.proxy(host).copy_legacy_config_volume
-    end
-
-    # The drain timeout the proxy is configured with, so a busy host is not cut
-    # off mid-request any more abruptly than a normal reboot would.
-    def replace_legacy_container
-      proxy = DASH.proxy(host)
-
-      execute *proxy.remove_legacy_container(timeout: DASH.config.drain_timeout)
-      execute *proxy.remove_legacy_holder_container
-    end
 end

data/lib/dash/cli/proxy/loadbalancer_reboot.rb

@@ -21,6 +21,14 @@ class Dash::Cli::Proxy::LoadbalancerReboot
     execute *DASH.registry.login
     ensure_network
 
+    # After the network the bridge attaches the legacy network's containers to,
+    # and before anything that could create the new container or let `docker
+    # run --volume` create dash-loadbalancer-config empty: adopt the legacy
+    # volume's routing table, dynamic domains and ACME cache. Carries the
+    # apps-config mkdir this reboot paid a round trip for anyway
+    # (zoolutions/dash#168, see Dash::Commands::Loadbalancer#legacy_rename).
+    execute *DASH.loadbalancer.prepare_boot
+
     info "Stopping and removing #{DASH.loadbalancer.container_name} on #{host}, if running..."
     execute *DASH.loadbalancer.stop, raise_on_non_zero_exit: false
     execute *DASH.loadbalancer.remove_container
@@ -32,7 +40,6 @@ class Dash::Cli::Proxy::LoadbalancerReboot
       execute *DASH.loadbalancer.remove_proxy_secrets_file, raise_on_non_zero_exit: false
     end
 
-    execute *DASH.loadbalancer.ensure_apps_config_directory
     Dash::Cli::Proxy::LoadbalancerClaim.new(host, sshkit).claim_run_config(replace: true)
     execute *DASH.loadbalancer.run
 

data/lib/dash/cli/proxy/reboot.rb

@@ -37,7 +37,12 @@ class Dash::Cli::Proxy::Reboot
 
     def replace_container
       execute *proxy.ensure_proxy_directory
-      execute *proxy.ensure_apps_config_directory
+      # Before the new container - and the new config volume `docker run
+      # --volume` would otherwise create empty - exists: bring a host still on
+      # pre-rename identity across. Carries the apps-config mkdir this reboot
+      # paid a round trip for anyway, so the bridge costs none here either
+      # (zoolutions/dash#168, see Dash::Cli::Proxy::LegacyRename).
+      execute *proxy.prepare_boot
       sync_proxy_secrets
 
       if proxy.port_holder?

data/lib/dash/cli/proxy.rb

@@ -22,7 +22,9 @@ class Dash::Cli::Proxy < Dash::Cli::Base
         execute *DASH.registry.login
 
         # Before anything reads the new container, volume or network: bring a
-        # host still on pre-rename identity across. A no-op once it has been.
+        # host still on pre-rename identity across, and make the apps-config
+        # directory in the same round trip. Nothing but a `test -f` once it has
+        # been - see Dash::Cli::Proxy::LegacyRename.
         Dash::Cli::Proxy::LegacyRename.new(host, self).run
 
         proxy = DASH.proxy(host)
@@ -34,7 +36,8 @@ class Dash::Cli::Proxy < Dash::Cli::Base
         else
           stale_hosts << host.to_s if drift.drifted?
 
-          version = capture_with_info(*proxy.version).strip.presence
+          # The tag off the inspect the drift check already made, not a read of its own.
+          version = drift.version
 
           if version && Dash::Utils.older_version?(version, Dash::Configuration::Proxy::Run::MINIMUM_VERSION)
             raise "dash-proxy version #{version} is too old, run `dash proxy reboot` in order to update to at least #{Dash::Configuration::Proxy::Run::MINIMUM_VERSION}"
@@ -48,7 +51,6 @@ class Dash::Cli::Proxy < Dash::Cli::Base
             execute *proxy.remove_proxy_secrets_file, raise_on_non_zero_exit: false
           end
 
-          execute *proxy.ensure_apps_config_directory
           execute *proxy.start_holder_or_run if proxy.port_holder?
           execute *proxy.start_or_run(digest: drift.expected_digest)
         end
@@ -77,10 +79,10 @@ class Dash::Cli::Proxy < Dash::Cli::Base
           execute *DASH.registry.login
 
           # Bring a pre-rename host across before the container can be created
-          # against an empty volume or a network nothing else joined. A no-op
-          # once it has been.
-          execute *DASH.docker.connect_legacy_network_containers
-          execute *DASH.loadbalancer.copy_legacy_config_volume
+          # against an empty volume or a network nothing else joined, and make
+          # the apps-config directory in the same round trip. Nothing but a
+          # `test -f` once it has been.
+          execute *DASH.loadbalancer.prepare_boot
 
           # The load balancer terminates TLS and owns the cache, so its host
           # needs the proxy secrets (acme credentials, cache store) just like
@@ -92,8 +94,6 @@ class Dash::Cli::Proxy < Dash::Cli::Base
             execute *DASH.loadbalancer.remove_proxy_secrets_file, raise_on_non_zero_exit: false
           end
 
-          execute *DASH.loadbalancer.ensure_apps_config_directory
-
           # TLS terminates at the load balancer, so the TLS material the app
           # hosts get - custom certificates and the mTLS client CA - must
           # reach this host too; the LB container reads it through the same
@@ -105,10 +105,11 @@ class Dash::Cli::Proxy < Dash::Cli::Base
           # The same drift detection the proxy hosts get: a loadbalancer booted
           # with a different config digest reboots below (or warns, when
           # automatic reboot is off) instead of serving a stale config forever.
-          container_id = capture_with_info(*DASH.loadbalancer.container_id, raise_on_non_zero_exit: false).strip
-          current_digest = capture_with_info(*DASH.loadbalancer.config_digest, raise_on_non_zero_exit: false).strip
+          state = Dash::Commands::Proxy::State.parse(
+            capture_with_info(*DASH.loadbalancer.inspect_state, raise_on_non_zero_exit: false)
+          )
 
-          if container_id.present? && current_digest != DASH.loadbalancer_config.run_config_digest
+          if state.exists? && state.digest.to_s != DASH.loadbalancer_config.run_config_digest
             if auto_reboot
               # Leave the old loadbalancer serving until its reboot below.
               lb_drifted << host.to_s
@@ -413,6 +414,12 @@ class Dash::Cli::Proxy < Dash::Cli::Base
       if DASH.config.proxy.load_balancing?
         on(DASH.config.proxy.effective_loadbalancer) do |host|
           execute *DASH.registry.login
+          # start_or_run falls through to `docker run` on a host with no
+          # container, so this is a volume-creating path too and gets the same
+          # bridge boot and reboot do (zoolutions/dash#168). It also makes the
+          # apps-config directory the bind mount would otherwise have docker
+          # create root-owned.
+          execute *DASH.loadbalancer.prepare_boot
           execute *DASH.loadbalancer.start_or_run
         end
       else

data/lib/dash/commands/app.rb

@@ -3,10 +3,16 @@ class Dash::Commands::App < Dash::Commands::Base
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  # Separates the two answers #boot_state returns. A container id is hex and a version is
-  # a name suffix, so neither can produce this line on its own.
+  # Separates the two answers #boot_state and #stale_state return. A container id is hex
+  # and a version is a name suffix, so neither can produce this line on its own.
   BOOT_STATE_SEPARATOR = "--%--"
 
+  # The two halves of a #boot_state or #stale_state capture, raw. Callers decide what an
+  # empty half means; the separator line itself is dropped.
+  def self.split_state(output)
+    output.to_s.partition(/^#{Regexp.escape(BOOT_STATE_SEPARATOR)}$/).values_at(0, 2)
+  end
+
   attr_reader :role, :host
 
   delegate :container_name, to: :role
@@ -54,6 +60,31 @@ class Dash::Commands::App < Dash::Commands::Base
     docker :exec, container_name(version), *shell([ role.healthcheck.exec ])
   end
 
+  # Waits on the host for the container to reach a status the poller accepts, so a boot
+  # pays one round trip for the wait however long the container takes to come up - the
+  # client-side poll paid one per attempt. Prints the status it stopped on to stdout: the
+  # moment it sees one of READY_STATUSES, or the last one it saw when the deadline passes.
+  # Progress goes to stderr once a second in between. Waiting through every other status is
+  # deliberate: docker reports a container `unhealthy` after three failed probes, which for
+  # an app slower than that is a state it recovers from.
+  #
+  # Reaching the deadline exits 0, because it is an answer - the poller phrases it. Only a
+  # status that could not be read at all exits non-zero, which is a broken command and
+  # SSHKit's to raise, exactly as it was when the read was a round trip of its own.
+  def wait_for_ready(version:, timeout:)
+    shell [
+      "started=$(date +%s);",
+      "while true; do",
+      *readiness_probe(version: version),
+      "case \"$status\" in #{READY_STATUSES.join("|")}) echo \"$status\"; exit 0;; esac;",
+      "elapsed=$(( $(date +%s) - started ));",
+      "if [ \"$elapsed\" -ge #{timeout.to_i} ]; then echo \"$status\"; exit 0; fi;",
+      "echo \"#{READINESS_PROGRESS_PREFIX} $elapsed $(( #{timeout.to_i} - elapsed )) $status\" 1>&2;",
+      "sleep 1;",
+      "done"
+    ]
+  end
+
   def stop(version: nil)
     pipe \
       version ? container_id_for_version(version) : current_running_container_id,
@@ -93,6 +124,16 @@ class Dash::Commands::App < Dash::Commands::Base
       current_running_version
   end
 
+  # Everything the stale check needs from a host: every version of the role that has a
+  # container, and the version running now - the difference is what is stale. Same shape
+  # as #boot_state, same separator, same reason for `;` over `&&`.
+  def stale_state
+    chain \
+      list_versions,
+      [ :echo, BOOT_STATE_SEPARATOR ],
+      current_running_version
+  end
+
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *container_filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
@@ -104,6 +145,28 @@ class Dash::Commands::App < Dash::Commands::Base
   end
 
   private
+    # The same two readiness sources #status and #health_probe cover, read into `$status`
+    # so the loop around them is the same either way. They differ in what a non-zero exit
+    # means. A probe that exits non-zero IS the answer "not ready", so its output is
+    # discarded and the loop goes on; an inspect that produced no answer at all - docker is
+    # unreachable, or the container is gone - takes the whole command down with it, with
+    # docker's complaint on stderr for SSHKit to put in the exception.
+    #
+    # An empty status is checked as well as the exit code, because the exit code alone is
+    # not portable: the read is a pipeline, so its status is xargs', and a `docker container
+    # ls` that failed pipes nothing. GNU xargs then runs `docker inspect` with no container
+    # and exits 123, but BSD and BusyBox xargs skip the utility entirely and exit 0. Both
+    # leave `$status` empty, and empty is not something a working `docker inspect --format`
+    # can print.
+    def readiness_probe(version:)
+      if role.healthcheck&.exec?
+        [ "if", *health_probe(version: version), ">/dev/null 2>&1;", "then status=healthy;", "else status=\"#{EXEC_PROBE_FAILED}\";", "fi;" ]
+      else
+        [ "status=#{substitute(*status(version: version))} || exit $?;",
+          "if [ -z \"$status\" ]; then echo \"could not read the status of #{container_name(version)}\" 1>&2; exit 1; fi;" ]
+      end
+    end
+
     def latest_image_id
       docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
     end

data/lib/dash/commands/base.rb

@@ -10,6 +10,22 @@ module Dash::Commands
 
     DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}#{NO_HEALTHCHECK}:{{.State.Status}}{{end}}'"
 
+    # The statuses a boot accepts as ready. Dash::Cli::Healthcheck::Poller decides what a
+    # status means; Dash::Commands::App#wait_for_ready only decides when to stop looking,
+    # and it stops on exactly these. The two must agree: a status the host loop returned
+    # early for that the poller would not accept fails a boot the old client-side poll
+    # would have waited out.
+    READY_STATUSES = [ "healthy", "#{NO_HEALTHCHECK}:running" ].freeze
+
+    # What a `healthcheck: exec:` probe reports when it exits non-zero. Produced by the
+    # host-side wait, read back by the poller, so it is a wire format, not a message.
+    EXEC_PROBE_FAILED = "exec probe exited non-zero"
+
+    # The line #wait_for_ready prints to stderr on every attempt, read back by
+    # Dash::Cli::Healthcheck::ProgressReporter. stderr, because a capture returns stdout
+    # alone - which keeps the captured value the final status and nothing else.
+    READINESS_PROGRESS_PREFIX = "dash-readiness"
+
     attr_accessor :config
 
     def initialize(config)
@@ -24,6 +40,18 @@ module Dash::Commands
       docker :container, :ls, *("--all" unless only_running), "--filter", "'name=^#{container_name}$'", "--quiet"
     end
 
+    # True only when `list_command`'s own output is confirmed empty - never inferred from
+    # a failure. `docker container inspect name > /dev/null 2>&1` (negated) cannot tell
+    # "no such container" from "the daemon could not be asked" - both exit non-zero - so a
+    # transient failure there reads as confirmed absence. A `list` exits 0 whichever way
+    # the match went and non-zero only on a genuine failure, so `result=$(list) && [ -z
+    # "$result" ]` fails closed: `result=$(list)` carries list's own exit status (POSIX;
+    # verified against sh and bash), so a failed list stops the chain before the test runs.
+    # `list_command` must be a listing (docker container/volume ls), never an inspect.
+    def confirmed_empty?(list_command)
+      [ "result=$(#{list_command.join(" ")})", "&&", "[", "-z", "\"$result\"", "]" ]
+    end
+
     def make_directory_for(remote_file)
       make_directory Pathname.new(remote_file).dirname.to_s
     end
@@ -98,6 +126,13 @@ module Dash::Commands
         combine *commands, by: ";"
       end
 
+      # One subshell around an && chain. Composing two builders that each mix && and ||
+      # cannot be done flat - the operators share precedence and associate left, so the
+      # second builder's guards re-associate across the first one's.
+      def group(*commands)
+        [ "(", *combine(*commands), ")" ]
+      end
+
       def pipe(*commands)
         combine *commands, by: "|"
       end
@@ -134,13 +169,19 @@ module Dash::Commands
         any \
           volume_exists(volume),
           negate(volume_exists(legacy)),
-          [ "(", *combine(docker(:volume, :create, volume), copy_between_volumes(legacy, volume, image: image)), ")" ]
+          group(docker(:volume, :create, volume), copy_between_volumes(legacy, volume, image: image))
       end
 
       def negate(command)
         [ "!", *command ]
       end
 
+      # The docker builders (network create, the stage-3c network bridge) for callers that
+      # compose them into a command of their own rather than executing them on their own.
+      def docker_commands
+        @docker_commands ||= Dash::Commands::Docker.new(config)
+      end
+
       def volume_exists(name)
         docker :volume, :inspect, name, ">", "/dev/null", "2>&1"
       end

data/lib/dash/commands/loadbalancer.rb

@@ -47,6 +47,28 @@ class Dash::Commands::Loadbalancer < Dash::Commands::Base
     copy_legacy_volume(legacy: legacy_config_volume_name, volume: config_volume_name, image: loadbalancer_config.run.image)
   end
 
+  # Everything this host needs before anything reads its container, volume or network,
+  # in the one round trip it already pays for the apps-config directory. Same shape as
+  # Dash::Commands::Proxy#prepare_boot, including why the guard has to be the first word.
+  def prepare_boot
+    combine legacy_rename, ensure_apps_config_directory
+  end
+
+  # The loadbalancer's half of the stage-3c bridge - network, then volume - skipped
+  # outright by a host that has already been through it. No legacy container is replaced
+  # here (a dedicated loadbalancer host never ran one under a name this gem knows), so
+  # the marker is verified on the volume instead: the new one exists, or there was never
+  # a legacy one to adopt. Stage 3d deletes this with the rest of the bridge.
+  def legacy_rename
+    any \
+      [ :test, "-f", legacy_rename_marker ],
+      group(
+        group(docker_commands.connect_legacy_network_containers),
+        group(copy_legacy_config_volume),
+        group(any(mark_legacy_renamed, [ :true ]))
+      )
+  end
+
   def deploy(targets: [])
     docker :exec, container_name, "dash-proxy", "deploy", loadbalancer_config.config.service,
       *loadbalancer_config.deploy_command_args(targets: targets)
@@ -76,6 +98,12 @@ class Dash::Commands::Loadbalancer < Dash::Commands::Base
     docker :inspect, container_name, "--format", Dash::Commands::Proxy::CONFIG_DIGEST_FORMAT
   end
 
+  # One read for container id, image tag and config digest - parsed by
+  # Dash::Commands::Proxy::State, same as the per-host proxy's.
+  def inspect_state
+    docker :inspect, container_name, "--format", Dash::Commands::Proxy::STATE_FORMAT
+  end
+
   def container_id(only_running: false)
     container_id_for(container_name: container_name, only_running: only_running)
   end
@@ -163,6 +191,32 @@ class Dash::Commands::Loadbalancer < Dash::Commands::Base
   end
 
   private
+    # Stage 3c. 3d deletes both of these with the rest of the bridge.
+    def legacy_rename_marker
+      File.join loadbalancer_config.directory, Dash::Configuration::Proxy::LEGACY_RENAME_MARKER
+    end
+
+    # Verified on the volume existing, not on a container being gone - the loadbalancer
+    # replaces no legacy container, so this is the only signal its bridge has. That makes
+    # it foolable in one specific way: if `dash-loadbalancer-config` comes to exist before
+    # this bridge ever runs on a host, the marker is written despite the legacy volume's
+    # routing table and ACME cache never having been copied. The fix is upstream of the
+    # heuristic rather than in it - every path that can create the container, and with it
+    # the volume `docker run --volume` auto-creates empty, runs the bridge first:
+    # `boot`, both reboots (Dash::Cli::Proxy::Reboot, Dash::Cli::Proxy::LoadbalancerReboot)
+    # and `dash proxy loadbalancer start` (zoolutions/dash#168).
+    #
+    # If a host is somehow in that state anyway - an operator's own `docker run`, a
+    # volume created by hand - recovery is to copy the legacy volume's contents over,
+    # then remove .legacy-renamed under this host's loadbalancer directory so this
+    # re-evaluates.
+    def mark_legacy_renamed
+      combine \
+        group(any(volume_exists(config_volume_name), negate(volume_exists(legacy_config_volume_name)))),
+        make_directory(loadbalancer_config.directory),
+        [ :touch, legacy_rename_marker ]
+    end
+
     def run_args
       loadbalancer_config.run_args
     end

data/lib/dash/commands/proxy/state.rb

@@ -0,0 +1,31 @@
+# What one `docker inspect` of a proxy container tells a boot: whether it exists, which
+# image tag it runs, and the config digest it was booted with. Three questions that used
+# to cost three round trips each (`container_id`, `version`, `config_digest`).
+#
+# Produced by Dash::Commands::Proxy#inspect_state and its loadbalancer twin, both of which
+# are captured with raise_on_non_zero_exit: false - a host with no container inspects to
+# empty output, which parses to a state that simply does not exist.
+class Dash::Commands::Proxy::State
+  attr_reader :id, :image, :digest
+
+  def self.parse(output)
+    id, image, digest = output.to_s.strip.split(" ", 3)
+    new(id: id, image: image, digest: digest)
+  end
+
+  def initialize(id: nil, image: nil, digest: nil)
+    @id = id.presence
+    @image = image.presence
+    @digest = digest.presence
+  end
+
+  def exists?
+    id.present?
+  end
+
+  # The tag, read the way Dash::Commands::Proxy#version reads it - everything past the
+  # LAST colon, so a registry host carrying a port does not get mistaken for the version.
+  def version
+    image&.split(":")&.last
+  end
+end

data/lib/dash/commands/proxy.rb

@@ -12,8 +12,13 @@ class Dash::Commands::Proxy < Dash::Commands::Base
   # Both the legacy constant and the fallback go away in stage 3d.
   LEGACY_CONFIG_DIGEST_LABEL = "org.kamal.proxy-config-digest"
 
-  CONFIG_DIGEST_FORMAT = "'{{ with index .Config.Labels \"#{CONFIG_DIGEST_LABEL}\" }}{{ . }}" \
-    "{{ else }}{{ index .Config.Labels \"#{LEGACY_CONFIG_DIGEST_LABEL}\" }}{{ end }}'"
+  CONFIG_DIGEST_TEMPLATE = "{{ with index .Config.Labels \"#{CONFIG_DIGEST_LABEL}\" }}{{ . }}" \
+    "{{ else }}{{ index .Config.Labels \"#{LEGACY_CONFIG_DIGEST_LABEL}\" }}{{ end }}"
+
+  CONFIG_DIGEST_FORMAT = "'#{CONFIG_DIGEST_TEMPLATE}'"
+
+  # Everything Dash::Cli::Proxy::Drift and the minimum-version gate need, in one format.
+  STATE_FORMAT = "'{{.Id}} {{.Config.Image}} #{CONFIG_DIGEST_TEMPLATE}'"
 
   def initialize(config, host:)
     super(config)
@@ -42,6 +47,41 @@ class Dash::Commands::Proxy < Dash::Commands::Base
   # destination not existing, so a second deploy is a no-op. Stage 3d deletes
   # them along with the legacy constants they read.
 
+  # Everything a proxy host needs before anything reads its container, volume or
+  # network, in the one round trip it already pays for the apps-config directory.
+  #
+  # `a || b && c` is `(a || b) && c`, so the mkdir runs whichever way the guard went -
+  # and the guard has to be the first word rather than a parenthesised group, because
+  # SSHKit prefixes the first word with /usr/bin/env and passes only `test` through.
+  # Stage 3d drops the legacy_rename half and leaves the mkdir.
+  def prepare_boot
+    combine legacy_rename, ensure_apps_config_directory
+  end
+
+  # The whole stage-3c bridge as one command, skipped outright by a host that has
+  # already been through it - or was installed fresh on 4.x and never had a kamal-proxy.
+  # The three steps keep their own bodies and their documented order (see
+  # Dash::Cli::Proxy::LegacyRename); each is wrapped in its own subshell because they
+  # all mix && and || at one precedence level, and composing them flat would
+  # re-associate across the volume copy's guard - the chain 4.0.0 got wrong.
+  #
+  # The marker is written on verified absence of both legacy containers, never on the
+  # chain's exit status: the two removals end in `|| true`, so a host whose stop failed
+  # would otherwise record itself as migrated and never retry. Its own `|| true` keeps
+  # that failure as quiet as it is today, while a failed volume copy still exits
+  # non-zero through the && chain and aborts the boot exactly as it does now.
+  def legacy_rename
+    any \
+      [ :test, "-f", legacy_rename_marker ],
+      group(
+        group(docker_commands.connect_legacy_network_containers),
+        group(copy_legacy_config_volume),
+        group(remove_legacy_container(timeout: config.drain_timeout)),
+        group(remove_legacy_holder_container),
+        group(any(mark_legacy_renamed, [ :true ]))
+      )
+  end
+
   # Copies the pre-rename config volume into the new one, before anything
   # starts. The volume holds the routing table and the ACME account and
   # certificate cache; losing it means re-issuing every certificate and
@@ -61,6 +101,16 @@ class Dash::Commands::Proxy < Dash::Commands::Base
   # first deploy. The subshell groups create-and-copy because `&&` and `||`
   # share precedence and associate left — without it a host that already has
   # the new volume would still run the copy over live state.
+  #
+  # The source volume is still mounted by the legacy container while this runs -
+  # deliberately, on every path (see Dash::Cli::Proxy::LegacyRename's step order), and
+  # safe because every writer into it renames into place: the routing table through
+  # writeFileAtomic, the dynamic domain and redirect state through their own temp +
+  # rename, the response cache through CreateTemp + Rename, and the ACME cache through
+  # autocert.DirCache. `cp -a` reads a complete file either way. Skew across files is
+  # possible and harmless - an unused certificate, or a route whose certificate reissues -
+  # and --recheck-targets-on-restore re-verifies the targets on the way back up
+  # (zoolutions/dash#169 review).
   def copy_legacy_config_volume(volume: Dash::Configuration::Proxy::CONFIG_VOLUME, legacy: Dash::Configuration::Proxy::LEGACY_CONFIG_VOLUME)
     copy_legacy_volume(legacy: legacy, volume: volume, image: proxy_image)
   end
@@ -116,6 +166,13 @@ class Dash::Commands::Proxy < Dash::Commands::Base
     docker :inspect, container_name, "--format", CONFIG_DIGEST_FORMAT
   end
 
+  # One read for container id, image tag and config digest - parsed by
+  # Dash::Commands::Proxy::State. Capture it with raise_on_non_zero_exit: false;
+  # a host with no proxy container inspects to nothing, which is an answer.
+  def inspect_state
+    docker :inspect, container_name, "--format", STATE_FORMAT
+  end
+
   def container_id(only_running: false)
     container_id_for(container_name: container_name, only_running: only_running)
   end
@@ -327,6 +384,21 @@ class Dash::Commands::Proxy < Dash::Commands::Base
       docker :container, :inspect, name, ">", "/dev/null", "2>&1"
     end
 
+    # Stage 3c. 3d deletes both of these with the rest of the bridge.
+    def legacy_rename_marker
+      File.join config.proxy_boot.host_directory, Dash::Configuration::Proxy::LEGACY_RENAME_MARKER
+    end
+
+    # confirmed_empty?, not a negated inspect: a docker error while checking must never
+    # read as "confirmed gone" (zoolutions/dash#167 review).
+    def mark_legacy_renamed
+      combine \
+        confirmed_empty?(container_id_for(container_name: Dash::Configuration::Proxy::LEGACY_CONTAINER_NAME)),
+        confirmed_empty?(container_id_for(container_name: Dash::Configuration::Proxy::LEGACY_HOLDER_CONTAINER_NAME)),
+        make_directory(config.proxy_boot.host_directory),
+        [ :touch, legacy_rename_marker ]
+    end
+
     # The image the volume copy borrows. The proxy this gem is pinned to is
     # already pulled by the time the copy runs, and `rake release` gates on
     # MINIMUM_VERSION being published, so this is always resolvable — unlike

data/lib/dash/commands/registry.rb

@@ -13,6 +13,20 @@ class Dash::Commands::Registry < Dash::Commands::Base
       "-p", sensitive(Dash::Utils.escape_shell_value(registry_config.password))
   end
 
+  # The login and whatever has to happen after it on the same host, in one round trip.
+  # `docker login` prints "Login Succeeded" to stdout, so its output is redirected away:
+  # a caller that captures this gets the folded command's answer and nothing else. A local
+  # registry needs no login at all, and the fold collapses to the commands alone.
+  #
+  # The credentials stay wrapped in sensitive(...) - composing keeps the array elements
+  # intact, so SSHKit redacts them here exactly as it does for a standalone login.
+  def login_then(*commands, registry_config: nil)
+    login = login(registry_config: registry_config)
+    login = [ *login, ">", "/dev/null" ] if login
+
+    combine login, *commands
+  end
+
   def logout(registry_config: nil)
     registry_config ||= config.registry
 

data/lib/dash/configuration/docs/role.yml

@@ -107,22 +107,22 @@ servers:
     # A `healthcheck` cannot be combined with `health-*` keys under `options`.
     #
     # `exec` is the escape hatch for an image whose HEALTHCHECK you cannot change,
-    # or for an emergency override without a rebuild. Kamal `docker exec`s it from
-    # the deploy host on every poll and gates the deploy on the exit code — no HTTP
+    # or for an emergency override without a rebuild. Dash `docker exec`s it on the
+    # deploy host once a second and gates the deploy on the exit code — no HTTP
     # server and no published port needed, and unlike `cmd` it may use `${...}`,
     # which is quoted through to the container. It is strictly worse than `cmd` in
     # the general case, so reach for it only when `cmd` is not available:
     #
     #   - deploy-time only. Docker never runs it, so `docker ps` never shows
     #     `(healthy)` and `docker inspect` keeps no probe history.
-    #   - each poll costs an SSH round trip plus a process spawn (~100-300ms), which
-    #     rules out sub-second polling.
+    #   - each attempt costs a process spawn on the host (the whole wait is one SSH
+    #     round trip, so the cost does not grow with how long the boot takes).
     #   - nothing outside a deploy ever runs it.
     #
     # `exec` replaces docker's healthcheck rather than configuring it, so it cannot
-    # be combined with `cmd`, `port`, `path`, or any of the duration keys. Polling
-    # follows the deploy's own backoff and gives up at `deploy_timeout`; a probe that
-    # never exits zero fails the boot and leaves the old container running.
+    # be combined with `cmd`, `port`, `path`, or any of the duration keys. The wait
+    # gives up at `deploy_timeout`; a probe that never exits zero fails the boot and
+    # leaves the old container running.
     #
     # A non-proxied role with neither a `healthcheck` nor a `health-cmd` option
     # warns on every deploy, because the readiness delay is the only thing standing

data/lib/dash/configuration/proxy.rb

@@ -23,6 +23,9 @@ class Dash::Configuration::Proxy
   LEGACY_LOADBALANCER_CONTAINER_NAME = "kamal-loadbalancer"
   LEGACY_HOLDER_CONTAINER_NAME = "kamal-proxy-net"
   LEGACY_NETWORK = "kamal"
+  # Written into the run directory once a host is verifiably past the 3c rename, so the
+  # bridge costs it nothing but a `test -f` on every deploy after. Deleted in stage 3d.
+  LEGACY_RENAME_MARKER = ".legacy-renamed"
   LEGACY_CONFIG_VOLUME = "kamal-proxy-config"
   LEGACY_LOADBALANCER_CONFIG_VOLUME = "kamal-loadbalancer-config"
   LEGACY_IMAGE_TITLE = "kamal-proxy"

data/lib/dash/version.rb

@@ -1,3 +1,3 @@
 module Dash
-  VERSION = "4.1.0"
+  VERSION = "4.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dash
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.1.1
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -201,6 +201,7 @@ files:
 - lib/dash/cli/healthcheck/drift_error.rb
 - lib/dash/cli/healthcheck/error.rb
 - lib/dash/cli/healthcheck/poller.rb
+- lib/dash/cli/healthcheck/progress_reporter.rb
 - lib/dash/cli/lock.rb
 - lib/dash/cli/main.rb
 - lib/dash/cli/main/migrate.rb
@@ -259,6 +260,7 @@ files:
 - lib/dash/commands/lock.rb
 - lib/dash/commands/proxy.rb
 - lib/dash/commands/proxy/cert_transfer.rb
+- lib/dash/commands/proxy/state.rb
 - lib/dash/commands/prune.rb
 - lib/dash/commands/registry.rb
 - lib/dash/commands/server.rb