dash 2.12.0 → 3.0.0

data/lib/kamal/cli/proxy/reboot.rb

@@ -0,0 +1,172 @@
+class Kamal::Cli::Proxy::Reboot
+  READY_TIMEOUT = 30
+
+  attr_reader :host, :sshkit
+  delegate :execute, :capture_with_info, :info, :upload!, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def run
+    execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
+    execute *KAMAL.registry.login
+
+    pull_image
+    replace_container
+    wait_until_ready
+    verify_services re_register_services
+  end
+
+  private
+    def proxy
+      @proxy ||= KAMAL.proxy(host)
+    end
+
+    def drift
+      @drift ||= Kamal::Cli::Proxy::Drift.new(host, sshkit)
+    end
+
+    # Pull before stopping the old container: a registry outage or rate limit
+    # can then no longer strand the host with no proxy at all.
+    def pull_image
+      info "Pulling kamal-proxy image on #{host}..."
+      execute *proxy.pull
+    end
+
+    def replace_container
+      execute *proxy.ensure_proxy_directory
+      execute *proxy.ensure_apps_config_directory
+
+      if proxy.port_holder?
+        replace_generation
+      else
+        stop_and_replace
+      end
+    end
+
+    # Phase 1 replacement: stop the old container, start the new one. Brief gap.
+    def stop_and_replace
+      info "Stopping and removing kamal-proxy on #{host}, if running..."
+      execute *proxy.stop(timeout: KAMAL.config.drain_timeout + 10), raise_on_non_zero_exit: false
+      execute *proxy.remove_container
+
+      if (run_config = proxy.proxy_run_config)&.secrets?
+        upload! run_config.secrets_io, run_config.secrets_path, mode: "0600"
+      else
+        # A host keeps no secrets it no longer needs.
+        execute *proxy.remove_proxy_secrets_file, raise_on_non_zero_exit: false
+      end
+
+      execute *proxy.run(digest: drift.expected_digest)
+    end
+
+    # Zero-downtime replacement: overlap a new generation on the port-holder's
+    # namespace, then drain the old one out.
+    def replace_generation
+      if holder_running?
+        if generation_running?
+          handoff_generation
+        else
+          execute *proxy.remove_container
+          execute *proxy.run(digest: drift.expected_digest)
+        end
+      else
+        migrate_to_holder
+      end
+    end
+
+    def handoff_generation
+      info "Handing off kamal-proxy on #{host} to a new generation (zero downtime)..."
+      execute *proxy.remove_stopped_container(name: proxy.next_container_name), raise_on_non_zero_exit: false
+      execute *proxy.run(digest: drift.expected_digest, name: proxy.next_container_name)
+      wait_until_ready(name: proxy.next_container_name)
+
+      execute *proxy.drain(timeout: KAMAL.config.drain_timeout)
+      execute *proxy.wait_for_exit
+      execute *proxy.remove_stopped_container
+      execute *proxy.promote_next_container
+    end
+
+    # The old proxy still owns the published ports, so the holder cannot bind
+    # them until it is gone. One final replacement with a brief gap.
+    def migrate_to_holder
+      info "Migrating kamal-proxy on #{host} to the port-holder architecture (brief gap)..."
+      execute *proxy.stop(timeout: KAMAL.config.drain_timeout + 10), raise_on_non_zero_exit: false
+      execute *proxy.remove_container
+
+      execute *proxy.run_holder
+      execute *proxy.run(digest: drift.expected_digest)
+    end
+
+    def holder_running?
+      capture_with_info(*proxy.holder_container_id, raise_on_non_zero_exit: false).strip.present?
+    end
+
+    def generation_running?
+      capture_with_info(*proxy.container_id(only_running: true), raise_on_non_zero_exit: false).strip.present?
+    end
+
+    def wait_until_ready(name: nil)
+      deadline = Time.now + READY_TIMEOUT
+
+      begin
+        capture_with_info(*proxy.list(**{ name: name }.compact), verbosity: :debug)
+      rescue SSHKit::Command::Failed
+        raise Kamal::Cli::BootError, "kamal-proxy on #{host} did not become ready within #{READY_TIMEOUT} seconds" if Time.now >= deadline
+        sleep 0.5
+        retry
+      end
+    end
+
+    # Re-register this app's services rather than relying solely on the
+    # proxy's own state restore — a proxy that came back with missing routes
+    # would otherwise 404 until the next deploy. Returns the registered
+    # service names for verify_services to check.
+    def re_register_services
+      registered_services = []
+
+      KAMAL.roles_on(host).select(&:running_proxy?).each do |role|
+        app = KAMAL.app(role: role, host: host)
+
+        version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip.presence
+        next unless version
+
+        endpoint = capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).strip.presence
+        next unless endpoint
+
+        info "Re-registering #{role.container_prefix} with kamal-proxy on #{host}..."
+        execute *app.deploy(target: endpoint)
+        registered_services << role.container_prefix
+      end
+
+      KAMAL.config.accessories.select(&:running_proxy?).each do |accessory_config|
+        next unless accessory_config.hosts.include?(host.to_s)
+
+        accessory = KAMAL.accessory(accessory_config.name)
+        target = capture_with_info(*accessory.container_id_for(container_name: accessory_config.service_name, only_running: true), raise_on_non_zero_exit: false).strip.presence
+        next unless target
+
+        info "Re-registering #{accessory_config.service_name} with kamal-proxy on #{host}..."
+        execute *accessory.deploy(target: target)
+        registered_services << accessory_config.service_name
+      end
+
+      registered_services
+    end
+
+    # `list --json` returns {"services": {"<name>": ...}} - exact key
+    # membership, where a substring match over the human table would find one
+    # service's name inside another's ("app" in "other-app").
+    def verify_services(registered_services)
+      return if registered_services.empty?
+
+      listed = JSON.parse(capture_with_info(*proxy.list(json: true))).fetch("services", {}).keys
+      missing = registered_services - listed
+
+      if missing.any?
+        raise Kamal::Cli::BootError, "kamal-proxy on #{host} is missing services after reboot: #{missing.join(", ")}"
+      end
+    end
+end

data/lib/kamal/cli/proxy.rb

@@ -14,24 +14,116 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
         proxy_hosts = proxy_hosts - [ KAMAL.config.proxy.effective_loadbalancer ]
       end
 
+      drifted_hosts = Concurrent::Array.new
+      stale_hosts = Concurrent::Array.new
+      auto_reboot = KAMAL.config.proxy.reboot_on_deploy?
+
       on(proxy_hosts) do |host|
         execute *KAMAL.registry.login
 
-        version = capture_with_info(*KAMAL.proxy(host).version).strip.presence
+        proxy = KAMAL.proxy(host)
+        drift = Kamal::Cli::Proxy::Drift.new(host, self)
+
+        if drift.drifted? && auto_reboot
+          # Leave the old proxy serving until its serial reboot slot below.
+          drifted_hosts << host.to_s
+        else
+          stale_hosts << host.to_s if drift.drifted?
+
+          version = capture_with_info(*proxy.version).strip.presence
 
-        if version && Kamal::Utils.older_version?(version, Kamal::Configuration::Proxy::Run::MINIMUM_VERSION)
-          raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Run::MINIMUM_VERSION}"
+          if version && Kamal::Utils.older_version?(version, Kamal::Configuration::Proxy::Run::MINIMUM_VERSION)
+            raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Run::MINIMUM_VERSION}"
+          end
+
+          if (run_config = proxy.proxy_run_config)&.secrets?
+            execute *proxy.ensure_proxy_directory
+            upload! run_config.secrets_io, run_config.secrets_path, mode: "0600"
+          else
+            # A host keeps no secrets it no longer needs.
+            execute *proxy.remove_proxy_secrets_file, raise_on_non_zero_exit: false
+          end
+
+          execute *proxy.ensure_apps_config_directory
+          execute *proxy.start_holder_or_run if proxy.port_holder?
+          execute *proxy.start_or_run(digest: drift.expected_digest)
         end
-        execute *KAMAL.proxy(host).ensure_apps_config_directory
-        execute *KAMAL.proxy(host).start_or_run
+      end
+
+      if stale_hosts.any?
+        say "kamal-proxy on #{stale_hosts.sort.join(", ")} is running with a configuration that no longer matches the deploy config. " \
+            "Automatic reboot is disabled (proxy: reboot_on_deploy: false) - run `kamal proxy reboot` to apply the new configuration.", :yellow
+      end
+
+      drifted_hosts.sort.each do |host|
+        say "kamal-proxy configuration changed, rebooting on #{host}...", :magenta
+        run_hook "pre-proxy-reboot", hosts: host
+        on(host) do |h|
+          Kamal::Cli::Proxy::Reboot.new(h, self).run
+        end
+        run_hook "post-proxy-reboot", hosts: host
       end
 
       if KAMAL.config.proxy.load_balancing?
+        lb_drifted = Concurrent::Array.new
+        lb_stale = Concurrent::Array.new
+
         on(KAMAL.config.proxy.effective_loadbalancer) do |host|
           info "Starting loadbalancer on #{host}..."
           execute *KAMAL.registry.login
+
+          # The load balancer terminates TLS and owns the cache, so its host
+          # needs the proxy secrets (acme credentials, cache store) just like
+          # the proxy hosts do.
+          if (lb_run = KAMAL.loadbalancer_config.run).secrets?
+            execute *KAMAL.loadbalancer.ensure_proxy_directory
+            upload! lb_run.secrets_io, lb_run.secrets_path, mode: "0600"
+          else
+            execute *KAMAL.loadbalancer.remove_proxy_secrets_file, raise_on_non_zero_exit: false
+          end
+
           execute *KAMAL.loadbalancer.ensure_apps_config_directory
-          execute *KAMAL.loadbalancer.start_or_run
+
+          # TLS terminates at the load balancer, so the TLS material the app
+          # hosts get - custom certificates and the mTLS client CA - must
+          # reach this host too; the LB container reads it through the same
+          # apps-config mount the per-host proxies use.
+          KAMAL.config.roles.select(&:running_proxy?).each do |role|
+            Kamal::Cli::App::SslCertificates.new(host, role, self).run
+          end
+
+          # The same drift detection the proxy hosts get: a loadbalancer booted
+          # with a different config digest reboots below (or warns, when
+          # automatic reboot is off) instead of serving a stale config forever.
+          container_id = capture_with_info(*KAMAL.loadbalancer.container_id, raise_on_non_zero_exit: false).strip
+          current_digest = capture_with_info(*KAMAL.loadbalancer.config_digest, raise_on_non_zero_exit: false).strip
+
+          if container_id.present? && current_digest != KAMAL.loadbalancer_config.run_config_digest
+            if auto_reboot
+              # Leave the old loadbalancer serving until its reboot below.
+              lb_drifted << host.to_s
+            else
+              lb_stale << host.to_s
+              execute *KAMAL.loadbalancer.start_or_run
+            end
+          else
+            Kamal::Cli::Proxy::LoadbalancerClaim.new(host, self).claim_run_config
+            execute *KAMAL.loadbalancer.start_or_run
+          end
+        end
+
+        if lb_stale.any?
+          say "The loadbalancer on #{lb_stale.sort.join(", ")} is running with a configuration that no longer matches the deploy config. " \
+              "Automatic reboot is disabled (proxy: reboot_on_deploy: false) - run `kamal proxy reboot` to apply the new configuration.", :yellow
+        end
+
+        lb_drifted.sort.each do |host|
+          say "Loadbalancer configuration changed, rebooting on #{host}...", :magenta
+          run_hook "pre-loadbalancer-reboot", hosts: host
+          on(host) do |h|
+            Kamal::Cli::Proxy::LoadbalancerReboot.new(h, self).run
+          end
+          run_hook "post-loadbalancer-reboot", hosts: host
         end
       end
     end
@@ -137,16 +229,8 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
           host_list = Array(hosts).join(",")
           run_hook "pre-proxy-reboot", hosts: host_list
           on(hosts) do |host|
-            proxy = KAMAL.proxy(host)
-            execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
-            execute *KAMAL.registry.login
-
-            info "Stopping and removing kamal-proxy on #{host}, if running..."
-            execute *proxy.stop, raise_on_non_zero_exit: false
-            execute *proxy.remove_container
-            execute *proxy.ensure_apps_config_directory
-
-            execute *proxy.run
+            info "Rebooting kamal-proxy on #{host}..."
+            Kamal::Cli::Proxy::Reboot.new(host, self).run
           end
           run_hook "post-proxy-reboot", hosts: host_list
         end
@@ -156,15 +240,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
           run_hook "pre-loadbalancer-reboot", hosts: lb_host
 
           on(lb_host) do |host|
-            execute *KAMAL.auditor.record("Rebooted loadbalancer"), verbosity: :debug
-            execute *KAMAL.registry.login
-
-            info "Stopping and removing #{KAMAL.loadbalancer.container_name} on #{host}, if running..."
-            execute *KAMAL.loadbalancer.stop, raise_on_non_zero_exit: false
-            execute *KAMAL.loadbalancer.remove_container
-            execute *KAMAL.loadbalancer.ensure_apps_config_directory
-
-            execute *KAMAL.loadbalancer.run
+            Kamal::Cli::Proxy::LoadbalancerReboot.new(host, self).run
           end
 
           run_hook "post-loadbalancer-reboot", hosts: lb_host
@@ -221,6 +297,13 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
         execute *KAMAL.auditor.record("Started proxy"), verbosity: :debug
         execute *KAMAL.proxy(host).start
       end
+
+      if KAMAL.config.proxy.load_balancing?
+        on(KAMAL.config.proxy.effective_loadbalancer) do
+          execute *KAMAL.auditor.record("Started loadbalancer"), verbosity: :debug
+          execute *KAMAL.loadbalancer.start, raise_on_non_zero_exit: false
+        end
+      end
     end
   end
 
@@ -231,6 +314,16 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
         execute *KAMAL.auditor.record("Stopped proxy"), verbosity: :debug
         execute *KAMAL.proxy(host).stop, raise_on_non_zero_exit: false
       end
+
+      # `docker container prune` only collects stopped containers, so leaving the
+      # loadbalancer running would also leave `kamal proxy remove` unable to
+      # remove it.
+      if KAMAL.config.proxy.load_balancing?
+        on(KAMAL.config.proxy.effective_loadbalancer) do
+          execute *KAMAL.auditor.record("Stopped loadbalancer"), verbosity: :debug
+          execute *KAMAL.loadbalancer.stop, raise_on_non_zero_exit: false
+        end
+      end
     end
   end
 
@@ -343,6 +436,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
         end
 
         on(KAMAL.config.proxy.effective_loadbalancer) do |host|
+          Kamal::Cli::Proxy::LoadbalancerClaim.new(host, self).claim_service
           info "Deploying to loadbalancer on #{host} with targets: #{targets.join(', ')}"
           execute *KAMAL.loadbalancer.deploy(targets: targets)
         end
@@ -354,6 +448,69 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
     end
   end
 
+  desc "cache SUBCOMMAND", "Manage the response cache (stats, purge)"
+  option :count, type: :boolean, default: false, desc: "stats: measure entries and bytes per service (walks a shared store's keyspace)"
+  option :json, type: :boolean, default: false, desc: "stats: print the raw report as JSON"
+  option :path_prefix, type: :string, default: nil, desc: "purge: drop only the entries below this path, e.g. /assets"
+  def cache(subcommand)
+    count, json, path_prefix = options[:count], options[:json], options[:path_prefix]
+
+    case subcommand
+    when "stats"
+      # The cache lives where its policy applies: at the loadbalancer when
+      # load balancing (cache is edge-only in the layering contract),
+      # otherwise on each proxy host.
+      if KAMAL.config.proxy.load_balancing?
+        on(KAMAL.config.proxy.effective_loadbalancer) do |host|
+          puts_by_host host, capture_with_info(*KAMAL.loadbalancer.cache_stats(count: count, json: json)), type: "Loadbalancer"
+        end
+      else
+        on(KAMAL.proxy_hosts) do |host|
+          puts_by_host host, capture_with_info(*KAMAL.proxy(host).cache_stats(count: count, json: json)), type: "Proxy"
+        end
+      end
+    when "purge"
+      if KAMAL.config.proxy.load_balancing?
+        # The loadbalancer registers one service under the bare service name.
+        on(KAMAL.config.proxy.effective_loadbalancer) do |host|
+          execute *KAMAL.auditor.record("Purged the response cache"), verbosity: :debug
+          puts_by_host host, capture_with_info(*KAMAL.loadbalancer.cache_purge(KAMAL.config.service, path_prefix: path_prefix)), type: "Loadbalancer"
+        end
+      else
+        # Each proxied role registered its own service, so purge walks them.
+        on(KAMAL.proxy_hosts) do |host|
+          execute *KAMAL.auditor.record("Purged the response cache"), verbosity: :debug
+
+          KAMAL.roles_on(host).select(&:running_proxy?).each do |role|
+            puts_by_host host, capture_with_info(*KAMAL.proxy(host).cache_purge(role.container_prefix, path_prefix: path_prefix)), type: "Proxy"
+          end
+        end
+      end
+    else
+      puts "Unknown cache subcommand: #{subcommand}. Available: stats, purge"
+    end
+  end
+
+  desc "domains SUBCOMMAND", "Manage dynamic TLS domains (refresh, list, stats)"
+  def domains(subcommand)
+    case subcommand
+    when "refresh", "list", "stats"
+      # TLS terminates at the load balancer when load balancing, so the
+      # dynamic domain set lives there rather than on the per-host proxies.
+      if KAMAL.config.proxy.load_balancing?
+        on(KAMAL.config.proxy.effective_loadbalancer) do |host|
+          puts_by_host host, capture_with_info(*KAMAL.loadbalancer.domains(subcommand)), type: "Loadbalancer"
+        end
+      else
+        on(KAMAL.proxy_hosts) do |host|
+          puts_by_host host, capture_with_info(*KAMAL.proxy(host).domains(subcommand)), type: "Proxy"
+        end
+      end
+    else
+      puts "Unknown domains subcommand: #{subcommand}. Available: refresh, list, stats"
+    end
+  end
+
   desc "remove_container", "Remove proxy container from servers", hide: true
   def remove_container
     modify(lock: true) do
@@ -394,12 +551,29 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
       on(KAMAL.proxy_hosts) do
         execute *KAMAL.proxy(host).remove_proxy_directory, raise_on_non_zero_exit: false
       end
+
+      # Otherwise the ownership registry outlives the load balancer and a later
+      # boot would fail against an owner that no longer exists.
+      if KAMAL.config.proxy.load_balancing?
+        on(KAMAL.config.proxy.effective_loadbalancer) do
+          execute *KAMAL.loadbalancer.remove_directory, raise_on_non_zero_exit: false
+        end
+      end
     end
   end
 
   private
+    # A shared load balancer tier is the exact case this guard exists for, so it
+    # has to cover the load balancer host too - `remove_container` and
+    # `remove_image` both act on it.
+    def removal_hosts
+      hosts = KAMAL.proxy_hosts.to_a
+      hosts |= [ KAMAL.config.proxy.effective_loadbalancer ] if KAMAL.config.proxy.load_balancing?
+      hosts
+    end
+
     def removal_allowed?(force)
-      on(KAMAL.proxy_hosts) do |host|
+      on(removal_hosts) do |host|
         app_count = capture_with_info(*KAMAL.server.app_directory_count).chomp.to_i
         raise "The are other applications installed on #{host}" if app_count > 0
       end

data/lib/kamal/cli/prune.rb

@@ -18,16 +18,19 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "containers", "Prune all stopped containers, except the last n (default 5)"
-  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain"
+  desc "containers", "Prune all stopped containers, except the last n per role (default 5)"
+  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain per role"
   def containers
     retain = options.fetch(:retain, KAMAL.config.retain_containers)
     raise "retain must be at least 1" if retain < 1
 
     modify(lock: true) do
-      on(KAMAL.hosts) do
+      on(KAMAL.hosts) do |host|
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
-        execute *KAMAL.prune.app_containers(retain: retain)
+
+        KAMAL.roles_on(host).each do |role|
+          execute *KAMAL.prune.app_containers(retain: retain, role: role)
+        end
       end
     end
   end

data/lib/kamal/cli/templates/sample_hooks/post-app-stop.sample

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Runs on each host just after the previous container has been stopped.
+#
+# NOTE: $KAMAL_VERSION is the version that was STOPPED, not the one being deployed.
+#
+# A failure here is logged and the deploy continues.
+
+echo "Stopped $KAMAL_ROLE version $KAMAL_VERSION on $KAMAL_HOSTS"

data/lib/kamal/cli/templates/sample_hooks/post-proxy-deploy.sample

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+echo "Routed traffic for $KAMAL_ROLE version $KAMAL_VERSION on $KAMAL_HOSTS"

data/lib/kamal/cli/templates/sample_hooks/pre-app-stop.sample

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+
+# Runs on each host just before the previous container is stopped, once the new
+# version is already live. Use it to drain the old container: unsubscribe a queue
+# group, deregister from service discovery, wait for in-flight jobs.
+#
+# NOTE: $KAMAL_VERSION is the version being STOPPED, not the one being deployed.
+#
+# A failure here is logged and the deploy continues -- the old container is
+# stopped either way.
+
+echo "Draining $KAMAL_ROLE version $KAMAL_VERSION on $KAMAL_HOSTS..."

data/lib/kamal/cli/templates/sample_hooks/pre-proxy-deploy.sample

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+echo "Routing traffic for $KAMAL_ROLE version $KAMAL_VERSION on $KAMAL_HOSTS..."

data/lib/kamal/cli.rb

@@ -3,6 +3,7 @@ module Kamal::Cli
   class HookError < StandardError; end
   class LockError < StandardError; end
   class DependencyError < StandardError; end
+  class DoctorError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

data/lib/kamal/commands/app/proxy.rb

@@ -9,6 +9,18 @@ module Kamal::Commands::App::Proxy
     proxy_exec :remove, role.container_prefix
   end
 
+  def rollout_deploy(target:)
+    proxy_exec :rollout, :deploy, role.container_prefix, *role.proxy.rollout_deploy_command_args(target: target)
+  end
+
+  def rollout_set(percent: nil, list: nil)
+    proxy_exec :rollout, :set, role.container_prefix, *role.proxy.rollout_set_command_args(percent: percent, list: list)
+  end
+
+  def rollout_stop
+    proxy_exec :rollout, :stop, role.container_prefix
+  end
+
   def live
     proxy_exec :resume, role.container_prefix
   end

data/lib/kamal/commands/app.rb

@@ -30,6 +30,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
       *role.asset_volume_args,
       *role.label_args,
       *role.option_args,
+      *role.healthcheck_args,
       config.absolute_image,
       role.cmd
   end
@@ -42,6 +43,13 @@ class Kamal::Commands::App < Kamal::Commands::Base
     pipe container_id_for_version(version), xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
   end
 
+  # The `healthcheck: exec:` probe, run from the deploy host against the container that is
+  # booting. Single-quoted by #shell, so the command expands inside the container rather
+  # than in the deploy host's shell. Non-zero exit means not ready.
+  def health_probe(version:)
+    docker :exec, container_name(version), *shell([ role.healthcheck.exec ])
+  end
+
   def stop(version: nil)
     pipe \
       version ? container_id_for_version(version) : current_running_container_id,

data/lib/kamal/commands/base.rb

@@ -2,7 +2,13 @@ module Kamal::Commands
   class Base
     delegate :sensitive, :argumentize, to: Kamal::Utils
 
-    DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
+    # Prefixed onto the docker state when the container declares no healthcheck at all, so
+    # "nothing is probing this container" stays distinguishable from "the probe passed" —
+    # both used to reach the poller as `running`. The state is kept after the prefix so a
+    # healthcheck-less container that died still reports why.
+    NO_HEALTHCHECK = "no-healthcheck"
+
+    DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}#{NO_HEALTHCHECK}:{{.State.Status}}{{end}}'"
 
     attr_accessor :config
 
@@ -34,6 +40,10 @@ module Kamal::Commands
       [ :rm, path ]
     end
 
+    def read_file(file, default: nil)
+      combine [ :cat, file, "2>", "/dev/null" ], [ :echo, "\"#{default}\"" ], by: "||"
+    end
+
     def ensure_docker_installed
       combine \
         ensure_local_docker_installed,

data/lib/kamal/commands/docker.rb

@@ -35,6 +35,11 @@ class Kamal::Commands::Docker < Kamal::Commands::Base
     [ "kill -HUP $PPID" ]
   end
 
+  # Checks that an image's manifest can be fetched from its registry. Fails if it's missing or we're unauthorized.
+  def manifest_available?(image)
+    docker :manifest, :inspect, image
+  end
+
   def create_network
     docker :network, :create, :kamal
   end