dash 2.12.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b02db379375fb5102b7464401e6d1d2ca56a0209df5965cd36b87face85a4cb7
-  data.tar.gz: 78e9cd7a40f9056abd03f39204e309a01bc4c884705c8a931d9aa5f622045863
+  metadata.gz: 6645870f4a0452741a5974d1d822e73fae0d4a04aec353196af3c02b3fb149d2
+  data.tar.gz: 23d1738ee101bf3eda8c03c363845dacc396403767271f7fbda7cd6656b9f0ff
 SHA512:
-  metadata.gz: 3a6c6bb668d700f596f330283d1c695b6d4120c653cc9bf8ff9be9f123f78b9d0725787d3dc276f8145c9f72cbee9786faac7fe68108337c3ccc622adb9c7ed5
-  data.tar.gz: 508b2b00b942d6f47abeb79f17a1d126b025e1b5aec08a65b9dc48088205b2967540f6877328067a64151a333a28c4d9b71cc3201fdad19555dd715d09e9e443
+  metadata.gz: 972ac9e339a4b56463c2f6caaf764a272cc01002101345e2696474fb01817d3a36dcb2c825b6f188521da82c764764323ff1b92b34ced280a330eb5a1d1bde0d
+  data.tar.gz: 7e698f1dbb093d79787f7c48aec3053742c90bfe08261b942d66a208b8051f53b8154969f796c82e29dac893b531dd9bd945a11015f598f4ac76577b271ce1fe

data/lib/kamal/cli/app/boot.rb

@@ -1,14 +1,16 @@
 class Kamal::Cli::App::Boot
-  attr_reader :host, :role, :version, :barrier, :sshkit
+  attr_reader :host, :role, :version, :barrier, :sshkit, :cli
   delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit
+  delegate :run_hook, to: :cli
   delegate :assets?, :running_proxy?, to: :role
 
-  def initialize(host, role, sshkit, version, barrier)
+  def initialize(host, role, sshkit, version, barrier, cli)
     @host = host
     @role = role
     @version = version
     @barrier = barrier
     @sshkit = sshkit
+    @cli = cli
   end
 
   def run
@@ -54,25 +56,65 @@ class Kamal::Cli::App::Boot
       if running_proxy?
         endpoint = capture_with_info(*app.container_id_for_version(version)).strip
         raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+
+        run_hook "pre-proxy-deploy", hosts: host.to_s, role: role.name
+        info "Deploying #{role} on #{host} via kamal-proxy (waiting up to #{KAMAL.config.deploy_timeout}s for it to become healthy)..."
         execute *app.deploy(target: endpoint)
+        run_hook "post-proxy-deploy", hosts: host.to_s, role: role.name
       else
-        Kamal::Cli::Healthcheck::Poller.wait_for_healthy { capture_with_info(*app.status(version: version)) }
+        Kamal::Cli::Healthcheck::Poller.wait_for_healthy(role: role) { health_status }
       end
     rescue => e
       error "Failed to boot #{role} on #{host}"
+      dump_diagnostics
       raise e
     end
 
+    # An exec probe is docker-invisible — the container declares no healthcheck, so
+    # `docker inspect` would only ever report its state. Poll the probe instead.
+    def health_status
+      role.healthcheck&.exec? ? exec_probe_status : capture_with_info(*app.status(version: version))
+    end
+
+    def exec_probe_status
+      execute *app.health_probe(version: version)
+      "healthy"
+    rescue SSHKit::Command::Failed
+      "exec probe exited non-zero"
+    end
+
+    # Every failed boot gets the container log, and the health probe history when the
+    # container declares a healthcheck — non-primary roles have no kamal-proxy report to fall back on.
+    def dump_diagnostics
+      error capture_with_info(*app.logs(container_id: app.container_id_for_version(version)))
+
+      health_log = capture_with_info(*app.container_health_log(version: version)).strip
+      error health_log unless health_log.empty? || health_log == "null"
+    rescue SSHKit::Command::Failed
+      error "Could not fetch logs for #{version}"
+    end
+
     def stop_new_version
       execute *app.stop(version: version), raise_on_non_zero_exit: false
     end
 
-    def stop_old_version(version)
-      execute *app.stop(version: version), raise_on_non_zero_exit: false
+    def stop_old_version(old_version)
+      run_stop_hook "pre-app-stop", old_version
+      execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+      run_stop_hook "post-app-stop", old_version
+
       execute *app.clean_up_assets if assets?
       execute *app.clean_up_error_pages if KAMAL.config.error_pages_path
     end
 
+    # The new version is already live by the time the old one is stopped, so a failing
+    # drain hook must not fail the deploy — warn and stop the old container anyway.
+    def run_stop_hook(hook, old_version)
+      run_hook hook, hosts: host.to_s, role: role.name, version: old_version
+    rescue Kamal::Cli::HookError => e
+      error "#{e.message}\nContinuing anyway: #{version} is already live for #{role} on #{host}."
+    end
+
     def release_barrier
       if barrier.open
         info "First #{KAMAL.primary_role} container is healthy on #{host}, booting any other roles"
@@ -91,12 +133,6 @@ class Kamal::Cli::App::Boot
     def close_barrier
       if barrier.close
         info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles"
-        begin
-          error capture_with_info(*app.logs(container_id: app.container_id_for_version(version)))
-          error capture_with_info(*app.container_health_log(version: version))
-        rescue SSHKit::Command::Failed
-          error "Could not fetch logs for #{version}"
-        end
       end
     end
 

data/lib/kamal/cli/app/rollout_boot.rb

@@ -0,0 +1,59 @@
+class Kamal::Cli::App::RolloutBoot
+  attr_reader :host, :role, :version, :sshkit
+  delegate :execute, :capture_with_info, :error, :upload!, to: :sshkit
+
+  def initialize(host, role, sshkit, version)
+    @host = host
+    @role = role
+    @version = version
+    @sshkit = sshkit
+  end
+
+  def run
+    ensure_not_already_deployed
+
+    begin
+      start_rollout_target
+    rescue
+      stop_rollout_target
+      raise
+    end
+  rescue Kamal::Cli::BootError => e
+    error e.message
+    raise
+  end
+
+  private
+    def ensure_not_already_deployed
+      if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+        raise Kamal::Cli::BootError, "Version #{version} is already deployed for #{role} on #{host}, roll out a different version"
+      end
+    end
+
+    def start_rollout_target
+      audit "Booted rollout target version #{version}"
+      hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
+
+      execute *app.ensure_env_directory
+      upload! role.secrets_io(host), role.secrets_path, mode: "0600"
+
+      execute *app.run(hostname: hostname)
+
+      endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+      raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+
+      execute *app.rollout_deploy(target: endpoint)
+    end
+
+    def stop_rollout_target
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+    end
+
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+
+    def audit(message)
+      execute *KAMAL.auditor(role: role).record(message), verbosity: :debug
+    end
+end

data/lib/kamal/cli/app/ssl_certificates.rb

@@ -9,9 +9,12 @@ class Kamal::Cli::App::SslCertificates
   end
 
   def run
-    if role.running_proxy? && role.proxy.custom_ssl_certificate?
-      info "Writing SSL certificates for #{role.name} on #{host}"
-      execute *app.create_ssl_directory
+    return unless role.running_proxy? && (role.proxy.custom_ssl_certificate? || role.proxy.client_ca?)
+
+    info "Writing SSL certificates for #{role.name} on #{host}"
+    execute *app.create_ssl_directory
+
+    if role.proxy.custom_ssl_certificate?
       if cert_content = role.proxy.certificate_pem_content
         upload!(StringIO.new(cert_content), role.proxy.host_tls_cert, mode: "0644")
       end
@@ -19,6 +22,12 @@ class Kamal::Cli::App::SslCertificates
         upload!(StringIO.new(key_content), role.proxy.host_tls_key, mode: "0644")
       end
     end
+
+    # The client CA arrives the same way the server certificate does: as
+    # secret content named in .kamal/secrets, resolved here at upload time.
+    if role.proxy.client_ca?
+      upload!(StringIO.new(role.proxy.client_ca_pem_content), role.proxy.host_client_ca, mode: "0644")
+    end
   end
 
   private

data/lib/kamal/cli/app.rb

@@ -18,17 +18,24 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
         # Primary hosts and roles are returned first, so they can open the barrier
         barrier = Kamal::Cli::Healthcheck::Barrier.new
+        cli = self
 
-        host_boot_groups.each do |hosts|
+        boot_groups = host_boot_groups
+
+        boot_groups.each_with_index do |hosts, index|
           host_list = Array(hosts).join(",")
           run_hook "pre-app-boot", hosts: host_list
 
-          on_roles(KAMAL.roles, hosts: hosts, parallel: KAMAL.config.boot.parallel_roles) do |host, role|
-            Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run
+          on_roles(KAMAL.roles, hosts: hosts, parallel: KAMAL.config.parallel_roles?, rolling: true) do |host, role|
+            Kamal::Cli::App::Boot.new(host, role, self, version, barrier, cli).run
           end
 
           run_hook "post-app-boot", hosts: host_list
-          sleep KAMAL.config.boot.wait if KAMAL.config.boot.wait
+
+          # `wait` paces one group against the next, so the last group has nothing to pace
+          # against. Without a `limit` there is only ever one group, and waiting at all
+          # would be a fixed delay on every deploy buying no staggering.
+          sleep KAMAL.config.boot.wait if KAMAL.config.boot.wait && index < boot_groups.size - 1
         end
 
         # Tag once the app booted on all hosts
@@ -43,7 +50,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "start", "Start existing app container on servers"
   def start
     modify(lock: true) do
-      on_roles(KAMAL.roles, hosts: KAMAL.app_hosts, parallel: KAMAL.config.boot.parallel_roles) do |host, role|
+      cli = self
+
+      on_roles(KAMAL.roles, hosts: KAMAL.app_hosts, parallel: KAMAL.config.parallel_roles?, rolling: true) do |host, role|
         app = KAMAL.app(role: role, host: host)
         execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
         execute *app.start, raise_on_non_zero_exit: false
@@ -53,7 +62,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
           endpoint = capture_with_info(*app.container_id_for_version(version)).strip
           raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
 
+          cli.run_hook "pre-proxy-deploy", hosts: host.to_s, role: role.name
           execute *app.deploy(target: endpoint)
+          cli.run_hook "post-proxy-deploy", hosts: host.to_s, role: role.name
         end
       end
     end
@@ -62,7 +73,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "stop", "Stop app container on servers"
   def stop
     modify(lock: true) do
-      on_roles(KAMAL.roles, hosts: KAMAL.app_hosts, parallel: KAMAL.config.boot.parallel_roles) do |host, role|
+      on_roles(KAMAL.roles, hosts: KAMAL.app_hosts, parallel: KAMAL.config.parallel_roles?) do |host, role|
         app = KAMAL.app(role: role, host: host)
         execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
 
@@ -272,6 +283,18 @@ class Kamal::Cli::App < Kamal::Cli::Base
     end
   end
 
+  desc "rollout <deploy|set|stop>", "Manage a canary rollout of a new version through kamal-proxy"
+  option :percent, type: :numeric, desc: "Percentage of traffic to send to the rollout target"
+  option :list, type: :array, desc: "Send requests whose kamal-rollout cookie matches these values to the rollout target"
+  def rollout(action)
+    case action
+    when "deploy" then start_rollout
+    when "set"    then set_rollout
+    when "stop"   then stop_rollout
+    else raise ArgumentError, "Unknown rollout action '#{action}', expected deploy, set or stop"
+    end
+  end
+
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
     modify(lock: true) do
@@ -323,6 +346,47 @@ class Kamal::Cli::App < Kamal::Cli::Base
   end
 
   private
+    def start_rollout
+      modify(lock: true) do
+        say "Get most recent version available as an image...", :magenta unless options[:version]
+        using_version(version_or_latest) do |version|
+          say "Start rollout target with version #{version} alongside the live version...", :magenta
+
+          on(KAMAL.proxy_hosts) do
+            KAMAL.roles_on(host).each do |role|
+              Kamal::Cli::App::Assets.new(host, role, self).run if role.running_proxy?
+            end
+          end
+
+          on_roles(KAMAL.roles, hosts: KAMAL.proxy_hosts, parallel: KAMAL.config.parallel_roles?, rolling: true) do |host, role|
+            Kamal::Cli::App::RolloutBoot.new(host, role, self, version).run if role.running_proxy?
+          end
+        end
+      end
+    end
+
+    def set_rollout
+      percent, list = options[:percent], options[:list]
+
+      if percent.nil? && list.blank?
+        raise ArgumentError, "No traffic split provided. You must specify --percent or --list."
+      end
+
+      modify(lock: true) do
+        on_roles(KAMAL.roles, hosts: KAMAL.proxy_hosts) do |host, role|
+          execute *KAMAL.app(role: role, host: host).rollout_set(percent: percent, list: list) if role.running_proxy?
+        end
+      end
+    end
+
+    def stop_rollout
+      modify(lock: true) do
+        on_roles(KAMAL.roles, hosts: KAMAL.proxy_hosts) do |host, role|
+          execute *KAMAL.app(role: role, host: host).rollout_stop if role.running_proxy?
+        end
+      end
+    end
+
     def hosts_removing_all_roles
       KAMAL.app_hosts.select { |host| KAMAL.roles_on(host).map(&:name).sort == KAMAL.config.host_roles(host.to_s).map(&:name).sort }
     end
@@ -363,6 +427,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
     end
 
     def host_boot_groups
-      KAMAL.config.boot.limit ? KAMAL.app_hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.app_hosts ]
+      hosts = KAMAL.app_hosts
+      limit = KAMAL.config.boot.limit_for(hosts)
+
+      limit ? hosts.each_slice(limit).to_a : [ hosts ]
     end
 end

data/lib/kamal/cli/base.rb

@@ -11,6 +11,10 @@ module Kamal::Cli
     class LockHeldError < StandardError; end
     class LockMissingError < StandardError; end
 
+    # Hooks set process-global state (the hook env and the SSHKit verbosity), so
+    # serialize them: some fire from inside SSHKit's per-host threads.
+    HOOK_MUTEX = Mutex.new
+
     def self.exit_on_failure?() true end
     def self.dynamic_command_class() Kamal::Cli::Alias::Command end
 
@@ -44,6 +48,43 @@ module Kamal::Cli
       initialize_commander unless KAMAL.configured?
     end
 
+    # Public so collaborators like Kamal::Cli::App::Boot can fire hooks, but not a Thor command.
+    no_commands do
+      def run_hook(hook, **extra_details)
+        if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
+          details = {
+            hosts: KAMAL.hosts.join(","),
+            roles: KAMAL.specific_roles&.join(","),
+            lock: KAMAL.holding_lock?.to_s,
+            command: command,
+            subcommand: subcommand
+          }.compact
+
+          hooks_output = KAMAL.config.hooks_output_for(hook)
+
+          # CLI flags override config: -q hides all, -v shows all
+          # Config setting :verbose forces output, :quiet forces silence
+          hook_verbosity = if KAMAL.verbosity == :info && hooks_output
+            VERBOSITY.fetch(hooks_output)
+          else
+            KAMAL.verbosity
+          end
+
+          HOOK_MUTEX.synchronize do
+            with_env KAMAL.hook.env(**details, **extra_details) do
+              KAMAL.with_verbosity(hook_verbosity) do
+                run_locally do
+                  execute *KAMAL.hook.run(hook)
+                end
+              end
+            rescue SSHKit::Command::Failed => e
+              raise HookError.new("Hook `#{hook}` failed:\n#{e.message}")
+            end
+          end
+        end
+      end
+    end
+
     private
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
@@ -221,38 +262,6 @@ module Kamal::Cli
         raise
       end
 
-      def run_hook(hook, **extra_details)
-        if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
-          details = {
-            hosts: KAMAL.hosts.join(","),
-            roles: KAMAL.specific_roles&.join(","),
-            lock: KAMAL.holding_lock?.to_s,
-            command: command,
-            subcommand: subcommand
-          }.compact
-
-          hooks_output = KAMAL.config.hooks_output_for(hook)
-
-          # CLI flags override config: -q hides all, -v shows all
-          # Config setting :verbose forces output, :quiet forces silence
-          hook_verbosity = if KAMAL.verbosity == :info && hooks_output
-            VERBOSITY.fetch(hooks_output)
-          else
-            KAMAL.verbosity
-          end
-
-          with_env KAMAL.hook.env(**details, **extra_details) do
-            KAMAL.with_verbosity(hook_verbosity) do
-              run_locally do
-                execute *KAMAL.hook.run(hook)
-              end
-            end
-          rescue SSHKit::Command::Failed => e
-            raise HookError.new("Hook `#{hook}` failed:\n#{e.message}")
-          end
-        end
-      end
-
       def on(*args, &block)
         pre_connect_if_required
 

data/lib/kamal/cli/doctor/config_checks.rb

@@ -0,0 +1,28 @@
+# Checks that only read the configuration — no SSH, no network. They report problems
+# that are visible in deploy.yml alone, so they still run when every host is unreachable.
+class Kamal::Cli::Doctor::ConfigChecks
+  def run
+    readiness_results
+  end
+
+  private
+    def result(check, target, status, detail)
+      Kamal::Cli::Doctor::Result.new(check, target, status, detail)
+    end
+
+    def readiness_results
+      KAMAL.roles.map { |role| readiness_check(role) }
+    end
+
+    def readiness_check(role)
+      if role.readiness_source != :none
+        result :readiness, role.name, :ok, role.readiness_description
+      elsif role.readiness_gated?
+        # readiness_gated? with no source left is `healthcheck: false` — the gap is deliberate.
+        result :readiness, role.name, :ok, "healthcheck: false — accepted #{role.readiness_delay}s after the container starts"
+      else
+        result :readiness, role.name, :warn, "no healthcheck — the old container stops #{role.readiness_delay}s after the new one starts; " \
+          "add a `healthcheck:` block, or opt out with `healthcheck: false`"
+      end
+    end
+end

data/lib/kamal/cli/doctor/endpoint_checks.rb

@@ -0,0 +1,114 @@
+require "resolv"
+require "socket"
+require "openssl"
+
+# Local (no SSH) readiness checks for the domains kamal-proxy will serve:
+# DNS resolution against the configured hosts and TLS certificate expiry.
+class Kamal::Cli::Doctor::EndpointChecks
+  CERTIFICATE_EXPIRY_WARN_DAYS = 14
+  SECONDS_PER_DAY = 86_400
+  TLS_CONNECT_TIMEOUT = 5
+
+  def run
+    dns_results + certificate_results
+  end
+
+  private
+    def result(check, target, status, detail)
+      Kamal::Cli::Doctor::Result.new(check, target, status, detail)
+    end
+
+    # Roles and accessories that run behind the proxy with custom domains.
+    def proxied_units
+      @proxied_units ||= (KAMAL.roles + KAMAL.config.proxy_accessories).select do |unit|
+        unit.running_proxy? && unit.proxy.hosts.any?
+      end
+    end
+
+    def dns_results
+      proxied_units.flat_map do |unit|
+        unit.proxy.hosts.map { |domain| dns_check(domain, unit.hosts) }
+      end
+    end
+
+    def certificate_results
+      proxied_units.flat_map do |unit|
+        next [] unless unit.proxy.ssl?
+
+        unit.proxy.hosts.map do |domain|
+          unit.proxy.custom_ssl_certificate? ? custom_certificate_check(unit.proxy, domain) : served_certificate_check(domain)
+        end
+      end
+    end
+
+    def dns_check(domain, unit_hosts)
+      resolved = Resolv.getaddresses(domain)
+      expected = expected_ips(unit_hosts)
+      matching = resolved & expected
+
+      if resolved.empty?
+        result :dns, domain, :fail, "does not resolve"
+      elsif matching.any?
+        result :dns, domain, :ok, "resolves to #{matching.join(", ")}"
+      else
+        result :dns, domain, :warn, "resolves to #{resolved.join(", ")}, expected one of #{expected.join(", ")} (fine when fronted by a CDN or load balancer)"
+      end
+    rescue StandardError => e
+      result :dns, domain, :fail, "could not resolve (#{e.message})"
+    end
+
+    def expected_ips(unit_hosts)
+      unit_hosts.flat_map do |unit_host|
+        if unit_host =~ Resolv::IPv4::Regex || unit_host =~ Resolv::IPv6::Regex
+          [ unit_host ]
+        else
+          Resolv.getaddresses(unit_host)
+        end
+      end.uniq
+    end
+
+    def custom_certificate_check(proxy, domain)
+      pem = proxy.certificate_pem_content
+      return result(:certificate, domain, :fail, "the certificate_pem secret is missing or empty") if pem.blank?
+
+      expiry_check domain, OpenSSL::X509::Certificate.new(pem), source: "configured certificate"
+    rescue OpenSSL::X509::CertificateError => e
+      result :certificate, domain, :fail, "certificate_pem is not a valid certificate (#{e.message})"
+    rescue StandardError => e
+      result :certificate, domain, :fail, "could not check the configured certificate (#{e.message})"
+    end
+
+    def served_certificate_check(domain)
+      expiry_check domain, peer_certificate(domain), source: "served certificate"
+    rescue StandardError => e
+      result :certificate, domain, :warn, "could not check TLS on #{domain}:443 (#{e.message}) - expected if the app isn't deployed yet"
+    end
+
+    def expiry_check(domain, certificate, source:)
+      days_left = ((certificate.not_after - Time.now) / SECONDS_PER_DAY).floor
+
+      if days_left.negative?
+        result :certificate, domain, :fail, "#{source} expired on #{certificate.not_after}"
+      elsif days_left < CERTIFICATE_EXPIRY_WARN_DAYS
+        result :certificate, domain, :warn, "#{source} expires in #{days_left} days (#{certificate.not_after})"
+      else
+        result :certificate, domain, :ok, "#{source} valid until #{certificate.not_after}"
+      end
+    end
+
+    def peer_certificate(domain, port = 443)
+      Socket.tcp(domain, port, connect_timeout: TLS_CONNECT_TIMEOUT) do |tcp|
+        context = OpenSSL::SSL::SSLContext.new
+        context.verify_mode = OpenSSL::SSL::VERIFY_NONE # only inspecting expiry, so self-signed is fine
+        ssl = OpenSSL::SSL::SSLSocket.new(tcp, context)
+        ssl.hostname = domain
+
+        begin
+          ssl.connect
+          ssl.peer_cert
+        ensure
+          ssl.close
+        end
+      end
+    end
+end