dash 2.12.0 → 3.0.0

data/lib/kamal/configuration/docs/role.yml

@@ -40,6 +40,37 @@ servers:
       - 172.1.0.4: experiment1
     cmd: "bin/jobs"
     stop_timeout: 30
+
+    # How long this role's container must keep running before the deploy accepts it,
+    # when there is no healthcheck to wait on. Overrides the root `readiness_delay`
+    # for this role only; roles with a healthcheck never wait on it.
+    readiness_delay: 4
+
+    # Rolling boot
+    #
+    # The root `boot` key paces the deploy across the whole cross-role host list, so it
+    # cannot serialize one role's hosts while another role keeps booting in parallel.
+    # Set `boot` on the role instead and its hosts are paced on their own — a second
+    # host in a worker role finally buys the redundancy it looks like it buys.
+    #
+    # `limit` is a count, or a percentage of the hosts this run is booting for the role
+    # (so `--roles` / `--hosts` narrow the denominator too); `limit: 1` boots them one at
+    # a time. `wait` is the pause between hosts or groups, and defaults to none. It needs
+    # `limit` to do anything — without one the role's hosts boot in a single group, leaving
+    # no gap to pause in, and Kamal warns. Nothing waits after the final group.
+    #
+    # A role using this key needs the role-first iteration of `boot/parallel_roles`.
+    # Kamal turns it on for you when you have not set it; an explicit
+    # `parallel_roles: false` is a conflict and raises instead.
+    boot:
+      limit: 25%
+      wait: 10
+
+    # Option values are shell-escaped before reaching `docker run`, but `${...}` is
+    # deliberately left for the deploy host's shell to expand. Role `env` is never
+    # defined in that shell, so `${MY_VAR}` resolves to empty. Use the bare `$MY_VAR`
+    # form, which is escaped and expands inside the container. Kamal rejects `${...}`
+    # in `health-*` options, where the empty value would only surface as a deploy timeout.
     options:
       memory: 2g
       cpus: 4
@@ -52,3 +83,58 @@ servers:
     env:
       ...
     asset_path: /public
+
+    # Healthcheck
+    #
+    # A role that does not run the proxy has no readiness gate of its own: the
+    # deploy only checks that the container is still running after
+    # `readiness_delay` seconds before stopping the old one. Declare a
+    # `healthcheck` and Kamal renders it into docker's own HEALTHCHECK flags, so
+    # the deploy waits for the container to report healthy instead.
+    #
+    # `port` is required unless you set `cmd`. The default HTTP check is
+    # `curl -f http://localhost:<port><path> || exit 1`, so the image needs curl.
+    #
+    # `cmd` runs inside the container via /bin/sh and can be anything that exits
+    # non-zero when unhealthy — no HTTP server and no published port needed. It
+    # may not contain `${...}`: that would expand on the deploy host, not in the
+    # container.
+    #
+    # Durations are seconds when given as a plain number, or any docker duration
+    # string ("90s", "1m30s"). `interval` defaults to 1 second so a deploy is not
+    # left waiting on docker's own 30 second default.
+    #
+    # A `healthcheck` cannot be combined with `health-*` keys under `options`.
+    #
+    # `exec` is the escape hatch for an image whose HEALTHCHECK you cannot change,
+    # or for an emergency override without a rebuild. Kamal `docker exec`s it from
+    # the deploy host on every poll and gates the deploy on the exit code — no HTTP
+    # server and no published port needed, and unlike `cmd` it may use `${...}`,
+    # which is quoted through to the container. It is strictly worse than `cmd` in
+    # the general case, so reach for it only when `cmd` is not available:
+    #
+    #   - deploy-time only. Docker never runs it, so `docker ps` never shows
+    #     `(healthy)` and `docker inspect` keeps no probe history.
+    #   - each poll costs an SSH round trip plus a process spawn (~100-300ms), which
+    #     rules out sub-second polling.
+    #   - nothing outside a deploy ever runs it.
+    #
+    # `exec` replaces docker's healthcheck rather than configuring it, so it cannot
+    # be combined with `cmd`, `port`, `path`, or any of the duration keys. Polling
+    # follows the deploy's own backoff and gives up at `deploy_timeout`; a probe that
+    # never exits zero fails the boot and leaves the old container running.
+    #
+    # A non-proxied role with neither a `healthcheck` nor a `health-cmd` option
+    # warns on every deploy, because the readiness delay is the only thing standing
+    # between a container that starts and a container that serves. Write
+    # `healthcheck: false` to accept that gap explicitly and silence the warning.
+    healthcheck:
+      port: 7434
+      path: /readyz
+      cmd: "pgrep -f bin/jobs"
+      exec: "bin/ready-check"
+      interval: 5
+      timeout: 3
+      retries: 3
+      start_period: 60
+      start_interval: 1

data/lib/kamal/configuration/loadbalancer.rb

@@ -10,19 +10,73 @@ class Kamal::Configuration::Loadbalancer < Kamal::Configuration::Proxy
     super
   end
 
-  def deploy_options
-    opts = super
-
-    opts[:host] = hosts if hosts.present?
-    opts[:tls] = proxy_config["ssl"].presence
-
-    opts
+  # The load balancer fans a single service out to many targets, so unlike the
+  # per-app proxy deploy (which takes one target) it takes the full list and
+  # joins them into a single --target flag, honouring app_port for each.
+  def deploy_command_args(targets:)
+    target_arg = targets.map { |target| "#{target}:#{app_port}" }.join(",")
+    optionize ({ target: target_arg }).merge(deploy_options), with: "="
   end
 
   def directory
     File.join config.run_directory, "loadbalancer"
   end
 
+  # The load balancer is a kamal-proxy container, so proxy/run applies to it
+  # exactly as it does to the per-host proxies. Without a run block it still
+  # needs the default surface (published ports, log rotation, the apps-config
+  # mount and the run command) to boot from, so fall back to an empty config
+  # rather than to nil.
+  def run
+    @run ||= Kamal::Configuration::Proxy::Run.new(config, run_config: {})
+  end
+
+  # Docker options for the load balancer container: publish/logging/options
+  # plus the mounts and env file the proxy's run surface brings along —
+  # notably the secrets env file, without which the edge cannot issue
+  # certificates even though it is the layer terminating TLS.
+  #
+  # The one holder-mode exception: port_holder suppresses publish_args on the
+  # proxy hosts (the holder owns the ports there), but the load balancer has
+  # no holder — its reboot path is stop->run — so it always publishes its own.
+  def run_args
+    [ *(run.publish_args if run.port_holder?), *run.docker_options_args ]
+  end
+
+  # Digest of what the load balancer container is booted with, so a second app
+  # pointing at the same host can tell whether its proxy/run agrees with
+  # whatever is already running there. Same composition as the proxy's own
+  # drift digest: image, run command, docker options and the acme credential
+  # names (their values deliberately stay out - see Proxy::Run#config_digest).
+  def run_config_digest
+    run.config_digest
+  end
+
+  # Kamal has no app identifier, so ownership of a service on a shared load
+  # balancer is expressed as repository plus destination-qualified service name:
+  # the repository separates two different apps, the destination separates two
+  # deployments of one app. The load balancer registers services under the bare
+  # service name, so both can collide.
+  def owner_token
+    [ config.service_and_destination, config.repository ].join(" ")
+  end
+
+  def run_config_record
+    [ owner_token, run_config_digest ].join(" ")
+  end
+
+  def run_config_file
+    File.join directory, "run_config"
+  end
+
+  def services_directory
+    File.join directory, "services"
+  end
+
+  def service_owner_file
+    File.join services_directory, config.service
+  end
+
   def container_name
     on_proxy_host? ? SHARED_CONTAINER_NAME : CONTAINER_NAME
   end
@@ -31,4 +85,13 @@ class Kamal::Configuration::Loadbalancer < Kamal::Configuration::Proxy
   def on_proxy_host?
     config.proxy_hosts.include?(config.proxy.effective_loadbalancer)
   end
+
+  private
+    # The edge half of the layering contract (see
+    # Kamal::Configuration::Proxy::DEPLOY_OPTION_DISPOSITIONS): the load
+    # balancer applies the edge and shared concerns, and leaves the per-app
+    # ones to the per-host proxies it forwards to.
+    def retained_dispositions
+      %i[ edge both ]
+    end
 end

data/lib/kamal/configuration/proxy/acme.rb

@@ -0,0 +1,69 @@
+# ACME configuration for the kamal-proxy container, including the DNS-01
+# challenge credentials.
+#
+# Everything here except the credentials becomes a `kamal-proxy run` flag, so it
+# lands in the run command and therefore in Proxy::Run#config_digest — changing
+# the block reboots the proxy on the next deploy.
+#
+# Credentials go the other way, through an env file uploaded at mode 0600. A DNS
+# API token can rewrite your zone; `docker run --env TOKEN=...` would put it in
+# the host's process listing and in kamal's own audit log.
+class Kamal::Configuration::Proxy::Acme
+  # The canonical provider names kamal-proxy MINIMUM_VERSION advertises. Kept in
+  # step with the proxy by test/proxy_flag_coverage_test.rb, which compares this
+  # list against the manifest bin/sync-proxy-flags generates from the image.
+  DNS_PROVIDERS = %w[
+    auto cloudflare digitalocean gcloud godaddy hetzner namecheap route53 vultr
+  ].freeze
+
+  # Short forms kamal-proxy's ParseProviderName accepts but does not advertise,
+  # so they cannot be generated from --help and are not drift-checked.
+  DNS_PROVIDER_ALIASES = %w[ cf do gcp google googledns gd hz nc aws r53 vr ].freeze
+
+  SUPPORTED_DNS_PROVIDERS = (DNS_PROVIDERS + DNS_PROVIDER_ALIASES).freeze
+
+  attr_reader :acme_config, :secrets
+
+  def initialize(acme_config:, secrets:)
+    @acme_config = acme_config || {}
+    @secrets = secrets
+  end
+
+  def configured?
+    acme_config.present?
+  end
+
+  def credential_names
+    Array(acme_config["credentials"])
+  end
+
+  def credentials?
+    credential_names.any?
+  end
+
+  # The resolved credentials, for the shared proxy secrets env file that
+  # Proxy::Run#secrets_io assembles (cache store and acme travel together).
+  def credentials_env
+    credential_names.to_h { |name| [ name, secrets[name] ] }
+  end
+
+  # Rendered with `=` rather than a space, unlike the rest of the run command:
+  # Cobra only reads a boolean flag's value in --flag=false form, so
+  # `--acme-http-fallback false` would set the flag true and leave "false" behind
+  # as a stray argument. Both booleans default to true in the proxy, which makes
+  # false the value an operator actually writes.
+  def run_command_args
+    Kamal::Utils.optionize(run_command_options, with: "=")
+  end
+
+  # compact, not compact_blank: false is a meaningful value for both booleans.
+  def run_command_options
+    {
+      "acme-email": acme_config["email"],
+      "acme-dns-provider": acme_config["dns_provider"],
+      "acme-directory": acme_config["directory"],
+      "acme-prefer-wildcard": acme_config["prefer_wildcard"],
+      "acme-http-fallback": acme_config["http_fallback"]
+    }.compact
+  end
+end

data/lib/kamal/configuration/proxy/run.rb

@@ -1,16 +1,49 @@
 class Kamal::Configuration::Proxy::Run
-  MINIMUM_VERSION = "v0.9.2.1"
+  MINIMUM_VERSION = "v1.0.0.1"
   DEFAULT_HTTP_PORT = 80
   DEFAULT_HTTPS_PORT = 443
   DEFAULT_LOG_MAX_SIZE = "10m"
 
+  # One env file for everything the proxy must know but nothing may print:
+  # ACME DNS credentials and the cache store URL. Also referenced by
+  # Kamal::Commands::Proxy#remove_proxy_secrets_file, which cleans it up when
+  # the config no longer needs it.
+  SECRETS_FILENAME = "secrets.env"
+
+  # Bump when the digest serialization changes, so every host converges with
+  # exactly one reboot after upgrading kamal.
+  DIGEST_SCHEMA_VERSION = "v1"
+
   attr_reader :config, :run_config
-  delegate :argumentize, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, :seconds_duration, to: Kamal::Utils
 
   def initialize(config, run_config:, context: "proxy/run")
     @config = config
     @run_config = run_config
     @context = context
+    ensure_no_conflicting_flags
+  end
+
+  def self.digest(*parts)
+    Digest::SHA256.hexdigest([ DIGEST_SCHEMA_VERSION, *parts ].join("\n"))
+  end
+
+  # Digest of the materialized run invocation, used to detect drift between
+  # the running proxy container and the current configuration.
+  #
+  # The secret *names* ride along because --env-file names a path, not the
+  # variables inside it: swapping one credential for another (or adding the
+  # cache store) would otherwise leave the digest unmoved and the old proxy
+  # running. The values deliberately stay out — the digest is published as a
+  # docker label, and hashing secret material into a world-readable label buys
+  # an offline guessing target for nothing. Rotating a credential's value or
+  # the store URL still needs an explicit `kamal proxy reboot`.
+  def config_digest
+    self.class.digest(image, run_command, *docker_options_args, *secret_names)
+  end
+
+  def acme
+    @acme ||= Kamal::Configuration::Proxy::Acme.new(acme_config: run_config["acme"], secrets: config.secrets)
   end
 
   def debug?
@@ -73,6 +106,29 @@ class Kamal::Configuration::Proxy::Run
     "kamal-proxy"
   end
 
+  # Zero-downtime reboots: a minimal long-lived holder container owns the
+  # published ports, and proxy generations join its network namespace so two
+  # can overlap on the same ports during a handoff.
+  def port_holder?
+    run_config.fetch("port_holder", false)
+  end
+
+  def holder_container_name
+    "kamal-proxy-net"
+  end
+
+  def network_args
+    if port_holder?
+      [ "--network", "container:#{holder_container_name}" ]
+    else
+      [ "--network", "kamal" ]
+    end
+  end
+
+  def holder_docker_args
+    [ *publish_args, *logging_args ].compact
+  end
+
   def options_args
     if args = run_config["options"]
       optionize args
@@ -80,7 +136,7 @@ class Kamal::Configuration::Proxy::Run
   end
 
   def run_command
-    [ "kamal-proxy", "run", *optionize(run_command_options) ].join(" ")
+    [ "kamal-proxy", "run", *optionize(run_command_options), *acme.run_command_args ].join(" ")
   end
 
   def metrics_port
@@ -88,19 +144,77 @@ class Kamal::Configuration::Proxy::Run
   end
 
   def run_command_options
-    { debug: debug? || nil, "metrics-port": metrics_port }.compact
+    named_run_command_options.merge(flags)
+  end
+
+  # Everything the gem has a key for, as opposed to whatever `flags` passes
+  # through. Kept separate so the two can be checked against each other.
+  def named_run_command_options
+    # recheck-targets-on-restore: after a reboot, re-verify restored targets
+    # with live health checks instead of trusting the saved state — a dead
+    # target demotes to 503 and self-heals rather than serving 502s forever.
+    # Available from MINIMUM_VERSION, so it is always safe to pass.
+    { debug: debug? || nil, "metrics-port": metrics_port, "recheck-targets-on-restore": true, "docker-socket": docker_socket }
+      .compact.merge(cache_options).merge(server_options)
+  end
+
+  # Anything kamal-proxy accepts that has no key of its own yet. Values are
+  # optionized as written - `true` is a bare flag, anything else takes a value -
+  # because the point of an escape hatch is that the gem holds no opinion about
+  # the flag it is forwarding.
+  #
+  # Note the asymmetry with `options`, which is a `docker run` passthrough. The
+  # names are deliberately different; most of why this gap went unnoticed is
+  # that `options` reads like it should do this.
+  def flags
+    (run_config["flags"] || {}).transform_keys(&:to_sym)
+  end
+
+  # What proxy/sleep needs in order to stop and start containers. Reaching this
+  # socket is root-equivalent on the host, so nothing turns it on implicitly -
+  # an operator has to name it, and naming it is also what mounts it (below).
+  def docker_socket
+    run_config["docker_socket"]
   end
 
   def docker_options_args
     [
       *apps_volume_args,
-      *publish_args,
+      *(publish_args unless port_holder?),
       *logging_args,
       *("--expose=#{metrics_port}" if metrics_port.present?),
+      *secrets_args,
+      *docker_socket_args,
       *options_args
     ].compact
   end
 
+  # The flag only tells kamal-proxy where to look inside its own container, so
+  # without this mount the socket is not there and a sleeping service never
+  # wakes - which the operator sees as one hung request, not as a misconfiguration.
+  def docker_socket_args
+    if docker_socket.present?
+      Kamal::Configuration::Volume.new(host_path: docker_socket, container_path: docker_socket).docker_args
+    end
+  end
+
+  # Where the proxy's secrets land on the host - the ACME DNS credentials and
+  # the cache store URL, which may embed one. Under the proxy's own directory
+  # rather than the app's env directory, because the container is host-scoped
+  # and shared by every app on the host - and so `kamal proxy remove` takes
+  # the secrets with it.
+  def secrets_path
+    File.join host_directory, SECRETS_FILENAME
+  end
+
+  def secrets?
+    acme.credentials? || cache_store.present?
+  end
+
+  def secrets_io
+    Kamal::EnvFile.new(acme.credentials_env.merge(cache_store_env)).to_io
+  end
+
   def host_directory
     File.join config.run_directory, "proxy"
   end
@@ -140,7 +254,82 @@ class Kamal::Configuration::Proxy::Run
     run_config.hash
   end
 
+  # Whether an operator turned on PROXY protocol without saying who may speak
+  # it. Kamal::Configuration warns about this: the header rewrites the
+  # connecting address that allow_ips and rate_limit key on, so honouring it
+  # from anywhere is a client-IP spoofing hole.
+  def proxy_protocol_unrestricted?
+    run_config["proxy_protocol"] && Array(run_config["proxy_protocol_allow_ips"]).empty?
+  end
+
   private
+    # The proxy's own listeners and process-level behaviour, as opposed to the
+    # per-service deadlines in Kamal::Configuration::Proxy#deploy_options. Zero
+    # disables each of the timeouts, so it is a value rather than an absence.
+    def server_options
+      {
+        "log-format": run_config["log_format"],
+        "trace-context": run_config["trace_context"],
+        "min-tls": run_config["min_tls"]&.to_s,
+        http3: run_config["http3"] ? true : nil,
+        # port_holder needs two generations bound to :80/:443 at once, so the
+        # holder topology forces the flag regardless of the standalone key.
+        "reuse-port": (run_config["reuse_port"] || port_holder?) ? true : nil,
+        "ignore-restore-errors": run_config["ignore_restore_errors"] ? true : nil,
+        "proxy-protocol": run_config["proxy_protocol"] ? true : nil,
+        "proxy-protocol-allow-ip": run_config["proxy_protocol_allow_ips"].presence,
+        "metrics-allow-ip": run_config["metrics_allow_ips"].presence,
+        "read-header-timeout": seconds_duration(run_config["read_header_timeout"]),
+        "read-timeout": seconds_duration(run_config["read_timeout"]),
+        "write-timeout": seconds_duration(run_config["write_timeout"]),
+        "idle-timeout": seconds_duration(run_config["idle_timeout"]),
+        "shutdown-timeout": seconds_duration(run_config["shutdown_timeout"])
+      }.compact
+    end
+
+    # A passthrough flag that a named key already emits would be optionized
+    # twice, and kamal-proxy would silently take the last one. Refuse instead.
+    # Only Proxy::Run knows which flags the named keys produce, so the check
+    # cannot live in the validator with the rest.
+    def ensure_no_conflicting_flags
+      conflicts = flags.keys.map(&:to_s) & named_run_command_options.keys.map(&:to_s)
+      return true if conflicts.empty?
+
+      raise Kamal::ConfigurationError,
+        "#{@context}/flags: #{conflicts.sort.join(", ")} is already set by a named key - remove one of them"
+    end
+
+    # Where cached responses are kept, which is a property of the proxy rather
+    # than of any one service - the policy that fills the cache is per service,
+    # in proxy/cache. The store itself is deliberately absent: a store URL may
+    # embed credentials, so it travels as CACHE_STORE in the secrets env file
+    # (kamal-proxy reads it as the --cache-store default) rather than on a
+    # command line that lands in process listings and the audit log.
+    def cache_options
+      cache = run_config["cache"] || {}
+
+      {
+        "cache-store-timeout": seconds_duration(cache["store_timeout"]),
+        "cache-memory-size": cache["memory_size"]
+      }.compact
+    end
+
+    def cache_store
+      run_config.dig("cache", "store")
+    end
+
+    def cache_store_env
+      cache_store.present? ? { "CACHE_STORE" => cache_store } : {}
+    end
+
+    def secret_names
+      [ *acme.credential_names, ("CACHE_STORE" if cache_store.present?) ].compact
+    end
+
+    def secrets_args
+      argumentize "--env-file", secrets_path if secrets?
+    end
+
     def format_bind_ip(ip)
       # Ensure IPv6 address inside square brackets - e.g. [::1]
       if ip =~ Resolv::IPv6::Regex && ip !~ /\A\[.*\]\z/