dash 4.1.3 → 4.2.0

data/lib/dash/commands/app.rb

@@ -5,7 +5,11 @@ class Dash::Commands::App < Dash::Commands::Base
 
   # Separates the two answers #boot_state and #stale_state return. A container id is hex
   # and a version is a name suffix, so neither can produce this line on its own.
-  BOOT_STATE_SEPARATOR = "--%--"
+  BOOT_STATE_SEPARATOR = SECTION_SEPARATOR
+
+  # Printed in place of the running containers when docker could not list them, so a boot
+  # can tell "nothing runs" from "could not ask" - the chain's `;` hides the exit status.
+  ACTIVE_CONTAINERS_UNREADABLE = "--unreadable--"
 
   # The two halves of a #boot_state or #stale_state capture, raw. Callers decide what an
   # empty half means; the separator line itself is dropped.
@@ -13,14 +17,30 @@ class Dash::Commands::App < Dash::Commands::Base
     output.to_s.partition(/^#{Regexp.escape(BOOT_STATE_SEPARATOR)}$/).values_at(0, 2)
   end
 
-  attr_reader :role, :host
+  # Every segment of a #boot_states capture, raw, in the order they were asked.
+  def self.split_states(output)
+    output.to_s.split(/^#{Regexp.escape(BOOT_STATE_SEPARATOR)}$/, -1)
+  end
 
-  delegate :container_name, to: :role
+  attr_reader :role, :host, :replica
 
-  def initialize(config, role: nil, host: nil)
+  # `replica` is the slot this instance reads and writes. Slot 1 is the container the role
+  # always had, so a role without `replicas:` is only ever slot 1 and its commands are
+  # exactly what they were before replicas existed.
+  def initialize(config, role: nil, host: nil, replica: 1)
     super(config)
     @role = role
     @host = host
+    @replica = replica
+  end
+
+  def container_name(version = nil)
+    role.replica_name(replica, version)
+  end
+
+  # The same role and host, another slot.
+  def for_replica(replica)
+    self.class.new(config, role: role, host: host, replica: replica)
   end
 
   def run(hostname: nil)
@@ -34,11 +54,13 @@ class Dash::Commands::App < Dash::Commands::Base
       "--env", "KAMAL_VERSION=\"#{config.version}\"",
       "--env", "KAMAL_HOST=\"#{host}\"",
       *([ "--env", "KAMAL_DESTINATION=\"#{config.destination}\"" ] if config.destination),
+      *([ "--env", "DASH_REPLICA=\"#{replica}\"" ] if slotted?),
       *role.env_args(host),
       *role.logging_args,
       *config.volume_args,
       *role.asset_volume_args,
       *role.label_args,
+      *(argumentize("--label", { "replica" => replica }) if slotted?),
       *role.option_args,
       *role.healthcheck_args,
       config.absolute_image,
@@ -92,7 +114,31 @@ class Dash::Commands::App < Dash::Commands::Base
   end
 
   def info
-    docker :ps, *container_filter_args
+    docker :ps, *container_filter_args(all_replicas: true)
+  end
+
+  # `docker kill --signal`: how a replica without the proxy is told to stop taking work
+  # before it is stopped.
+  def signal(signal, version:)
+    pipe container_id_for_version(version), xargs(docker(:kill, "--signal=#{signal}"))
+  end
+
+  # The slot's container hostname (`--hostname` at boot), which Sidekiq reports as its
+  # process's `hostname` - how a drain asks Sidekiq what that container is still working on.
+  def container_hostname(version:)
+    pipe container_id_for_version(version), xargs(docker(:inspect, "--format", "'{{.Config.Hostname}}'"))
+  end
+
+  # Every running container of the role with docker's status line, for `dash scale status`.
+  def replica_status
+    active_containers_ps format: '"{{.Names}}\t{{.Status}}"'
+  end
+
+  # The names of every running container of the role, whatever its slot. A slot's version
+  # and whether it counts as running both come out of the name, so this is all a boot needs
+  # to know how many replicas a host runs.
+  def active_containers
+    active_containers_ps format: '"{{.Names}}"'
   end
 
 
@@ -101,7 +147,7 @@ class Dash::Commands::App < Dash::Commands::Base
   end
 
   def container_id_for_version(version, only_running: false)
-    container_id_for(container_name: container_name(version), only_running: only_running)
+    container_id_for(container_name: container_name_pattern(version), only_running: only_running)
   end
 
   def current_running_version
@@ -127,6 +173,23 @@ class Dash::Commands::App < Dash::Commands::Base
   # Everything the stale check needs from a host: every version of the role that has a
   # container, and the version running now - the difference is what is stale. Same shape
   # as #boot_state, same separator, same reason for `;` over `&&`.
+  # #boot_state for the whole role on this host, still one round trip: slot 1's answers,
+  # then every running container of the role (which carries each other slot's version and
+  # the replica count), then whether slots 2..max already hold the version being deployed.
+  # A role without replicas asks for the running containers too, so a deploy after
+  # `replicas` was lowered still finds the slots it has to stop.
+  def boot_states(version)
+    clashes = role.replica_numbers.drop(1).flat_map do |replica|
+      [ [ :echo, BOOT_STATE_SEPARATOR ], for_replica(replica).container_id_for_version(version) ]
+    end
+
+    chain \
+      boot_state(version),
+      [ :echo, BOOT_STATE_SEPARATOR ],
+      [ *active_containers, "||", :echo, ACTIVE_CONTAINERS_UNREADABLE ],
+      *clashes
+  end
+
   def stale_state
     chain \
       list_versions,
@@ -145,6 +208,12 @@ class Dash::Commands::App < Dash::Commands::Base
   end
 
   private
+    # The one `docker ps` both #active_containers and #replica_status ask: every running
+    # container of the role, whatever its slot.
+    def active_containers_ps(format:)
+      docker :ps, *container_filter_args(statuses: ACTIVE_DOCKER_STATUSES, all_replicas: true), "--format", format
+    end
+
     # The same two readiness sources #status and #health_probe cover, read into `$status`
     # so the loop around them is the same either way. They differ in what a non-zero exit
     # means. A probe that exits non-zero IS the answer "not ready", so its output is
@@ -185,8 +254,8 @@ class Dash::Commands::App < Dash::Commands::Base
       docker :ps, "--latest", *format, *container_filter_args(statuses: ACTIVE_DOCKER_STATUSES), argumentize("--filter", filters)
     end
 
-    def container_filter_args(statuses: nil)
-      argumentize "--filter", container_filters(statuses: statuses)
+    def container_filter_args(statuses: nil, all_replicas: false)
+      argumentize "--filter", container_filters(statuses: statuses, all_replicas: all_replicas)
     end
 
     def image_filter_args
@@ -195,13 +264,24 @@ class Dash::Commands::App < Dash::Commands::Base
 
     def extract_version_from_name
       # Extract SHA from "service-role-dest-SHA"
-      %(while read line; do echo ${line##{role.container_prefix}-}; done)
+      %(while read line; do echo ${line##{role.replica_prefix(replica)}-}; done)
+    end
+
+    # The exact-name lookup for this slot's container of a version. Slot 1 keeps the name as
+    # it always was; slot n escapes its prefix (see Role#replica_name_pattern).
+    def container_name_pattern(version)
+      replica > 1 ? "#{role.replica_name_pattern(replica)}-#{version}" : container_name(version)
+    end
+
+    def slotted?
+      role.present? && role.replica_scoped?(replica)
     end
 
-    def container_filters(statuses: nil)
+    def container_filters(statuses: nil, all_replicas: false)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}"
         filters << "label=role=#{role}" if role
+        filters << role.replica_name_filter(replica) if slotted? && !all_replicas
         statuses&.each do |status|
           filters << "status=#{status}"
         end

data/lib/dash/commands/auditor.rb

@@ -24,8 +24,8 @@ class Dash::Commands::Auditor < Dash::Commands::Base
     combine record(line, **details), *commands
   end
 
-  def reveal
-    [ :tail, "-n", 50, audit_log_file ]
+  def reveal(lines: 50)
+    [ :tail, "-n", Integer(lines.to_s, 10), audit_log_file ]
   end
 
   private

data/lib/dash/commands/autoscale.rb

@@ -0,0 +1,133 @@
+require "base64"
+
+# The autoscaling controller's state files, under `<app_directory>/autoscale/` on the
+# state host (Dash::Autoscale::StateStore.host): `heartbeat.json`, `state.json`, `decisions.jsonl`
+# and one `pause/<role>.json` file per paused role. JSON travels base64-encoded and is decoded
+# on the host, so no value is ever shell-quoted; a file is written beside its target and
+# moved into place, so a reader never sees half of one. See Dash::Autoscale::StateStore.
+class Dash::Commands::Autoscale < Dash::Commands::Base
+  # What docker allows in a container name, which every role name ends up in.
+  ROLE_NAME = /\A[a-zA-Z0-9_][a-zA-Z0-9_.-]*\z/
+  PAUSE_EXTENSION = ".json"
+
+  def ensure_directory
+    make_directory pause_directory
+  end
+
+  def read_heartbeat
+    read_file heartbeat_file
+  end
+
+  def write_heartbeat(json)
+    write_json json, heartbeat_file
+  end
+
+  # Prints `taken` and writes nothing when the heartbeat names another controller;
+  # otherwise - missing, naming `id`, or corrupted - writes it and prints `held`. A write
+  # that fails exits non-zero, as any failed command does. The check and the write are one
+  # command, so a takeover between them is the width of a `grep`, not of a tick.
+  def write_heartbeat_if_held(id, json)
+    raise ArgumentError, "#{id.inspect} is not a controller id" unless /\A\h+\z/.match?(id.to_s)
+
+    any \
+      combine(
+        [ :grep, "-qF", Dash::Utils.single_quote(%("controller_id":")), heartbeat_file, "2>", "/dev/null" ],
+        [ "!", :grep, "-qF", Dash::Utils.single_quote(%("controller_id":"#{id}")), heartbeat_file ],
+        [ :echo, "taken" ]),
+      [ "(", *combine(write_json(json, heartbeat_file), [ :echo, "held" ]), ")" ]
+  end
+
+  def read_state
+    read_file state_file
+  end
+
+  def write_state(json)
+    write_json json, state_file
+  end
+
+  # `lines` are JSON objects, one per line.
+  def append_decisions(lines)
+    append decode(lines.map { |line| "#{line}\n" }.join), decisions_file
+  end
+
+  # The last `lines` entries, or the last `lines` of these roles' (the log writes `role`
+  # first), filtered on the host before the tail so other roles cannot crowd them out.
+  def read_decisions(lines:, roles: nil)
+    lines = Integer(lines.to_s, 10)
+
+    if roles
+      patterns = roles.flat_map { |role| [ "-e", Dash::Utils.single_quote(%("role":"#{role_name(role)}")) ] }
+      any pipe([ :grep, "-F", *patterns, decisions_file, "2>", "/dev/null" ], [ :tail, "-n", lines ]), [ :true ]
+    else
+      any [ :tail, "-n", lines, decisions_file, "2>", "/dev/null" ], [ :true ]
+    end
+  end
+
+  def count_decisions
+    any [ :wc, "-l", "<", decisions_file, "2>", "/dev/null" ], [ :echo, 0 ]
+  end
+
+  def trim_decisions(keep:)
+    combine \
+      write([ :tail, "-n", Integer(keep.to_s, 10), decisions_file ], temporary(decisions_file)),
+      [ :mv, temporary(decisions_file), decisions_file ]
+  end
+
+  def write_pause(role, json)
+    write_json json, pause_file(role)
+  end
+
+  def remove_pause(role)
+    [ :rm, "-f", pause_file(role) ]
+  end
+
+  # `<path>:<json>` per file in the pause directory.
+  def read_pauses
+    any [ :grep, "-H", "\"\"", "#{pause_directory}/*", "2>", "/dev/null" ], [ :true ]
+  end
+
+  private
+    def write_json(json, file)
+      combine \
+        write(decode(json), temporary(file)),
+        [ :mv, temporary(file), file ]
+    end
+
+    def decode(content)
+      pipe [ :echo, "\"#{Base64.strict_encode64(content)}\"" ], [ :base64, "-d" ]
+    end
+
+    def temporary(file)
+      "#{file}.tmp"
+    end
+
+    def directory
+      File.join(config.app_directory, "autoscale")
+    end
+
+    def pause_directory
+      File.join(directory, "pause")
+    end
+
+    def heartbeat_file
+      File.join(directory, "heartbeat.json")
+    end
+
+    def state_file
+      File.join(directory, "state.json")
+    end
+
+    def decisions_file
+      File.join(directory, "decisions.jsonl")
+    end
+
+    def pause_file(role)
+      File.join(pause_directory, "#{role_name(role)}#{PAUSE_EXTENSION}")
+    end
+
+    def role_name(role)
+      raise ArgumentError, "#{role.to_s.inspect} is not a role name dash can keep autoscale state for" unless ROLE_NAME.match?(role.to_s)
+
+      role.to_s
+    end
+end

data/lib/dash/commands/base.rb

@@ -8,6 +8,10 @@ module Dash::Commands
     # healthcheck-less container that died still reports why.
     NO_HEALTHCHECK = "no-healthcheck"
 
+    # Separates the answers of several commands chained into one round trip. Each half is
+    # docker or /proc output, none of which prints this line on its own.
+    SECTION_SEPARATOR = "--%--"
+
     DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}#{NO_HEALTHCHECK}:{{.State.Status}}{{end}}'"
 
     # The statuses a boot accepts as ready. Dash::Cli::Healthcheck::Poller decides what a

data/lib/dash/commands/proxy/services.rb

@@ -0,0 +1,10 @@
+require "json"
+
+# What `dash-proxy list --json` answers, for the per-host proxy and the load balancer
+# alike: {"services": {"<name>": {"host":, "targets": [...], ...}}}. One parser for the
+# reboots' verification and the diagnostics, so they cannot read the list differently.
+module Dash::Commands::Proxy::Services
+  def self.parse(output)
+    JSON.parse(output).fetch("services", {})
+  end
+end

data/lib/dash/commands/prune.rb

@@ -19,9 +19,13 @@ class Dash::Commands::Prune < Dash::Commands::Base
   # candidate, and once it is gone every wake 404s. With `retain >= 1` a role's
   # newest container always survives, and the slept one is always the newest —
   # sleeping happens to the current release.
-  def app_containers(retain:, role:)
+  #
+  # Scoped to one replica slot as well when the role has several, so `retain` holds per
+  # slot: a rollback needs every slot's container of the version, and a role-wide window
+  # would keep 1/N as many versions.
+  def app_containers(retain:, role:, replica: 1)
     pipe \
-      docker(:ps, "-q", "-a", *service_filter, *destination_filter, *role_filter(role), *stopped_containers_filters),
+      docker(:ps, "-q", "-a", *service_filter, *destination_filter, *role_filter(role), *replica_filter(role, replica), *stopped_containers_filters),
       "tail -n +#{retain + 1}",
       "while read container_id; do docker rm $container_id; done"
   end
@@ -49,4 +53,8 @@ class Dash::Commands::Prune < Dash::Commands::Base
     def role_filter(role)
       [ "--filter", "label=role=#{role}" ]
     end
+
+    def replica_filter(role, replica)
+      role.replica_scoped?(replica) ? [ "--filter", role.replica_name_filter(replica) ] : []
+    end
 end

data/lib/dash/commands/server.rb

@@ -9,6 +9,22 @@ class Dash::Commands::Server < Dash::Commands::Base
       [ :wc, "-l" ]
   end
 
+  # Load, CPU count, memory, disk for / and Docker's data root, and uptime, one section
+  # each in one round trip. df -Pk rather than -B1, which busybox df lacks. getconf is
+  # POSIX; where it is missing the CPU count is nil and the rest still answers.
+  def stats
+    sections = [
+      [ :cat, "/proc/loadavg" ],
+      [ :getconf, "_NPROCESSORS_ONLN" ],
+      [ :grep, "-E", "'^(MemTotal|MemAvailable|SwapTotal|SwapFree):'", "/proc/meminfo" ],
+      [ :df, "-Pk", "/" ],
+      [ :df, "-Pk", %("$(docker info --format '{{.DockerRootDir}}')") ],
+      [ :cat, "/proc/uptime" ]
+    ]
+
+    chain(*sections.flat_map { |section| [ section, [ :echo, SECTION_SEPARATOR ] ] }[0...-1])
+  end
+
   # Lists TCP listeners on the given port, one line per listener, no header.
   def listeners_on(port)
     [ :ss, "-ltnH", :sport, "=", ":#{port}" ]

data/lib/dash/configuration/accessory.rb

@@ -246,9 +246,9 @@ class Dash::Configuration::Accessory
 
     def hosts_from_roles
       if accessory_config.key?("role")
-       config.role(accessory_config["role"])&.hosts
+       config.role(accessory_config["role"])&.baseline_hosts
       elsif accessory_config.key?("roles")
-        accessory_config["roles"].flat_map { |role| config.role(role)&.hosts }
+        accessory_config["roles"].flat_map { |role| config.role(role)&.baseline_hosts }
       end
     end
 

data/lib/dash/configuration/autoscale.rb

@@ -0,0 +1,123 @@
+require "active_support/values/time_zone"
+
+# The `autoscale:` block: which provider holds the pool members of scaled roles, how the
+# controller (`dash autoscale run`) ticks, and where its signals and guards read from
+# (`redis`, `postgres`, `prometheus`). Only the shape is checked here; credentials and URLs
+# are secrets and are resolved on first use, like the registry password, so loading
+# deploy.yml never needs them.
+class Dash::Configuration::Autoscale
+  include Dash::Configuration::Validation
+
+  DEFAULT_INTERVAL = 10
+  DEFAULT_LOCK_WAIT_TIMEOUT = 60
+  DEFAULT_TIMEZONE = "UTC"
+
+  # redis-client connects to the URL's host and verifies the certificate against that same
+  # name, so through a tunnel (127.0.0.1) a TLS certificate can never verify.
+  REDISS_VIA_MESSAGE = "rediss:// cannot be verified through via: redis-client checks the certificate against the address it " \
+    "connects to, which a tunnel makes 127.0.0.1. Reach Redis directly, or use redis:// over the tunnel"
+
+  def self.unverifiable_through_tunnel?(source, url)
+    source.to_s == "redis" && url.is_a?(String) && url.downcase.start_with?("rediss://")
+  end
+
+  attr_reader :autoscale_config
+
+  def initialize(config:)
+    @autoscale_config = config.raw_config.autoscale
+    @secrets = config.secrets
+    validate! autoscale_config, with: Dash::Configuration::Validator::Autoscale unless autoscale_config.nil?
+  end
+
+  def configured?
+    !autoscale_config.nil?
+  end
+
+  def provider_name
+    provider_config&.keys&.first&.to_s
+  end
+
+  # The ops host that runs `dash autoscale run` and keeps its state, or nil to keep it on
+  # the primary role's first baseline host (see Dash::Autoscale::StateStore.host).
+  def controller
+    autoscale_config&.fetch("controller", nil)
+  end
+
+  # Seconds between the controller's ticks.
+  def interval
+    autoscale_config&.fetch("interval", nil) || DEFAULT_INTERVAL
+  end
+
+  # Seconds a tick waits for the deploy lock before holding with `lock_busy`.
+  def lock_wait_timeout
+    autoscale_config&.fetch("lock_wait_timeout", nil) || DEFAULT_LOCK_WAIT_TIMEOUT
+  end
+
+  # The zone schedule crons are evaluated in.
+  def timezone
+    autoscale_config&.fetch("timezone", nil) || DEFAULT_TIMEZONE
+  end
+
+  def time_zone
+    ActiveSupport::TimeZone[timezone]
+  end
+
+  def upcloud_username
+    lookup(provider_config.dig("upcloud", "username"), "provider/upcloud/username")
+  end
+
+  def upcloud_password
+    lookup(provider_config.dig("upcloud", "password"), "provider/upcloud/password")
+  end
+
+  def exec_script(action)
+    provider_config.dig("exec", action.to_s)
+  end
+
+  # Whether `autoscale/<source>` (redis, postgres or prometheus) is set.
+  def source?(source)
+    !source_config(source).nil?
+  end
+
+  def source_url(source)
+    lookup(source_config(source)["url"], "#{source}/url").tap do |url|
+      if source_via(source) && self.class.unverifiable_through_tunnel?(source, url)
+        raise Dash::ConfigurationError, "autoscale/#{source}: #{REDISS_VIA_MESSAGE}"
+      end
+    end
+  end
+
+  # The host to tunnel the source's connection through over SSH, or nil to connect directly.
+  def source_via(source)
+    source_config(source)&.fetch("via", nil)
+  end
+
+  def prometheus_token
+    reference = source_config(:prometheus)&.fetch("token", nil)
+    lookup(reference, "prometheus/token") unless reference.nil?
+  end
+
+  private
+    attr_reader :secrets
+
+    def provider_config
+      autoscale_config&.fetch("provider", nil)
+    end
+
+    def source_config(source)
+      autoscale_config&.fetch(source.to_s, nil)
+    end
+
+    # A one-element array names a secret; anything else is the value as written.
+    def lookup(reference, path)
+      value = reference.is_a?(Array) ? secrets[reference.first] : reference
+      return value if value.present?
+
+      if reference.is_a?(Array)
+        raise Dash::ConfigurationError, "autoscale/#{path}: secret '#{reference.first}' resolved to an empty value — " \
+          "if your secrets file forwards it from the environment (#{reference.first}=$#{reference.first}), export the variable before running dash"
+      else
+        raise Dash::ConfigurationError, "autoscale/#{path}: is blank"
+      end
+    end
+end

data/lib/dash/configuration/docs/autoscale.yml

@@ -0,0 +1,141 @@
+# Autoscale
+#
+# Where the pool members of scaled roles (see `scale` in dash docs role) live. dash asks the
+# provider for them at the start of a command — the provider is the source of truth for
+# which extra hosts are running, so there is no state file to drift.
+
+# UpCloud
+#
+# Members are UpCloud servers carrying the `dash.service`, `dash.destination` and
+# `dash.role` labels. The API user's credentials are secrets: a one-element array names
+# the secret to read, as for `registry/password`. They are sent to the UpCloud API as
+# HTTP basic auth and nowhere else.
+#
+# Set exactly one provider.
+autoscale:
+  provider:
+    upcloud:
+      username:
+        - UPCLOUD_USERNAME
+      password:
+        - UPCLOUD_PASSWORD
+
+# Exec
+#
+# Your own scripts, for any other cloud. Each runs locally with the role's labels as
+# DASH_SERVICE, DASH_DESTINATION (`-` without a destination) and DASH_ROLE in its
+# environment, plus DASH_MEMBER_ID and DASH_HOST for the member it acts on, and
+# DASH_STOP_TIMEOUT (seconds) for `stop`. Only `members`, `start` and `stop` are
+# required; `create` and `destroy` are for `members: create` roles.
+#
+# `members` prints one JSON object per line —
+# `{"id": "...", "host": "10.0.0.22", "role": "payments", "state": "started"}`, where
+# state is started, stopped, maintenance or error — and `create` prints the new
+# member the same way. A script that exits non-zero fails the command, and the last
+# line of its stderr is the error dash shows.
+autoscale:
+  provider:
+    exec:
+      members: bin/pool-members
+      start: bin/pool-start
+      stop: bin/pool-stop
+      create: bin/pool-create
+      destroy: bin/pool-destroy
+
+# Controller
+#
+# How `dash autoscale run` ticks. Every `interval` seconds (default 10, at least 5) it reads
+# the pool and the running containers, and scales each role with a `scale/schedule` or a
+# `scale/signal` toward its target through `dash scale set`, under the deploy lock. A tick waits up to
+# `lock_wait_timeout` seconds (default 60) for a lock someone else holds, then holds the
+# role until the next tick. Schedule crons are evaluated in `timezone`, an IANA name
+# (default UTC; this example evaluates them in Stockholm time). `controller` names the
+# ops host that runs it and keeps its state (see below); without it, the state lives on
+# the primary role's first host.
+#
+# `redis`, `postgres` and `prometheus` are where signals and guards read from (see
+# `scale/signal` and `scale/hold_when` in dash docs role): a sidekiq signal reads Redis, a
+# pgbus signal and a postgres guard read Postgres. Each `url` is a secret (a one-element
+# array) or a literal URL, read from the machine the controller runs on. `via` names a host
+# to tunnel the connection through over SSH, for a database only the app servers reach;
+# `rediss://` cannot go through `via` (the certificate cannot be verified through a
+# tunnel). Postgres reads run in a read-only transaction. `token` is an optional bearer
+# token for Prometheus. The controller's Gemfile needs `redis-client` for Redis and `pg` for
+# Postgres; dash does not depend on either.
+autoscale:
+  provider:
+    ...
+  controller: 10.0.0.50
+  interval: 10
+  lock_wait_timeout: 60
+  timezone: Europe/Stockholm
+  redis:
+    url:
+      - REDIS_URL
+    via: 10.0.0.31
+  postgres:
+    url:
+      - DATABASE_URL
+  prometheus:
+    url: http://10.0.0.9:9090
+    token:
+      - PROMETHEUS_TOKEN
+
+# Running the controller
+#
+# `dash autoscale run` is a long-lived process on an ops host - a machine with a checkout
+# of the app, its `.dash/secrets` and SSH access to the servers. Name that host in
+# `controller` and run the controller there: it keeps its state on that host, under the
+# app's run directory - a heartbeat, the decision log, the pauses and what it remembers
+# between ticks (cooldowns, warmups, unreachable members) - away from the app servers that
+# are busiest when the app is in trouble. It reaches its own state over SSH, like every
+# other host, so the ops host needs an SSH key (and `ssh/user`) that logs into itself.
+# Every operator's `dash autoscale status`, `history` and `explain`, `pause` and `resume`,
+# `dash doctor` and `dash mcp` read and write it there too.
+#
+# Without `controller`, the state lives on the primary role's first host. Setting or
+# changing `controller` migrates nothing: the controller uses whatever state the selected
+# host holds - none on a new host, so cooldowns, warmups and pauses start fresh; the files
+# it had before, pauses included, on a host it used earlier. The old host's files stay
+# where they were. Stop the old controller first, then check `dash autoscale status` on
+# the new host for pauses that are missing or stale.
+#
+# One controller per app and destination: a second one refuses to start while the first
+# one's heartbeat is fresh (three intervals), unless started with `--takeover`, and the
+# first stops at its next tick. TERM or INT lets the tick in progress finish - a member
+# joining included - before it exits, so give the service manager more time to stop it
+# than the slowest tick takes. A tick acts on its roles one after another, and each
+# action may first wait `lock_wait_timeout` for the deploy lock and then join members one
+# at a time, each within `boot_timeout` plus the app's boot - budget for the roles that
+# can scale out together.
+#
+# ```ini
+# # /etc/systemd/system/dash-autoscale.service
+# [Unit]
+# Description=dash autoscale controller
+# After=network-online.target
+#
+# [Service]
+# WorkingDirectory=/srv/myapp
+# ExecStart=/usr/local/bin/bundle exec dash autoscale run --metrics-port 9394
+# Restart=always
+# TimeoutStopSec=900
+#
+# [Install]
+# WantedBy=multi-user.target
+# ```
+#
+# Or in a container, with the checkout and an SSH key mounted:
+#
+# ```sh
+# docker run -d --restart unless-stopped --name dash-autoscale \
+#   -v /srv/myapp:/app -w /app -v ~/.ssh:/root/.ssh:ro \
+#   ruby:3.4 sh -c "bundle install && bundle exec dash autoscale run"
+# ```
+#
+# `--once` runs a single tick and exits, for cron; the state on the controller host carries
+# cooldowns and warmups from one run to the next. `--dry-run` decides and prints without
+# acting or writing the controller's state (only the `--record` file, when given). `--metrics-port` serves Prometheus metrics at `/metrics`
+# (on 127.0.0.1 unless `--metrics-bind` says otherwise). `dash autoscale explain ROLE`
+# shows what the next tick would decide and why; `dash autoscale pause ROLE --for 2h`
+# keeps the controller's hands off a role, `resume ROLE` gives it back.