dash 4.1.3 → 4.2.0

data/lib/dash/configuration.rb

@@ -16,17 +16,19 @@ class Dash::Configuration
   delegate :argumentize, :optionize, to: Dash::Utils
 
   attr_reader :destination, :raw_config, :secrets
-  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :output, :proxy, :proxy_boot, :report, :servers, :ssh, :sshkit, :registry
+  attr_reader :accessories, :aliases, :autoscale, :boot, :builder, :env, :logging, :output, :proxy, :proxy_boot, :report, :servers, :ssh, :sshkit, :registry
 
   include Validation
 
   class << self
-    def create_from(config_file:, destination: nil, version: nil)
+    # `explicit_hosts` / `explicit_roles` are --hosts and --roles, which the pool needs before
+    # Commander can filter by them: filtering reads all_hosts, and that asks the pool.
+    def create_from(config_file:, destination: nil, version: nil, explicit_hosts: nil, explicit_roles: nil)
       ENV["DASH_DESTINATION"] = ENV["KAMAL_DESTINATION"] = destination
 
       raw_config = load_raw_config(config_file: config_file, destination: destination)
 
-      new raw_config, destination: destination, version: version
+      new raw_config, destination: destination, version: version, explicit_hosts: explicit_hosts, explicit_roles: explicit_roles
     end
 
     def load_raw_config(config_file:, destination: nil)
@@ -55,10 +57,12 @@ class Dash::Configuration
       end
   end
 
-  def initialize(raw_config, destination: nil, version: nil, validate: true)
+  def initialize(raw_config, destination: nil, version: nil, validate: true, explicit_hosts: nil, explicit_roles: nil)
     @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
     @destination = destination
     @declared_version = version
+    @explicit_hosts = explicit_hosts
+    @explicit_roles = explicit_roles
 
     validate! raw_config, example: validation_yml.symbolize_keys, context: "", with: Dash::Configuration::Validator::Configuration
 
@@ -70,6 +74,7 @@ class Dash::Configuration
 
     @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || []
     @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
+    @autoscale = Autoscale.new(config: self)
     @boot = Boot.new(config: self)
     @builder = Builder.new(config: self)
     @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
@@ -102,6 +107,12 @@ class Dash::Configuration
     ensure_sleep_has_a_docker_socket
     ensure_proxy_protocol_names_its_peers
     ensure_max_idle_conns_meaningful
+    ensure_replicas_fit_their_roles
+    ensure_scaled_roles_have_a_provider
+    ensure_upcloud_templates_are_complete
+    ensure_scaled_proxied_roles_have_a_loadbalancer
+    ensure_signal_sources_are_configured
+    ensure_pgbus_signals_have_no_drain_signal
   end
 
   # Resolves every secret the deploy will need so a missing secret fails fast,
@@ -113,7 +124,9 @@ class Dash::Configuration
     builder.secrets
 
     roles.each do |role|
-      role.secrets_io(role.hosts.first) if role.hosts.any?
+      # A baseline host when there is one: the same secrets, without asking the pool.
+      host = role.baseline_hosts.first || role.hosts.first
+      role.secrets_io(host) if host
 
       if role.running_proxy?
         role.proxy.run&.secrets_io
@@ -213,6 +226,16 @@ class Dash::Configuration
     roles.select(&:running_proxy?)
   end
 
+  def scaled_roles
+    roles.select(&:scaled?)
+  end
+
+  # Built on first use, never while the configuration loads: asking the provider is a
+  # network call, and loading deploy.yml must not need the network.
+  def pool
+    @pool ||= Dash::Autoscale::Pool.new(config: self, explicit_hosts: @explicit_hosts, explicit_roles: @explicit_roles)
+  end
+
   def load_balancing?
     proxy&.load_balancing?
   end
@@ -229,6 +252,12 @@ class Dash::Configuration
     (proxy_roles.flat_map(&:hosts) + proxy_accessories.flat_map(&:hosts)).uniq
   end
 
+  # proxy_hosts without the pool members, for what must not ask the provider - where the
+  # load balancer runs is decided by deploy.yml alone.
+  def baseline_proxy_hosts
+    (proxy_roles.flat_map(&:baseline_hosts) + proxy_accessories.flat_map(&:hosts)).uniq
+  end
+
   def image
     name = raw_config&.image.presence
     name ||= raw_config&.service if registry.local?
@@ -363,8 +392,9 @@ class Dash::Configuration
   def to_h
     {
       roles: role_names,
-      hosts: all_hosts,
-      primary_host: primary_host,
+      hosts: baseline_hosts,
+      members: scaled_roles.to_h { |role| [ role.name.to_s, "#{role.scale.members} members from #{autoscale.provider_name}, #{role.scale.min}-#{role.scale.max} hosts" ] }.presence,
+      primary_host: primary_role&.baseline_hosts&.first,
       version: version,
       repository: repository,
       absolute_image: absolute_image,
@@ -402,13 +432,13 @@ class Dash::Configuration
           raise Dash::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
         end
 
-        if primary_role.hosts.empty?
+        if primary_role.baseline_hosts.empty?
           raise Dash::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
         end
 
         unless allow_empty_roles?
           roles.each do |role|
-            if role.hosts.empty?
+            if role.baseline_hosts.empty?
               raise Dash::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
             end
           end
@@ -458,7 +488,7 @@ class Dash::Configuration
     # accepts the container as ready once it is merely still running after readiness_delay,
     # then stops the old one. Warn-only for now so existing configs keep deploying.
     def ensure_unproxied_roles_are_readiness_gated
-      offenders = roles.reject { |role| role.hosts.empty? || role.running_proxy? || role.readiness_gated? }
+      offenders = roles.reject { |role| role.baseline_hosts.empty? || role.running_proxy? || role.readiness_gated? }
       return true if offenders.empty?
 
       warn "Non-proxied role(s) #{offenders.map(&:name).join(", ")} have no healthcheck. " \
@@ -475,7 +505,7 @@ class Dash::Configuration
     # port hands that address to whoever asks. Warn rather than raise: a proxy on
     # a private network with no other route in is a legitimate configuration.
     def ensure_proxy_protocol_names_its_peers
-      offenders = all_hosts.select { |host| proxy_run(host)&.proxy_protocol_unrestricted? }
+      offenders = baseline_hosts.select { |host| proxy_runs(host, baseline: true).first&.proxy_protocol_unrestricted? }
       return true if offenders.empty?
 
       warn "Host(s) #{offenders.sort.join(", ")}: proxy_protocol is enabled without proxy_protocol_allow_ips, " \
@@ -508,6 +538,79 @@ class Dash::Configuration
         "through the container runtime socket, and it is not mounted into the proxy without it"
     end
 
+    # Run once every role and the root proxy exist: a role's merged proxy reads the root
+    # one, which Servers.new builds roles ahead of.
+    def ensure_replicas_fit_their_roles
+      roles.each { |role| role.replicas.ensure_fits!(role, volumes: Array(raw_config.volumes)) }
+    end
+
+    # A scaled role's extra hosts come from the provider; without one there is nothing to
+    # power on, and every command would fail at the first `hosts` read instead of here.
+    def ensure_scaled_roles_have_a_provider
+      return true if autoscale.configured?
+
+      if (role = scaled_roles.first)
+        raise Dash::ConfigurationError, "servers/#{role.name}/scale: needs a provider for its members, set autoscale/provider (see dash docs autoscale)"
+      end
+
+      true
+    end
+
+    # UpCloud builds a server from the template; an exec `create` script reads its own.
+    def ensure_upcloud_templates_are_complete
+      return true unless autoscale.provider_name == "upcloud"
+
+      scaled_roles.select { |role| role.scale.create? }.each do |role|
+        missing = %w[ storage plan zone ].select { |key| role.scale.template[key].blank? }
+
+        if missing.any?
+          raise Dash::ConfigurationError, "servers/#{role.name}/scale/template: #{missing.join(", ")} #{missing.one? ? "is" : "are"} required to create an UpCloud server"
+        end
+      end
+
+      true
+    end
+
+    # A member of a proxied role reaches traffic through the load balancer's target list -
+    # that is the only way a host joins or leaves a dash-proxy service across hosts.
+    def ensure_scaled_proxied_roles_have_a_loadbalancer
+      return true unless proxy.loadbalancer == false
+
+      if (role = scaled_roles.find(&:running_proxy?))
+        raise Dash::ConfigurationError, "servers/#{role.name}/scale: a scaled role behind dash-proxy needs the load balancer, " \
+          "remove proxy/loadbalancer: false"
+      end
+
+      true
+    end
+
+    # A signal or guard reads through its autoscale/<connection> block; without one the
+    # controller would hold the role with missing_data forever instead of failing here.
+    def ensure_signal_sources_are_configured
+      scaled_roles.select { |role| role.scale.signal? }.each do |role|
+        readers = [ [ "signal", role.scale.signal ] ] + role.scale.guards.each_with_index.map { |guard, index| [ "hold_when/#{index}", guard ] }
+
+        readers.each do |key, reader|
+          next if autoscale.source?(reader.connection)
+
+          raise Dash::ConfigurationError, "servers/#{role.name}/scale/#{key}: #{reader.source} reads #{reader.connection.capitalize}, " \
+            "set autoscale/#{reader.connection} (see dash docs autoscale)"
+        end
+      end
+
+      true
+    end
+    # pgbus has no quiet mode: a drain signal changes nothing, and the drain wait only delays
+    # the stop that drains it. Legal, so warn.
+    def ensure_pgbus_signals_have_no_drain_signal
+      scaled_roles.select { |role| role.scale.signal&.source == "pgbus" && role.drain_signal }.each do |role|
+        warn "Role #{role.name}: a pgbus signal with drain/signal #{role.drain_signal} - pgbus has no quiet mode and ignores it. " \
+          "It drains its jobs on docker stop, so give the role a stop timeout of at least pgbus's drain_timeout (30s by default)."
+      end
+
+      true
+    end
+
     # dash-proxy resolves a zero max_idle_conns to its default of 100
     # (target_pool.go resolves defaults from zeros), so the one value an
     # operator writes to mean "keep none" is the one value that cannot mean it.
@@ -629,16 +732,24 @@ class Dash::Configuration
     end
 
     def ensure_no_conflicting_proxy_runs
-      all_hosts.each do |host|
-        run_configs = proxy_runs(host)
+      baseline_hosts.each do |host|
+        run_configs = proxy_runs(host, baseline: true)
         if run_configs.uniq.size > 1
           raise Dash::ConfigurationError, "Conflicting proxy run configurations for host #{host}"
         end
       end
     end
 
-    def proxy_runs(host)
-      (host_roles(host) + host_accessories(host)).map(&:proxy).compact.map(&:run).compact
+    # `baseline: true` for the checks run while the configuration loads, which must not ask
+    # the autoscale provider. A member shares its role's proxy run, so they lose nothing.
+    def proxy_runs(host, baseline: false)
+      host_roles = baseline ? roles.select { |role| role.baseline_hosts.include?(host) } : host_roles(host)
+      (host_roles + host_accessories(host)).map(&:proxy).compact.map(&:run).compact
+    end
+
+    # all_hosts without the pool members.
+    def baseline_hosts
+      (roles.flat_map(&:baseline_hosts) + accessories.flat_map(&:hosts)).uniq
     end
 
     # `loadbalancer: true` resolves to the primary role's first host, so an

data/lib/dash/diagnostics/audit.rb

@@ -0,0 +1,28 @@
+# The last lines of this deploy's audit log on each host, parsed into when, who, the tags
+# the line was recorded with (destination, role, ...) and the message.
+class Dash::Diagnostics::Audit < Dash::Diagnostics::Base
+  # "[2026-10-05T10:00:00Z] [ops@example.com] [staging] [web] Booted app version 999"
+  LINE = /\A\[(?<recorded_at>[^\]]*)\] \[(?<performer>[^\]]*)\](?<tags>(?: \[[^\]]*\])*) ?(?<message>.*)\z/
+
+  def initialize(hosts: DASH.hosts, lines: 50)
+    @hosts = hosts
+    @lines = Dash::Diagnostics::Lines.bounded(lines)
+  end
+
+  private
+    def snapshot
+      { hosts: per_host(@hosts) { |backend, _host| { entries: entries(backend.capture_with_info(*DASH.auditor.reveal(lines: @lines), raise_on_non_zero_exit: false)) } } }
+    end
+
+    def entries(output)
+      output.lines.map(&:chomp).reject(&:blank?).map { |line| entry(line) }
+    end
+
+    def entry(line)
+      if (match = LINE.match(line))
+        { recorded_at: match[:recorded_at], performer: match[:performer], tags: match[:tags].scan(/\[([^\]]*)\]/).flatten, message: match[:message] }
+      else
+        { message: line }
+      end
+    end
+end

data/lib/dash/diagnostics/autoscale_decisions.rb

@@ -0,0 +1,18 @@
+# `dash autoscale history` and the MCP `autoscale_decisions` tool: the tail of the
+# controller's decision log on the state host, oldest first - of some roles, or of all.
+# Each entry is what the controller decided, why, and every input it used.
+class Dash::Diagnostics::AutoscaleDecisions < Dash::Diagnostics::Base
+  DEFAULT_LINES = 50
+
+  # `roles`: role names, or nil for every role.
+  def initialize(roles: nil, lines: DEFAULT_LINES)
+    @roles, @lines = roles, Dash::Diagnostics::Lines.bounded(lines)
+  end
+
+  private
+    def snapshot
+      per_host([ Dash::Autoscale::StateStore.host ]) do |backend, _host|
+        { decisions: Dash::Autoscale::StateStore.new(backend).decisions(lines: @lines, roles: @roles) }
+      end.first
+    end
+end

data/lib/dash/diagnostics/autoscale_explain.rb

@@ -0,0 +1,30 @@
+# `dash autoscale explain ROLE` and the MCP `autoscale_explain` tool: one live policy
+# evaluation for a scaled role, from the same inputs the controller's next tick reads -
+# the running containers, the pool, and the state and pauses on the state host - with
+# every input and `eligible_at`. Decides, never acts, writes nothing, needs no running
+# controller. State that cannot be read is `state_error`, and the role is evaluated as if
+# the controller remembered nothing.
+class Dash::Diagnostics::AutoscaleExplain < Dash::Diagnostics::Base
+  def initialize(role:, now: Time.now.utc)
+    raise ArgumentError, "#{role} has no scale, so the autoscale controller never touches it" unless role.scaled?
+    # Read on some of its hosts, the role's count would be too low and the decision wrong.
+    raise ArgumentError, "explain observes every host of #{role}; it cannot be narrowed with --hosts" if DASH.specific_hosts
+
+    @role, @now = role, now
+  end
+
+  private
+    def snapshot
+      host = Dash::Autoscale::StateStore.host
+      read = per_host([ host ]) do |backend, _host|
+        store = Dash::Autoscale::StateStore.new(backend)
+        { state: store.state, pauses: store.pauses }
+      end.first
+
+      state = Dash::Autoscale::RoleState.from(read.dig(:state, "roles", @role.name))
+      pause = Dash::Autoscale::Pause.from(read.dig(:pauses, @role.name))
+      decision = Dash::Autoscale::Evaluation.new(role: @role, now: @now, state: state, pause: pause).decide
+
+      { role: @role.name, controlled: @role.controlled?, state_host: host, state_error: read[:error], decision: decision.to_h }.compact
+    end
+end

data/lib/dash/diagnostics/autoscale_signal.rb

@@ -0,0 +1,82 @@
+require "socket"
+
+# `dash autoscale signal ROLE`, the MCP `signal_sample` tool and the doctor's signal check:
+# one live sample of a role's `scale.signal` and of each `scale.hold_when` guard, through
+# the controller's own readers (Dash::Autoscale::Signal, Dash::Autoscale::Guard), each read
+# timed. The fields are those of a decision's `inputs[:signal]` and `inputs[:guards]`, so
+# a sample compares directly with what the controller logged.
+#
+# Read from this machine: a laptop reaches the sources by another path than the controller
+# host (a direct URL, or its own `via:` tunnel), so `from` names it. Read-only, as the
+# readers are: Redis LINDEX, SMEMBERS and HMGET, Postgres inside a rolled-back read-only
+# transaction, a Prometheus instant query. A read that fails - a configuration error
+# included - is the sample's error; nothing raises.
+class Dash::Diagnostics::AutoscaleSignal < Dash::Diagnostics::Base
+  # `connections` given are the caller's to close; without, the sample opens its own and
+  # closes them when done.
+  def initialize(role:, now: Time.now.utc, connections: nil)
+    @role, @now, @connections = role, now, connections
+  end
+
+  private
+    def snapshot
+      return { role: @role.name, configured: false, message: "#{@role} has no scale/signal, so there is nothing to sample" } unless @role.scaled? && @role.scale.signal?
+
+      with_connections do |connections|
+        signal = timed { Dash::Autoscale::Signal.for(@role, connections).sample(at: @now) }
+        guards = @role.scale.guards.each_with_index.map { |guard, index| guard_reading(guard, index, connections) }
+
+        Dash::Autoscale::Decision.json_safe(role: @role.name, configured: true, from: Socket.gethostname,
+          signal: { source: scale.signal.source }.merge(signal[:result].to_h, latency_ms: signal[:latency_ms]),
+          position: position(signal[:result]), guards: guards, band: band)
+      end
+    end
+
+    def scale
+      @role.scale
+    end
+
+    def with_connections
+      return yield @connections if @connections
+
+      connections = Dash::Autoscale::Connections.new
+      begin
+        yield connections
+      ensure
+        connections.close
+      end
+    end
+
+    # A guard read on its own, so it has its own latency.
+    def guard_reading(guard, index, connections)
+      read = timed { Dash::Autoscale::Guard.read([ guard ], connections, at: @now).first.sample }
+      reading = Dash::Autoscale::Guard::Reading.new(index: index, source: guard.source, above: guard.above, sample: read[:result])
+
+      reading.to_h.merge(holding: reading.holding?, latency_ms: read[:latency_ms])
+    end
+
+    def position(sample)
+      return if sample.missing?
+
+      if sample.value > scale.up_above then "above"
+      elsif sample.value < scale.down_below then "below"
+      else "inside"
+      end
+    end
+
+    def band
+      { up_above: scale.up_above, up_for: scale.up_for, down_below: scale.down_below, down_for: scale.down_for }
+    end
+
+    # { result: the Sample, latency_ms: }; a configuration error becomes the Sample's error.
+    def timed
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      result = begin
+        yield
+      rescue Dash::ConfigurationError => e
+        Dash::Autoscale::Sample.new(value: nil, at: @now, error: "#{e.class}: #{e.message}")
+      end
+
+      { result: result, latency_ms: ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round(1) }
+    end
+end

data/lib/dash/diagnostics/base.rb

@@ -0,0 +1,53 @@
+require "dash/sshkit_with_ext"
+
+# What every diagnostic shares: a JSON-safe `to_h` stamped with when it was taken, and
+# per-host isolation. A host that cannot be reached, or answers with something that does
+# not parse, becomes `{ host:, error: }` in the snapshot - a debugging tool that dies on
+# the broken host is useless exactly when it is needed (same contract as Dash::Diagnostics::Doctor).
+#
+# Read-only by contract: a diagnostic only ever captures, never takes the deploy lock,
+# never writes an audit line, never fires a hook. The pre-connect hook is the caller's
+# (the CLI command or `dash mcp` at boot), as with Doctor.
+class Dash::Diagnostics::Base
+  include SSHKit::DSL
+
+  def to_h
+    { generated_at: Time.now.utc.iso8601 }.merge(snapshot)
+  end
+
+  private
+    def snapshot
+      raise NotImplementedError
+    end
+
+    # [ { host:, **yield } | { host:, error: } ] in the order the hosts were given. The
+    # block runs inside each host's SSHKit backend and gets it as its argument.
+    #
+    # StandardError rather than the SSHKit errors alone, deliberately: a failed connection
+    # raises anything from Net::SSH errors to Errno and DNS resolution errors, and none of
+    # them may take the other hosts' answers down with them.
+    def per_host(hosts, &block)
+      results = {}
+      mutex = Mutex.new
+
+      on(hosts) do |host|
+        result = begin
+          block.call(self, host.to_s)
+        rescue StandardError => e
+          { error: "#{e.class}: #{e.message}" }
+        end
+
+        mutex.synchronize { results[host.to_s] = result }
+      end
+
+      hosts.map { |host| { host: host.to_s }.merge(results.fetch(host.to_s) { { error: "no answer" } }) }
+    end
+
+    # per_host for every accessory on its own hosts within scope: [ { host:, accessory:, ... } ].
+    def per_accessory(accessories)
+      accessories.flat_map do |accessory|
+        per_host(accessory.hosts & DASH.accessory_hosts) { |backend, host| yield accessory, backend, host }
+          .map { |entry| { host: entry[:host], accessory: accessory.name }.merge(entry.except(:host)) }
+      end
+    end
+end

data/lib/dash/diagnostics/config.rb

@@ -0,0 +1,32 @@
+# The effective configuration as `dash config` prints it (secrets as their redaction),
+# plus the topology an agent would otherwise have to derive: which hosts each role runs
+# on, which roles are proxied, and where the proxies and the load balancer are.
+class Dash::Diagnostics::Config < Dash::Diagnostics::Base
+  private
+    def snapshot
+      config = DASH.config
+
+      { service: config.service, destination: config.destination, roles: config.roles.map { |role| role_topology(role) },
+        proxy: { hosts: config.proxy_hosts, loadbalancer: (config.proxy.effective_loadbalancer.presence if config.proxy.load_balancing?) },
+        accessories: config.accessories.map { |accessory| { name: accessory.name, hosts: accessory.hosts } },
+        config: json_safe(Dash::Utils.redacted(config.to_h)) }
+    end
+
+    # config.to_h is built for YAML: symbols, and an SSH jump host as a Net::SSH object.
+    def json_safe(value)
+      case value
+      when Hash then value.transform_values { |nested| json_safe(nested) }
+      when Array then value.map { |element| json_safe(element) }
+      when String, Numeric, true, false, nil then value
+      when Symbol then value.to_s
+      when Net::SSH::Proxy::Jump then "jump #{value.jump_proxies}"
+      when Net::SSH::Proxy::Command then "command #{value.command_line_template}"
+      else value.class.name
+      end
+    end
+
+    def role_topology(role)
+      { name: role.name, hosts: role.hosts, primary: role.primary?, proxied: role.running_proxy?,
+        replicas: { min: role.replicas.min, max: role.replicas.max } }
+    end
+end

data/lib/dash/diagnostics/container_stats.rb

@@ -0,0 +1,67 @@
+require "json"
+
+# Per app host, the resource use of every running container of this service: CPU, memory,
+# network and block I/O and PIDs, with its role, replica slot and version - then each
+# accessory's container on its own hosts. A point-in-time
+# sample - `docker stats --no-stream` measures over about a second, per host, in parallel.
+#
+# Every number keeps its raw string under `raw:`; a value docker printed in a form Units
+# cannot read becomes nil there, never a wrong number.
+class Dash::Diagnostics::ContainerStats < Dash::Diagnostics::Base
+  # Printed when `docker stats` fails after `docker ps` answered (a container removed in
+  # between makes the whole sample fail): the containers stay, `stats_error` says why.
+  STATS_UNREADABLE = "--stats-unreadable--"
+
+  # Raised inside per_host, which turns it into the host's error entry.
+  class DockerUnreadable < StandardError
+    def message = "docker ps failed: the docker daemon could not be asked"
+  end
+
+  # `accessories: []` for the app alone, `hosts: []` for accessories alone.
+  def initialize(hosts: DASH.app_hosts, accessories: DASH.config.accessories)
+    @hosts = hosts
+    @accessories = accessories
+  end
+
+  private
+    def snapshot
+      { hosts: per_host(@hosts) { |backend, _host| sample(backend.capture_with_info(*DASH.app.stats_json, raise_on_non_zero_exit: false)) },
+        accessories: per_accessory(@accessories) do |accessory, backend, _host|
+          sample(backend.capture_with_info(*DASH.accessory(accessory.name).stats_json, raise_on_non_zero_exit: false), accessory: accessory.name)
+        end }
+    end
+
+    # { containers: } for one host, plus `stats_error:` when docker stats itself failed.
+    def sample(output, accessory: nil)
+      ps, stats = output.to_s.split(/^#{Regexp.escape(Dash::Commands::Base::SECTION_SEPARATOR)}$/, 2)
+      raise DockerUnreadable if ps.to_s.lines.map(&:strip).include?(Dash::Commands::App::ACTIVE_CONTAINERS_UNREADABLE)
+
+      stats_lines = stats.to_s.lines.map(&:strip)
+      by_id = stats_lines.filter_map { |line| JSON.parse(line) if line.start_with?("{") }.index_by { |sample| sample["ID"].to_s[0, 12] }
+
+      containers = Dash::Diagnostics::DockerPs.containers(ps, accessory: accessory).map do |container|
+        sample = by_id[container[:id].to_s[0, 12]]
+        container.merge(stats: sample && stats_of(sample))
+      end
+
+      # Containers listed but not one sample: docker stats failed in a way its exit status
+      # hid (the second docker ps failing hands xargs -r nothing, and xargs exits 0).
+      failed = stats_lines.include?(STATS_UNREADABLE) || (containers.any? && by_id.empty?)
+
+      { containers: containers }.tap do |entry|
+        entry[:stats_error] = "docker stats failed, so these containers have no sample" if failed
+      end
+    end
+
+    def stats_of(sample)
+      memory, memory_limit = Dash::Diagnostics::Units.pair(sample["MemUsage"])
+      net_rx, net_tx = Dash::Diagnostics::Units.pair(sample["NetIO"])
+      block_read, block_write = Dash::Diagnostics::Units.pair(sample["BlockIO"])
+
+      { cpu_percent: Dash::Diagnostics::Units.percent(sample["CPUPerc"]),
+        memory_bytes: memory, memory_limit_bytes: memory_limit, memory_percent: Dash::Diagnostics::Units.percent(sample["MemPerc"]),
+        net_rx_bytes: net_rx, net_tx_bytes: net_tx, block_read_bytes: block_read, block_write_bytes: block_write,
+        pids: Integer(sample["PIDs"].to_s, 10, exception: false),
+        raw: { cpu: sample["CPUPerc"], memory: sample["MemUsage"], memory_percent: sample["MemPerc"], net_io: sample["NetIO"], block_io: sample["BlockIO"], pids: sample["PIDs"] } }
+    end
+end

data/lib/dash/diagnostics/containers.rb

@@ -0,0 +1,20 @@
+# Per app host, every container of this service and destination: role, replica slot,
+# version, state and health. One `docker ps` per host, whatever the roles and slots. Then
+# each accessory's container on its own hosts.
+class Dash::Diagnostics::Containers < Dash::Diagnostics::Base
+  # `accessories: []` for the app alone, `hosts: []` for accessories alone.
+  def initialize(hosts: DASH.app_hosts, accessories: DASH.config.accessories)
+    @hosts = hosts
+    @accessories = accessories
+  end
+
+  private
+    def snapshot
+      { hosts: per_host(@hosts) { |backend, _host| { containers: Dash::Diagnostics::DockerPs.containers(backend.capture_with_info(*DASH.app.list_containers_json)) } },
+        accessories: per_accessory(@accessories) { |accessory, backend, _host| { containers: accessory_containers(accessory, backend) } } }
+    end
+
+    def accessory_containers(accessory, backend)
+      Dash::Diagnostics::DockerPs.containers(backend.capture_with_info(*DASH.accessory(accessory.name).list_containers_json), accessory: accessory.name)
+    end
+end

data/lib/dash/diagnostics/controller_status.rb

@@ -0,0 +1,44 @@
+# `dash autoscale status`, the MCP `controller_status` tool and the doctor's controller
+# check: the heartbeat on the state host - which controller, where, which dash, how long
+# since its last tick - read as `running`, `stale` (no tick for three of its intervals) or
+# `stopped`, plus the pauses still in force. `controller` is nil when none ever ran.
+class Dash::Diagnostics::ControllerStatus < Dash::Diagnostics::Base
+  # `roles`: the roles this run is about (the doctor's scope); the controlled ones and their pauses.
+  def initialize(now: Time.now.utc, roles: DASH.specific_roles || DASH.config.roles)
+    @now, @roles = now, roles
+  end
+
+  private
+    def snapshot
+      host = Dash::Autoscale::StateStore.host
+      roles = @roles.select(&:controlled?).map(&:name)
+      read = per_host([ host ]) do |backend, _host|
+        store = Dash::Autoscale::StateStore.new(backend)
+        heartbeat = store.heartbeat
+        # A corrupted heartbeat is not the same as none: say so instead of "never ran".
+        next { error: "heartbeat.json is not valid JSON" } if store.malformed?("heartbeat.json")
+
+        { heartbeat: heartbeat, pauses: store.pauses }
+      end.first
+
+      return { state_host: host, roles: roles, error: read[:error] } if read[:error]
+
+      { state_host: host, roles: roles, controller: controller(read[:heartbeat]), pauses: pauses(read[:pauses].slice(*@roles.map(&:name))) }
+    end
+
+    def controller(heartbeat)
+      return if heartbeat.empty?
+
+      lease = Dash::Autoscale::Lease.new(heartbeat, now: @now)
+      status = if lease.stopped? then "stopped" elsif lease.alive? then "running" else "stale" end
+
+      heartbeat.symbolize_keys.merge(status: status, age_seconds: lease.age)
+    end
+
+    def pauses(pauses)
+      pauses.filter_map do |role, hash|
+        pause = Dash::Autoscale::Pause.from(hash)
+        { role: role }.merge(pause.to_h.symbolize_keys) if pause.active?(@now)
+      end.sort_by { |pause| pause[:role] }
+    end
+end