dash 4.1.3 → 4.2.0

data/lib/dash/diagnostics/docker_ps.rb

@@ -0,0 +1,40 @@
+require "json"
+
+# One line of the app's `docker ps` in Commands::App::Containers::PS_WITH_ROLE_FORMAT - the
+# container's JSON, a tab, docker's own value of its `role` label - as a container of this
+# deploy. The role comes from that label value, never from the name alone (a sibling role
+# whose name extends another's would be misattributed) and never from the rendered `Labels`
+# string (a comma in another label's value could pass for a role there).
+module Dash::Diagnostics::DockerPs
+  HEALTH = /\((healthy|unhealthy|health: starting)\)/
+
+  module_function
+
+  # An accessory's containers carry only its `service` label, so they are named for the
+  # accessory the caller asked about rather than for a role; their lines are plain JSON.
+  def containers(output, accessory: nil)
+    output.to_s.lines.filter_map do |line|
+      next if line.strip.empty?
+
+      ps, role = line.chomp.split("\t", 2)
+      accessory ? accessory_container(JSON.parse(ps), accessory) : container(JSON.parse(ps), role && JSON.parse(role))
+    end
+  end
+
+  def accessory_container(ps, accessory)
+    { name: ps["Names"], id: ps["ID"], accessory: accessory, state: ps["State"], status: ps["Status"], health: ps["Status"].to_s[HEALTH, 1],
+      image: ps["Image"], created_at: ps["CreatedAt"] }
+  end
+
+  def container(ps, role_name)
+    role = DASH.config.role(role_name) if role_name.present?
+    name = ps["Names"]
+
+    {
+      name: name, id: ps["ID"], role: role_name.presence,
+      replica: role&.replica_from_name(name), version: role&.version_from_name(name),
+      state: ps["State"], status: ps["Status"], health: ps["Status"].to_s[HEALTH, 1],
+      image: ps["Image"], created_at: ps["CreatedAt"]
+    }
+  end
+end

data/lib/dash/{cli → diagnostics}/doctor/config_checks.rb

@@ -1,17 +1,17 @@
 # Checks that only read the configuration — no SSH, no network. They report problems
 # that are visible in deploy.yml alone, so they still run when every host is unreachable.
-class Dash::Cli::Doctor::ConfigChecks
+class Dash::Diagnostics::Doctor::ConfigChecks
   def run
     readiness_results + dockerfile_results
   end
 
   private
     def result(check, target, status, detail)
-      Dash::Cli::Doctor::Result.new(check, target, status, detail)
+      Dash::Diagnostics::Doctor::Result.new(check, target, status, detail)
     end
 
     def readiness_results
-      DASH.roles.map { |role| readiness_check(role) }
+      Dash::Diagnostics::Doctor.scoped_roles.map { |role| readiness_check(role) }
     end
 
     def readiness_check(role)

data/lib/dash/diagnostics/doctor/controller_checks.rb

@@ -0,0 +1,46 @@
+# Whether the autoscale controller is running, for apps that need one: a role with a
+# `scale.schedule` or `scale.signal` changes capacity only while `dash autoscale run` ticks. A missing,
+# stale or stopped heartbeat warns, and so does every pause still in force. A state host
+# that does not answer is a warning, never an exception.
+class Dash::Diagnostics::Doctor::ControllerChecks
+  def initialize(now: Time.now.utc)
+    @now = now
+  end
+
+  # Within the run's --roles, like the other checks.
+  def run
+    roles = Dash::Diagnostics::Doctor.scoped_roles
+    return [] if roles.none?(&:controlled?)
+
+    status = Dash::Diagnostics::ControllerStatus.new(now: @now, roles: roles).to_h
+    return [ result(status[:state_host], :warn, "could not read the controller heartbeat (#{status[:error]})") ] if status[:error]
+
+    [ controller_result(status), *status[:pauses].map { |pause| pause_result(pause) } ]
+  end
+
+  private
+    def result(target, status, detail)
+      Dash::Diagnostics::Doctor::Result.new(:controller, target, status, detail)
+    end
+
+    def controller_result(status)
+      host, controller = status[:state_host], status[:controller]
+      return result(host, :warn, "no controller has run; #{status[:roles].join(", ")} have a schedule or signal that only dash autoscale run carries out") unless controller
+
+      name = "controller #{controller[:controller_id]} on #{controller[:hostname]}"
+
+      case controller[:status]
+      when "running"
+        result host, :ok, "#{name} (dash #{controller[:version]}) ticked #{controller[:age_seconds]}s ago, for #{status[:roles].join(", ")}"
+      when "stopped"
+        result host, :warn, "#{name} stopped at #{controller[:stopped_at]}"
+      else
+        result host, :warn, "#{name} last ticked #{controller[:age_seconds]}s ago (interval #{controller[:interval]}s), it looks stopped or stuck"
+      end
+    end
+
+    def pause_result(pause)
+      until_text = pause[:until] == Dash::Autoscale::Pause::INDEFINITE ? "until resumed" : "until #{pause[:until]}"
+      result pause[:role], :warn, "paused by #{pause[:by] || "unknown"} #{until_text}"
+    end
+end

data/lib/dash/{cli → diagnostics}/doctor/endpoint_checks.rb

@@ -4,7 +4,7 @@ require "openssl"
 
 # Local (no SSH) readiness checks for the domains dash-proxy will serve:
 # DNS resolution against the configured hosts and TLS certificate expiry.
-class Dash::Cli::Doctor::EndpointChecks
+class Dash::Diagnostics::Doctor::EndpointChecks
   CERTIFICATE_EXPIRY_WARN_DAYS = 14
   SECONDS_PER_DAY = 86_400
   TLS_CONNECT_TIMEOUT = 5
@@ -15,22 +15,30 @@ class Dash::Cli::Doctor::EndpointChecks
 
   private
     def result(check, target, status, detail)
-      Dash::Cli::Doctor::Result.new(check, target, status, detail)
+      Dash::Diagnostics::Doctor::Result.new(check, target, status, detail)
     end
 
     # Roles and accessories that run behind the proxy with custom domains.
     def proxied_units
-      @proxied_units ||= (DASH.roles + DASH.config.proxy_accessories).select do |unit|
+      @proxied_units ||= (Dash::Diagnostics::Doctor.scoped_roles + DASH.config.proxy_accessories).select do |unit|
         unit.running_proxy? && unit.proxy.hosts.any?
       end
     end
 
     def dns_results
       proxied_units.flat_map do |unit|
-        unit.proxy.hosts.map { |domain| dns_check(domain, unit.hosts, acme_issued: acme_issued?(unit)) }
+        unit.proxy.hosts.map { |domain| dns_check_of(unit, domain) }
       end
     end
 
+    # A scaled role's hosts come from the autoscale pool; when it cannot be read, its DNS
+    # cannot be compared, but its certificates still can.
+    def dns_check_of(unit, domain)
+      dns_check(domain, unit.hosts, acme_issued: acme_issued?(unit))
+    rescue Dash::Autoscale::ProviderError => e
+      result :dns, domain, :fail, "not compared, the hosts of #{unit.name} are unknown (#{e.message})"
+    end
+
     # Whether this unit's certificate comes from ACME rather than the operator.
     # Only then does where the domain points decide whether a certificate can
     # exist — a supplied certificate is unaffected by DNS.

data/lib/dash/{cli → diagnostics}/doctor/host_checks.rb

@@ -3,14 +3,15 @@
 # Every check rescues StandardError at its boundary: the doctor's contract is
 # to report broken environments, not crash on them, and a failed connection can
 # raise anything from Net::SSH errors to Errno and DNS resolution errors.
-class Dash::Cli::Doctor::HostChecks
+class Dash::Diagnostics::Doctor::HostChecks
   attr_reader :host, :sshkit, :proxy_host
   delegate :execute, :capture_with_info, to: :sshkit
 
-  def initialize(host, sshkit, proxy_host:)
+  def initialize(host, sshkit, proxy_host:, registry: true)
     @host = host
     @sshkit = sshkit
     @proxy_host = proxy_host
+    @registry = registry
   end
 
   def run
@@ -18,7 +19,7 @@ class Dash::Cli::Doctor::HostChecks
 
     if checks[:ssh].ok?
       checks[:docker] = docker_check
-      checks[:registry] = registry_check
+      checks[:registry] = registry_check if @registry
       checks.merge!(proxy_checks) if proxy_host
     end
 
@@ -27,7 +28,7 @@ class Dash::Cli::Doctor::HostChecks
 
   private
     def result(check, status, detail)
-      Dash::Cli::Doctor::Result.new(check, host, status, detail)
+      Dash::Diagnostics::Doctor::Result.new(check, host, status, detail)
     end
 
     def ssh_check

data/lib/dash/diagnostics/doctor/pool_checks.rb

@@ -0,0 +1,65 @@
+# Checks on the pool members of scaled roles: whether the provider answers, whether a
+# `power` role has enough members to reach its max, which members are stuck in a state
+# that is neither on nor off, and whether a scaled role behind dash-proxy is load balanced.
+# A provider failure is a failing result, never an exception.
+class Dash::Diagnostics::Doctor::PoolChecks
+  def run
+    DASH.config.scaled_roles.flat_map { |role| role_results(role) }
+  end
+
+  private
+    def result(target, status, detail)
+      Dash::Diagnostics::Doctor::Result.new(:pool, target, status, detail)
+    end
+
+    def role_results(role)
+      members = role.members
+      return [ unverified_result(role, members), loadbalancer_result(role) ].compact if members.any? { |member| !member.verified }
+
+      [ provider_result(role, members), capacity_result(role, members), *stuck_results(role, members), loadbalancer_result(role) ].compact
+    rescue Dash::Autoscale::ProviderError => e
+      [ result(role.name, :fail, e.message), loadbalancer_result(role) ].compact
+    end
+
+    def provider_result(role, members)
+      active = members.count(&:started?)
+      result role.name, :ok, "#{DASH.config.autoscale.provider_name} answered: #{members.size} #{"member".pluralize(members.size)}, #{active} started"
+    end
+
+    # Only --hosts produces unverified members, and only when the provider did not answer.
+    def unverified_result(role, members)
+      result role.name, :warn, "#{DASH.config.autoscale.provider_name} did not answer: #{members.size} #{"member".pluralize(members.size)} " \
+        "taken from --hosts, unverified (#{members.map(&:host).join(", ")})"
+    end
+
+    def capacity_result(role, members)
+      if role.scale.create?
+        result role.name, :ok, "creates members from its template, up to #{role.scale.max} hosts"
+      else
+        needed = role.scale.max - role.baseline_hosts.size
+
+        if members.size >= needed
+          result role.name, :ok, "#{members.size} #{"member".pluralize(members.size)} for the #{needed} it can scale out to (max #{role.scale.max} hosts)"
+        else
+          result role.name, :warn, "#{members.size} of the #{needed} members max #{role.scale.max} needs carry its labels " \
+            "(#{DASH.config.pool.labels_for(role).map { |key, value| "#{key}=#{value}" }.join(", ")}); it cannot scale past #{role.baseline_hosts.size + members.size} hosts"
+        end
+      end
+    end
+
+    def stuck_results(role, members)
+      members.select(&:transitional?).map do |member|
+        result "#{role.name} #{member.id}", :warn, "member #{member.host || member.id} is #{member.state}, dash leaves it alone until it is started or stopped"
+      end
+    end
+
+    def loadbalancer_result(role)
+      return unless role.running_proxy?
+
+      if DASH.config.proxy.load_balancing?
+        result role.name, :ok, "load balanced by #{DASH.config.proxy.effective_loadbalancer}"
+      else
+        result role.name, :fail, "a scaled role behind dash-proxy needs the load balancer"
+      end
+    end
+end

data/lib/dash/diagnostics/doctor/signal_checks.rb

@@ -0,0 +1,54 @@
+# Whether each signalled role's `scale.signal` source and `scale.hold_when` guards answer,
+# through Dash::Diagnostics::AutoscaleSignal - the reads the controller makes. Each result
+# names the value, the latency and the machine it was read from: a doctor run on a laptop
+# reaches the sources by another path than the controller host does. A source that does
+# not answer fails; one that answers without a number, and a guard holding a scale-out, warn.
+class Dash::Diagnostics::Doctor::SignalChecks
+  # Within the run's --roles, like the other checks. No signalled role: no result.
+  # One set of connections (and `via:` tunnels) for every role: the sources are the app's.
+  def run
+    roles = Dash::Diagnostics::Doctor.scoped_roles.select { |role| role.scaled? && role.scale.signal? }
+    return [] if roles.empty?
+
+    connections = Dash::Autoscale::Connections.new
+    begin
+      roles.flat_map { |role| role_results(role, connections) }
+    ensure
+      connections.close
+    end
+  end
+
+  private
+    def role_results(role, connections)
+      sample = Dash::Diagnostics::AutoscaleSignal.new(role: role, connections: connections).to_h
+      from = "from this machine (#{sample[:from]})"
+
+      [ source_result(role.name, sample[:signal], from),
+        *sample[:guards].map { |guard| guard_result("#{role.name} hold_when/#{guard[:index]}", guard, from) } ]
+    end
+
+    def source_result(target, read, from)
+      if read[:error]
+        result target, :fail, "#{read[:source]} did not answer #{from}: #{read[:error]}"
+      elsif read[:value].nil?
+        result target, :warn, "#{read[:source]} answered with no value in #{read[:latency_ms]}ms, #{from}"
+      else
+        result target, :ok, "#{read[:source]} answered #{read[:value]} in #{read[:latency_ms]}ms, #{from}"
+      end
+    end
+
+    def guard_result(target, guard, from)
+      return source_result(target, guard, from) if guard[:error]
+      return result(target, :warn, "#{guard[:source]} answered with no value in #{guard[:latency_ms]}ms, #{from}; without a value it holds a scale-out") if guard[:value].nil?
+
+      if guard[:holding]
+        result target, :warn, "#{guard[:source]} answered #{guard[:value]} in #{guard[:latency_ms]}ms, #{from}; above #{guard[:above]}, so it holds a scale-out"
+      else
+        source_result(target, guard, from)
+      end
+    end
+
+    def result(target, status, detail)
+      Dash::Diagnostics::Doctor::Result.new(:signal, target, status, detail)
+    end
+end

data/lib/dash/diagnostics/doctor.rb

@@ -0,0 +1,182 @@
+# `dash doctor`: deploy readiness checks, collected as results without ever raising on a
+# broken environment - failures become failing results instead. `to_h` is the same as data.
+#
+# Every check reads, except the registry check: it runs `docker login` on every host,
+# which writes the registry credentials there. `registry: false` leaves it out, and
+# `dash mcp` asks for that.
+class Dash::Diagnostics::Doctor < Dash::Diagnostics::Base
+  REGISTRY_SKIPPED = { check: "registry", reason: "runs docker login on each host, which writes credentials there; run `dash doctor` to include it" }.freeze
+
+  HOST_CHECKS = %i[ ssh docker registry proxy_image proxy_version proxy_socket ports ]
+
+  CHECK_TITLES = {
+    ssh: "SSH",
+    docker: "Docker",
+    registry: "Registry",
+    proxy_image: "Proxy image",
+    proxy_version: "Proxy version",
+    proxy_socket: "Proxy docker socket",
+    ports: "Ports",
+    dns: "DNS",
+    certificate: "Certificates",
+    readiness: "Readiness",
+    dockerfile: "Dockerfile",
+    drift: "Drift",
+    pool: "Pool",
+    controller: "Controller",
+    signal: "Signals"
+  }.freeze
+
+  STATUS_COLORS = { ok: :green, warn: :yellow, fail: :red }.freeze
+
+  Result = Struct.new(:check, :target, :status, :detail) do
+    def ok?
+      status == :ok
+    end
+
+    def warn?
+      status == :warn
+    end
+
+    def fail?
+      status == :fail
+    end
+
+    def title
+      CHECK_TITLES[check]
+    end
+
+    def to_s
+      "#{status.to_s.upcase} #{target}: #{detail}"
+    end
+  end
+
+  attr_reader :results
+
+  # The roles this run is about. DASH.roles builds the host scope, which asks the autoscale
+  # pool - so it is only used when --hosts narrowed the run, and the pool has been read
+  # already to filter by them. Otherwise --roles, or every role, straight from deploy.yml.
+  def self.scoped_roles
+    DASH.specific_hosts ? DASH.roles : (DASH.specific_roles || DASH.config.roles)
+  end
+
+  def initialize(registry: true)
+    @results = []
+    @registry = registry
+  end
+
+  # Pool checks first: when the provider cannot be asked, the member hosts are unknown, so
+  # the checks that need every host report that once instead of crashing on it.
+  def run
+    pool = pool_check_results
+    @results = without_pool(:ssh) { host_check_results } + without_pool(:dns) { endpoint_check_results } +
+      config_check_results + pool + controller_check_results + signal_check_results + drift_check_results
+  end
+
+  def failures
+    results.select(&:fail?)
+  end
+
+  def warnings
+    results.select(&:warn?)
+  end
+
+  def successful?
+    failures.none?
+  end
+
+  private
+    def snapshot
+      run
+
+      { successful: successful?, results: results.map { |result| result_h(result) }, skipped: (@registry ? [] : [ REGISTRY_SKIPPED ]) }
+    end
+
+    def result_h(result)
+      { check: result.check.to_s, target: result.target.to_s, status: result.status.to_s, detail: result.detail.to_s }
+    end
+
+    def host_check_results
+      results_by_host = collect_host_checks
+
+      HOST_CHECKS.flat_map do |check|
+        DASH.hosts.filter_map { |host| results_by_host.dig(host, check) }
+      end
+    end
+
+    def collect_host_checks
+      results_by_host = {}
+      mutex = Mutex.new
+      proxy_hosts = DASH.proxy_hosts
+      registry = @registry
+      error = nil
+
+      begin
+        on(DASH.hosts) do |host|
+          checks = Dash::Diagnostics::Doctor::HostChecks.new(host.hostname, self, proxy_host: proxy_hosts.include?(host.hostname), registry: registry).run
+          mutex.synchronize { results_by_host[host.hostname] = checks }
+        end
+      # Only ExecuteError: sshkit 1.25 has no MultipleExecuteError, and naming
+      # a constant that does not exist turns "a host's checks failed" into a
+      # NameError - the opposite of the doctor's never-crash contract.
+      rescue SSHKit::Runner::ExecuteError => e
+        # Per-check errors are captured inside HostChecks; getting here means a
+        # host's checks never completed at all. Record those as SSH failures below.
+        error = e
+      end
+
+      DASH.hosts.each do |host|
+        results_by_host[host] ||= { ssh: Result.new(:ssh, host, :fail, "could not run checks (#{error&.message || "connection failed"})") }
+      end
+
+      results_by_host
+    end
+
+    def endpoint_check_results
+      Dash::Diagnostics::Doctor::EndpointChecks.new.run
+    end
+
+    def config_check_results
+      Dash::Diagnostics::Doctor::ConfigChecks.new.run
+    end
+
+    def pool_check_results
+      Dash::Diagnostics::Doctor::PoolChecks.new.run
+    end
+
+    def controller_check_results
+      Dash::Diagnostics::Doctor::ControllerChecks.new.run
+    rescue StandardError => e
+      [ Result.new(:controller, "controller", :warn, "could not check the autoscale controller (#{e.class}: #{e.message})") ]
+    end
+
+    def signal_check_results
+      Dash::Diagnostics::Doctor::SignalChecks.new.run
+    rescue StandardError => e
+      [ Result.new(:signal, "signals", :warn, "could not check the autoscale signals (#{e.class}: #{e.message})") ]
+    end
+
+    def without_pool(check)
+      yield
+    rescue Dash::Autoscale::ProviderError => e
+      [ Result.new(check, "pool", :fail, "not run, the pool members are unknown (#{e.message})") ]
+    end
+
+    # Whether the proxies route to what runs. A target that is not running, or a host the
+    # load balancer should forward to and does not, fails; the rest warn.
+    def drift_check_results
+      drift = Dash::Diagnostics::Drift.take
+      unread = drift.unread.map { |host| Result.new(:drift, host[:host], :warn, "could not read #{host[:source]} (#{host[:error]}), so it was not compared") }
+
+      if drift.entries.empty? && unread.empty?
+        [ Result.new(:drift, "proxy", :ok, "proxy targets match the running containers") ]
+      else
+        unread + drift.entries.map do |entry|
+          status = Dash::Diagnostics::Drift::FAILURES.include?(entry[:code]) ? :fail : :warn
+          Result.new(:drift, entry[:host] || entry[:role], status, "#{entry[:code]}: #{entry[:detail]}")
+        end
+      end
+    rescue StandardError => e
+      [ Result.new(:drift, "proxy", :warn, "could not compare proxy targets with containers (#{e.class}: #{e.message})") ]
+    end
+end

data/lib/dash/diagnostics/drift.rb

@@ -0,0 +1,168 @@
+# Compares what should be running with what is: the containers on each host against the
+# targets dash-proxy routes to there, against the load balancer's targets. Pure - it reads
+# the Containers, ProxyServices and Lock snapshots and touches nothing; `.take` captures
+# them first.
+#
+# Each mismatch is `{ code:, host:, role:, detail: }`, with a code an agent can branch on:
+#
+#   proxy_target_not_running     the proxy routes to a container that is not running
+#   running_not_targeted         a running container of a proxied role is not a target
+#   loadbalancer_target_missing  a host the load balancer should forward to is not a target
+#   loadbalancer_target_extra    the load balancer forwards to a host it should not
+#   version_mismatch             a role's hosts run different versions
+#   multiple_running_versions    one host runs more than one version of a role
+#   member_not_targeted          a started pool member the load balancer does not forward to
+#                                (reported instead of loadbalancer_target_missing for it)
+#   member_orphan                a started pool member runs nothing of its role
+#
+# A host whose snapshot is an error is not compared; it is listed under `unread` instead,
+# and the fleet is not called consistent.
+# While the deploy lock is held the version codes are skipped too - a deploy in flight runs
+# two versions on purpose - and `lock_held` says so (nil when the lock was not read).
+class Dash::Diagnostics::Drift < Dash::Diagnostics::Base
+  FAILURES = %w[ proxy_target_not_running loadbalancer_target_missing ].freeze
+  REPLACED_SUFFIX = /_replaced_\h{16}\z/
+  TARGET_KEYS = %w[ targets reader_targets rollout_targets ].freeze
+
+  # The lock is only read when the primary host is in scope; narrowed away from it, the
+  # version codes are reported whether or not a deploy is in flight.
+  def self.take
+    lock = Dash::Diagnostics::Lock.new.to_h if DASH.hosts.include?(DASH.config.primary_host)
+
+    new(containers: Dash::Diagnostics::Containers.new(accessories: []).to_h, proxy_services: Dash::Diagnostics::ProxyServices.new.to_h, lock: lock)
+  end
+
+  def initialize(containers:, proxy_services:, lock: nil, config: nil)
+    @containers = containers
+    @proxy_services = proxy_services
+    @lock = lock
+    @config = config
+  end
+
+  def entries
+    @entries ||= proxy_entries + loadbalancer_entries + member_entries + version_entries
+  end
+
+  # The hosts whose containers or routes could not be read, so were not compared. A fleet
+  # with any is not reported consistent: no drift found is not the same as none there.
+  def unread
+    @unread ||= [ [ "containers", Array(@containers[:hosts]) ], [ "proxy_services", [ *@proxy_services[:hosts], @proxy_services[:loadbalancer] ].compact ] ]
+      .flat_map { |source, hosts| hosts.select { |host| host[:error] }.map { |host| { host: host[:host], source: source, error: host[:error] } } }
+  end
+
+  private
+    def snapshot
+      { consistent: entries.empty? && unread.empty?, lock_held: @lock&.dig(:lock, :held), drift: entries, unread: unread }
+    end
+
+    def proxy_entries
+      Array(@proxy_services[:hosts]).reject { |host| host[:error] }.flat_map do |proxy_host|
+        host = proxy_host[:host]
+        next [] unless running.key?(host)
+
+        proxied_roles_on(host).flat_map { |role| role_proxy_entries(role, host, targets(proxy_host[:services].to_h[role.container_prefix])) }
+      end
+    end
+
+    def role_proxy_entries(role, host, targets)
+      containers = running[host].select { |container| container[:role] == role.name }
+
+      not_running = targets.reject { |target| containers.any? { |container| container[:id].to_s.start_with?(target) } }.map do |target|
+        entry "proxy_target_not_running", host, role, "#{role.container_prefix} routes to #{target}, which is not a running container"
+      end
+
+      not_targeted = containers.reject { |container| targets.any? { |target| container[:id].to_s.start_with?(target) } }.map do |container|
+        entry "running_not_targeted", host, role, "#{container[:name]} (replica #{container[:replica]}) runs but #{role.container_prefix} does not route to it"
+      end
+
+      not_running + not_targeted
+    end
+
+    def loadbalancer_entries
+      loadbalancer = @proxy_services[:loadbalancer]
+      return [] if loadbalancer.nil? || loadbalancer[:error]
+
+      expected = Dash::Configuration::Loadbalancer.new(config: config, proxy_config: config.proxy.proxy_config, secrets: config.secrets).target_hosts.uniq
+      actual = targets(loadbalancer[:services].to_h[config.service])
+      host = loadbalancer[:host]
+
+      (expected - actual).map { |target| missing_target_entry(host, target) } +
+        (actual - expected).map { |target| entry "loadbalancer_target_extra", host, nil, "the load balancer forwards to #{target}, which no proxied role runs on" }
+    end
+
+    # A member that joined but never reached the load balancer serves nobody: capacity paid
+    # for and wasted, not a broken route - so it is its own code, not a failure.
+    def missing_target_entry(host, target)
+      if (member = started_members.find { |candidate| candidate.host == target })
+        role = config.roles.find { |candidate| candidate.name == member.role }
+        entry "member_not_targeted", host, role, "member #{member.id} (#{target}) is started but the load balancer does not forward to it"
+      else
+        entry "loadbalancer_target_missing", host, nil, "the load balancer does not forward to #{target}"
+      end
+    end
+
+    def member_entries
+      config.scaled_roles.flat_map do |role|
+        role.active_members.filter_map do |member|
+          next unless running.key?(member.host)
+          next if running[member.host].any? { |container| container[:role] == role.name }
+
+          entry "member_orphan", member.host, role, "member #{member.id} is started but runs no #{role} container"
+        end
+      end
+    end
+
+    def started_members
+      @started_members ||= config.scaled_roles.flat_map(&:active_members)
+    end
+
+    def version_entries
+      return [] if @lock&.dig(:lock, :held)
+
+      config.roles.flat_map do |role|
+        by_host = running.transform_values { |containers| versions(containers.select { |container| container[:role] == role.name }) }.reject { |_, versions| versions.empty? }
+
+        multiple = by_host.select { |_, versions| versions.size > 1 }.map do |host, versions|
+          entry "multiple_running_versions", host, role, "#{role} runs #{versions.join(", ")} on #{host}"
+        end
+
+        mismatch = if by_host.values.uniq.size > 1
+          [ entry("version_mismatch", nil, role, "#{role} runs " + by_host.map { |host, versions| "#{versions.join("+")} on #{host}" }.join(", ")) ]
+        end
+
+        multiple + mismatch.to_a
+      end
+    end
+
+    def config
+      @config ||= DASH.config
+    end
+
+    def running
+      @running ||= Array(@containers[:hosts]).reject { |host| host[:error] }.to_h do |host|
+        [ host[:host], Array(host[:containers]).select { |container| container[:state] == "running" } ]
+      end
+    end
+
+    def proxied_roles_on(host)
+      config.roles.select { |role| role.running_proxy? && role.hosts.include?(host) }
+    end
+
+    # dash-proxy lists targets as "<container id or host>:<port>"; the id is the short one
+    # dash deployed with, the host is what the load balancer forwards to.
+    def targets(service)
+      return [] unless service
+
+      listed = TARGET_KEYS.flat_map { |key| Array(service[key]) }
+      listed = service["target"].to_s.split(",") if listed.empty?
+      listed.map { |target| target.strip.sub(/:\d+\z/, "") }.reject(&:empty?).uniq
+    end
+
+    def versions(containers)
+      containers.map { |container| container[:version].to_s.sub(REPLACED_SUFFIX, "") }.uniq.sort
+    end
+
+    def entry(code, host, role, detail)
+      { code: code, host: host, role: role&.name, detail: detail }
+    end
+end