dash 4.1.2 → 4.2.0

data/lib/dash/cli/proxy.rb

@@ -1,5 +1,8 @@
 class Dash::Cli::Proxy < Dash::Cli::Base
+  include Inspection
+
   desc "boot", "Boot proxy on servers"
+  option :skip_loadbalancer, type: :boolean, default: false, desc: "Boot the per-host proxies only, leaving the load balancer alone (dash scale set uses it for a joining member)"
   def boot
     modify(lock: true, server_lock: true) do
       on(network_hosts) do |host|
@@ -70,7 +73,9 @@ class Dash::Cli::Proxy < Dash::Cli::Base
         run_hook "post-proxy-reboot", hosts: host
       end
 
-      if DASH.config.proxy.load_balancing?
+      # A member joining with `dash scale set` holds the server lock on itself only, so the
+      # shared load balancer - which a drifted config would reboot here - is not its to touch.
+      if boots_loadbalancer?
         lb_drifted = Concurrent::Array.new
         lb_stale = Concurrent::Array.new
 
@@ -443,12 +448,8 @@ class Dash::Cli::Proxy < Dash::Cli::Base
       end
     when "deploy"
       if DASH.config.proxy.load_balancing?
-        targets = DASH.loadbalancer_config.target_hosts
-
         on(DASH.config.proxy.effective_loadbalancer) do |host|
-          Dash::Cli::Proxy::LoadbalancerClaim.new(host, self).claim_service
-          info "Deploying to loadbalancer on #{host} with targets: #{targets.join(', ')}"
-          execute *DASH.loadbalancer.deploy(targets: targets)
+          Dash::Cli::Proxy::LoadbalancerDeploy.new(host, self).run
         end
       else
         puts "Load balancing is not configured"
@@ -667,10 +668,14 @@ class Dash::Cli::Proxy < Dash::Cli::Base
     # network there (zoolutions/dash#140).
     def network_hosts
       hosts = DASH.hosts
-      hosts |= [ DASH.config.proxy.effective_loadbalancer ] if DASH.config.proxy.load_balancing?
+      hosts |= [ DASH.config.proxy.effective_loadbalancer ] if boots_loadbalancer?
       hosts
     end
 
+    def boots_loadbalancer?
+      DASH.config.proxy.load_balancing? && !options[:skip_loadbalancer]
+    end
+
     # The host that owns TLS, and so the certificate store: the loadbalancer
     # host when load balancing (TLS terminates at the edge), else the primary
     # host - the same host `loadbalancer: true` would resolve to.

data/lib/dash/cli/prune.rb

@@ -16,8 +16,8 @@ class Dash::Cli::Prune < Dash::Cli::Base
     end
   end
 
-  desc "containers", "Prune all stopped containers, except the last n per role (default 5)"
-  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain per role"
+  desc "containers", "Prune all stopped containers, except the last n per role replica (default 5)"
+  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain per role replica"
   def containers
     retain = options.fetch(:retain, DASH.config.retain_containers)
     raise "retain must be at least 1" if retain < 1
@@ -27,7 +27,9 @@ class Dash::Cli::Prune < Dash::Cli::Base
         # One round trip per host, whatever it runs: a host with no app roles still
         # records that the sweep reached it.
         execute *DASH.auditor.record_then("Pruned containers",
-          *DASH.roles_on(host).map { |role| DASH.prune.app_containers(retain: retain, role: role) })
+          *DASH.roles_on(host).flat_map { |role|
+            role.replica_numbers.map { |replica| DASH.prune.app_containers(retain: retain, role: role, replica: replica) }
+          })
       end
     end
   end

data/lib/dash/cli/report.rb

@@ -13,8 +13,11 @@ class Dash::Cli::Report < Dash::Cli::Base
 
   desc "show", "Print the last saved deploy report"
   option :last, type: :numeric, banner: "N", desc: "Print a trend table over the last N reports instead"
+  option :json, type: :boolean, default: false, desc: "Print the saved reports as JSON, most recent first (at most #{Dash::Diagnostics::Reports::MAX})"
   def show
-    if (last = options[:last])
+    if options[:json]
+      puts_json(ssh: false) { Dash::Diagnostics::Reports.new(last: options[:last] || 1, directory: reports_directory).to_h }
+    elsif (last = options[:last])
       return say "--last takes a positive number of reports, got #{last}", :red unless count?(last)
 
       print_trend saved.recent(last.to_i)

data/lib/dash/cli/scale/host_plan.rb

@@ -0,0 +1,63 @@
+# How `dash scale set` reaches a container count on a scaled role, across hosts. Pure: it is
+# handed the role's baseline hosts, its started members (in the order they would leave
+# last-first), the target and both bounds, and says how many
+# members join, which leave, and how many containers each host ends with.
+#
+# Hosts are kept as few as the count allows, never below scale min or the baseline (so
+# scale min joins members too). Every
+# host gets replicas min; the rest fills the baseline first, then members in order - so a
+# scale-out fills the hosts it has before a member joins, and a scale-in empties members
+# down to min, then lets them leave, before the baseline gives anything up.
+class Dash::Cli::Scale::HostPlan
+  attr_reader :joins, :leaving, :counts, :join_counts
+
+  # [ slots to add, slots to remove ] to take a host from `slots` to `count` containers:
+  # the lowest free slots in, the highest out.
+  def self.slot_changes(slots, count)
+    slots = slots.sort
+
+    if count > slots.size
+      [ ((1..count + slots.size).to_a - slots).first(count - slots.size), [] ]
+    else
+      [ [], slots.reverse.first(slots.size - count) ]
+    end
+  end
+
+  def initialize(baseline:, members:, target:, replicas_min:, replicas_max:, hosts_min:, hosts_max:)
+    @min, @max = replicas_min, replicas_max
+    current = baseline + members
+    hosts = host_count(baseline.size, target, hosts_min)
+
+    raise ArgumentError, "#{target} containers need #{hosts} hosts, scale max is #{hosts_max}" if hosts > hosts_max
+    if hosts * @min > target || hosts * @max < target
+      raise ArgumentError, "#{target} containers cannot be split over #{hosts} hosts with #{@min} to #{@max} per host"
+    end
+
+    @joins = [ hosts - current.size, 0 ].max
+    @leaving = members.last([ current.size - hosts, 0 ].max)
+
+    kept = current - @leaving
+    spread = spread(target, kept.size + @joins)
+    @counts = kept.zip(spread).to_h
+    @join_counts = spread.drop(kept.size)
+  end
+
+  private
+    # As few hosts as the count fits on, never below the baseline or scale min - which is
+    # hosts kept running, so it joins members as well as keeping them.
+    def host_count(baseline, target, hosts_min)
+      [ (target.to_f / @max).ceil, hosts_min, baseline ].max
+    end
+
+    # Each host min, then the rest to the earliest hosts up to max.
+    def spread(target, hosts)
+      counts = Array.new(hosts, @min)
+      remaining = target - (hosts * @min)
+
+      counts.map do |count|
+        extra = [ remaining, @max - count ].min
+        remaining -= extra
+        count + extra
+      end
+    end
+end

data/lib/dash/cli/scale/join.rb

@@ -0,0 +1,128 @@
+# Brings one pool member into a scaled role: powers a stopped member on (or creates one
+# from the template), waits for SSH, then runs what a deploy would run on a new host -
+# bootstrap (created members only), registry login, the stale-container sweep, the proxy
+# for a proxied role and the app boot at the version the role already runs - narrowed to
+# that host and role. A proxied member joins the load balancer last, once it is healthy.
+#
+# The sweep comes before the boot on purpose: a member powered off outside dash resumes
+# its old container under `restart: unless-stopped`, and a worker would consume jobs with
+# old code until the boot replaced it.
+#
+# Anything failing after the power-on powers the member off again (or destroys it), unless
+# --keep-on-failure, so a failed join never leaves a running host with nothing on it.
+class Dash::Cli::Scale::Join
+  SSH_RETRY_INTERVAL = 5
+
+  attr_reader :role, :cli, :version, :count
+
+  def initialize(role, cli, version:, count:, member: nil, keep_on_failure: false)
+    @role = role
+    @cli = cli
+    @member = member
+    @version = version
+    @count = count
+    @keep_on_failure = keep_on_failure
+  end
+
+  # The member's host, once it serves.
+  def run
+    deadline = monotonic_now + role.scale.boot_timeout
+
+    begin
+      member = power_on
+      host = joined_host(member)
+      wait_for_ssh(host, deadline)
+      cli.run_hook "pre-scale-out", role: role.name, hosts: host, replicas: count.to_s
+
+      cli.narrowed_to(host, role) do
+        cli.invoke_narrowed "dash:cli:server:bootstrap", Dash::Cli::Server if created?
+        cli.invoke_narrowed "dash:cli:registry:login", Dash::Cli::Registry, skip_local: true
+        cli.invoke_narrowed "dash:cli:app:stale_containers", Dash::Cli::App, stop: true
+        cli.invoke_narrowed "dash:cli:proxy:boot", Dash::Cli::Proxy, skip_loadbalancer: true if role.running_proxy?
+        cli.invoke_narrowed "dash:cli:app:boot", Dash::Cli::App, version: version
+      end
+
+      if role.running_proxy?
+        cli.deploy_loadbalancer
+        @in_loadbalancer = host
+      end
+      cli.run_hook "post-scale-out", role: role.name, hosts: host, replicas: count.to_s
+      cli.report "Joined #{member.id} (#{host}) to #{role}"
+      host
+    rescue StandardError => e
+      # The created server, or the member started; nil when a create failed before the
+      # provider returned one, and then there is nothing to power off.
+      member = @created || @member
+
+      unless @keep_on_failure
+        # Out of the load balancer before it goes, or it would forward to a stopped host.
+        cli.deploy_loadbalancer(except: @in_loadbalancer) if @in_loadbalancer
+        power_off(member) if member
+      end
+
+      raise e
+    end
+  end
+
+  private
+    def created?
+      @member.nil?
+    end
+
+    def power_on
+      if created?
+        cli.report "Creating a #{role} member...", :magenta
+        @created = provider.create(labels: pool.labels_for(role), template: role.scale.template, address: role.scale.address)
+      else
+        cli.report "Powering on #{@member.id} (#{@member.host}) for #{role}...", :magenta
+        provider.start(@member)
+      end
+
+      provider.wait_until(@created || @member, state: "started", timeout: role.scale.boot_timeout)
+      @created || @member
+    end
+
+    # A created server may not have its address until it runs, so the host is read back
+    # from the provider - which also puts the member into role.hosts for the narrowing.
+    def joined_host(member)
+      pool.refresh!
+      role.active_members.find { |candidate| candidate.id == member.id }&.host ||
+        raise(Dash::Autoscale::ProviderError, "#{member.id} is started but #{provider.name} does not list it among #{role}'s members")
+    end
+
+    # The bounded handshake of Dash::Cli::Base#on (zoolutions/dash#152), retried until the
+    # member's boot_timeout runs out.
+    def wait_for_ssh(host, deadline)
+      cli.report "Waiting for #{host} to answer SSH...", :magenta
+
+      begin
+        cli.on_hosts([ host ]) { execute :true }
+      rescue SSHKit::Runner::ExecuteError => e
+        if monotonic_now >= deadline
+          raise Dash::Cli::BootError, "#{host} did not answer SSH within #{role} scale/boot_timeout (#{role.scale.boot_timeout}s): #{e.message}"
+        end
+
+        sleep SSH_RETRY_INTERVAL
+        retry
+      end
+    end
+
+    def power_off(member)
+      cli.report "Join failed, powering #{member.id} off again (--keep-on-failure keeps it)", :red
+      Dash::Cli::Scale::PowerOff.new(role, member, provider).run(destroy: created?)
+    rescue StandardError => e
+      cli.report "Could not power #{member.id} off: #{e.message}", :red
+    end
+
+    def pool
+      DASH.config.pool
+    end
+
+    def provider
+      pool.provider
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+end

data/lib/dash/cli/scale/leave.rb

@@ -0,0 +1,58 @@
+# Takes one pool member out of a scaled role. A proxied member leaves the load balancer
+# first - the redeploy without it drains its in-flight requests - and its containers stop
+# after. A worker member is drained slot by slot (drain signal, drain wait, stop) the way a
+# replica scale-in is. Then the provider powers it off, or deletes it for `members: create`.
+# A baseline host never leaves: dash scale does not stop what deploy.yml lists.
+#
+# `reachable: false` replaces a member that stopped answering SSH (the autoscale controller,
+# after `scale.boot_timeout`): nothing is run on it - no drain, no `docker stop` - but it
+# still leaves the load balancer and the provider still powers it off (or deletes it).
+class Dash::Cli::Scale::Leave
+  attr_reader :role, :member, :cli, :count
+
+  # `running` is { replica => version } on the member.
+  def initialize(role, member, cli, running:, count:, reachable: true)
+    @role = role
+    @member = member
+    @cli = cli
+    @running = running
+    @count = count
+    @reachable = reachable
+  end
+
+  def run
+    host = member.host
+    raise ArgumentError, "#{host} is a baseline host of #{role}, dash scale never stops it" if role.baseline_hosts.include?(host)
+
+    cli.run_hook "pre-scale-in", role: role.name, hosts: host, replicas: count.to_s
+    cli.report "Removing #{member.id} (#{host}) from #{role}#{", unreachable" unless @reachable}...", :magenta
+
+    cli.deploy_loadbalancer(except: (host unless shared_with_another_proxied_role?(host))) if role.running_proxy?
+    stop_containers(host) if @reachable
+    Dash::Cli::Scale::PowerOff.new(role, member, DASH.config.pool.provider).run(destroy: role.scale.create?)
+    DASH.config.pool.refresh!
+
+    cli.run_hook "post-scale-in", role: role.name, hosts: host, replicas: count.to_s
+    cli.report "Removed #{member.id} (#{host}) from #{role}"
+  end
+
+  private
+    # The load balancer has one target list for every proxied role of the app.
+    def shared_with_another_proxied_role?(host)
+      DASH.config.roles.any? { |other| other != role && other.running_proxy? && other.hosts.include?(host) }
+    end
+
+    def stop_containers(host)
+      role, running = self.role, @running
+
+      cli.on_hosts([ host ]) do
+        if running.empty?
+          # An orphan: nothing of the role to stop or drain.
+        elsif role.running_proxy?
+          running.each { |replica, version| execute *DASH.app(role: role, host: host, replica: replica).stop(version: version) }
+        else
+          Dash::Cli::Scale::ReplicaLeave.new(host, role, running, self, running: running.keys).run
+        end
+      end
+    end
+end

data/lib/dash/cli/scale/placement.rb

@@ -0,0 +1,54 @@
+# Where `dash scale set` adds or removes containers to reach a total. Pure: it is handed the
+# slots running on each host and returns the slots to add and remove per host.
+#
+# Adds go to the host with the fewest replicas, in its lowest free slot; removals come from
+# the host with the most, highest slot first. Ties go to the earlier host for adds and the
+# later host for removals, so repeated scaling out and in keeps the same hosts fuller.
+class Dash::Cli::Scale::Placement
+  attr_reader :additions, :removals
+
+  def initialize(running:, target:, max:)
+    @slots = running.transform_values { |slots| slots.sort }
+    @max = max
+    @additions = Hash.new { |hash, host| hash[host] = [] }
+    @removals = Hash.new { |hash, host| hash[host] = [] }
+
+    plan(target)
+
+    @additions = @additions.to_h
+    @removals = @removals.to_h
+  end
+
+  private
+    # A host above max (max was lowered since it scaled) sheds its surplus first: the next
+    # deploy would clamp it anyway, and the count asked for would quietly not hold.
+    def plan(target)
+      @slots.each_key do |host|
+        remove_from(host) while @slots[host].size > @max
+      end
+
+      total = @slots.values.sum(&:size)
+
+      (target - total).times { add_one } if target > total
+      (total - target).times { remove_one } if target < total
+    end
+
+    def add_one
+      host = @slots.keys.select { |candidate| @slots[candidate].size < @max }.min_by { |candidate| @slots[candidate].size }
+      slot = ((1..@max).to_a - @slots[host]).first
+
+      @slots[host] = (@slots[host] + [ slot ]).sort
+      @additions[host] << slot
+    end
+
+    def remove_one
+      remove_from @slots.keys.reverse.max_by { |candidate| @slots[candidate].size }
+    end
+
+    def remove_from(host)
+      slot = @slots[host].max
+
+      @slots[host] = @slots[host] - [ slot ]
+      @removals[host] << slot
+    end
+end

data/lib/dash/cli/scale/power_off.rb

@@ -0,0 +1,26 @@
+# Powers a pool member off through the provider and waits for it to be stopped, then
+# deletes it when the role creates its members. The provider's soft stop gets the role's
+# stop budget plus room for the operating system to shut down.
+class Dash::Cli::Scale::PowerOff
+  SHUTDOWN_ALLOWANCE = 30
+  DEFAULT_STOP_TIMEOUT = 30
+
+  def initialize(role, member, provider)
+    @role = role
+    @member = member
+    @provider = provider
+  end
+
+  def run(destroy:)
+    timeout = (@role.stop_timeout || DEFAULT_STOP_TIMEOUT) + SHUTDOWN_ALLOWANCE
+
+    # A member a failed join left stopped refuses a second stop (UpCloud answers
+    # SERVER_STATE_ILLEGAL), which would skip the destroy.
+    unless @provider.state(@member) == "stopped"
+      @provider.stop(@member, timeout: timeout)
+      @provider.wait_until(@member, state: "stopped", timeout: timeout + SHUTDOWN_ALLOWANCE)
+    end
+
+    @provider.destroy(@member) if destroy
+  end
+end

data/lib/dash/cli/scale/replica_join.rb

@@ -0,0 +1,74 @@
+# Adds replica slots of one role on one host, at the version the host already runs (the
+# caller sets it on the config). A web replica joins the proxy pool only once healthy:
+# dash-proxy waits for every target before it swaps. On failure the new containers are
+# stopped and the replicas already running are left alone.
+class Dash::Cli::Scale::ReplicaJoin
+  SHORT_CONTAINER_ID_LENGTH = Dash::Cli::App::Boot::SHORT_CONTAINER_ID_LENGTH
+
+  attr_reader :host, :role, :replicas, :sshkit
+  delegate :execute, :capture_with_info, :upload!, :error, to: :sshkit
+
+  # `running` is the slots already running on the host, which stay in the pool.
+  def initialize(host, role, replicas, sshkit, running:)
+    @host = host
+    @role = role
+    @replicas = replicas
+    @sshkit = sshkit
+    @running = running
+  end
+
+  def run
+    execute *DASH.auditor(role: role).record_then("Scaled out to replicas #{replicas.join(", ")} at #{DASH.config.version}", app.ensure_env_directory)
+    upload! role.secrets_io(host), role.secrets_path, mode: "0600"
+
+    begin
+      replicas.each { |replica| start(replica) }
+      role.running_proxy? ? join_proxy : wait_for_ready
+    rescue => e
+      error "Failed to scale out #{role} on #{host}"
+      replicas.each { |replica| execute *app(replica).stop(version: DASH.config.version), raise_on_non_zero_exit: false }
+      raise e
+    end
+  end
+
+  private
+    # A scale-in stops a replica and leaves its container, so the slot's name can still be
+    # taken by a stopped container of this version. `docker rm` refuses a running one, and
+    # the slot is not running - that is why it is being added.
+    def start(replica)
+      execute *app(replica).remove_container(version: DASH.config.version), raise_on_non_zero_exit: false
+
+      hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
+      capture_with_info(*app(replica).run(hostname: hostname))
+    end
+
+    # The whole pool, old replicas and new: a deploy replaces the service's target list, so a
+    # missing id would drop a running replica from it.
+    def join_proxy
+      pool = (@running + replicas).sort
+      targets = Dash::Cli::App::RunningTargets.new(sshkit, role: role, host: host).container_ids(replicas: pool)
+
+      if targets.size != pool.size
+        raise Dash::Cli::BootError, "Found #{targets.size} of the #{pool.size} replicas of #{role} on #{host} to pool, not changing the proxy"
+      end
+
+      execute *app.deploy(targets: targets.map { |id| id[0, SHORT_CONTAINER_ID_LENGTH] })
+    end
+
+    def wait_for_ready
+      replicas.each do |replica|
+        Dash::Cli::Healthcheck::Poller.wait_for_healthy(role: role) do |mode, seconds_left = nil|
+          if mode == :confirm
+            capture_with_info(*app(replica).status(version: DASH.config.version))
+          else
+            capture_with_info *app(replica).wait_for_ready(version: DASH.config.version, timeout: seconds_left),
+              interaction_handler: Dash::Cli::Healthcheck::ProgressReporter.new
+          end
+        end
+      end
+    end
+
+    def app(replica = 1)
+      DASH.app(role: role, host: host, replica: replica)
+    end
+end

data/lib/dash/cli/scale/replica_leave.rb

@@ -0,0 +1,121 @@
+# Removes replica slots of one role on one host, gracefully. A web replica leaves the proxy
+# pool first - the redeploy without it drains its in-flight requests - and only then stops.
+# A worker replica is sent its drain signal and given its drain wait before `docker stop`,
+# whose own timeout is the role's stop budget. A role with a Sidekiq signal ends the wait as
+# soon as Sidekiq reports nothing busy in the leaving containers.
+class Dash::Cli::Scale::ReplicaLeave
+  SHORT_CONTAINER_ID_LENGTH = Dash::Cli::App::Boot::SHORT_CONTAINER_ID_LENGTH
+  BUSY_POLL_INTERVAL = 2
+
+  attr_reader :host, :role, :leaving, :sshkit
+  delegate :execute, :capture_with_info, :info, to: :sshkit
+
+  # What can say how many jobs a container is still working on: the role's signal, when its
+  # source can (only Sidekiq). Nil for every other role.
+  def self.busy_reader_for(role)
+    return unless role.scaled? && role.scale.signal?
+
+    Dash::Autoscale::Signal.for(role, Dash::Autoscale::Connections.shared).then { |reader| reader if reader.busy? }
+  end
+
+  # `leaving` is { replica => running version }; `running` is every slot running on the host.
+  def initialize(host, role, leaving, sshkit, running:, busy_reader: self.class.busy_reader_for(role))
+    @host = host
+    @role = role
+    @leaving = leaving
+    @sshkit = sshkit
+    @running = running
+    @busy_reader = busy_reader
+  end
+
+  def run
+    execute *DASH.auditor(role: role).record("Scaled in replicas #{leaving.keys.join(", ")}"), verbosity: :debug
+
+    if role.running_proxy?
+      leave_proxy
+    else
+      drain
+    end
+
+    # A stop that fails must fail the scale-in: the replica would still run while the
+    # command reported the count reached.
+    leaving.each do |replica, version|
+      execute *app(replica).stop(version: version)
+    end
+  end
+
+  private
+    def leave_proxy
+      staying = @running - leaving.keys
+      targets = Dash::Cli::App::RunningTargets.new(sshkit, role: role, host: host).container_ids(replicas: staying)
+      execute *app.deploy(targets: targets.map { |id| id[0, SHORT_CONTAINER_ID_LENGTH] })
+    end
+
+    # The signal is optional; the wait is not tied to it, so `drain: { wait: N }` alone
+    # still gives a replica N seconds before `docker stop`. The hostnames are read first:
+    # a container told to stop may not answer an inspect for long.
+    def drain
+      hostnames = leaving_hostnames if @busy_reader && role.drain_wait.positive?
+
+      if role.drain_signal
+        leaving.each do |replica, version|
+          execute *app(replica).signal(role.drain_signal, version: version), raise_on_non_zero_exit: false
+        end
+      end
+
+      await_drain(hostnames)
+    end
+
+    # The full drain wait, or - with the hostnames to ask about - until nothing is busy in
+    # them, polling every BUSY_POLL_INTERVAL seconds. A busy count that cannot be read waits
+    # out the rest of the drain.
+    def await_drain(hostnames)
+      return unless role.drain_wait.positive?
+
+      info "Waiting up to #{role.drain_wait}s for #{role} replicas #{leaving.keys.join(", ")} on #{host} to drain..."
+      return sleep(role.drain_wait) if hostnames.blank?
+
+      deadline = monotonic + role.drain_wait
+      loop do
+        busy = busy_count(hostnames)
+        return info("#{role} replicas #{leaving.keys.join(", ")} on #{host} have nothing busy") if busy&.zero?
+
+        remaining = deadline - monotonic
+        return unless remaining.positive?
+
+        sleep(busy.nil? ? remaining : [ BUSY_POLL_INTERVAL, remaining ].min)
+        return if busy.nil?
+      end
+    end
+
+    # Nil unless every leaving container's hostname is read: one missing would drop out of
+    # what the busy count covers.
+    def leaving_hostnames
+      hostnames = leaving.map { |replica, version| capture_with_info(*app(replica).container_hostname(version: version)).strip }
+      return hostnames if hostnames.none?(&:empty?)
+
+      info "Could not read the hostname of every #{role} replica #{leaving.keys.join(", ")} on #{host}, so the drain waits its full #{role.drain_wait}s"
+      nil
+    rescue SSHKit::Command::Failed => e
+      info "Could not read the hostnames of #{role} replicas #{leaving.keys.join(", ")} on #{host} (#{e.message}), so the drain waits its full #{role.drain_wait}s"
+      nil
+    end
+
+    # Like a signal sample, a failed read never fails the scale-in: it only loses the early end.
+    def busy_count(hostnames)
+      @busy_reader.busy(hostnames).tap do |busy|
+        info "Sidekiq reports no process in #{role} replicas #{leaving.keys.join(", ")} on #{host}, so the drain waits out drain.wait" if busy.nil?
+      end
+    rescue StandardError => e
+      info "Could not read what #{role} replicas #{leaving.keys.join(", ")} on #{host} are working on (#{e.class}: #{e.message}), so the drain waits out drain.wait"
+      nil
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def app(replica = 1)
+      DASH.app(role: role, host: host, replica: replica)
+    end
+end