dash 4.1.2 → 4.2.0

data/lib/dash/cli/scale.rb

@@ -0,0 +1,276 @@
+# Changes how many containers of a role run, at runtime, at the version already running -
+# no build and no deploy. The count is in containers across the role's hosts, like Heroku
+# dynos, spread evenly within each host's `replicas` bounds. A later deploy keeps it,
+# because a deploy boots as many replicas as are running.
+#
+# A role with `scale:` also spans hosts: when its hosts are full, pool members join (see
+# Dash::Cli::Scale::Join), and they leave again before the baseline gives up a container
+# (Dash::Cli::Scale::HostPlan).
+class Dash::Cli::Scale < Dash::Cli::Base
+  desc "set ROLE COUNT", "Run COUNT containers of ROLE across its hosts, within its replicas bounds (and scale bounds, joining or removing pool members)"
+  option :keep_on_failure, type: :boolean, default: false, desc: "Leave a member that failed to join powered on, for debugging"
+  def set(role_name, count)
+    role = scalable_role(role_name)
+    count = Integer(count, 10)
+    ensure_within_bounds(role, count)
+
+    modify(lock: true) do
+      next scale_across_hosts(role, count) if role.scaled?
+
+      running = running_replicas(role)
+      placement = Dash::Cli::Scale::Placement.new(running: running.transform_values(&:keys), target: count, max: role.replicas.max)
+
+      if placement.additions.empty? && placement.removals.empty?
+        say "#{role} already runs #{count} containers", :magenta
+      else
+        scale_out(role, placement.additions, running, count) if placement.additions.any?
+        scale_in(role, placement.removals, running, count) if placement.removals.any?
+        say "#{role} now runs #{count} containers", :magenta
+      end
+    end
+  end
+
+  desc "status [ROLE]", "Show the containers of each role per host (or one role)"
+  option :json, type: :boolean, default: false, desc: "Print the status as JSON"
+  def status(role_name = nil)
+    roles = role_name ? [ scalable_role(role_name) ] : DASH.config.roles
+    scale = -> { Dash::Diagnostics::Scale.new(roles: roles).to_h }
+
+    if options[:json]
+      puts_json(&scale)
+    else
+      pre_connect_if_required
+      scale.call[:roles].each { |role| print_status(role) }
+    end
+  end
+
+  no_commands do
+    # Run a sub-command on one host and role. Sub-commands read the run's scope from DASH,
+    # not from their own --hosts/--roles once DASH is configured, so the scope is set here.
+    def narrowed_to(host, role)
+      hosts, roles = DASH.specific_hosts, DASH.specific_roles&.map(&:name)
+      DASH.specific_hosts = [ host ]
+      DASH.specific_roles = [ role.name ]
+      yield
+    ensure
+      DASH.specific_roles = roles
+      DASH.specific_hosts = hosts
+    end
+
+    def invoke_narrowed(command, cli_class, **command_options)
+      invoke command, [], options.except("keep_on_failure").merge(command_options)
+      reset_invocation(cli_class)
+    end
+
+    def on_hosts(hosts, &block)
+      on(hosts, &block)
+    end
+
+    def report(message, color = nil)
+      say message, color
+    end
+
+    # Takes out a started member that no longer answers SSH, under the deploy lock, without
+    # a command reaching it (see Dash::Cli::Scale::Leave). Its run directory is never
+    # ensured: acquiring the lock would otherwise SSH to every host of the app, it included.
+    # `count` is the role's container count, for the scale-in hooks.
+    def replace_unreachable(role, member, count:)
+      skipped = !role.baseline_hosts.include?(member.host) && !DASH.run_directory_ensured_on.include?(member.host)
+      DASH.run_directory_ensured_on << member.host if skipped
+
+      modify(lock: true) do
+        Dash::Cli::Scale::Leave.new(role, member, self, running: {}, count: count, reachable: false).run
+      end
+    ensure
+      # The same host may come back as a powered-on member, whose run directory is then real.
+      DASH.run_directory_ensured_on.delete(member.host) if skipped
+    end
+
+    def deploy_loadbalancer(except: nil)
+      targets = DASH.loadbalancer_config.target_hosts - Array(except)
+
+      on(DASH.config.proxy.effective_loadbalancer) do |host|
+        Dash::Cli::Proxy::LoadbalancerDeploy.new(host, self, targets: targets).run
+      end
+    end
+  end
+
+  private
+    def scalable_role(role_name)
+      DASH.config.role(role_name) || raise(ArgumentError, "No role named #{role_name}, expected one of #{DASH.config.roles.map(&:name).join(", ")}")
+    end
+
+    def ensure_within_bounds(role, count)
+      return ensure_within_scale_bounds(role, count) if role.scaled?
+
+      hosts = role.hosts.size
+      bounds = (hosts * role.replicas.min)..(hosts * role.replicas.max)
+
+      unless bounds.cover?(count)
+        raise ArgumentError, "#{role} runs #{bounds.min} to #{bounds.max} containers on its #{hosts} #{"host".pluralize(hosts)} " \
+          "(replicas min #{role.replicas.min}, max #{role.replicas.max} per host), not #{count}"
+      end
+    end
+
+    def ensure_within_scale_bounds(role, count)
+      scale, replicas = role.scale, role.replicas
+      bounds = (scale.min * replicas.min)..(scale.max * replicas.max)
+
+      unless bounds.cover?(count)
+        raise ArgumentError, "#{role} runs #{bounds.min} to #{bounds.max} containers on #{scale.min} to #{scale.max} hosts " \
+          "(replicas min #{replicas.min}, max #{replicas.max} per host), not #{count}"
+      end
+    end
+
+    # Joins first, so the count never dips on the way; then the slots on the hosts that
+    # stay, adds before removals; members leave between the two, so a scale-in empties
+    # members before the baseline gives anything up.
+    def scale_across_hosts(role, count)
+      if DASH.config.pool.degraded?(role)
+        raise ArgumentError, "the #{role} pool could not be read, so dash scale set cannot count the containers it runs; " \
+          "try again once the provider answers"
+      end
+
+      running = running_replicas(role)
+      members = role.active_members
+      plan = Dash::Cli::Scale::HostPlan.new(baseline: role.baseline_hosts, members: members.map(&:host), target: count,
+        replicas_min: role.replicas.min, replicas_max: role.replicas.max, hosts_min: role.scale.min, hosts_max: role.scale.max)
+
+      ensure_members_can_move(role, members, plan)
+      joined = join_members(role, plan, running, count)
+      running = running_replicas(role) if joined.any?
+
+      counts = plan.counts.merge(joined.zip(plan.join_counts).to_h)
+      changes = counts.to_h { |host, target| [ host, Dash::Cli::Scale::HostPlan.slot_changes(running.fetch(host, {}).keys, target) ] }
+      additions = changes.transform_values(&:first).reject { |_, slots| slots.empty? }
+      removals = changes.transform_values(&:last).reject { |_, slots| slots.empty? }
+
+      if joined.empty? && plan.leaving.empty? && additions.empty? && removals.empty?
+        return say "#{role} already runs #{count} containers", :magenta
+      end
+
+      scale_out(role, additions, running, count) if additions.any?
+      plan.leaving.each do |host|
+        Dash::Cli::Scale::Leave.new(role, members.find { |member| member.host == host }, self, running: running.fetch(host, {}), count: count).run
+      end
+      scale_in(role, removals, running, count) if removals.any?
+
+      say "#{role} now runs #{count} containers on #{counts.size} #{"host".pluralize(counts.size)}", :magenta
+    end
+
+    def ensure_members_can_move(role, members, plan)
+      return if plan.joins.zero? && plan.leaving.empty?
+
+      if role.scale.power? && (stopped = role.members.count(&:stopped?)) < plan.joins
+        raise ArgumentError, "#{role} needs #{plan.joins} more #{"host".pluralize(plan.joins)} but has #{stopped} stopped " \
+          "#{"member".pluralize(stopped)} to power on (see dash doctor for the labels a member carries)"
+      end
+    end
+
+    def join_members(role, plan, running, count)
+      return [] if plan.joins.zero?
+
+      version = join_version(role, running)
+      stopped = role.members.select(&:stopped?)
+
+      plan.joins.times.map do |index|
+        Dash::Cli::Scale::Join.new(role, self, member: (stopped[index] if role.scale.power?), version: version, count: count,
+          keep_on_failure: options[:keep_on_failure]).run
+      end
+    end
+
+    # The version the role runs, baseline first - never this checkout's version.
+    def join_version(role, running)
+      replicas = role.hosts.lazy.map { |host| running.fetch(host, {}) }.find(&:any?)
+      raise Dash::Cli::BootError, "No running version of #{role} to join a member at; run dash deploy first" unless replicas
+
+      host_version(role, replicas)
+    end
+
+    # { host => { replica => version } } for every running container of the role, one
+    # `docker ps` per host. Hosts run in parallel threads, so the hash is guarded.
+    def running_replicas(role)
+      running = {}
+      mutex = Mutex.new
+
+      on(role.hosts) do |host|
+        names = capture_with_info(*DASH.app(role: role, host: host).active_containers).lines.map(&:strip)
+        replicas = names.filter_map { |name| [ role.replica_from_name(name), role.version_from_name(name) ] if role.replica_from_name(name) }.reverse.to_h
+
+        mutex.synchronize { running[host.to_s] = replicas }
+      end
+
+      role.hosts.to_h { |host| [ host, running.fetch(host, {}) ] }
+    end
+
+    # New replicas boot at the version their host already runs. Hosts are grouped by that
+    # version because the image a container runs comes from the one shared config.
+    def scale_out(role, additions, running, count)
+      hosts = role.hosts & additions.keys
+      run_hook "pre-scale-out", role: role.name, hosts: hosts.join(","), replicas: count.to_s
+
+      hosts.group_by { |host| host_version(role, running[host]) }.each do |version, group|
+        using_version(version) do
+          on(group) do |host|
+            Dash::Cli::Scale::ReplicaJoin.new(host, role, additions.fetch(host.to_s), self, running: running[host.to_s].keys).run
+          end
+        end
+
+        group.each { |host| additions[host].each { |replica| say "Added #{role.replica_name(replica, version)} on #{host}" } }
+      end
+
+      run_hook "post-scale-out", role: role.name, hosts: hosts.join(","), replicas: count.to_s
+    end
+
+    def scale_in(role, removals, running, count)
+      hosts = role.hosts & removals.keys
+      run_hook "pre-scale-in", role: role.name, hosts: hosts.join(","), replicas: count.to_s
+
+      on(hosts) do |host|
+        leaving = removals.fetch(host.to_s).to_h { |replica| [ replica, running[host.to_s][replica] ] }
+        Dash::Cli::Scale::ReplicaLeave.new(host, role, leaving, self, running: running[host.to_s].keys).run
+      end
+
+      removals.each { |host, replicas| replicas.each { |replica| say "Removed #{role.replica_name(replica, running[host][replica])} on #{host}" } }
+
+      run_hook "post-scale-in", role: role.name, hosts: hosts.join(","), replicas: count.to_s
+    end
+
+    # The version names the image, so a container a boot renamed out of the way
+    # (`<version>_replaced_<hex>`) still runs `<version>`.
+    def host_version(role, replicas)
+      version = replicas[1] || replicas.values.first || raise(Dash::Cli::BootError, "Nothing of #{role} runs on a host to scale from, deploy it first")
+      version.sub(/_replaced_\h{16}\z/, "")
+    end
+
+    def using_version(version)
+      old_version = DASH.config.version
+      DASH.config.version = version
+      yield
+    ensure
+      DASH.config.version = old_version
+    end
+
+    def print_status(role)
+      return say("#{role[:role]}: #{role[:error]}", :red) if role[:error]
+
+      hosts = " on #{role[:scale][:min]} to #{role[:scale][:max]} hosts" if role[:scale]
+      say "#{role[:role]}: #{role[:total]} #{"container".pluralize(role[:total])} (replicas min #{role[:min]}, max #{role[:max]} per host#{hosts})"
+
+      role[:hosts].each do |host, replicas|
+        say format("  %-20s %-3s %s", host, "-", "no replicas running") if replicas.empty?
+
+        replicas.each do |replica|
+          say format("  %-20s %-3s %-20s %s", host, replica[:replica], replica[:version], replica[:status])
+        end
+      end
+
+      role[:unread].each { |host| say format("  %-20s %-3s %s", host[:host], "-", "could not read (#{host[:error]})"), :red }
+      Array(role[:members]).each { |member| print_member(member) }
+    end
+
+    def print_member(member)
+      orphan = " - orphan: started, but runs nothing of the role" if member[:orphan]
+      say format("  member %-12s %-20s %s%s", member[:id] || "?", member[:host], member[:state], orphan), (:yellow if member[:orphan])
+    end
+end

data/lib/dash/cli/server.rb

@@ -1,4 +1,6 @@
 class Dash::Cli::Server < Dash::Cli::Base
+  HOST_STATS_COLUMNS = "%-20s %-16s %4s %-20s %-16s %-12s %-12s %s".freeze
+
   desc "exec", "Run a custom command on the server (use --help to show options)"
   option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
   option :raw, type: :boolean, default: false, desc: "Output raw, unmodified stdout"
@@ -34,6 +36,16 @@ class Dash::Cli::Server < Dash::Cli::Base
     end
   end
 
+  desc "stats", "Show load, memory, swap, disk and uptime of the servers"
+  option :json, type: :boolean, default: false, desc: "Print the stats per host as JSON"
+  def stats
+    return puts_json { Dash::Diagnostics::HostStats.new.to_h } if options[:json]
+
+    pre_connect_if_required
+    puts format(HOST_STATS_COLUMNS, "HOST", "LOAD 1/5/15", "CPUS", "MEMORY USED", "SWAP USED", "DISK /", "DISK DOCKER", "UPTIME")
+    Dash::Diagnostics::HostStats.new.to_h[:hosts].each { |host| puts host_stats_line(host) }
+  end
+
   desc "bootstrap", "Set up Docker to run dash apps"
   def bootstrap
     modify(lock: true) do
@@ -67,4 +79,33 @@ class Dash::Cli::Server < Dash::Cli::Base
       run_hook "docker-setup"
     end
   end
+
+  private
+    def host_stats_line(host)
+      return format("%-20s ERROR %s", host[:host], host[:error]) if host[:error]
+
+      load = host[:load] ? host[:load].values_at(:one, :five, :fifteen).join("/") : "-"
+      memory = host[:memory] ? "#{host[:memory][:used_percent]}% of #{Dash::Utils.human_bytes(host[:memory][:total_bytes])}" : "-"
+      format(HOST_STATS_COLUMNS, host[:host], load, host[:cpus] || "-", memory, swap_used(host[:swap]), disk_used(host[:disk][:root]), disk_used(host[:disk][:docker_root]), uptime(host[:uptime_seconds]))
+    end
+
+    # A host with no swap configured reads "none" rather than a 0% that suggests headroom.
+    def swap_used(swap)
+      return "-" unless swap
+      return "none" if swap[:total_bytes].to_i.zero?
+
+      used = ((swap[:total_bytes] - swap[:free_bytes]) * 100.0 / swap[:total_bytes]).round
+      "#{used}% of #{Dash::Utils.human_bytes(swap[:total_bytes])}"
+    end
+
+    def disk_used(disk)
+      disk ? "#{disk[:used_percent].round}%" : "-"
+    end
+
+    def uptime(seconds)
+      return "-" unless seconds
+
+      days, rest = seconds.divmod(86_400)
+      days > 0 ? "#{days}d #{rest / 3600}h" : "#{rest / 3600}h #{rest % 3600 / 60}m"
+    end
 end

data/lib/dash/cli/stats_table.rb

@@ -0,0 +1,34 @@
+# The human table for `dash app stats` and `dash accessory stats`: one block per host, a
+# row per container. The JSON is the ContainerStats diagnostic's own `to_h`.
+module Dash::Cli::StatsTable
+  COLUMNS = "  %-28s %-10s %4s %8s %21s %21s %21s %6s".freeze
+
+  module_function
+
+  # `host` is one entry of ContainerStats#to_h's `hosts` or `accessories`.
+  def lines(type, host)
+    heading = "#{type} Host: #{host[:host]}"
+    return [ heading, "  ERROR #{host[:error]}" ] if host[:error]
+    return [ heading, "  no containers running" ] if host[:containers].empty?
+
+    [ heading, format(COLUMNS, "CONTAINER", host[:accessory] ? "ACCESSORY" : "ROLE", "SLOT", "CPU", "MEMORY / LIMIT", "NET IN / OUT", "BLOCK READ / WRITE", "PIDS"),
+      *host[:containers].map { |container| row(container) }, *("  WARN #{host[:stats_error]}" if host[:stats_error]) ]
+  end
+
+  def row(container)
+    stats = container[:stats] || {}
+
+    format COLUMNS, container[:name], container[:role] || container[:accessory], container[:replica] || "-",
+      percent(stats[:cpu_percent]), memory(stats), stats.dig(:raw, :net_io) || "-", stats.dig(:raw, :block_io) || "-", stats[:pids] || "-"
+  end
+
+  def percent(value)
+    value ? format("%.1f%%", value) : "-"
+  end
+
+  def memory(stats)
+    return stats.dig(:raw, :memory) || "-" unless stats[:memory_bytes] && stats[:memory_limit_bytes]
+
+    "#{Dash::Utils.human_bytes(stats[:memory_bytes])} / #{Dash::Utils.human_bytes(stats[:memory_limit_bytes])}"
+  end
+end

data/lib/dash/cli/templates/sample_hooks/post-scale-in.sample

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+echo "Scaled $DASH_ROLE in to $DASH_REPLICAS containers on $DASH_HOSTS"

data/lib/dash/cli/templates/sample_hooks/post-scale-out.sample

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+echo "Scaled $DASH_ROLE out to $DASH_REPLICAS containers on $DASH_HOSTS"

data/lib/dash/cli/templates/sample_hooks/pre-scale-in.sample

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+# Runs before `dash scale set` removes containers of a role. $DASH_HOSTS are the hosts
+# that lose replicas; $DASH_REPLICAS is the total the role will run afterwards.
+
+echo "Scaling $DASH_ROLE in to $DASH_REPLICAS containers on $DASH_HOSTS..."

data/lib/dash/cli/templates/sample_hooks/pre-scale-out.sample

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+# Runs before `dash scale set` adds containers of a role. $DASH_HOSTS are the hosts that
+# get new replicas; $DASH_REPLICAS is the total the role will run afterwards.
+
+echo "Scaling $DASH_ROLE out to $DASH_REPLICAS containers on $DASH_HOSTS..."

data/lib/dash/commander.rb

@@ -49,6 +49,19 @@ class Dash::Commander
     @config, @config_kwargs = nil, kwargs
   end
 
+  # A fresh read of deploy.yml for a long-lived process (`dash mcp`), one per question:
+  # the builders bound to the old Configuration and any --hosts/--roles narrowing go,
+  # `connected` stays, so the pre-connect hook does not fire again. The output loggers are
+  # closed, since loading the config broadcasts to them again.
+  def reconfigure(**kwargs)
+    configure(**kwargs)
+    @output_logger&.close
+    @output_logger = nil
+    @commands = {}
+    @loadbalancer_config = nil
+    @specifics = @specific_hosts = @specific_roles = nil
+  end
+
   def configured?
     @config || @config_kwargs
   end
@@ -91,8 +104,8 @@ class Dash::Commander
     config.accessories&.collect(&:name) || []
   end
 
-  def app(role: nil, host: nil)
-    Dash::Commands::App.new(config, role: role, host: host)
+  def app(role: nil, host: nil, replica: 1)
+    Dash::Commands::App.new(config, role: role, host: host, replica: replica)
   end
 
   def accessory(name)
@@ -238,7 +251,8 @@ class Dash::Commander
 
       SSHKit.config.output = Dash::Output::Formatter.new($stdout, output_logger)
 
-      at_exit { @output_logger&.close }
+      # Once per commander: a long-lived `dash mcp` loads the config once per question.
+      @output_logger_closer ||= at_exit { @output_logger&.close }
     rescue => e
       $stderr.puts "Output logger setup failed: #{e.class}: #{e.message}"
       $stderr.puts e.backtrace.join("\n") if ENV["VERBOSE"]

data/lib/dash/commands/accessory/proxy.rb

@@ -2,7 +2,7 @@ module Dash::Commands::Accessory::Proxy
   delegate :container_name, to: :"config.proxy_boot", prefix: :proxy
 
   def deploy(target:)
-    proxy_exec :deploy, service_name, *proxy.deploy_command_args(target: target)
+    proxy_exec :deploy, service_name, *proxy.deploy_command_args(targets: [ target ])
   end
 
   def remove

data/lib/dash/commands/accessory.rb

@@ -1,3 +1,5 @@
+require "shellwords"
+
 class Dash::Commands::Accessory < Dash::Commands::Base
   include Proxy
 
@@ -37,14 +39,31 @@ class Dash::Commands::Accessory < Dash::Commands::Base
     docker :container, :stop, service_name
   end
 
+  # The accessory's container as `docker ps` JSON, running or not.
+  def list_containers_json
+    docker :ps, "--all", *service_filter, "--format", "'{{json .}}'"
+  end
+
+  # Same shape as Commands::App#stats_json: the labels (or the unreadable marker), the
+  # separator, then `docker stats` behind `xargs -r`, which keeps an accessory that is not
+  # running from reporting every container on the host.
+  def stats_json
+    filters = [ *service_filter, *Dash::Commands::App::ACTIVE_DOCKER_STATUSES.flat_map { |status| [ "--filter", "status=#{status}" ] } ]
+
+    chain \
+      [ *docker(:ps, *filters, "--format", "'{{json .}}'"), "||", :echo, Dash::Commands::App::ACTIVE_CONTAINERS_UNREADABLE ],
+      [ :echo, SECTION_SEPARATOR ],
+      pipe(docker(:ps, "--quiet", *filters), [ :xargs, "-r", :docker, :stats, "--no-stream", "--format", "'{{json .}}'", "||", :echo, Dash::Diagnostics::ContainerStats::STATS_UNREADABLE ])
+  end
+
   def info(all: false, quiet: false)
     docker :ps, *("-a" if all), *("-q" if quiet), *service_filter
   end
 
   def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
     pipe \
-      docker(:logs, service_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), ("--timestamps" if timestamps), "2>&1"),
-      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
+      docker(:logs, service_name, (" --since #{since.to_s.shellescape}" if since), (" --tail #{Integer(lines.to_s, 10)}" if lines), ("--timestamps" if timestamps), "2>&1"),
+      ("grep #{Dash::Utils.single_quote(grep)}#{" #{grep_options}" if grep_options}" if grep)
   end
 
   def follow_logs(timestamps: true, grep: nil, grep_options: nil)

data/lib/dash/commands/app/containers.rb

@@ -1,17 +1,42 @@
 module Dash::Commands::App::Containers
+  # A container's `docker ps` JSON, a tab, then docker's own value of its `role` label. The
+  # JSON's `Labels` is "k=v,k=v" with no escaping, so a comma in another label's value could
+  # pass for a role there; `.Label "role"` cannot be faked that way. JSON never holds a raw tab.
+  PS_WITH_ROLE_FORMAT = %q('{{json .}}{{"\t"}}{{json (.Label "role")}}')
+
   DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
 
   def list_containers
     docker :container, :ls, "--all", *container_filter_args
   end
 
+  # Every container of the service in this destination, one per line in PS_WITH_ROLE_FORMAT.
+  # Called without a role it covers every role and every replica slot on the host.
+  def list_containers_json
+    docker :container, :ls, "--all", *container_filter_args(all_replicas: true), "--format", PS_WITH_ROLE_FORMAT
+  end
+
+  # The running containers in PS_WITH_ROLE_FORMAT, then the separator,
+  # then their `docker stats`. `xargs -r` matters: `docker stats` given no container
+  # reports every container on the host, other apps' included. A docker ps that fails
+  # prints ACTIVE_CONTAINERS_UNREADABLE, since the chain's `;` hides its exit status and
+  # "could not ask" would otherwise read as "nothing runs".
+  def stats_json
+    filters = container_filter_args(statuses: Dash::Commands::App::ACTIVE_DOCKER_STATUSES, all_replicas: true)
+
+    chain \
+      [ *docker(:ps, *filters, "--format", PS_WITH_ROLE_FORMAT), "||", :echo, Dash::Commands::App::ACTIVE_CONTAINERS_UNREADABLE ],
+      [ :echo, Dash::Commands::Base::SECTION_SEPARATOR ],
+      pipe(docker(:ps, "--quiet", *filters), [ :xargs, "-r", :docker, :stats, "--no-stream", "--format", "'{{json .}}'", "||", :echo, Dash::Diagnostics::ContainerStats::STATS_UNREADABLE ])
+  end
+
   def list_container_names
     [ *list_containers, "--format", "'{{ .Names }}'" ]
   end
 
   def remove_container(version:)
     pipe \
-      container_id_for(container_name: container_name(version)),
+      container_id_for(container_name: container_name_pattern(version)),
       xargs(docker(:container, :rm))
   end
 
@@ -20,12 +45,12 @@ module Dash::Commands::App::Containers
   end
 
   def remove_containers
-    docker :container, :prune, "--force", *container_filter_args
+    docker :container, :prune, "--force", *container_filter_args(all_replicas: true)
   end
 
   def container_health_log(version:)
     pipe \
-      container_id_for(container_name: container_name(version)),
+      container_id_for(container_name: container_name_pattern(version)),
       xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
   end
 end

data/lib/dash/commands/app/logging.rb

@@ -1,9 +1,20 @@
+require "shellwords"
+
 module Dash::Commands::App::Logging
+  # `since` and `grep` are quoted and `lines` must be a number, so none of them can end
+  # the command they sit in. `dash mcp` hands them over from an agent, not an operator.
+  # `grep_options` is the operator's own extra flags and stays raw.
   def logs(container_id: nil, timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
     pipe \
       container_id_command(container_id),
-      "xargs docker logs#{" --timestamps" if timestamps}#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
-      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
+      "xargs docker logs#{" --timestamps" if timestamps}#{" --since #{since.to_s.shellescape}" if since}#{" --tail #{Integer(lines.to_s, 10)}" if lines} 2>&1",
+      ("grep #{Dash::Utils.single_quote(grep)}#{" #{grep_options}" if grep_options}" if grep)
+  end
+
+  # Every replica slot's logs in one round trip, each slot's after a `separator` line. Log
+  # lines are anyone's text, so the caller picks a separator no log line can guess.
+  def replica_logs(separator:, **options)
+    chain(*role.replica_numbers.flat_map { |replica| [ [ :echo, separator ], for_replica(replica).logs(**options) ] })
   end
 
   def follow_logs(host:, container_id: nil, timestamps: true, lines: nil, grep: nil, grep_options: nil)

data/lib/dash/commands/app/proxy.rb

@@ -1,16 +1,16 @@
 module Dash::Commands::App::Proxy
   delegate :container_name, to: :"config.proxy_boot", prefix: :proxy
 
-  def deploy(target:)
-    proxy_exec :deploy, role.container_prefix, *role.proxy.deploy_command_args(target: target)
+  def deploy(targets:)
+    proxy_exec :deploy, role.container_prefix, *role.proxy.deploy_command_args(targets: targets)
   end
 
   def remove
     proxy_exec :remove, role.container_prefix
   end
 
-  def rollout_deploy(target:)
-    proxy_exec :rollout, :deploy, role.container_prefix, *role.proxy.rollout_deploy_command_args(target: target)
+  def rollout_deploy(targets:)
+    proxy_exec :rollout, :deploy, role.container_prefix, *role.proxy.rollout_deploy_command_args(targets: targets)
   end
 
   def rollout_set(percent: nil, list: nil)