dash 4.1.2 → 4.2.0

data/lib/dash/cli/app/boot.rb

@@ -18,77 +18,84 @@ class Dash::Cli::App::Boot
     @cli = cli
   end
 
+  # Every replica of the role on this host is swapped together: the old containers keep
+  # serving until all the new ones are ready, and a failure stops every new one.
   def run
     DASH.timings.phase("#{role} #{host}", depth: 1) do |timing|
       @timing = timing
 
-      old_version = old_version_renamed_if_clashing
+      @state = capture_boot_state
+      announce_replicas if role.replicas.scalable?
+      old_versions = old_versions_renamed_if_clashing
 
       wait_at_barrier if queuer?
 
       begin
-        start_new_version
+        start_new_versions
       rescue => e
         close_barrier if gatekeeper?
-        stop_new_version
+        stop_new_versions
         raise
       end
 
       release_barrier if gatekeeper?
 
-      if old_version
-        stop_old_version(old_version)
-      end
+      stop_old_versions(old_versions)
     end
   end
 
   private
-    # Both answers come back from one round trip, which means the running version is read
-    # before any rename happens. When the clashing container IS the running one, the
+    attr_reader :state
+
+    # Every answer comes back from one round trip, which means the running versions are
+    # read before any rename happens. When a clashing container IS the running one, the
     # version to stop later is the name it was renamed to - the name that was read now
     # belongs to the container this boot is about to start.
-    def old_version_renamed_if_clashing
-      clashing_container_id, old_version = capture_boot_state
+    def old_versions_renamed_if_clashing
+      old_versions = state.replicas.to_h { |replica| [ replica, state.running_version(replica) ] }
 
-      if clashing_container_id.present?
+      state.replicas.select { |replica| state.clashing?(replica) }.each do |replica|
         renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
-        info "Renaming container #{version} to #{renamed_version} as already deployed on #{host}"
+        info "Renaming container #{app(replica).container_name(version)} to #{renamed_version} as already deployed on #{host}"
         execute *auditor.record_then("Renaming container #{version} to #{renamed_version}",
-          app.rename_container(version: version, new_version: renamed_version))
+          app(replica).rename_container(version: version, new_version: renamed_version))
 
-        old_version = renamed_version if old_version == version
+        old_versions[replica] = renamed_version if old_versions[replica] == version
       end
 
-      old_version
+      old_versions.merge(state.surplus_replicas.to_h { |replica| [ replica, state.running_version(replica) ] }).compact
     end
 
-    def capture_boot_state
-      clashing, running = Dash::Commands::App.split_state(capture_with_info(*app.boot_state(version), raise_on_non_zero_exit: false))
-
-      [ clashing.strip.presence, running.strip.presence ]
+    # A slot that crashed between min and the runtime count is not running, so it is not
+    # counted - say so, rather than letting the count drop silently.
+    def announce_replicas
+      info "Booting #{state.count} #{"replica".pluralize(state.count)} of #{role} on #{host} (#{state.running_count} of #{role.replicas.max} slots running, min #{role.replicas.min})"
     end
 
-    def start_new_version
-      hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
+    def capture_boot_state
+      Dash::Cli::App::BootState.new(role, capture_with_info(*app.boot_states(version), raise_on_non_zero_exit: false))
+    end
 
+    def start_new_versions
       execute *auditor.record_then("Booted app version #{version}", app.ensure_env_directory)
       upload! role.secrets_io(host), role.secrets_path, mode: "0600"
 
-      # `docker run --detach` prints the id of the container it just started, so the
-      # proxy target comes out of the run itself — asking docker for it again was a round
-      # trip spent re-reading something the host had already said.
-      container_id = capture_with_info(*app.run(hostname: hostname)).strip
+      container_ids = state.replicas.map { |replica| run_replica(replica) }
 
       if running_proxy?
-        endpoint = container_id[0, SHORT_CONTAINER_ID_LENGTH]
-        raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+        endpoints = container_ids.map { |container_id| container_id[0, SHORT_CONTAINER_ID_LENGTH] }
+        raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoints.any?(&:empty?)
 
         run_hook "pre-proxy-deploy", hosts: host.to_s, role: role.name
         info "Deploying #{role} on #{host} via dash-proxy (waiting up to #{DASH.config.deploy_timeout}s for it to become healthy)..."
-        timing_healthy { execute *app.deploy(target: endpoint) }
+        timing_healthy { execute *app.deploy(targets: endpoints) }
         run_hook "post-proxy-deploy", hosts: host.to_s, role: role.name
       else
-        timing_healthy { Dash::Cli::Healthcheck::Poller.wait_for_healthy(role: role, &method(:readiness_status)) }
+        timing_healthy do
+          state.replicas.each do |replica|
+            Dash::Cli::Healthcheck::Poller.wait_for_healthy(role: role) { |*args| readiness_status(replica, *args) }
+          end
+        end
       end
     rescue => e
       error "Failed to boot #{role} on #{host}"
@@ -96,6 +103,14 @@ class Dash::Cli::App::Boot
       raise e
     end
 
+    # `docker run --detach` prints the id of the container it just started, so the proxy
+    # target comes out of the run itself — asking docker for it again was a round trip
+    # spent re-reading something the host had already said.
+    def run_replica(replica)
+      hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
+      capture_with_info(*app(replica).run(hostname: hostname)).strip
+    end
+
     # A role behind the proxy lets `dash-proxy deploy` block on the host until the
     # container is healthy; a role without one now does the same, waiting in a shell loop
     # on the host that streams its progress back rather than being polled from here once
@@ -104,11 +119,11 @@ class Dash::Cli::App::Boot
     #
     # Neither capture suppresses a non-zero exit: a status that cannot be read is a broken
     # command, and it has always failed the boot on the spot rather than being waited out.
-    def readiness_status(mode, seconds_left = nil)
+    def readiness_status(replica, mode, seconds_left = nil)
       if mode == :confirm
-        capture_with_info(*app.status(version: version))
+        capture_with_info(*app(replica).status(version: version))
       else
-        capture_with_info *app.wait_for_ready(version: version, timeout: seconds_left),
+        capture_with_info *app(replica).wait_for_ready(version: version, timeout: seconds_left),
           interaction_handler: Dash::Cli::Healthcheck::ProgressReporter.new
       end
     end
@@ -116,22 +131,30 @@ class Dash::Cli::App::Boot
     # Every failed boot gets the container log, and the health probe history when the
     # container declares a healthcheck — non-primary roles have no dash-proxy report to fall back on.
     def dump_diagnostics
-      error capture_with_info(*app.logs(container_id: app.container_id_for_version(version)))
+      state.replicas.each do |replica|
+        error capture_with_info(*app(replica).logs(container_id: app(replica).container_id_for_version(version)))
 
-      health_log = capture_with_info(*app.container_health_log(version: version)).strip
-      error health_log unless health_log.empty? || health_log == "null"
+        health_log = capture_with_info(*app(replica).container_health_log(version: version)).strip
+        error health_log unless health_log.empty? || health_log == "null"
+      end
     rescue SSHKit::Command::Failed
       error "Could not fetch logs for #{version}"
     end
 
-    def stop_new_version
-      execute *app.stop(version: version), raise_on_non_zero_exit: false
+    def stop_new_versions
+      state.replicas.each do |replica|
+        execute *app(replica).stop(version: version), raise_on_non_zero_exit: false
+      end
     end
 
-    def stop_old_version(old_version)
-      run_stop_hook "pre-app-stop", old_version
-      execute *app.stop(version: old_version), raise_on_non_zero_exit: false
-      run_stop_hook "post-app-stop", old_version
+    def stop_old_versions(old_versions)
+      return if old_versions.empty?
+
+      old_versions.each do |replica, old_version|
+        run_stop_hook "pre-app-stop", old_version
+        execute *app(replica).stop(version: old_version), raise_on_non_zero_exit: false
+        run_stop_hook "post-app-stop", old_version
+      end
 
       execute *app.clean_up_assets if assets?
       execute *app.clean_up_error_pages if DASH.config.error_pages_path
@@ -172,15 +195,17 @@ class Dash::Cli::App::Boot
       started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       yield
     ensure
-      @timing.detail = format("healthy after %.1fs", Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+      replicas = ", #{state.count} replicas" if role.replicas.scalable?
+      @timing.detail = format("healthy after %.1fs%s", Process.clock_gettime(Process::CLOCK_MONOTONIC) - started, replicas)
     end
 
     def barrier_role?
       role == DASH.primary_role
     end
 
-    def app
-      @app ||= DASH.app(role: role, host: host)
+    def app(replica = 1)
+      @apps ||= {}
+      @apps[replica] ||= DASH.app(role: role, host: host, replica: replica)
     end
 
     def auditor

data/lib/dash/cli/app/boot_state.rb

@@ -0,0 +1,70 @@
+# What one host said about a role before a boot starts anything, parsed out of the single
+# Commands::App#boot_states capture: which slots already hold the version being deployed,
+# which version each slot runs now, and so how many replicas to boot.
+#
+# The replica count is never stored. It is the number of slots with a running container,
+# kept within the role's bounds, so `dash scale set` changes it and a deploy reads it back.
+class Dash::Cli::App::BootState
+  attr_reader :role
+
+  def initialize(role, output)
+    @role = role
+
+    clash, running, names, *clashes = Dash::Commands::App.split_states(output).map(&:strip)
+
+    # The whole line, never a substring: a version may contain the marker's characters.
+    if names&.lines&.any? { |line| line.strip == Dash::Commands::App::ACTIVE_CONTAINERS_UNREADABLE }
+      raise Dash::Cli::BootError, "Could not list the running containers of #{role}, not booting it"
+    end
+
+    @clashes = { 1 => clash.presence }.merge(clashes.each.with_index(2).to_h { |id, replica| [ replica, id.presence ] })
+    @running = running_by_replica(running.presence, names&.lines&.map(&:strip)&.reject(&:empty?))
+  end
+
+  # How many replicas this boot starts: the slots running now, never fewer than min (so a
+  # crash cannot shrink the role below its floor) nor more than max.
+  def count
+    role.replicas.clamp(running_count)
+  end
+
+  def replicas
+    (1..count).to_a
+  end
+
+  def running_count
+    @running.size
+  end
+
+  def clashing?(replica)
+    @clashes[replica].present?
+  end
+
+  def running_version(replica)
+    @running[replica]
+  end
+
+  # Slots running above the count being booted - a lowered max, or a role whose
+  # `replicas` key went away. Their containers are stopped with the old version.
+  def surplus_replicas
+    @running.keys.select { |replica| replica > count }.sort
+  end
+
+  private
+    # Every slot's version comes out of its container's name. Slot 1's own `--latest` read
+    # wins for slot 1 (it prefers the latest image, as it always has), unless it is really
+    # another slot's whole name: on a role that was scalable and no longer is, `--latest`
+    # over the role labels can return a leftover slot's container.
+    def running_by_replica(slot_one, names)
+      by_replica = Array(names).reverse.to_h { |name| [ role.replica_from_name(name), role.version_from_name(name) ] }.except(nil)
+
+      if slot_one && !foreign?(slot_one)
+        by_replica[1] = slot_one
+      end
+
+      by_replica
+    end
+
+    def foreign?(version)
+      (owner = role.replica_from_name(version)) && owner != 1
+    end
+end

data/lib/dash/cli/app/rollout_boot.rb

@@ -25,32 +25,51 @@ class Dash::Cli::App::RolloutBoot
 
   private
     def ensure_not_already_deployed
-      if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+      if replicas.any? { |replica| capture_with_info(*app(replica).container_id_for_version(version), raise_on_non_zero_exit: false).present? }
         raise Dash::Cli::BootError, "Version #{version} is already deployed for #{role} on #{host}, roll out a different version"
       end
     end
 
+    # The rollout target is as many replicas as the live version runs, so a canary split
+    # compares like with like.
     def start_rollout_target
       audit "Booted rollout target version #{version}"
-      hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
 
       execute *app.ensure_env_directory
       upload! role.secrets_io(host), role.secrets_path, mode: "0600"
 
-      execute *app.run(hostname: hostname)
+      endpoints = replicas.map do |replica|
+        hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
+        execute *app(replica).run(hostname: hostname)
 
-      endpoint = capture_with_info(*app.container_id_for_version(version)).strip
-      raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+        capture_with_info(*app(replica).container_id_for_version(version)).strip.tap do |endpoint|
+          raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+        end
+      end
 
-      execute *app.rollout_deploy(target: endpoint)
+      execute *app.rollout_deploy(targets: endpoints)
     end
 
     def stop_rollout_target
-      execute *app.stop(version: version), raise_on_non_zero_exit: false
+      replicas.each do |replica|
+        execute *app(replica).stop(version: version), raise_on_non_zero_exit: false
+      end
+    end
+
+    # A role without replicas is one slot, and asks nothing more than it always did.
+    def replicas
+      @replicas ||=
+        if role.replicas.scalable?
+          names = capture_with_info(*app.active_containers).lines.map(&:strip)
+          (1..role.replicas.clamp(names.filter_map { |name| role.replica_from_name(name) }.uniq.size)).to_a
+        else
+          [ 1 ]
+        end
     end
 
-    def app
-      @app ||= DASH.app(role: role, host: host)
+    def app(replica = 1)
+      @apps ||= {}
+      @apps[replica] ||= DASH.app(role: role, host: host, replica: replica)
     end
 
     def audit(message)

data/lib/dash/cli/app/running_targets.rb

@@ -0,0 +1,39 @@
+# The dash-proxy targets of one role on one host: the running container of every slot, in
+# slot order. A role without replicas has one slot, so this asks what re-registering a
+# single container always asked, plus which slots run at all.
+class Dash::Cli::App::RunningTargets
+  attr_reader :sshkit, :role, :host
+  delegate :capture_with_info, to: :sshkit
+
+  def initialize(sshkit, role:, host:)
+    @sshkit = sshkit
+    @role = role
+    @host = host
+  end
+
+  def container_ids(only_running: false, replicas: self.replicas)
+    replicas.filter_map do |replica|
+      app = DASH.app(role: role, host: host, replica: replica)
+
+      read = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false)
+
+      if (version = Dash::Cli::App::SlotVersion.new(sshkit, app).resolve(read))
+        capture_with_info(*app.container_id_for_version(version, only_running: only_running), raise_on_non_zero_exit: false).strip.presence
+      end
+    end
+  end
+
+  # The configured slots, plus any slot still running above them - a lowered max, or a
+  # role whose `replicas` went away - until the next deploy stops it.
+  def replicas
+    @replicas ||= (role.replica_numbers | running_replicas).sort
+  end
+
+  private
+    # A listing that fails raises: an empty answer would read as "no surplus slots", and the
+    # caller would stop or register an incomplete set.
+    def running_replicas
+      names = capture_with_info(*DASH.app(role: role, host: host).active_containers).lines.map(&:strip)
+      names.filter_map { |name| role.replica_from_name(name) }
+    end
+end

data/lib/dash/cli/app/slot_version.rb

@@ -0,0 +1,29 @@
+# The version one replica slot runs, from a `current_running_version` read that may have
+# answered for another slot. A role without replicas reads slot 1 over the role labels alone,
+# so while slots left over from when it had them still run, `--latest` can return one of
+# theirs - an unstripped name like `app-web.2-123`. Taking that as slot 1's version would
+# make the live container look stale, or drop it from the proxy targets. Only then is the
+# role's running containers read, once more, to find the slot's own.
+class Dash::Cli::App::SlotVersion
+  attr_reader :sshkit, :app
+  delegate :role, :replica, to: :app
+  delegate :capture_with_info, to: :sshkit
+
+  def initialize(sshkit, app)
+    @sshkit = sshkit
+    @app = app
+  end
+
+  def resolve(read)
+    version = read.to_s.strip.presence
+    return version unless foreign?(version)
+
+    names = capture_with_info(*app.active_containers).lines.map(&:strip)
+    names.filter_map { |name| role.version_from_name(name) if role.replica_from_name(name) == replica }.first
+  end
+
+  # A listed "version" that is really another slot's whole container name.
+  def foreign?(version)
+    (owner = version && role.replica_from_name(version)) && owner != replica
+  end
+end

data/lib/dash/cli/app.rb

@@ -55,15 +55,23 @@ class Dash::Cli::App < Dash::Cli::Base
       on_roles(DASH.roles, hosts: DASH.app_hosts, parallel: DASH.config.parallel_roles?, rolling: true) do |host, role|
         app = DASH.app(role: role, host: host)
         execute *DASH.auditor.record("Started app version #{DASH.config.version}"), verbosity: :debug
-        execute *app.start, raise_on_non_zero_exit: false
+
+        # Every slot's container of the version, up to max: a stopped app does not know
+        # how many replicas it ran, and a slot that never existed just fails to start.
+        role.replica_numbers.each do |replica|
+          execute *DASH.app(role: role, host: host, replica: replica).start, raise_on_non_zero_exit: false
+        end
 
         if role.running_proxy?
-          version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-          endpoint = capture_with_info(*app.container_id_for_version(version)).strip
-          raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+          endpoints = Dash::Cli::App::RunningTargets.new(self, role: role, host: host).container_ids
+          raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoints.empty?
+
+          if endpoints.size < role.replicas.min
+            raise Dash::Cli::BootError, "Found #{endpoints.size} of the #{role.replicas.min} replicas of #{role} on #{host}, did every container start?"
+          end
 
           cli.run_hook "pre-proxy-deploy", hosts: host.to_s, role: role.name
-          execute *app.deploy(target: endpoint)
+          execute *app.deploy(targets: endpoints)
           cli.run_hook "post-proxy-deploy", hosts: host.to_s, role: role.name
         end
       end
@@ -77,15 +85,15 @@ class Dash::Cli::App < Dash::Cli::Base
         app = DASH.app(role: role, host: host)
         execute *DASH.auditor.record("Stopped app", role: role), verbosity: :debug
 
-        if role.running_proxy?
-          version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-          endpoint = capture_with_info(*app.container_id_for_version(version)).strip
-          if endpoint.present?
-            execute *app.remove, raise_on_non_zero_exit: false
-          end
+        targets = Dash::Cli::App::RunningTargets.new(self, role: role, host: host)
+
+        if role.running_proxy? && targets.container_ids.any?
+          execute *app.remove, raise_on_non_zero_exit: false
         end
 
-        execute *app.stop, raise_on_non_zero_exit: false
+        targets.replicas.each do |replica|
+          execute *DASH.app(role: role, host: host, replica: replica).stop, raise_on_non_zero_exit: false
+        end
       end
     end
   end
@@ -102,6 +110,7 @@ class Dash::Cli::App < Dash::Cli::Base
   desc "exec [CMD...]", "Execute a custom command on servers within the app container (use --help to show options)"
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
+  option :replica, type: :numeric, default: 1, desc: "Replica slot whose container --reuse runs in"
   option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
   option :detach, type: :boolean, default: false, desc: "Execute command in a detached container"
   option :raw, type: :boolean, default: false, desc: "Output raw, unmodified stdout"
@@ -127,12 +136,17 @@ class Dash::Cli::App < Dash::Cli::Base
       env = options[:env]
       detach = options[:detach]
       quiet = options[:quiet]
+      replica = options[:replica]
       case
       when options[:interactive] && options[:reuse]
+        unless DASH.primary_role.replica_numbers.include?(replica)
+          raise ArgumentError, "#{DASH.primary_role} has no replica #{replica}, it runs at most #{DASH.primary_role.replicas.max}"
+        end
+
         say "Get current version of running container...", :magenta unless options[:version]
         using_version(options[:version] || current_running_version) do |version|
           say "Launching interactive command with version #{version} via SSH from existing container on #{DASH.primary_host}...", :magenta
-          run_locally { exec DASH.app(role: DASH.primary_role, host: DASH.primary_host).execute_in_existing_container_over_ssh(cmd, env: env) }
+          run_locally { exec DASH.app(role: DASH.primary_role, host: DASH.primary_host, replica: replica).execute_in_existing_container_over_ssh(cmd, env: env) }
         end
 
       when options[:interactive]
@@ -151,8 +165,10 @@ class Dash::Cli::App < Dash::Cli::Base
           say "Launching command with version #{version} from existing container...", :magenta
 
           on_roles(DASH.roles, hosts: DASH.app_hosts) do |host, role|
+            next unless role.replica_numbers.include?(replica)
+
             execute *DASH.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
-            puts_by_host host, capture_with_info(*DASH.app(role: role, host: host).execute_in_existing_container(cmd, env: env), strip: !raw), quiet: quiet, raw: raw
+            puts_by_host host, capture_with_info(*DASH.app(role: role, host: host, replica: replica).execute_in_existing_container(cmd, env: env), strip: !raw), quiet: quiet, raw: raw
           end
         end
 
@@ -172,11 +188,23 @@ class Dash::Cli::App < Dash::Cli::Base
   end
 
   desc "containers", "Show app containers on servers"
+  option :json, type: :boolean, default: false, desc: "Print every container per host as JSON (role, replica, version, state, health)"
   def containers
+    return puts_json { Dash::Diagnostics::Containers.new(accessories: []).to_h } if options[:json]
+
     quiet = options[:quiet]
     on(DASH.app_hosts) { |host| puts_by_host host, capture_with_info(*DASH.app.list_containers), quiet: quiet }
   end
 
+  desc "stats", "Show CPU, memory, network and block I/O of the app containers (a one-second sample per host)"
+  option :json, type: :boolean, default: false, desc: "Print the stats per host as JSON"
+  def stats
+    return puts_json { Dash::Diagnostics::ContainerStats.new(accessories: []).to_h } if options[:json]
+
+    pre_connect_if_required
+    Dash::Diagnostics::ContainerStats.new(accessories: []).to_h[:hosts].each { |host| puts Dash::Cli::StatsTable.lines("App", host) }
+  end
+
   desc "stale_containers", "Detect app stale containers"
   option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
   def stale_containers
@@ -185,16 +213,23 @@ class Dash::Cli::App < Dash::Cli::Base
 
     with_lock_if_stopping do
       on_roles(DASH.roles, hosts: DASH.app_hosts) do |host, role|
-        app = DASH.app(role: role, host: host)
-        listed, running = Dash::Commands::App.split_state(capture_with_info(*app.stale_state, raise_on_non_zero_exit: false))
-        versions = listed.strip.split("\n") - [ running.strip ]
-
-        versions.each do |version|
-          if stop
-            puts_by_host host, "Stopping stale container for role #{role} with version #{version}", quiet: quiet
-            execute *app.stop(version: version), raise_on_non_zero_exit: false
-          else
-            puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `dash app stale_containers --stop` to stop)", quiet: quiet
+        role.replica_numbers.each do |replica|
+          app = DASH.app(role: role, host: host, replica: replica)
+          listed, running = Dash::Commands::App.split_state(capture_with_info(*app.stale_state, raise_on_non_zero_exit: false))
+          slot_version = Dash::Cli::App::SlotVersion.new(self, app)
+          running = slot_version.resolve(running)
+          next unless running # nothing of this slot runs, so nothing is stale relative to it
+
+          versions = listed.strip.split("\n").reject { |version| slot_version.foreign?(version) } - [ running ]
+          name = replica == 1 ? "role #{role}" : "role #{role} replica #{replica}"
+
+          versions.each do |version|
+            if stop
+              puts_by_host host, "Stopping stale container for #{name} with version #{version}", quiet: quiet
+              execute *app.stop(version: version), raise_on_non_zero_exit: false
+            else
+              puts_by_host host,  "Detected stale container for #{name} with version #{version} (use `dash app stale_containers --stop` to stop)", quiet: quiet
+            end
           end
         end
       end
@@ -215,6 +250,7 @@ class Dash::Cli::App < Dash::Cli::Base
   option :follow, aliases: "-f", desc: "Follow log on primary server (or specific host set by --hosts)"
   option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output"
   option :container_id, desc: "Docker container ID to fetch logs"
+  option :replica, type: :numeric, desc: "Replica slot to show (default: every replica, or replica 1 with --follow)"
   def logs
     # FIXME: Catch when app containers aren't running
 
@@ -225,8 +261,15 @@ class Dash::Cli::App < Dash::Cli::Base
     timestamps = !options[:skip_timestamps]
     quiet = options[:quiet]
 
+    replica = options[:replica]
+
     if options[:follow]
       lines = options[:lines].presence || ((since || grep) ? nil : 10) # Default to 10 lines if since or grep isn't set
+      follow_role = DASH.specific_roles&.first || DASH.primary_role
+
+      if replica && !follow_role.replica_numbers.include?(replica)
+        raise ArgumentError, "#{follow_role} has no replica #{replica}, it runs at most #{follow_role.replicas.max}"
+      end
 
       run_locally do
         info "Following logs on #{DASH.primary_host}..."
@@ -234,7 +277,7 @@ class Dash::Cli::App < Dash::Cli::Base
         DASH.specific_roles ||= [ DASH.primary_role.name ]
         role = DASH.roles_on(DASH.primary_host).first
 
-        app = DASH.app(role: role, host: host)
+        app = DASH.app(role: role, host: host, replica: replica || 1)
         info app.follow_logs(host: DASH.primary_host, container_id: container_id, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options)
         exec app.follow_logs(host: DASH.primary_host, container_id: container_id, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options)
       end
@@ -242,10 +285,19 @@ class Dash::Cli::App < Dash::Cli::Base
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on_roles(DASH.roles, hosts: DASH.app_hosts) do |host, role|
-        begin
-          puts_by_host host, capture_with_info(*DASH.app(role: role, host: host).logs(container_id: container_id, timestamps: timestamps, since: since, lines: lines, grep: grep, grep_options: grep_options)), quiet: quiet
-        rescue SSHKit::Command::Failed
-          puts_by_host host, "Nothing found", quiet: quiet
+        # A --container-id names one container, whatever the slot.
+        slots = replica || container_id ? [ replica || 1 ] & role.replica_numbers : role.replica_numbers
+
+        slots.each do |slot|
+          # Each replica's lines under a header of their own, so they are not mistaken for one stream.
+          # A --container-id names a container whose slot is unknown, so claim none.
+          type = role.replicas.scalable? && (replica || !container_id) ? "App (replica #{slot})" : "App"
+
+          begin
+            puts_by_host host, capture_with_info(*DASH.app(role: role, host: host, replica: slot).logs(container_id: container_id, timestamps: timestamps, since: since, lines: lines, grep: grep, grep_options: grep_options)), type: type, quiet: quiet
+          rescue SSHKit::Command::Failed
+            puts_by_host host, "Nothing found", type: type, quiet: quiet
+          end
         end
       end
     end
@@ -301,6 +353,11 @@ class Dash::Cli::App < Dash::Cli::Base
       on_roles(DASH.roles, hosts: DASH.app_hosts) do |host, role|
         execute *DASH.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
         execute *DASH.app(role: role, host: host).remove_container(version: version)
+
+        # A slot that never ran the version has nothing to remove.
+        role.replica_numbers.drop(1).each do |replica|
+          execute *DASH.app(role: role, host: host, replica: replica).remove_container(version: version), raise_on_non_zero_exit: false
+        end
       end
     end
   end
@@ -408,10 +465,10 @@ class Dash::Cli::App < Dash::Cli::Base
     def current_running_version(host: DASH.primary_host)
       version = nil
       on(host) do
-        role = DASH.roles_on(host).first
-        version = capture_with_info(*DASH.app(role: role, host: host).current_running_version).strip
+        app = DASH.app(role: DASH.roles_on(host).first, host: host)
+        version = Dash::Cli::App::SlotVersion.new(self, app).resolve(capture_with_info(*app.current_running_version))
       end
-      version.presence
+      version
     end
 
     def version_or_latest