dash 4.0.8 → 4.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a417156134dd79a4724f87192f70baa55fdbc4478a115e021845e3742bacee88
-  data.tar.gz: ebdbb8d7f192b692c31c8cdaa9834584c01dec3653b744b1a481672fee179d28
+  metadata.gz: bb6c271cec9b9ab118649ffeaf503aa5ba31bd8d0f8057ce0a61adb93e130ef4
+  data.tar.gz: ce7f60b8e4a764e06ab868663f825134d8f73cac0e11f663793e6a12471ab490
 SHA512:
-  metadata.gz: 3b263730b7ccf4eeeb857304477945fd9d2476bdf52cdb30a964b12b09143b36b5f585ae672d776eac29a6e009606add151aa6e9dd98385f9146980bbc40a9db
-  data.tar.gz: a3ebb3e0141a0f692bd70205aa3d438f9a3cbd1022d1b92c229bf1daf19294c4826a7420c0fefbb077f5a6d6cb46016ee6ab3926bb08f11383bb7a6eb0aaa23e
+  metadata.gz: 35204403e1680a42ce0b0726225d68c55119a74b387a9cd45189822499e5aa225c250ad82190944cbd4a5ff7aa976b3dcf02600e44b92a8a4ed165f86123c96b
+  data.tar.gz: cb520eb7f9028fad75e4537afcdcc1c7aea71aa6cb58c35a33b9c8d185781a4de9ad7a9f3d1f1f8daba8b73390b279bbd292ac1a82d531a303f34086c69c28d3

data/lib/dash/build/progress_parser.rb

@@ -0,0 +1,136 @@
+# Reads `docker buildx build --progress=plain` as it streams past and turns it into a
+# per-step report. It is an SSHKit interaction handler, so it sees the same bytes SSHKit
+# is already printing — no second process, no `docker buildx history`, no extra command.
+#
+# The stream is line-oriented but arrives in chunks (the SSH backend splits on packet
+# boundaries, not newlines), so data is buffered and only whole lines are parsed. Every
+# line belongs to a vertex — `#12` — and the first line for a vertex names it; the rest
+# are events on it (DONE, CACHED, ERROR) or its own stdout, which is ignored.
+#
+# Nothing here may raise into a build. A parse error stops the parsing and is reported
+# once by the caller; the deploy keeps whatever was collected up to that point.
+class Dash::Build::ProgressParser
+  VERTEX = /\A#(?<number>\d+)(?: (?<rest>.*))?\z/
+
+  # `[linux/amd64 build 2/3] RUN …` — the platform prefix only appears on multi-platform
+  # builds, and the stage name only when the Dockerfile named it with AS.
+  STEP = %r{\A\[(?:(?<platform>[a-z0-9]+/[a-z0-9][\w./-]*) )?(?:(?<stage>[A-Za-z0-9][\w.-]*) )?(?<ordinal>\d+)/(?<steps_in_stage>\d+)\] (?<instruction>.+)\z}
+  INTERNAL = /\A\[(?:\S+ )?internal\] (?<what>.+)\z/
+  DONE = /\ADONE (?<seconds>\d+(?:\.\d+)?)s\z/
+  ERROR = /\AERROR: (?<message>.+)\z/
+  TRANSFERRING_CONTEXT = /\Atransferring context: (?<size>[\d.]+)(?<unit>[a-zA-Z]+)(?: (?<seconds>\d+(?:\.\d+)?)s)? done\z/
+  PUSHING = /\Apushing .*?(?<seconds>\d+(?:\.\d+)?)s done\z/
+
+  BYTE_UNITS = { "b" => 1, "kb" => 1_000, "mb" => 1_000_000, "gb" => 1_000_000_000, "tb" => 1_000_000_000_000 }.freeze
+
+  attr_reader :error
+
+  def initialize
+    @steps = {}
+    @order = []
+    @buffer = +""
+    @push_seconds = 0.0
+    @mutex = Mutex.new
+  end
+
+  # SSHKit's interaction-handler contract.
+  def on_data(_command, _stream_name, data, _channel)
+    @mutex.synchronize do
+      next if @error
+
+      @buffer << data.to_s
+      while (newline = @buffer.index("\n"))
+        parse_line @buffer.slice!(0..newline).chomp
+      end
+    rescue StandardError => e
+      @error = e
+    end
+  end
+
+  # The last line of a build has no trailing newline when the command dies mid-write.
+  def finish
+    @mutex.synchronize do
+      next if @error
+
+      parse_line @buffer.chomp unless @buffer.empty?
+      @buffer = +""
+    rescue StandardError => e
+      @error = e
+    end
+  end
+
+  def result
+    @mutex.synchronize do
+      Dash::Build::Report.new(steps: @order.map { |number| @steps[number] }, push_seconds: @push_seconds)
+    end
+  end
+
+  private
+    def parse_line(line)
+      match = VERTEX.match(line) or return
+      rest = match[:rest].to_s.strip
+
+      # `#3 ...` is buildx saying the vertex is deferred, not a step of its own.
+      return if rest.empty? || rest == "..."
+
+      number = match[:number].to_i
+      step = @steps[number]
+
+      unless step
+        step = @steps[number] = build_step(number, rest)
+        @order << number
+      end
+
+      update step, rest
+    end
+
+    def build_step(number, rest)
+      Dash::Build::Step.new(number, kind: :other, name: rest).tap do |step|
+        if (match = STEP.match(rest))
+          step.platform = match[:platform]
+          step.stage = match[:stage]
+          step.ordinal = match[:ordinal].to_i
+          step.steps_in_stage = match[:steps_in_stage].to_i
+          step.instruction = match[:instruction].squeeze(" ").strip
+          step.kind = step.instruction.start_with?("FROM ") ? :from : :instruction
+        elsif (match = INTERNAL.match(rest))
+          step.kind = internal_kind(match[:what])
+        elsif rest.start_with?("exporting cache")
+          step.kind = :cache_export
+        elsif rest.match?(/\A(exporting|pushing|writing image)/)
+          step.kind = :export
+        end
+      end
+    end
+
+    def internal_kind(what)
+      case what
+      when /\Aload build context/ then :context
+      when /\Aload metadata for/ then :metadata
+      else :other
+      end
+    end
+
+    # Later lines for a vertex are its result, its output, or its progress. buildx
+    # re-reports DONE per platform on a multi-platform build, so the last one wins.
+    def update(step, rest)
+      if (match = DONE.match(rest))
+        step.seconds = match[:seconds].to_f
+      elsif rest == "CACHED"
+        step.cached = true
+        step.seconds = 0.0
+      elsif (match = ERROR.match(rest))
+        step.error = match[:message]
+      elsif (match = TRANSFERRING_CONTEXT.match(rest)) && step.kind == :context
+        step.bytes = bytes(match[:size], match[:unit])
+        step.seconds ||= match[:seconds]&.to_f
+      elsif (match = PUSHING.match(rest)) && step.kind == :export
+        @push_seconds += match[:seconds].to_f
+      end
+    end
+
+    # buildx reports decimal units — 25.18MB is 25,180,000 bytes, not 25.18 MiB.
+    def bytes(size, unit)
+      (size.to_f * BYTE_UNITS.fetch(unit.downcase, 1)).round
+    end
+end

data/lib/dash/build/report.rb

@@ -0,0 +1,104 @@
+# What the build actually spent its time on, derived from the buildx progress stream.
+#
+# "Steps" here means the operator's own Dockerfile steps — the vertices buildx numbered
+# `[stage k/m]`. BuildKit's own bookkeeping (booting, auth tokens, metadata lookups) is
+# kept in `steps` because it is still time the build took, but it is never counted as a
+# step the operator wrote.
+class Dash::Build::Report
+  attr_reader :steps, :push_seconds
+
+  # Rebuilds a report from what #to_h exported. Only the steps and the push are restored:
+  # every other number in the export is derived from them, so recomputing keeps a
+  # hand-edited file from claiming a total its own steps do not add up to.
+  def self.from_h(hash)
+    hash = hash.transform_keys(&:to_sym)
+
+    new steps: Array(hash[:steps]).map { |step| Dash::Build::Step.from_h(step) }, push_seconds: hash[:push_seconds].to_f
+  end
+
+  def initialize(steps: [], push_seconds: 0.0)
+    @steps = steps
+    @push_seconds = push_seconds
+  end
+
+  def any?
+    steps.any?
+  end
+
+  def dockerfile_steps
+    steps.select(&:dockerfile_step?)
+  end
+
+  def instruction_steps
+    steps.select { |step| step.kind == :instruction }
+  end
+
+  def cached_steps
+    dockerfile_steps.select(&:cached)
+  end
+
+  def uncached_steps
+    dockerfile_steps.reject(&:cached)
+  end
+
+  # The rows worth printing: the operator's own instructions, longest first. A FROM or a
+  # metadata lookup is not something they can speed up by editing the Dockerfile.
+  def slowest(count)
+    instruction_steps.reject(&:cached).sort_by { |step| -step.seconds.to_f }.first(count)
+  end
+
+  def context_step
+    steps.find { |step| step.kind == :context }
+  end
+
+  def context_bytes
+    context_step&.bytes
+  end
+
+  def context_seconds
+    context_step&.seconds
+  end
+
+  def export_seconds
+    seconds_for(:export)
+  end
+
+  def cache_export_seconds
+    seconds_for(:cache_export)
+  end
+
+  def total_step_seconds
+    steps.sum { |step| step.seconds.to_f }
+  end
+
+  def errors
+    steps.select(&:error)
+  end
+
+  # The errors worth putting in front of an operator. BuildKit reports a cache-import
+  # miss as an ERROR on its own vertex — the first build against a fresh cache always
+  # has one — and a row saying "error" for something that did not fail the build teaches
+  # people to ignore the column.
+  def failed_steps
+    errors.select { |step| step.dockerfile_step? || step.kind == :export }
+  end
+
+  def to_h
+    {
+      context_bytes: context_bytes,
+      context_seconds: context_seconds,
+      cached_steps: cached_steps.size,
+      total_steps: dockerfile_steps.size,
+      export_seconds: export_seconds,
+      cache_export_seconds: cache_export_seconds,
+      push_seconds: push_seconds,
+      total_step_seconds: total_step_seconds,
+      steps: steps.map(&:to_h)
+    }
+  end
+
+  private
+    def seconds_for(kind)
+      steps.sum { |step| step.kind == kind ? step.seconds.to_f : 0.0 }
+    end
+end

data/lib/dash/build/step.rb

@@ -0,0 +1,49 @@
+# One buildx vertex. A vertex is a unit of work BuildKit reports on: a Dockerfile
+# instruction, the context transfer, a metadata lookup, the export to the registry.
+#
+# Only the ones with an ordinal (`[build 5/9]`) are the operator's own steps — the rest
+# are BuildKit's own bookkeeping, and counting them would make "cached steps 9 of 22"
+# say something nobody asked.
+class Dash::Build::Step
+  attr_reader :number
+  attr_accessor :kind, :name, :platform, :stage, :ordinal, :steps_in_stage, :instruction, :seconds, :cached, :error, :bytes
+
+  def self.from_h(step)
+    step = step.transform_keys(&:to_sym)
+
+    new(step[:number], kind: step[:kind]&.to_sym || :other, name: step[:label]).tap do |rebuilt|
+      rebuilt.platform, rebuilt.stage = step[:platform], step[:stage]
+      rebuilt.ordinal, rebuilt.steps_in_stage = step[:ordinal], step[:steps_in_stage]
+      rebuilt.instruction, rebuilt.seconds = step[:instruction], step[:seconds]
+      rebuilt.cached, rebuilt.bytes, rebuilt.error = !!step[:cached], step[:bytes], step[:error]
+    end
+  end
+
+  def initialize(number, kind: :other, name: nil)
+    @number = number
+    @kind = kind
+    @name = name
+    @cached = false
+  end
+
+  # What buildx itself printed for this vertex, minus the platform prefix it adds on a
+  # multi-platform build — the operator matches these against their Dockerfile, and the
+  # platform is already its own column in the data.
+  def label
+    ordinal ? "[#{[ stage, "#{ordinal}/#{steps_in_stage}" ].compact.join(" ")}] #{instruction}" : name.to_s
+  end
+
+  def dockerfile_step?
+    !ordinal.nil?
+  end
+
+  # `label` is here for whatever reads the JSON — it is the string buildx printed and the
+  # one a human matches against their Dockerfile — and the parts it is built from are here
+  # so #from_h can rebuild it rather than trusting a field a hand-edited file may disagree
+  # with.
+  def to_h
+    { number: number, kind: kind, label: label, platform: platform, stage: stage, ordinal: ordinal,
+      steps_in_stage: steps_in_stage, instruction: instruction, seconds: seconds, cached: cached,
+      bytes: bytes, error: error }
+  end
+end

data/lib/dash/cli/app/boot.rb

@@ -1,4 +1,9 @@
 class Dash::Cli::App::Boot
+  # What `docker container ls --quiet` prints, and so what dash-proxy has always been
+  # handed as a target. `docker run --detach` prints the full 64-character id, so the
+  # target is its first twelve characters rather than a round trip of its own.
+  SHORT_CONTAINER_ID_LENGTH = 12
+
   attr_reader :host, :role, :version, :barrier, :sshkit, :cli
   delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit
   delegate :run_hook, to: :cli
@@ -38,27 +43,44 @@ class Dash::Cli::App::Boot
   end
 
   private
+    # Both answers come back from one round trip, which means the running version is read
+    # before any rename happens. When the clashing container IS the running one, the
+    # version to stop later is the name it was renamed to - the name that was read now
+    # belongs to the container this boot is about to start.
     def old_version_renamed_if_clashing
-      if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+      clashing_container_id, old_version = capture_boot_state
+
+      if clashing_container_id.present?
         renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
         info "Renaming container #{version} to #{renamed_version} as already deployed on #{host}"
-        audit("Renaming container #{version} to #{renamed_version}")
-        execute *app.rename_container(version: version, new_version: renamed_version)
+        execute *auditor.record_then("Renaming container #{version} to #{renamed_version}",
+          app.rename_container(version: version, new_version: renamed_version))
+
+        old_version = renamed_version if old_version == version
       end
 
-      capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip.presence
+      old_version
+    end
+
+    def capture_boot_state
+      clashing, running = Dash::Commands::App.split_state(capture_with_info(*app.boot_state(version), raise_on_non_zero_exit: false))
+
+      [ clashing.strip.presence, running.strip.presence ]
     end
 
     def start_new_version
-      audit "Booted app version #{version}"
       hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
 
-      execute *app.ensure_env_directory
+      execute *auditor.record_then("Booted app version #{version}", app.ensure_env_directory)
       upload! role.secrets_io(host), role.secrets_path, mode: "0600"
 
-      execute *app.run(hostname: hostname)
+      # `docker run --detach` prints the id of the container it just started, so the
+      # proxy target comes out of the run itself — asking docker for it again was a round
+      # trip spent re-reading something the host had already said.
+      container_id = capture_with_info(*app.run(hostname: hostname)).strip
+
       if running_proxy?
-        endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+        endpoint = container_id[0, SHORT_CONTAINER_ID_LENGTH]
         raise Dash::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
 
         run_hook "pre-proxy-deploy", hosts: host.to_s, role: role.name
@@ -66,7 +88,7 @@ class Dash::Cli::App::Boot
         timing_healthy { execute *app.deploy(target: endpoint) }
         run_hook "post-proxy-deploy", hosts: host.to_s, role: role.name
       else
-        timing_healthy { Dash::Cli::Healthcheck::Poller.wait_for_healthy(role: role) { health_status } }
+        timing_healthy { Dash::Cli::Healthcheck::Poller.wait_for_healthy(role: role, &method(:readiness_status)) }
       end
     rescue => e
       error "Failed to boot #{role} on #{host}"
@@ -74,17 +96,21 @@ class Dash::Cli::App::Boot
       raise e
     end
 
-    # An exec probe is docker-invisible — the container declares no healthcheck, so
-    # `docker inspect` would only ever report its state. Poll the probe instead.
-    def health_status
-      role.healthcheck&.exec? ? exec_probe_status : capture_with_info(*app.status(version: version))
-    end
-
-    def exec_probe_status
-      execute *app.health_probe(version: version)
-      "healthy"
-    rescue SSHKit::Command::Failed
-      "exec probe exited non-zero"
+    # A role behind the proxy lets `dash-proxy deploy` block on the host until the
+    # container is healthy; a role without one now does the same, waiting in a shell loop
+    # on the host that streams its progress back rather than being polled from here once
+    # per attempt. The poller asks for the wait, and — only for an unchecked container it
+    # has just let through its readiness delay — for a plain confirming read.
+    #
+    # Neither capture suppresses a non-zero exit: a status that cannot be read is a broken
+    # command, and it has always failed the boot on the spot rather than being waited out.
+    def readiness_status(mode, seconds_left = nil)
+      if mode == :confirm
+        capture_with_info(*app.status(version: version))
+      else
+        capture_with_info *app.wait_for_ready(version: version, timeout: seconds_left),
+          interaction_handler: Dash::Cli::Healthcheck::ProgressReporter.new
+      end
     end
 
     # Every failed boot gets the container log, and the health probe history when the
@@ -161,10 +187,6 @@ class Dash::Cli::App::Boot
       @auditor = DASH.auditor(role: role)
     end
 
-    def audit(message)
-      execute *auditor.record(message), verbosity: :debug
-    end
-
     def gatekeeper?
       barrier && barrier_role?
     end

data/lib/dash/cli/app.rb

@@ -40,8 +40,8 @@ class Dash::Cli::App < Dash::Cli::Base
 
         # Tag once the app booted on all hosts
         on(DASH.app_hosts) do |host|
-          execute *DASH.auditor.record("Tagging #{DASH.config.absolute_image} as the latest image"), verbosity: :debug
-          execute *DASH.app.tag_latest_image
+          execute *DASH.auditor.record_then("Tagging #{DASH.config.absolute_image} as the latest image",
+            DASH.app.tag_latest_image)
         end
       end
     end
@@ -186,8 +186,8 @@ class Dash::Cli::App < Dash::Cli::Base
     with_lock_if_stopping do
       on_roles(DASH.roles, hosts: DASH.app_hosts) do |host, role|
         app = DASH.app(role: role, host: host)
-        versions = capture_with_info(*app.list_versions, raise_on_non_zero_exit: false).split("\n")
-        versions -= [ capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip ]
+        listed, running = Dash::Commands::App.split_state(capture_with_info(*app.stale_state, raise_on_non_zero_exit: false))
+        versions = listed.strip.split("\n") - [ running.strip ]
 
         versions.each do |version|
           if stop

data/lib/dash/cli/base.rb

@@ -1,4 +1,5 @@
 require "thor"
+require "time"
 require "dash/sshkit_with_ext"
 
 module Dash::Cli
@@ -145,13 +146,111 @@ module Dash::Cli
       def print_runtime
         started_at = Time.now
         @print_runtime_depth = @print_runtime_depth.to_i + 1
+        record_startup_timing if @print_runtime_depth == 1
         yield
         Time.now - started_at
+      rescue StandardError => e
+        # Kept so the saved report says how the run ended. `setup` nests, and the inner
+        # deploy sets it first — the outermost writer reads the same error either way.
+        @report_error = e
+        raise
       ensure
         @print_runtime_depth -= 1
         runtime = Time.now - started_at
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
-        puts DASH.timings.lines if @print_runtime_depth.zero? && DASH.timings.any?
+        finish_report(started_at, runtime) if @print_runtime_depth.zero? && DASH.timings.any?
+      end
+
+      # Trends, then the table, then the saved JSON. Each half is guarded on its own so a
+      # report that cannot be written still prints, and a table that cannot be compared is
+      # still a table — none of this is allowed to be why a deploy ends badly.
+      def finish_report(started_at, runtime)
+        run = nil
+
+        guarded_report do
+          run = report_run(started_at, runtime)
+          DASH.report.advice += report_trends(run)
+        end
+
+        puts DASH.report.lines
+
+        guarded_report { write_report(run) } if run
+      end
+
+      # What this invocation was, for the saved report and for the trend rules that
+      # compare it with the invocations before it.
+      def report_run(started_at, runtime)
+        {
+          command: [ command, subcommand ].compact.join(" "),
+          service: DASH.config.service, destination: DASH.config.destination, version: DASH.config.version,
+          started_at: started_at.getutc.iso8601, runtime: runtime.round(1),
+          status: @report_error ? "failed" : "succeeded",
+          error: @report_error && { class: @report_error.class.name, message: @report_error.message }
+        }.compact
+      end
+
+      def report_trends(run)
+        return [] unless DASH.config.report.advice?
+
+        history = Dash::Report::History.new(reports_directory, destination: DASH.config.destination)
+
+        Dash::Report::Trends.new(run.merge(phases: DASH.timings.to_h),
+          history: history.recent(DASH.config.report.history), ignore: DASH.config.report.ignore).findings
+      end
+
+      def write_report(run)
+        @report_path = Dash::Report::Writer.new(DASH.report,
+          run: run, keep: DASH.config.report.history, directory: reports_directory).write
+
+        puts "  Report written to #{@report_path}" if @report_path
+      end
+
+      def reports_directory
+        Dash::ProjectDirectory.join("reports")
+      end
+
+      # Summary numbers for the post-deploy hook, so a hook can page on a build that
+      # doubled or a warning that appeared without re-deriving any of it. Phases that did
+      # not run contribute nothing rather than a zero that reads as "instant".
+      def report_hook_details
+        guarded_report({}) do
+          {
+            build_runtime: phase_runtime(Dash::Report::Trends::BUILD_PHASE),
+            boot_runtime: phase_runtime(Dash::Report::Trends::BOOT_PHASE),
+            advice_count: DASH.report.advice.size.to_s,
+            advice_warnings: DASH.report.advice.count(&:warn?).to_s,
+            report_path: @report_path
+          }.compact
+        end
+      end
+
+      def phase_runtime(name)
+        DASH.timings.seconds_for(name)&.round(1)&.to_s
+      end
+
+      # Everything that happened before the first phase could be timed: requiring the gem,
+      # Zeitwerk, Thor parsing the command line, and building the config. It is the one row
+      # an operator cannot influence from deploy.yml, which is exactly why it has to be
+      # visible — a five-second startup is dash's problem, not theirs.
+      def record_startup_timing
+        DASH.timings.record "Startup (load, config)", Process.clock_gettime(Process::CLOCK_MONOTONIC) - Dash::PROCESS_STARTED_AT
+      end
+
+      # Advice about the Dockerfile, run once the build report is in so measured rules can
+      # name real seconds. Nothing here is allowed to end a deploy: a report is a courtesy
+      # printed next to the work, and the work already succeeded or failed on its own.
+      def analyze_report
+        guarded_report { DASH.report.analyze!(DASH.config) }
+      end
+
+      # `fallback` is what the caller gets when the report could not be produced, for the
+      # callers that need a value rather than a side effect.
+      def guarded_report(fallback = nil)
+        yield
+      rescue StandardError => e
+        say "Deploy report unavailable: #{e.class}: #{e.message}", :yellow
+        say e.backtrace.join("\n"), :yellow if ENV["VERBOSE"]
+        fallback
       end
 
       def timed(name, depth: 0, &block)
@@ -204,6 +303,10 @@ module Dash::Cli
       # in one `on(hosts)` sweep would leave the locks we did win in place, and
       # every retry would then collide with itself and wait out the timeout.
       def acquire_server_lock
+        timed("Acquire server lock") { acquire_server_lock_now }
+      end
+
+      def acquire_server_lock_now
         ensure_run_directory
 
         timeout = DASH.lock_wait_timeout
@@ -343,6 +446,10 @@ module Dash::Cli
       end
 
       def acquire_lock
+        timed("Acquire deploy lock") { acquire_lock_now }
+      end
+
+      def acquire_lock_now
         ensure_run_directory
 
         if DASH.lock_wait
@@ -470,10 +577,18 @@ module Dash::Cli
         instance_variable_get("@_invocations")[cli_class].pop
       end
 
+      # Every lock acquire wants the run directory to exist, but the sweep is idempotent
+      # and a process only needs it once per host - the deploy lock and the server lock
+      # were paying for it twice.
       def ensure_run_directory
-        on(DASH.hosts) do
+        pending = DASH.hosts.map(&:to_s) - DASH.run_directory_ensured_on
+        return if pending.empty?
+
+        on(pending) do
           execute(*DASH.server.ensure_run_directory)
         end
+
+        DASH.run_directory_ensured_on.concat(pending)
       end
 
       def with_env(env)