dash 4.0.8 → 4.1.1

data/lib/dash/sshkit_with_ext.rb

@@ -6,6 +6,27 @@ require "json"
 require "resolv"
 require "concurrent/atomic/semaphore"
 
+# Deploy-report timing reaches into dash's runtime from code that runs for everyone else
+# using SSHKit in this process too, and this file can be required on its own — without the
+# gem's autoloader and without the DASH commander. So every reach is optional, and it is
+# funnelled through here rather than repeated at each hook, where one site would sooner or
+# later forget the guard and take down every parallel run with a NameError.
+module DashTimings
+  class << self
+    def timings
+      DASH.timings if defined?(DASH)
+    end
+
+    def current_entry
+      Dash::Timings.current_entry if defined?(Dash::Timings)
+    end
+
+    def current_entry=(entry)
+      Dash::Timings.current_entry = entry if defined?(Dash::Timings)
+    end
+  end
+end
+
 class SSHKit::Backend::Abstract
   def capture_with_info(*args, **kwargs)
     capture(*args, **kwargs, verbosity: Logger::INFO)
@@ -64,6 +85,22 @@ class SSHKit::Backend::Abstract
     end
   end
   prepend CommandEnvMerge
+
+  # Attributes the wall time of every command to the timing entry that is current on
+  # this thread, so a phase can report how much of its total was spent waiting on round
+  # trips rather than on the app. Nothing is executed that would not have run anyway —
+  # this only stamps the commands dash was already issuing.
+  module TimedCommands
+    private
+      def create_command_and_execute(args, options)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        super
+      ensure
+        DashTimings.timings&.attribute_command(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started, local: !!host&.local?)
+      end
+  end
+  prepend TimedCommands
 end
 
 class SSHKit::Backend::Netssh::Configuration
@@ -171,6 +208,21 @@ class SSHKit::Backend::Netssh
   end
   prepend LimitConcurrentStartsInstance
 
+  # Prepended last, so it is in front of the concurrency limiter and the DNS retries:
+  # what a phase pays for a connection includes queueing behind max_concurrent_starts.
+  # The pool only calls through on a cache miss, so this measures real connects.
+  module TimedConnects
+    private
+      def connect_ssh(...)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        super
+      ensure
+        DashTimings.timings&.attribute_connect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+      end
+  end
+  prepend TimedConnects
+
   # A pooled session that sat idle while dash was busy elsewhere (a server-lock
   # wait, a loadbalancer reboot) gets dropped by NATs and cloud networks without
   # either end noticing: net-ssh only sends keepalives from inside its event
@@ -235,9 +287,15 @@ class SSHKit::Runner::Parallel
   # problem occurs on multiple hosts.
   module CompleteAll
     def execute
+      # A new thread starts with none of its parent's thread-locals, so the timing entry
+      # has to be handed over explicitly or every command run on a host would be
+      # attributed to no phase at all.
+      timing_entry = DashTimings.current_entry
+
       threads = hosts.map do |host|
         Thread.new(host) do |h|
           Thread.current.report_on_exception = false
+          DashTimings.current_entry = timing_entry
           backend(h, &block).run
         rescue ::StandardError => e
           e2 = SSHKit::Runner::ExecuteError.new e
@@ -315,9 +373,13 @@ module SSHKitDslRoles
   #   end
   def on_roles(roles, hosts:, parallel: true, rolling: false, &block)
     if parallel
+      # See CompleteAll#execute: thread-locals do not cross Thread.new.
+      timing_entry = DashTimings.current_entry
+
       threads = roles.filter_map do |role|
         if (role_hosts = role.hosts & hosts).any?
           Thread.new do
+            DashTimings.current_entry = timing_entry
             on(role_hosts, rolling ? role.boot_runner_options(role_hosts) : {}) { |host| instance_exec(host, role, &block) }
           rescue StandardError => e
             raise SSHKit::Runner::ExecuteError.new(e), "Exception while executing on #{role}: #{e.message}"

data/lib/dash/timings.rb

@@ -5,40 +5,193 @@
 # Entries land in start order and carry a depth, so a parent phase (Boot) prints above
 # the host entries it wraps even though the hosts finish first. Boot runs one thread per
 # host, so every mutation is behind the mutex.
+#
+# Each entry also accounts for the commands issued while it was the current phase, which
+# is what separates "the boot took 55s" from "the boot spent 41s of that in forty serial
+# SSH round trips". Attribution is by thread-local: `phase` marks its entry as current for
+# the duration of the block, and `lib/dash/sshkit_with_ext.rb` stamps every command and
+# every SSH connect onto whatever entry is current on that thread. Nothing extra is
+# executed — dash only measures the round trips it was already making.
 class Dash::Timings
-  Entry = Struct.new(:name, :seconds, :detail, :depth)
+  # The entry the current thread is inside. SSHKit's per-host threads inherit it from
+  # the thread that spawned them (see CompleteAll#execute and SSHKitDslRoles#on_roles),
+  # so a command issued on a boot thread lands on that host's row.
+  CURRENT_KEY = :dash_timing_entry
+
+  Entry = Struct.new(:name, :seconds, :detail, :depth, :parent, :commands, :command_seconds, :connect_seconds, :local)
+
+  # Read and written from lib/dash/sshkit_with_ext.rb, which hands the entry across the
+  # per-host and per-role threads SSHKit spawns. Class-level because the patches run
+  # without a commander in reach.
+  class << self
+    def current_entry
+      Thread.current[CURRENT_KEY]
+    end
 
-  def initialize
-    @entries = []
+    def current_entry=(entry)
+      Thread.current[CURRENT_KEY] = entry
+    end
+
+    # Rebuilds a table from what #to_h exported, so `dash report` can print a saved
+    # deploy the way the deploy printed it. The entries come back parentless: their
+    # counters are already the subtree totals #to_h computed, and re-nesting them would
+    # roll those totals up a second time.
+    def from_h(phases)
+      new entries: Array(phases).map { |phase| entry_from(phase.transform_keys(&:to_sym)) }
+    end
+
+    private
+      def entry_from(phase)
+        Entry.new \
+          phase[:name], phase[:seconds].to_f, phase[:detail], phase[:depth].to_i, nil,
+          phase[:commands].to_i, phase[:command_seconds].to_f, phase[:connect_seconds].to_f,
+          phase[:local].nil? ? true : phase[:local]
+      end
+  end
+
+  def initialize(entries: [])
+    @entries = entries
     @mutex = Mutex.new
   end
 
   # Times the block. The entry is yielded so the block can annotate it — a boot can say
   # how long of its total was spent waiting for the container to become healthy.
   def phase(name, depth: 0)
-    entry = Entry.new(name, nil, nil, depth)
+    entry = new_entry(name, depth: depth)
     @mutex.synchronize { @entries << entry }
     started = clock
 
-    yield entry
+    previous = self.class.current_entry
+    self.class.current_entry = entry
+
+    begin
+      yield entry
+    ensure
+      self.class.current_entry = previous
+    end
   ensure
     entry.seconds = clock - started
   end
 
+  # For time that was measured elsewhere — the gem load and config parse that happened
+  # before there was a Timings to record into.
+  def record(name, seconds, depth: 0, detail: nil)
+    new_entry(name, depth: depth).tap do |entry|
+      entry.seconds = seconds
+      entry.detail = detail
+      @mutex.synchronize { @entries << entry }
+    end
+  end
+
+  def current
+    self.class.current_entry
+  end
+
+  def attribute_command(seconds, local:)
+    entry = current or return
+
+    @mutex.synchronize do
+      entry.commands += 1
+      entry.command_seconds += seconds
+      # A phase that issued even one remote command is reported as ssh: the local
+      # label is only honest when nothing left the machine.
+      entry.local &&= local
+    end
+  end
+
+  def attribute_connect(seconds)
+    entry = current or return
+
+    @mutex.synchronize { entry.connect_seconds += seconds }
+  end
+
   def any?
     @mutex.synchronize { @entries.any? }
   end
 
+  # Where an entry's row sits in #lines, so Dash::Report can splice its build rows in
+  # under the phase they belong to. Identity, not equality: two phases of the same name
+  # and duration are equal as Structs but are not the same row.
+  # The row at a position, so a consumer that saved an index (Dash::Report, reattaching
+  # its build rows to the phase they hung under) can find the entry again.
+  def entry_at(index)
+    @mutex.synchronize { @entries[index] } if index
+  end
+
+  def index_of(entry)
+    @mutex.synchronize { @entries.index { |candidate| candidate.equal?(entry) } }
+  end
+
   def lines
-    @mutex.synchronize do
-      @entries.map do |entry|
-        line = format("  %s%-36s %6.1fs", "  " * entry.depth, entry.name, entry.seconds.to_f)
-        entry.detail ? "#{line} (#{entry.detail})" : line
-      end
+    with_totals do |entry, totals|
+      line = format("  %s%-36s %6.1fs", "  " * entry.depth, entry.name, entry.seconds.to_f)
+      line += format(" %3d %-5s %5.1fs", totals[:commands], totals[:local] ? "local" : "ssh", totals[:command_seconds]) if totals[:commands] > 0
+      entry.detail ? "#{line} (#{entry.detail})" : line
+    end
+  end
+
+  # A top-level phase's wall time by name, for consumers that know a phase by what it is
+  # called rather than by where it sits — the post-deploy hook's build and boot runtimes.
+  # Depth-0 only: a per-host row inside Boot is named after the host, but a role could be
+  # named "Boot" and must not be mistaken for the phase.
+  def seconds_for(name)
+    @mutex.synchronize { @entries.find { |entry| entry.name == name && entry.depth.to_i.zero? }&.seconds }
+  end
+
+  # The command counts here are subtree totals, matching what the table prints — phases
+  # nest, so a consumer must not sum them across depths.
+  def to_h
+    with_totals do |entry, totals|
+      {
+        name: entry.name,
+        depth: entry.depth,
+        seconds: entry.seconds.to_f,
+        detail: entry.detail,
+        commands: totals[:commands],
+        command_seconds: totals[:command_seconds],
+        connect_seconds: totals[:connect_seconds],
+        local: totals[:local]
+      }
     end
   end
 
   private
+    def new_entry(name, depth:)
+      Entry.new(name, nil, nil, depth, current, 0, 0.0, 0.0, true)
+    end
+
+    def with_totals
+      @mutex.synchronize do
+        totals = subtree_totals
+        @entries.map { |entry| yield entry, totals[entry.object_id] }
+      end
+    end
+
+    # A parent phase issues few commands of its own — Boot opens one thread per host and
+    # then waits — so its own counters say nothing. Roll each entry's counters up through
+    # its ancestors at render time rather than at record time, so the host rows keep
+    # their own numbers while the parent shows what the phase cost in total.
+    def subtree_totals
+      totals = @entries.to_h { |entry| [ entry.object_id, { commands: 0, command_seconds: 0.0, connect_seconds: 0.0, local: true } ] }
+
+      @entries.each do |entry|
+        node = entry
+
+        while node
+          if (total = totals[node.object_id])
+            total[:commands] += entry.commands
+            total[:command_seconds] += entry.command_seconds
+            total[:connect_seconds] += entry.connect_seconds
+            total[:local] &&= entry.local if entry.commands > 0
+          end
+
+          node = node.parent
+        end
+      end
+
+      totals
+    end
+
     def clock
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end

data/lib/dash/utils.rb

@@ -134,6 +134,13 @@ module Dash::Utils
     end
   end
 
+  # buildx reports decimal units, so dash does too — an operator comparing a report row
+  # with what buildx printed should see the same number.
+  def human_bytes(bytes)
+    divisor, unit = [ [ 1_000_000_000, "GB" ], [ 1_000_000, "MB" ], [ 1_000, "kB" ] ].find { |size, _| bytes >= size }
+    divisor ? format("%.1f%s", bytes.to_f / divisor, unit) : "#{bytes}B"
+  end
+
   def older_version?(version, other_version)
     Gem::Version.new(version.delete_prefix("v")) < Gem::Version.new(other_version.delete_prefix("v"))
   end

data/lib/dash/version.rb

@@ -1,3 +1,3 @@
 module Dash
-  VERSION = "4.0.8"
+  VERSION = "4.1.1"
 end

data/lib/dash.rb

@@ -1,5 +1,9 @@
 module Dash
   class ConfigurationError < StandardError; end
+
+  # Stamped before the first require so the Startup row in the deploy report covers what
+  # it actually costs to get here: gem load, Zeitwerk, and everything bin/dash pulls in.
+  PROCESS_STARTED_AT = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 end
 
 require "active_support"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dash
 version: !ruby/object:Gem::Version
-  version: 4.0.8
+  version: 4.1.1
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -177,6 +177,9 @@ files:
 - README.md
 - bin/dash
 - lib/dash.rb
+- lib/dash/build/progress_parser.rb
+- lib/dash/build/report.rb
+- lib/dash/build/step.rb
 - lib/dash/cli.rb
 - lib/dash/cli/accessory.rb
 - lib/dash/cli/alias/command.rb
@@ -198,6 +201,7 @@ files:
 - lib/dash/cli/healthcheck/drift_error.rb
 - lib/dash/cli/healthcheck/error.rb
 - lib/dash/cli/healthcheck/poller.rb
+- lib/dash/cli/healthcheck/progress_reporter.rb
 - lib/dash/cli/lock.rb
 - lib/dash/cli/main.rb
 - lib/dash/cli/main/migrate.rb
@@ -209,6 +213,7 @@ files:
 - lib/dash/cli/proxy/reboot.rb
 - lib/dash/cli/prune.rb
 - lib/dash/cli/registry.rb
+- lib/dash/cli/report.rb
 - lib/dash/cli/secrets.rb
 - lib/dash/cli/server.rb
 - lib/dash/cli/templates/deploy.yml
@@ -255,6 +260,7 @@ files:
 - lib/dash/commands/lock.rb
 - lib/dash/commands/proxy.rb
 - lib/dash/commands/proxy/cert_transfer.rb
+- lib/dash/commands/proxy/state.rb
 - lib/dash/commands/prune.rb
 - lib/dash/commands/registry.rb
 - lib/dash/commands/server.rb
@@ -273,6 +279,7 @@ files:
 - lib/dash/configuration/docs/output.yml
 - lib/dash/configuration/docs/proxy.yml
 - lib/dash/configuration/docs/registry.yml
+- lib/dash/configuration/docs/report.yml
 - lib/dash/configuration/docs/role.yml
 - lib/dash/configuration/docs/servers.yml
 - lib/dash/configuration/docs/ssh.yml
@@ -287,6 +294,7 @@ files:
 - lib/dash/configuration/proxy/boot.rb
 - lib/dash/configuration/proxy/run.rb
 - lib/dash/configuration/registry.rb
+- lib/dash/configuration/report.rb
 - lib/dash/configuration/role.rb
 - lib/dash/configuration/role/healthcheck.rb
 - lib/dash/configuration/servers.rb
@@ -305,6 +313,31 @@ files:
 - lib/dash/configuration/validator/servers.rb
 - lib/dash/configuration/volume.rb
 - lib/dash/docker.rb
+- lib/dash/dockerfile/analyzer.rb
+- lib/dash/dockerfile/context.rb
+- lib/dash/dockerfile/dockerignore.rb
+- lib/dash/dockerfile/document.rb
+- lib/dash/dockerfile/finding.rb
+- lib/dash/dockerfile/hadolint.rb
+- lib/dash/dockerfile/instruction.rb
+- lib/dash/dockerfile/parser.rb
+- lib/dash/dockerfile/rules/apt_hygiene.rb
+- lib/dash/dockerfile/rules/base.rb
+- lib/dash/dockerfile/rules/cache_busting_arg.rb
+- lib/dash/dockerfile/rules/cache_export_cost.rb
+- lib/dash/dockerfile/rules/context_size.rb
+- lib/dash/dockerfile/rules/copy_before_install.rb
+- lib/dash/dockerfile/rules/curl_pipe_shell.rb
+- lib/dash/dockerfile/rules/dockerignore_gaps.rb
+- lib/dash/dockerfile/rules/inline_env_blob.rb
+- lib/dash/dockerfile/rules/latest_base.rb
+- lib/dash/dockerfile/rules/missing_dockerignore.rb
+- lib/dash/dockerfile/rules/no_cache_mount.rb
+- lib/dash/dockerfile/rules/root_user.rb
+- lib/dash/dockerfile/rules/secret_in_build_arg.rb
+- lib/dash/dockerfile/rules/single_stage_build_deps.rb
+- lib/dash/dockerfile/rules/uncached_install.rb
+- lib/dash/dockerfile/stage.rb
 - lib/dash/env_file.rb
 - lib/dash/git.rb
 - lib/dash/otel_shipper.rb
@@ -313,6 +346,10 @@ files:
 - lib/dash/output/formatter.rb
 - lib/dash/output/otel_logger.rb
 - lib/dash/project_directory.rb
+- lib/dash/report.rb
+- lib/dash/report/history.rb
+- lib/dash/report/trends.rb
+- lib/dash/report/writer.rb
 - lib/dash/secrets.rb
 - lib/dash/secrets/adapters.rb
 - lib/dash/secrets/adapters/aws_secrets_manager.rb