dash 4.0.7 → 4.1.0

data/lib/dash/report/trends.rb

@@ -0,0 +1,129 @@
+# Compares this run with the last few saved reports of the same command and destination.
+#
+# The Dockerfile rules can only see what one build looked like; these see what a project's
+# deploys usually look like, which is the only way to tell "the build takes 84 seconds"
+# from "the build suddenly takes twice what it used to". Everything here is informational:
+# a slow deploy is a fact about today, not a defect to fix.
+#
+# The phase names are the ones a deploy records in Dash::Cli::Main and Dash::Cli::Base.
+# They are pinned from the other side by test/cli/main_test.rb, which asserts the table a
+# real deploy prints, so a rename cannot quietly turn these rules off.
+class Dash::Report::Trends
+  BUILD_PHASE = "Build and push app image".freeze
+  BOOT_PHASE = "Boot".freeze
+  # Everything a deploy spends before and around the work itself: loading the gem, parsing
+  # the config, resolving secrets, and taking the locks.
+  OVERHEAD_PHASES = [ "Startup (load, config)", "Validate config and secrets",
+                      "Acquire deploy lock", "Acquire server lock" ].freeze
+
+  # Two deploys are an anecdote. Three are the fewest that can have a median worth
+  # comparing against.
+  MINIMUM_HISTORY = 3
+  WINDOW = 5
+  FACTOR = 1.5
+  # Overhead this large is worth naming even to a project whose deploys have always
+  # carried it — "normal" is not the same as "cheap".
+  OVERHEAD_FLOOR = 10.0
+
+  LOCATION = "deploy history".freeze
+
+  def initialize(document, history: [], ignore: [])
+    @document = document
+    @history = comparable(history)
+    @ignore = Array(ignore).map(&:to_s)
+  end
+
+  def findings
+    return [] if @history.size < MINIMUM_HISTORY
+
+    [ build_finding, boot_finding, overhead_finding, total_finding ].compact.reject { |finding| @ignore.include?(finding.rule) }
+  end
+
+  private
+    def comparable(history)
+      Array(history)
+        .map { |document| document.transform_keys(&:to_sym) }
+        .select { |document| document[:command] == @document[:command] && document[:status] == "succeeded" }
+        .first(WINDOW)
+    end
+
+    def build_finding
+      phase_finding "trend-build", "build", BUILD_PHASE
+    end
+
+    def boot_finding
+      phase_finding "trend-boot", "boot", BOOT_PHASE
+    end
+
+    def phase_finding(rule, label, name)
+      current = phase_seconds(@document, name) or return
+      past = @history.filter_map { |document| phase_seconds(document, name) }
+      return if past.size < MINIMUM_HISTORY || current <= median(past) * FACTOR
+
+      note rule, "#{label} #{seconds(current)} vs median #{seconds(median(past))} over the last #{past.size} deploys"
+    end
+
+    def total_finding
+      current = @document[:runtime].to_f
+      past = @history.map { |document| document[:runtime].to_f }
+      return if current <= median(past) * FACTOR
+
+      note "trend-total", "total #{seconds(current)} vs median #{seconds(median(past))} over the last #{past.size} deploys"
+    end
+
+    # Fires on the absolute number as well as the trend, and names the row responsible —
+    # ten seconds before the first docker command is nearly always one slow thing (a
+    # secrets adapter shelling out, a lock waiting on a sweep) rather than a diffuse cost.
+    def overhead_finding
+      current = overhead_seconds(@document)
+      past = @history.map { |document| overhead_seconds(document) }
+      return if current < OVERHEAD_FLOOR && current <= median(past) * FACTOR
+
+      message = "dash overhead #{seconds(current)} vs median #{seconds(median(past))} over the last #{past.size} deploys"
+      if (slowest = slowest_overhead_phase)
+        message += "; #{slowest[:name]} was #{seconds(slowest[:seconds])} of it"
+      end
+
+      note "trend-overhead", message, "a secrets adapter or a lock sweep is the usual cause"
+    end
+
+    def overhead_seconds(document)
+      overhead_phases(document).sum { |phase| phase[:seconds].to_f }
+    end
+
+    def slowest_overhead_phase
+      overhead_phases(@document).max_by { |phase| phase[:seconds].to_f }
+    end
+
+    def overhead_phases(document)
+      Array(document[:phases])
+        .map { |phase| phase.transform_keys(&:to_sym) }
+        .select { |phase| OVERHEAD_PHASES.include?(phase[:name]) }
+    end
+
+    # Only depth-0 rows: a per-host row inside Boot is named after the host, but a role
+    # could be named "Boot" and would otherwise be counted as the phase.
+    def phase_seconds(document, name)
+      phase = Array(document[:phases])
+        .map { |candidate| candidate.transform_keys(&:to_sym) }
+        .find { |candidate| candidate[:name] == name && candidate[:depth].to_i.zero? }
+
+      phase[:seconds].to_f if phase
+    end
+
+    def median(values)
+      sorted = values.sort
+      middle = sorted.size / 2
+
+      sorted.size.odd? ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2.0
+    end
+
+    def seconds(value)
+      format("%.1fs", value.to_f)
+    end
+
+    def note(rule, message, suggestion = nil)
+      Dash::Dockerfile::Finding.new \
+        rule: rule, severity: :info, location: LOCATION, message: message, suggestion: suggestion
+    end
+end

data/lib/dash/report/writer.rb

@@ -0,0 +1,142 @@
+require "fileutils"
+require "json"
+
+# Writes one JSON report per run under `.dash/reports`, so the next deploy has something
+# to compare itself against and CI has something to archive.
+#
+# A failed deploy is written too, with whatever was measured before it broke: the run an
+# operator most wants to look at afterwards is the one that went wrong.
+class Dash::Report::Writer
+  GITIGNORE = "*\n!.gitignore\n".freeze
+
+  # A destination is an operator-supplied string and a command can carry a subcommand, so
+  # neither is trusted to be a filename.
+  UNSAFE = /[^\w.-]+/
+
+  attr_reader :report, :run, :keep, :directory
+
+  # `run` is the metadata Dash::Cli::Base assembled for this invocation — the same hash
+  # the trend rules compared against, so the file and the advice cannot disagree.
+  def initialize(report, run:, keep:, directory: Dash::ProjectDirectory.join("reports"))
+    @report = report
+    @run = run
+    @keep = keep
+    @directory = directory
+  end
+
+  # Returns the path it wrote, or nil when the operator turned history off.
+  def write
+    return if keep.zero?
+
+    prepare_directory
+    @path = publish JSON.pretty_generate(report.to_h(**run))
+    Dash::Report::History.new(directory, destination: run[:destination]).prune(keep)
+
+    @path
+  end
+
+  private
+    # The name and the content arrive together, always. A hard link publishes a file
+    # that is already complete, atomically, and only if the name is free — so two runs
+    # racing for one name (started_at is recorded to the second, and `safe` maps
+    # `eu/west` and `eu-west` onto the same string) each keep their own report, and
+    # there is never a moment where a report exists empty or half-written. That last
+    # part matters more than it sounds: every reader skips a file it cannot parse, and
+    # the prune only counts the files it could read, so anything left behind here would
+    # stay in the directory forever.
+    def publish(content)
+      scratch = File.join(directory, ".#{base_name}.#{Process.pid}.tmp")
+      File.write(scratch, content)
+
+      begin
+        linked(scratch)
+      rescue SystemCallError
+        created(scratch)
+      end
+    ensure
+      File.delete(scratch) if scratch && File.exist?(scratch)
+    end
+
+    def linked(scratch)
+      claim do |candidate|
+        File.link(scratch, candidate)
+        candidate
+      rescue Errno::EEXIST
+        nil
+      end
+    end
+
+    # For a filesystem with no hard links. The name is claimed with an exclusive create —
+    # which never replaces, so another run's report is safe — and the completed scratch
+    # file is then renamed onto that placeholder, atomically, so no reader ever sees a
+    # half-written report. The placeholder is the one thing that can be left behind
+    # here, and it is the one thing nothing would ever clean up, so it comes down in an
+    # `ensure`: a full disk and the operator's Ctrl-C (an Interrupt, not a
+    # StandardError) both take the claimed name with them.
+    def created(scratch)
+      claim do |candidate|
+        next unless placeholder?(candidate)
+
+        filled(candidate, scratch)
+      end
+    end
+
+    def placeholder?(candidate)
+      File.open(candidate, File::WRONLY | File::CREAT | File::EXCL) { }
+      true
+    rescue Errno::EEXIST
+      false
+    end
+
+    # The cleanup asks the filesystem what happened rather than trusting a flag set after
+    # the fact: the rename consumes the scratch file, so a scratch that is still there is
+    # a rename that did not happen. A flag would have its own window — an interrupt
+    # between a successful rename and the assignment would delete a published report.
+    def filled(candidate, scratch)
+      File.rename(scratch, candidate)
+
+      candidate
+    ensure
+      File.delete(candidate) if File.exist?(scratch) && File.exist?(candidate)
+    end
+
+    # Walks the candidate names until the block takes one, yielding nil for a name that
+    # was already gone. Suffixed names sort after the one they collided with, which
+    # Dash::Report::History#order_key relies on to read them back in run order.
+    def claim
+      suffix = 1
+      candidate = File.join(directory, "#{base_name}.json")
+
+      until (claimed = yield candidate)
+        candidate = File.join(directory, "#{base_name}-#{suffix += 1}.json")
+      end
+
+      claimed
+    end
+
+    def base_name
+      "#{timestamp}-#{safe(destination)}-#{safe(run[:command])}"
+    end
+
+    # The started_at the report already carries, with the colons a filename cannot have.
+    def timestamp
+      safe(run[:started_at].to_s.tr(":", "-"))
+    end
+
+    def destination
+      run[:destination].presence || "default"
+    end
+
+    def safe(part)
+      part.to_s.gsub(UNSAFE, "-")
+    end
+
+    # The .gitignore is written next to the reports rather than left to the operator: a
+    # project that commits `.dash/` would otherwise start committing a deploy report on
+    # every deploy, and nobody asked for that in their diff.
+    def prepare_directory
+      FileUtils.mkdir_p(directory)
+      gitignore = File.join(directory, ".gitignore")
+      File.write(gitignore, GITIGNORE) unless File.exist?(gitignore)
+    end
+end

data/lib/dash/report.rb

@@ -0,0 +1,170 @@
+require "active_support/core_ext/string/filters"
+require "active_support/core_ext/module/delegation"
+
+# The deploy report: the phase table plus everything measured inside a phase that the
+# table itself has no column for. Today that is the build; the Dockerfile advice and the
+# JSON export hang off the same object.
+#
+# It owns the rendering rather than Timings because the build rows have to be spliced
+# into the middle of the table, under the phase they belong to, and Timings has no
+# business knowing what a buildx vertex is.
+class Dash::Report
+  delegate :human_bytes, to: Dash::Utils
+
+  # Depth-1 rows, so build steps line up with the per-host rows a Boot phase prints.
+  INDENT = "    "
+  # Wider than the phase table's 36-column name, because a Dockerfile instruction is the
+  # whole point of the row and most of them are longer than that. The build rows line up
+  # with each other as their own block under the phase; 4 + 60 + 8 still fits 80 columns.
+  NAME_WIDTH = 60
+  VALUE_WIDTH = 7
+  SLOWEST_STEPS = 5
+
+  # Severity, then the file or key to open, then the sentence. The suggestion hangs under
+  # the sentence so the eye can skip the whole block or read one finding in full.
+  SEVERITY_WIDTH = 4
+  LOCATION_WIDTH = 15
+  SEVERITY_COLORS = { warn: "\e[33m" }.freeze
+
+  # The version of the JSON documents #to_h writes and #from_h reads. Bumping it is a
+  # promise to whatever reads .dash/reports, so a reader that does not recognise the
+  # number skips the file rather than guessing.
+  SCHEMA = 1
+
+  attr_reader :timings
+  attr_accessor :build, :build_entry, :advice
+
+  # Rebuilds a saved report so `dash report` prints it the way the deploy printed it —
+  # same table, same build rows under the same phase, same advice.
+  def self.from_h(document)
+    document = document.transform_keys(&:to_sym)
+    timings = Dash::Timings.from_h(document[:phases])
+
+    new(timings: timings).tap do |report|
+      report.build_entry = timings.entry_at(document[:build_phase])
+      report.build = Dash::Build::Report.from_h(document[:build]) if document[:build]
+      report.advice = Array(document[:advice]).map { |finding| Dash::Dockerfile::Finding.from_h(finding) }
+    end
+  end
+
+  def initialize(timings:)
+    @timings = timings
+    @advice = []
+  end
+
+  # The run's own facts (command, service, destination, version, timings of the whole
+  # thing) belong to the caller that knows them; the report contributes what it measured.
+  # `build_phase` is the row the build rows hang under, by position, so a reader can put
+  # them back without matching on a phase name dash is free to reword.
+  def to_h(**run)
+    {
+      schema: SCHEMA, dash_version: Dash::VERSION, **run,
+      phases: timings.to_h, build_phase: build_entry && timings.index_of(build_entry),
+      build: build&.to_h, advice: advice.map(&:to_h)
+    }.compact
+  end
+
+  def lines
+    lines = timings.lines
+    rows = build_lines
+
+    unless rows.empty?
+      index = build_entry && timings.index_of(build_entry)
+      lines = index ? lines.insert(index + 1, *rows) : lines + rows
+    end
+
+    lines + advice_lines
+  end
+
+  # Runs the Dockerfile rules against the file this deploy would build, upgraded with what
+  # the build measured when there was one. Silent about a Dockerfile that is not there:
+  # a --skip-push deploy never looks at one, and a missing file is `dash doctor`'s finding
+  # to report, not a deploy's.
+  #
+  # `build_directory` is where the Dockerfile and context are read from: the git clone for
+  # a `push`, the working directory for a `dev` build that never clones.
+  def analyze!(config, build: @build, build_directory: config.builder.build_directory)
+    @advice = []
+    return unless config.report.advice?
+
+    dockerfile = File.expand_path(config.builder.dockerfile, build_directory)
+    return unless File.exist?(dockerfile)
+
+    @advice = Dash::Dockerfile::Analyzer.new(
+      document: Dash::Dockerfile::Parser.parse(File.read(dockerfile)),
+      path: config.builder.dockerfile,
+      file: dockerfile,
+      context_dir: File.expand_path(config.builder.context, build_directory),
+      build: build,
+      builder: config.builder,
+      ignore: config.report.ignore,
+      hadolint: config.report.hadolint?
+    ).findings
+  end
+
+  # Also printed on their own by `dash build push`, for the same reason the build rows are.
+  def advice_lines
+    return [] if advice.blank?
+
+    [ "  Advice", *advice.flat_map { |finding| advice_rows(finding) } ]
+  end
+
+  # Also printed on their own by `dash build push`, which has no phase table to sit under.
+  def build_lines
+    return [] unless build&.any?
+
+    rows = []
+    rows << context_row if build.context_bytes
+    build.slowest(SLOWEST_STEPS).select(&:seconds).each { |step| rows << row(step.label, seconds(step.seconds)) }
+    rows << row("cached steps", "#{build.cached_steps.size} of #{build.dockerfile_steps.size}") if build.dockerfile_steps.any?
+    rows << export_row if export_and_push_seconds > 0
+    # A multi-platform build runs the same instruction once per platform, so one broken
+    # step fails once per platform with the same message. Print that once.
+    build.failed_steps.uniq { |step| [ step.label, step.error ] }.each { |step| rows << row(step.label, "error", step.error) }
+    rows
+  end
+
+  private
+    # The suggestion hangs under the message rather than under the row, so a location
+    # longer than its column (a Dockerfile somewhere deep in the tree) shifts both.
+    def advice_rows(finding)
+      prefix = format("%s%-#{SEVERITY_WIDTH}s  %-#{LOCATION_WIDTH}s ", INDENT, finding.severity, finding.location)
+
+      rows = [ colorize(finding.severity, "#{prefix}#{finding.message}") ]
+      rows << "#{" " * prefix.length}→ #{finding.suggestion}" if finding.suggestion.present?
+      rows
+    end
+
+    # Colour is for the terminal only: a report piped to a file or asserted in a test
+    # should be the same text without the escape codes.
+    def colorize(severity, line)
+      color = SEVERITY_COLORS[severity] if $stdout.tty?
+      color ? "#{color}#{line}\e[0m" : line
+    end
+
+    # A build killed between the context transfer and that vertex's DONE has a size but
+    # no duration. "0.0s" would be a measurement nobody took.
+    def context_row
+      row "build context", build.context_seconds ? seconds(build.context_seconds) : "n/a", human_bytes(build.context_bytes)
+    end
+
+    def export_row
+      note = "cache export #{seconds(build.cache_export_seconds)}" if build.cache_export_seconds > 0
+      row "export + push", seconds(export_and_push_seconds), note
+    end
+
+    def export_and_push_seconds
+      build.export_seconds + build.push_seconds
+    end
+
+    # Indented like a Dash::Timings depth-1 row, with a wider name column of its own; a
+    # step longer than that is cut rather than allowed to run off the line.
+    def row(name, value, detail = nil)
+      line = format("%s%-#{NAME_WIDTH}s %#{VALUE_WIDTH}s", INDENT, name.truncate(NAME_WIDTH), value)
+      detail ? "#{line} (#{detail})" : line
+    end
+
+    def seconds(value)
+      format("%.1fs", value.to_f)
+    end
+end

data/lib/dash/sshkit_with_ext.rb

@@ -6,6 +6,27 @@ require "json"
 require "resolv"
 require "concurrent/atomic/semaphore"
 
+# Deploy-report timing reaches into dash's runtime from code that runs for everyone else
+# using SSHKit in this process too, and this file can be required on its own — without the
+# gem's autoloader and without the DASH commander. So every reach is optional, and it is
+# funnelled through here rather than repeated at each hook, where one site would sooner or
+# later forget the guard and take down every parallel run with a NameError.
+module DashTimings
+  class << self
+    def timings
+      DASH.timings if defined?(DASH)
+    end
+
+    def current_entry
+      Dash::Timings.current_entry if defined?(Dash::Timings)
+    end
+
+    def current_entry=(entry)
+      Dash::Timings.current_entry = entry if defined?(Dash::Timings)
+    end
+  end
+end
+
 class SSHKit::Backend::Abstract
   def capture_with_info(*args, **kwargs)
     capture(*args, **kwargs, verbosity: Logger::INFO)
@@ -64,6 +85,22 @@ class SSHKit::Backend::Abstract
     end
   end
   prepend CommandEnvMerge
+
+  # Attributes the wall time of every command to the timing entry that is current on
+  # this thread, so a phase can report how much of its total was spent waiting on round
+  # trips rather than on the app. Nothing is executed that would not have run anyway —
+  # this only stamps the commands dash was already issuing.
+  module TimedCommands
+    private
+      def create_command_and_execute(args, options)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        super
+      ensure
+        DashTimings.timings&.attribute_command(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started, local: !!host&.local?)
+      end
+  end
+  prepend TimedCommands
 end
 
 class SSHKit::Backend::Netssh::Configuration
@@ -171,6 +208,21 @@ class SSHKit::Backend::Netssh
   end
   prepend LimitConcurrentStartsInstance
 
+  # Prepended last, so it is in front of the concurrency limiter and the DNS retries:
+  # what a phase pays for a connection includes queueing behind max_concurrent_starts.
+  # The pool only calls through on a cache miss, so this measures real connects.
+  module TimedConnects
+    private
+      def connect_ssh(...)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        super
+      ensure
+        DashTimings.timings&.attribute_connect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+      end
+  end
+  prepend TimedConnects
+
   # A pooled session that sat idle while dash was busy elsewhere (a server-lock
   # wait, a loadbalancer reboot) gets dropped by NATs and cloud networks without
   # either end noticing: net-ssh only sends keepalives from inside its event
@@ -181,8 +233,15 @@ class SSHKit::Backend::Netssh
   # The block is rerun, so a drop *mid-command* runs the command twice. That
   # is accepted: the errors here are the idle-drop ones, where nothing reached
   # the server, and the alternative is the deploy failing on the next host.
+  #
+  # Net::SSH::Timeout (a Disconnect subclass) is deliberately excluded. net-ssh
+  # raises it when a host has ignored `keepalive_maxcount` keepalives in a row,
+  # which is a host that stopped answering mid-command, not a dropped idle
+  # socket. Retrying that reconnects to a host that has just proven unresponsive
+  # and, if it accepts the connection but never finishes the handshake or the
+  # command, hangs the deploy with no further guard. Let it fail.
   module ReconnectOnStaleConnection
-    STALE_CONNECTION_ERRORS = [ Errno::ECONNRESET, Errno::EPIPE, Net::SSH::Disconnect, Net::SSH::Timeout ].freeze
+    STALE_CONNECTION_ERRORS = [ Errno::ECONNRESET, Errno::EPIPE, Net::SSH::Disconnect ].freeze
 
     private
       def with_ssh
@@ -191,12 +250,12 @@ class SSHKit::Backend::Netssh
         begin
           super do |ssh|
             yield ssh
-          rescue *STALE_CONNECTION_ERRORS
-            evict_stale_session(ssh)
+          rescue *STALE_CONNECTION_ERRORS => e
+            evict_stale_session(ssh) if stale_connection_error?(e)
             raise
           end
         rescue *STALE_CONNECTION_ERRORS => e
-          raise if reconnected
+          raise if reconnected || !stale_connection_error?(e)
 
           reconnected = true
           SSHKit.config.output.warn("Reconnecting to #{host}: #{e.message}")
@@ -204,6 +263,10 @@ class SSHKit::Backend::Netssh
         end
       end
 
+      def stale_connection_error?(error)
+        !error.is_a?(Net::SSH::Timeout)
+      end
+
       # `close` waits for channel-close acknowledgements, which never come over
       # a dead socket. `shutdown!` just closes the socket, and a closed session
       # is what makes the pool drop it instead of caching it again.
@@ -224,9 +287,15 @@ class SSHKit::Runner::Parallel
   # problem occurs on multiple hosts.
   module CompleteAll
     def execute
+      # A new thread starts with none of its parent's thread-locals, so the timing entry
+      # has to be handed over explicitly or every command run on a host would be
+      # attributed to no phase at all.
+      timing_entry = DashTimings.current_entry
+
       threads = hosts.map do |host|
         Thread.new(host) do |h|
           Thread.current.report_on_exception = false
+          DashTimings.current_entry = timing_entry
           backend(h, &block).run
         rescue ::StandardError => e
           e2 = SSHKit::Runner::ExecuteError.new e
@@ -304,9 +373,13 @@ module SSHKitDslRoles
   #   end
   def on_roles(roles, hosts:, parallel: true, rolling: false, &block)
     if parallel
+      # See CompleteAll#execute: thread-locals do not cross Thread.new.
+      timing_entry = DashTimings.current_entry
+
       threads = roles.filter_map do |role|
         if (role_hosts = role.hosts & hosts).any?
           Thread.new do
+            DashTimings.current_entry = timing_entry
             on(role_hosts, rolling ? role.boot_runner_options(role_hosts) : {}) { |host| instance_exec(host, role, &block) }
           rescue StandardError => e
             raise SSHKit::Runner::ExecuteError.new(e), "Exception while executing on #{role}: #{e.message}"