dash 4.0.8 → 4.1.0

data/lib/dash/cli/doctor.rb

@@ -17,7 +17,8 @@ class Dash::Cli::Doctor
     ports: "Ports",
     dns: "DNS",
     certificate: "Certificates",
-    readiness: "Readiness"
+    readiness: "Readiness",
+    dockerfile: "Dockerfile"
   }.freeze
 
   STATUS_COLORS = { ok: :green, warn: :yellow, fail: :red }.freeze

data/lib/dash/cli/main.rb

@@ -26,16 +26,23 @@ class Dash::Cli::Main < Dash::Cli::Base
         print_config_banner
 
         say "Validate configuration and secrets...", :magenta
-        DASH.config.validate_secrets!(include_accessories: boot_accessories)
+        timed("Validate config and secrets") { DASH.config.validate_secrets!(include_accessories: boot_accessories) }
 
         if options[:skip_push]
           say "Pull app image...", :magenta
           timed("Pull app image") { invoke "dash:cli:build:pull", [], invoke_options }
         else
           say "Build and push app image...", :magenta
-          timed("Build and push app image") { invoke "dash:cli:build:deliver", [], invoke_options }
+          timed("Build and push app image") do |entry|
+            DASH.report.build_entry = entry
+            invoke "dash:cli:build:deliver", [], invoke_options
+          end
         end
 
+        # Before the boot, so the advice still prints when a boot fails — a slow build is
+        # exactly the kind of thing an operator wants to see on a deploy that went wrong.
+        analyze_report
+
         modify(lock: true) do
           run_hook "pre-deploy", secrets: true
 
@@ -59,7 +66,7 @@ class Dash::Cli::Main < Dash::Cli::Base
         end
       end
 
-      run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
+      run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s, **report_hook_details
     end
   end
 
@@ -74,16 +81,23 @@ class Dash::Cli::Main < Dash::Cli::Base
         print_config_banner
 
         say "Validate configuration and secrets...", :magenta
-        DASH.config.validate_secrets!
+        timed("Validate config and secrets") { DASH.config.validate_secrets! }
 
         if options[:skip_push]
           say "Pull app image...", :magenta
           timed("Pull app image") { invoke "dash:cli:build:pull", [], invoke_options }
         else
           say "Build and push app image...", :magenta
-          timed("Build and push app image") { invoke "dash:cli:build:deliver", [], invoke_options }
+          timed("Build and push app image") do |entry|
+            DASH.report.build_entry = entry
+            invoke "dash:cli:build:deliver", [], invoke_options
+          end
         end
 
+        # Before the boot, so the advice still prints when a boot fails — a slow build is
+        # exactly the kind of thing an operator wants to see on a deploy that went wrong.
+        analyze_report
+
         modify(lock: true) do
           run_hook "pre-deploy", secrets: true
 
@@ -99,7 +113,7 @@ class Dash::Cli::Main < Dash::Cli::Base
         end
       end
 
-      run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
+      run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s, **report_hook_details
     end
   end
 
@@ -125,7 +139,7 @@ class Dash::Cli::Main < Dash::Cli::Base
         end
       end
 
-      run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s if rolled_back
+      run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s, **report_hook_details if rolled_back
     end
   end
 
@@ -299,6 +313,9 @@ class Dash::Cli::Main < Dash::Cli::Base
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Dash::Cli::Prune
 
+  desc "report", "Read the deploy reports saved under .dash/reports"
+  subcommand "report", Dash::Cli::Report
+
   desc "registry", "Login and -out of the image registry"
   subcommand "registry", Dash::Cli::Registry
 

data/lib/dash/cli/prune.rb

@@ -11,9 +11,7 @@ class Dash::Cli::Prune < Dash::Cli::Base
   def images
     modify(lock: true, server_lock: true) do
       on(DASH.hosts) do
-        execute *DASH.auditor.record("Pruned images"), verbosity: :debug
-        execute *DASH.prune.dangling_images
-        execute *DASH.prune.tagged_images
+        execute *DASH.auditor.record_then("Pruned images", DASH.prune.dangling_images, DASH.prune.tagged_images)
       end
     end
   end
@@ -26,11 +24,10 @@ class Dash::Cli::Prune < Dash::Cli::Base
 
     modify(lock: true, server_lock: true) do
       on(DASH.hosts) do |host|
-        execute *DASH.auditor.record("Pruned containers"), verbosity: :debug
-
-        DASH.roles_on(host).each do |role|
-          execute *DASH.prune.app_containers(retain: retain, role: role)
-        end
+        # One round trip per host, whatever it runs: a host with no app roles still
+        # records that the sweep reached it.
+        execute *DASH.auditor.record_then("Pruned containers",
+          *DASH.roles_on(host).map { |role| DASH.prune.app_containers(retain: retain, role: role) })
       end
     end
   end

data/lib/dash/cli/report.rb

@@ -0,0 +1,97 @@
+require "json"
+
+# Reads the JSON reports every deploy leaves under `.dash/reports`.
+#
+# Entirely local and read-only: no lock, no SSH, nothing that can change a server. It is
+# the command to reach for after a deploy has finished and the table has scrolled away,
+# and the one that answers "was it always this slow?".
+class Dash::Cli::Report < Dash::Cli::Base
+  default_command :show
+
+  TREND_HEADINGS = %w[ started version total build boot advice ].freeze
+  COLUMNS = "  %-20s %-10s %8s %8s %8s  %s".freeze
+
+  desc "show", "Print the last saved deploy report"
+  option :last, type: :numeric, banner: "N", desc: "Print a trend table over the last N reports instead"
+  def show
+    if (last = options[:last])
+      return say "--last takes a positive number of reports, got #{last}", :red unless count?(last)
+
+      print_trend saved.recent(last.to_i)
+    else
+      print_latest saved.recent(1).first
+    end
+  end
+
+  desc "path", "Print the directory saved reports are written to"
+  def path
+    puts reports_directory
+  end
+
+  private
+    # Thor's :numeric happily hands over -1 or 2.5, which Array#first turns into a
+    # backtrace. A typo in a flag deserves a sentence, not a stack trace.
+    def count?(value)
+      value.to_i == value && value.to_i > 0
+    end
+
+    def saved
+      Dash::Report::History.new(reports_directory, destination: DASH.config.destination)
+    end
+
+    def print_latest(document)
+      return say_nothing_saved unless document
+
+      say "Deploy report for #{subject}", :magenta
+      puts summary_line(document)
+      puts Dash::Report.from_h(document).lines
+    end
+
+    def print_trend(documents)
+      return say_nothing_saved if documents.empty?
+
+      say "Last #{documents.size} #{"report".pluralize(documents.size)} for #{subject}", :magenta
+      puts format(COLUMNS, *TREND_HEADINGS)
+      documents.reverse_each { |document| puts trend_row(document) }
+    end
+
+    def subject
+      [ DASH.config.service, ("to #{DASH.config.destination}" if DASH.config.destination) ].compact.join(" ")
+    end
+
+    def summary_line(document)
+      "  #{document[:command]} #{document[:status]} in #{seconds(document[:runtime])} at #{document[:started_at]}" \
+        "#{" (version #{document[:version]})" if document[:version]}#{error_note(document)}"
+    end
+
+    def error_note(document)
+      " — #{document.dig(:error, :class)}: #{document.dig(:error, :message)}" if document[:error]
+    end
+
+    def trend_row(document)
+      format COLUMNS, document[:started_at], (document[:version] || "").to_s[0...10],
+        seconds(document[:runtime]), phase(document, Dash::Report::Trends::BUILD_PHASE),
+        phase(document, Dash::Report::Trends::BOOT_PHASE), advice_count(document)
+    end
+
+    def phase(document, name)
+      found = Array(document[:phases]).find { |candidate| candidate[:name] == name && candidate[:depth].to_i.zero? }
+
+      found ? seconds(found[:seconds]) : "-"
+    end
+
+    def advice_count(document)
+      findings = Array(document[:advice])
+      warnings = findings.count { |finding| finding[:severity] == "warn" }
+
+      "#{findings.size}#{" (#{warnings} warn)" if warnings > 0}"
+    end
+
+    def seconds(value)
+      format("%.1fs", value.to_f)
+    end
+
+    def say_nothing_saved
+      say "No saved reports for #{subject} in #{reports_directory}", :yellow
+    end
+end

data/lib/dash/cli/templates/sample_hooks/post-deploy.sample

@@ -10,5 +10,10 @@
 # DASH_ROLES (if set)
 # DASH_DESTINATION (if set)
 # DASH_RUNTIME
+# DASH_BUILD_RUNTIME (if the deploy built an image)
+# DASH_BOOT_RUNTIME (if the deploy booted containers)
+# DASH_ADVICE_COUNT
+# DASH_ADVICE_WARNINGS
+# DASH_REPORT_PATH (unless report/history is 0)
 
 echo "$DASH_PERFORMER deployed $DASH_VERSION to $DASH_DESTINATION in $DASH_RUNTIME seconds"

data/lib/dash/commander.rb

@@ -6,7 +6,13 @@ require "active_support/notifications"
 
 class Dash::Commander
   attr_accessor :verbosity, :holding_lock, :holding_server_lock, :connected, :logging, :lock_wait, :lock_wait_timeout, :lock_wait_interval
-  attr_reader :specific_roles, :specific_hosts, :timings
+  attr_reader :specific_roles, :specific_hosts, :timings, :report
+
+  # Hosts whose run directory this process has already swept, so the second lock acquire
+  # of a command does not re-run the migration everywhere. Per host rather than a flag:
+  # `dash upgrade` narrows the host set between acquires, and a host that was never in
+  # scope has never been swept.
+  attr_reader :run_directory_ensured_on
   delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :app_hosts, :proxy_hosts, :accessory_hosts, to: :specifics
 
   def initialize
@@ -24,10 +30,12 @@ class Dash::Commander
     self.lock_wait_interval = 15
     @modify_depth = 0
     @timings = Dash::Timings.new
+    @report = Dash::Report.new(timings: @timings)
     @specifics = @specific_roles = @specific_hosts = nil
     @config = @config_kwargs = nil
     @output_logger = nil
     @commands = {}
+    @run_directory_ensured_on = []
   end
 
   def config
@@ -170,7 +178,7 @@ class Dash::Commander
     @logging = true
     if modify_started
       ActiveSupport::Notifications.instrument("modify.kamal",
-        command: command, subcommand: subcommand, destination: config.destination, hosts: hosts) { yield }
+        command: command, subcommand: subcommand, destination: config.destination, hosts: hosts, report: report) { yield }
     else
       yield
     end

data/lib/dash/commands/app.rb

@@ -3,6 +3,10 @@ class Dash::Commands::App < Dash::Commands::Base
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
+  # Separates the two answers #boot_state returns. A container id is hex and a version is
+  # a name suffix, so neither can produce this line on its own.
+  BOOT_STATE_SEPARATOR = "--%--"
+
   attr_reader :role, :host
 
   delegate :container_name, to: :role
@@ -75,6 +79,20 @@ class Dash::Commands::App < Dash::Commands::Base
       extract_version_from_name
   end
 
+  # Everything a boot needs to know about a host before it starts anything: whether a
+  # container for the version being deployed already exists (so it can be renamed out of
+  # the way) and which version is running now (so it can be stopped once the new one is
+  # live). Two questions, one round trip, answers split on BOOT_STATE_SEPARATOR.
+  #
+  # Chained with `;` rather than `&&`: an empty answer to either is a normal result, not
+  # a failure, and the second question must be asked whatever the first one said.
+  def boot_state(version)
+    chain \
+      container_id_for_version(version),
+      [ :echo, BOOT_STATE_SEPARATOR ],
+      current_running_version
+  end
+
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *container_filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),

data/lib/dash/commands/auditor.rb

@@ -14,6 +14,16 @@ class Dash::Commands::Auditor < Dash::Commands::Base
       append([ :echo, escape_shell_value(audit_line(line, **details)) ], audit_log_file)
   end
 
+  # The audit line and the action it describes in one round trip, still in that order:
+  # the log is written first, and `&&` means a failed write aborts the action exactly as
+  # a failed standalone audit would have.
+  #
+  # Only ever fold in commands the caller would `execute`. A `capture` folded in here
+  # would come back with nothing to distinguish the audit's own output from the answer.
+  def record_then(line, *commands, **details)
+    combine record(line, **details), *commands
+  end
+
   def reveal
     [ :tail, "-n", 50, audit_log_file ]
   end

data/lib/dash/commands/builder/base.rb

@@ -14,9 +14,27 @@ class Dash::Commands::Builder::Base < Dash::Commands::Base
     docker :image, :rm, "--force", config.absolute_image
   end
 
+  # Dropping the old image is housekeeping - a host that never had it is not an error -
+  # so it must not short-circuit whatever it shares a round trip with.
+  #
+  # The `|| true` is parenthesised because `&&` and `||` bind equally and associate left:
+  # ungrouped, an `audit && clean || true && pull` chain lets a FAILED audit fall into the
+  # same `|| true` and pull anyway, exit status 0. The group confines it to the clean.
+  #
+  # Composed only, never executed on its own: SSHKit's command map prefixes an unknown
+  # first word with /usr/bin/env, and the first word here is `(`.
+  def clean_then_pull
+    combine [ "(", *any(clean, [ :true ]), ")" ], pull
+  end
+
   def push(export_action = "registry", tag_as_dirty: false, no_cache: false)
     docker :buildx, :build,
       "--output=type=#{export_action}",
+      # Plain progress is what dash parses into the build rows of the deploy report
+      # (Dash::Build::ProgressParser). buildx already falls back to it when stdout is a
+      # pipe, which it always is under SSHKit — this only pins it so the format cannot
+      # change under us.
+      "--progress=plain",
       *platform_options(arches),
       *([ "--builder", builder_name ] unless docker_driver?),
       *build_tag_options(tag_as_dirty: tag_as_dirty),

data/lib/dash/commands/builder.rb

@@ -2,7 +2,7 @@ require "active_support/core_ext/string/filters"
 
 class Dash::Commands::Builder < Dash::Commands::Base
   delegate \
-    :create, :remove, :dev, :push, :clean, :pull, :info, :inspect_builder,
+    :create, :remove, :dev, :push, :clean, :pull, :clean_then_pull, :info, :inspect_builder,
     :validate_image, :first_mirror, :login_to_registry_locally?, :push_env,
     to: :target
 

data/lib/dash/configuration/docs/configuration.yml

@@ -229,6 +229,12 @@ logging:
 output:
   ...
 
+# Deploy report
+#
+# Advice printed under the deploy timing table, see dash docs report:
+report:
+  ...
+
 # Aliases
 #
 # Alias configuration, see dash docs alias:

data/lib/dash/configuration/docs/report.yml

@@ -0,0 +1,39 @@
+# Deploy report
+#
+# Every deploy, redeploy, setup, rollback and standalone build prints a report: a table of
+# where the time went, the build steps underneath it, and advice about the Dockerfile and
+# the build context. The `report` key controls the advice half of that; the table always
+# prints.
+
+# Report options
+#
+# The options are specified under the report key in the configuration file.
+report:
+
+  # Advice
+  #
+  # Print the Advice block under the table. The measurements themselves are always
+  # collected — this only decides whether dash says anything about them.
+  advice: true
+
+  # hadolint
+  #
+  # Supplement dash's own rules with hadolint's when it is installed.
+  #
+  # `auto` runs it when `hadolint` is on PATH and stays silent when it is not; `false`
+  # never runs it. Any other value is a configuration error rather than a silent off.
+  # hadolint's exit status never affects the deploy, and its rule codes (DL3008 and
+  # friends) can be silenced through `ignore` like any other rule.
+  hadolint: auto
+
+  # History
+  #
+  # How many JSON reports to keep per destination under .dash/reports. Set to 0 to write
+  # none.
+  history: 20
+
+  # Ignore
+  #
+  # Rule ids to silence. The id is the string printed with each piece of advice.
+  ignore:
+    - root-user

data/lib/dash/configuration/report.rb

@@ -0,0 +1,65 @@
+# The `report:` block: how much of the deploy report to print, and how much of it to keep.
+#
+# Every key is optional and the defaults are what an operator who has never heard of the
+# block gets — the table always prints, the advice under it prints, and hadolint joins in
+# only if they already have it installed.
+class Dash::Configuration::Report
+  include Dash::Configuration::Validation
+
+  DEFAULT_HISTORY = 20
+  HADOLINT_AUTO = "auto".freeze
+
+  attr_reader :report_config
+
+  HADOLINT_SETTINGS = [ HADOLINT_AUTO, true, false ].freeze
+
+  def initialize(config:)
+    @report_config = config.raw_config.report || {}
+    validate! @report_config unless @report_config.empty?
+    ensure_valid_hadolint_setting
+    ensure_valid_history
+  end
+
+  def advice?
+    report_config.fetch("advice", true)
+  end
+
+  def hadolint
+    report_config.fetch("hadolint", HADOLINT_AUTO)
+  end
+
+  # "auto" (or true) means run it when it is on PATH — the availability check itself
+  # lives in Dash::Dockerfile::Hadolint, because only it knows what running costs.
+  def hadolint?
+    hadolint != false
+  end
+
+  def history
+    report_config.fetch("history", DEFAULT_HISTORY).to_i
+  end
+
+  def ignore
+    Array(report_config["ignore"]).map(&:to_s)
+  end
+
+  def to_h
+    report_config
+  end
+
+  private
+    # A misspelling must not read as "off": the operator would lose findings and never
+    # learn why.
+    def ensure_valid_hadolint_setting
+      return if HADOLINT_SETTINGS.include?(hadolint)
+
+      raise Dash::ConfigurationError, "report/hadolint: must be auto or false, got #{hadolint.inspect}"
+    end
+
+    # Same reasoning: a negative count is a typo, and reading it as "keep none" would
+    # quietly stop saving the reports the operator was configuring.
+    def ensure_valid_history
+      return if history >= 0
+
+      raise Dash::ConfigurationError, "report/history: must be 0 or more, got #{report_config["history"].inspect}"
+    end
+end

data/lib/dash/configuration.rb

@@ -16,7 +16,7 @@ class Dash::Configuration
   delegate :argumentize, :optionize, to: Dash::Utils
 
   attr_reader :destination, :raw_config, :secrets
-  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :output, :proxy, :proxy_boot, :servers, :ssh, :sshkit, :registry
+  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :output, :proxy, :proxy_boot, :report, :servers, :ssh, :sshkit, :registry
 
   include Validation
 
@@ -76,6 +76,7 @@ class Dash::Configuration
 
     @logging = Logging.new(logging_config: @raw_config.logging)
     @output = Output.new(config: self)
+    @report = Report.new(config: self)
     @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy, secrets: secrets)
     @proxy_boot = Proxy::Boot.new(config: self)
     @ssh = Ssh.new(config: self)

data/lib/dash/dockerfile/analyzer.rb

@@ -0,0 +1,66 @@
+# Runs the rule set over a parsed Dockerfile and returns the advice.
+#
+# Rules that need measurements (`build:`) stay silent without them, so the same analyzer
+# serves `dash doctor` — static, no build, no SSH — and the block printed under a deploy,
+# where the numbers upgrade a static hint into "this cost you 84.1 seconds".
+class Dash::Dockerfile::Analyzer
+  RULES = [
+    Dash::Dockerfile::Rules::CopyBeforeInstall,
+    Dash::Dockerfile::Rules::ContextSize,
+    Dash::Dockerfile::Rules::MissingDockerignore,
+    Dash::Dockerfile::Rules::DockerignoreGaps,
+    Dash::Dockerfile::Rules::LatestBase,
+    Dash::Dockerfile::Rules::SecretInBuildArg,
+    Dash::Dockerfile::Rules::SingleStageBuildDeps,
+    Dash::Dockerfile::Rules::AptHygiene,
+    Dash::Dockerfile::Rules::CacheBustingArg,
+    Dash::Dockerfile::Rules::CacheExportCost,
+    Dash::Dockerfile::Rules::CurlPipeShell,
+    Dash::Dockerfile::Rules::InlineEnvBlob,
+    Dash::Dockerfile::Rules::NoCacheMount,
+    Dash::Dockerfile::Rules::UncachedInstall,
+    Dash::Dockerfile::Rules::RootUser
+  ].freeze
+
+  # Reads and parses the file. Missing is the caller's problem to report: `dash doctor`
+  # fails the check, a deploy stays quiet (a --skip-push deploy has no Dockerfile and no
+  # business complaining about it).
+  def self.for_file(path, **options)
+    new document: Dash::Dockerfile::Parser.parse(File.read(path)), path: path, file: path, **options
+  end
+
+  # `path` is what findings print (the operator's own `builder: dockerfile:`); `file` is
+  # where the file actually is, for the one rule that has to open it again.
+  def initialize(document:, context_dir: nil, build: nil, builder: nil, path: "Dockerfile", file: path, ignore: [], hadolint: false)
+    # A context that is not a local directory (a git URL, or a clone that has not been
+    # prepared yet) is not something the .dockerignore rules can say anything about.
+    directory = context_dir if context_dir && File.directory?(context_dir)
+
+    @context = Dash::Dockerfile::Context.new \
+      document: document, context_dir: directory, dockerignore: (Dash::Dockerfile::Dockerignore.in(directory) if directory),
+      build: build, builder: builder, path: path
+    @file = file
+    @ignore = Array(ignore).map(&:to_s)
+    @hadolint = hadolint
+  end
+
+  # Warnings first, then the informational findings, each group in rule order — an
+  # operator reading the block top down sees what is costing them before what is merely
+  # worth knowing.
+  def findings
+    (rule_findings + hadolint_findings)
+      .reject { |finding| @ignore.include?(finding.rule) }
+      .sort_by.with_index { |finding, index| [ finding.warn? ? 0 : 1, index ] }
+  end
+
+  private
+    def rule_findings
+      RULES.flat_map { |rule| rule.new(@context).findings }
+    end
+
+    def hadolint_findings
+      return [] unless @hadolint
+
+      Dash::Dockerfile::Hadolint.new(path: @context.path, file: @file).findings
+    end
+end