dash 4.0.8 → 4.1.0

data/lib/dash/dockerfile/rules/cache_export_cost.rb

@@ -0,0 +1,20 @@
+# Measured only: mode=max exports every intermediate layer to the cache. On a persistent
+# builder that is often pure overhead — but only the numbers from this build can say so.
+class Dash::Dockerfile::Rules::CacheExportCost < Dash::Dockerfile::Rules::Base
+  SHARE = 0.2
+  SUGGESTION = "try mode=min under builder: cache: options:"
+
+  def findings
+    return [] unless build && mode_max?
+
+    exported, total = build.cache_export_seconds, build.total_step_seconds
+    return [] if total.zero? || exported <= total * SHARE
+
+    [ note("builder.cache", format("exporting the build cache took %.1fs of %.1fs with mode=max", exported, total), SUGGESTION) ]
+  end
+
+  private
+    def mode_max?
+      builder&.cache_to.to_s.split(",").map(&:strip).include?("mode=max")
+    end
+end

data/lib/dash/dockerfile/rules/context_size.rb

@@ -0,0 +1,20 @@
+# Measured only: how many megabytes this build actually shipped to the builder before it
+# ran a single instruction.
+class Dash::Dockerfile::Rules::ContextSize < Dash::Dockerfile::Rules::Base
+  THRESHOLD_BYTES = 50_000_000
+  SUGGESTION = "trim the build context with .dockerignore, or point builder: context: at a smaller directory"
+
+  def findings
+    bytes = build&.context_bytes
+    return [] if bytes.to_i <= THRESHOLD_BYTES
+
+    [ warning("build context", "the build shipped #{Dash::Utils.human_bytes(bytes)}#{elapsed} of build context to the builder", SUGGESTION) ]
+  end
+
+  private
+    # A build killed mid-transfer has a size but no duration; "in 0.0s" would be a
+    # measurement nobody took.
+    def elapsed
+      format(" in %.1fs", build.context_seconds) if build.context_seconds
+    end
+end

data/lib/dash/dockerfile/rules/copy_before_install.rb

@@ -0,0 +1,34 @@
+# The single most expensive Dockerfile mistake: copying the whole tree before installing
+# dependencies, so every commit — a README typo included — reinstalls them.
+class Dash::Dockerfile::Rules::CopyBeforeInstall < Dash::Dockerfile::Rules::Base
+  SUGGESTION = "copy the dependency manifests first (Gemfile, package.json, lockfiles), install, then copy the rest of the tree"
+
+  def findings
+    document.stages.filter_map { |stage| finding_for(stage) }
+  end
+
+  private
+    def finding_for(stage)
+      copy = stage.instructions.find { |instruction| context.broad_copy?(instruction) } or return
+      install = stage.instructions.find do |instruction|
+        instruction.line > copy.line && context.dependency_install?(instruction)
+      end
+      return unless install
+
+      warning at(copy), message(copy, install), SUGGESTION
+    end
+
+    def message(copy, install)
+      [ "#{copy.name} #{copy.args} runs before `#{context.install_command(install)}` (line #{install.line}), ",
+        "so dependencies reinstall on every commit", measured(install) ].compact.join
+    end
+
+    # Only when the build that just ran proves it: a cached install step costs nothing,
+    # and advice that names a number nobody measured is worse than advice that doesn't.
+    def measured(install)
+      step = context.build_step_for(install)
+      return if step.nil? || step.cached || step.seconds.to_f.zero?
+
+      format(" (measured %.1fs uncached)", step.seconds)
+    end
+end

data/lib/dash/dockerfile/rules/curl_pipe_shell.rb

@@ -0,0 +1,16 @@
+# Piping a download straight into a shell runs whatever the server sends today, and there
+# is no version in the Dockerfile to say what that was.
+class Dash::Dockerfile::Rules::CurlPipeShell < Dash::Dockerfile::Rules::Base
+  # `sudo` may carry flags with or without arguments (`-E`, `-u root`) before the shell.
+  SHELL = /(?:sudo\s+(?:-\S+(?:\s+[^-\s]\S*)?\s+)*)?(?:ba|z|k)?sh\b/
+  PIPE_TO_SHELL = /\b(?:curl|wget)\b[^|]*\|\s*#{SHELL.source}|#{SHELL.source}\s+<\(\s*(?:curl|wget)\b/
+  SUGGESTION = "download to a file, verify a checksum, then run it — and pin the version"
+
+  def findings
+    document.instructions.filter_map do |instruction|
+      next unless instruction.name == "RUN" && instruction.shell_command.match?(PIPE_TO_SHELL)
+
+      note at(instruction), "a download is piped straight into a shell, so the build runs unverified code", SUGGESTION
+    end
+  end
+end

data/lib/dash/dockerfile/rules/dockerignore_gaps.rb

@@ -0,0 +1,36 @@
+# A .dockerignore that misses the obvious offenders. .git is always worth naming; the
+# rest only when the directory actually exists in the context.
+class Dash::Dockerfile::Rules::DockerignoreGaps < Dash::Dockerfile::Rules::Base
+  ALWAYS = ".git".freeze
+  WHEN_PRESENT = %w[ node_modules tmp storage coverage log .env* ].freeze
+  SUGGESTION = "add them to .dockerignore so they are not sent to the builder"
+
+  def findings
+    return [] if dockerignore.nil?
+    return [] if (gaps = gaps()).empty?
+
+    [ finding(severity, Dash::Dockerfile::Dockerignore::FILENAME, message(gaps), SUGGESTION) ]
+  end
+
+  private
+    def gaps
+      candidates = [ ALWAYS, *WHEN_PRESENT.flat_map { |name| context.context_entries(name) } ]
+      candidates.uniq.reject { |name| dockerignore.covers?(name) }
+    end
+
+    # A context big enough to be worth seconds of every build turns the same gap from a
+    # tidiness note into something that is costing the operator time.
+    def severity
+      big_context? ? :warn : :info
+    end
+
+    def message(gaps)
+      measured = " — the measured build context was #{Dash::Utils.human_bytes(build.context_bytes)}" if big_context?
+
+      "the build context ships #{gaps.join(", ")}#{measured}"
+    end
+
+    def big_context?
+      build&.context_bytes.to_i > Dash::Dockerfile::Rules::ContextSize::THRESHOLD_BYTES
+    end
+end

data/lib/dash/dockerfile/rules/inline_env_blob.rb

@@ -0,0 +1,23 @@
+# A wall of inline assignments in front of a command means editing any one value
+# invalidates the layer — and makes the instruction unreadable in the bargain.
+class Dash::Dockerfile::Rules::InlineEnvBlob < Dash::Dockerfile::Rules::Base
+  ASSIGNMENT = /\A[A-Za-z_]\w*=(?:"[^"]*"|'[^']*'|\\.|\S)*\s+/
+  THRESHOLD = 20
+  SUGGESTION = "read them from an env file at runtime, or set the build-time ones in one ENV block, so the command line stops changing"
+
+  def findings
+    document.instructions.filter_map do |instruction|
+      next unless instruction.name == "RUN"
+      next unless (count = leading_assignments(instruction.shell_command.dup)) > THRESHOLD
+
+      note at(instruction), "#{count} inline environment assignments precede the command", SUGGESTION
+    end
+  end
+
+  private
+    def leading_assignments(command)
+      count = 0
+      count += 1 while command.sub!(ASSIGNMENT, "")
+      count
+    end
+end

data/lib/dash/dockerfile/rules/latest_base.rb

@@ -0,0 +1,24 @@
+# An unpinned base image makes a build unreproducible: the same Dockerfile builds a
+# different image tomorrow, and nothing in the deploy says so.
+class Dash::Dockerfile::Rules::LatestBase < Dash::Dockerfile::Rules::Base
+  SUGGESTION = "pin a specific tag, or a digest for a build that must be reproducible"
+
+  def findings
+    document.stages.filter_map { |stage| finding_for(stage) }
+  end
+
+  private
+    def finding_for(stage)
+      return if built_on_another_stage?(stage) || stage.scratch? || stage.digest || stage.interpolated_tag?
+
+      if stage.tag.nil?
+        warning at(stage.instructions.first), "FROM #{stage.base} has no tag, so it resolves to :latest", SUGGESTION
+      elsif stage.tag == "latest"
+        warning at(stage.instructions.first), "FROM #{stage.base} is not pinned; the base image can change between builds", SUGGESTION
+      end
+    end
+
+    def built_on_another_stage?(stage)
+      document.stages.any? { |other| other.named? && other.name == stage.base }
+    end
+end

data/lib/dash/dockerfile/rules/missing_dockerignore.rb

@@ -0,0 +1,11 @@
+# No .dockerignore means the whole working tree is shipped to the builder: .git, test
+# artifacts, node_modules, and whatever else happens to be lying around.
+class Dash::Dockerfile::Rules::MissingDockerignore < Dash::Dockerfile::Rules::Base
+  SUGGESTION = "add a .dockerignore covering .git and any build or dependency directories"
+
+  def findings
+    return [] if context_dir.nil? || dockerignore
+
+    [ warning(Dash::Dockerfile::Dockerignore::FILENAME, "the build context has no .dockerignore, so everything in it is sent to the builder", SUGGESTION) ]
+  end
+end

data/lib/dash/dockerfile/rules/no_cache_mount.rb

@@ -0,0 +1,26 @@
+# Without a cache mount, an install that misses the layer cache downloads every package
+# again. With one, it reuses whatever the last build left behind.
+class Dash::Dockerfile::Rules::NoCacheMount < Dash::Dockerfile::Rules::Base
+  def findings
+    document.instructions.filter_map do |instruction|
+      next unless installs?(instruction)
+      next if Array(instruction.flags["mount"]).any? { |mount| mount.include?("type=cache") }
+
+      note at(instruction), "#{command_for(instruction)} runs without a cache mount, so a cache miss downloads everything again",
+        "add --mount=type=cache,target=#{target_for(instruction)}"
+    end
+  end
+
+  private
+    def installs?(instruction)
+      context.dependency_install?(instruction) || context.apt_install?(instruction)
+    end
+
+    def command_for(instruction)
+      context.install_command(instruction) || "apt-get install"
+    end
+
+    def target_for(instruction)
+      context.install_cache_target(instruction) || Dash::Dockerfile::Context::APT_INSTALL.last
+    end
+end

data/lib/dash/dockerfile/rules/root_user.rb

@@ -0,0 +1,12 @@
+# A final stage that sets no USER runs as whatever its base image left it as — root, for
+# nearly every official image — and so does anything that gets a shell in it.
+class Dash::Dockerfile::Rules::RootUser < Dash::Dockerfile::Rules::Base
+  SUGGESTION = "create a non-root user and add a USER line before the entrypoint, unless the base image already switched"
+
+  def findings
+    stage = document.final_stage or return []
+    return [] if stage.instructions.any? { |instruction| instruction.name == "USER" }
+
+    [ note(at(stage.instructions.first), "the final stage sets no USER, so the container runs as whatever the base image does — usually root", SUGGESTION) ]
+  end
+end

data/lib/dash/dockerfile/rules/secret_in_build_arg.rb

@@ -0,0 +1,30 @@
+# Build args and ENV values are readable in the image history forever. A secret that has
+# to be present at build time belongs in a mount that leaves no layer behind.
+class Dash::Dockerfile::Rules::SecretInBuildArg < Dash::Dockerfile::Rules::Base
+  SECRETISH = /(PASSWORD|SECRET|TOKEN|_KEY)\b/i
+  # Rails' own placeholder: it exists precisely so no real key is needed at build time.
+  ALLOWED = %w[ SECRET_KEY_BASE_DUMMY ].freeze
+  SUGGESTION = "pass it with --mount=type=secret and list it under builder: secrets: in deploy.yml"
+
+  def findings
+    document.instructions.flat_map do |instruction|
+      next [] unless %w[ ARG ENV ].include?(instruction.name)
+
+      names(instruction).filter_map do |name|
+        next if ALLOWED.include?(name) || !name.match?(SECRETISH)
+
+        warning at(instruction), "#{instruction.name} #{name} bakes a secret into the image history", SUGGESTION
+      end
+    end
+  end
+
+  private
+    # `ENV KEY value` (legacy form) declares exactly one name, whatever the value holds;
+    # `ENV A=1 B=2` and `ARG NAME[=default]` declare one per assignment.
+    def names(instruction)
+      first = instruction.args.split(/\s+/).first.to_s
+      return [ first ] unless first.include?("=")
+
+      instruction.args.scan(/(?:\A|\s)([A-Za-z_]\w*)=/).flatten
+    end
+end

data/lib/dash/dockerfile/rules/single_stage_build_deps.rb

@@ -0,0 +1,19 @@
+# Compilers and header packages installed in the only stage ship to production: a bigger
+# image, a bigger attack surface, and nothing gained once the gems are built.
+class Dash::Dockerfile::Rules::SingleStageBuildDeps < Dash::Dockerfile::Rules::Base
+  # A lookahead for whitespace or a shell separator rather than `\b`: `+` is not a word
+  # character, so `g++\b` never matches.
+  BUILD_PACKAGES = /\b(build-essential|gcc|g\+\+|make|[\w.+-]+-dev)(?=[\s;&|)]|$)/
+  SUGGESTION = "split into a build stage and a runtime stage, and COPY --from the build output"
+
+  def findings
+    return [] unless document.stages.one?
+
+    document.stages.first.instructions.filter_map do |instruction|
+      next unless instruction.name == "RUN"
+      next unless (packages = instruction.shell_command.scan(BUILD_PACKAGES).map(&:first)).any?
+
+      warning at(instruction), "the only stage installs build tooling (#{packages.uniq.join(", ")}), which ships in the image", SUGGESTION
+    end
+  end
+end

data/lib/dash/dockerfile/rules/uncached_install.rb

@@ -0,0 +1,20 @@
+# Measured only: a dependency install that missed the cache and cost real time, where no
+# broad copy above it explains why. Something else invalidated the layer, and the operator
+# is the only one who can say what.
+class Dash::Dockerfile::Rules::UncachedInstall < Dash::Dockerfile::Rules::Base
+  THRESHOLD_SECONDS = 10
+  SUGGESTION = "check what changed above it — a COPY, an ARG, or a base image that moved"
+
+  def findings
+    return [] unless build
+
+    document.instructions.filter_map do |instruction|
+      next unless context.dependency_install?(instruction) && !context.busted_by_broad_copy?(instruction)
+
+      step = context.build_step_for(instruction)
+      next if step.nil? || step.cached || step.seconds.to_f <= THRESHOLD_SECONDS
+
+      note at(instruction), format("`%s` ran %.1fs uncached", context.install_command(instruction), step.seconds), SUGGESTION
+    end
+  end
+end

data/lib/dash/dockerfile/stage.rb

@@ -0,0 +1,65 @@
+# One build stage: everything from a FROM up to the next one.
+#
+# `shipped?` is the distinction that matters for advice. A stage only reachable through
+# `COPY --from=` contributes files, not layers, so apt hygiene and a missing USER in it
+# say nothing about the image that ends up on the server. The last stage is shipped, and
+# so is anything it (transitively) builds FROM.
+class Dash::Dockerfile::Stage
+  # A registry may carry a port (`localhost:5000/app`), so the tag is only what follows a
+  # colon in the last path segment.
+  IMAGE_REF = %r{\A(?<image>(?:[^/@\s]+/)*[^:/@\s]+)(?::(?<tag>[^@\s]+))?(?:@(?<digest>\S+))?\z}
+  INTERPOLATION = /\$\{?\w+\}?/
+  # Docker's reserved empty base: nothing to pin, nothing to resolve.
+  SCRATCH = "scratch".freeze
+
+  attr_reader :name, :index, :base, :instructions
+  attr_writer :shipped
+
+  def initialize(name:, index:, base:, from:, named: true)
+    @name = name
+    @named = named
+    @index = index
+    @base = base
+    @from = from
+    @instructions = [ from ]
+    @shipped = false
+  end
+
+  def shipped?
+    @shipped
+  end
+
+  # An explicit `AS` name, as opposed to the `stage-N` label dash generates.
+  def named?
+    @named
+  end
+
+  def scratch?
+    base == SCRATCH
+  end
+
+  def image
+    ref[:image]
+  end
+
+  def tag
+    ref[:tag]
+  end
+
+  def digest
+    ref[:digest]
+  end
+
+  # `FROM ruby:$RUBY_VERSION` pins a version through an ARG, so it is not the unpinned
+  # base the latest-base rule is looking for. With no tag at all, `FROM $IMAGE` might be
+  # carrying one inside the variable — unknowable, so it passes too. An explicit `:latest`
+  # is explicit whatever the registry in front of it was.
+  def interpolated_tag?
+    (tag || image).to_s.match?(INTERPOLATION)
+  end
+
+  private
+    def ref
+      @ref ||= base.match(IMAGE_REF) || {}
+    end
+end

data/lib/dash/otel_shipper.rb

@@ -131,10 +131,11 @@ class Dash::OtelShipper
 
     def typed_value(v)
       case v
-      when Integer then { intValue: v }
-      when Float   then { doubleValue: v }
-      when Array   then { arrayValue: { values: v.map { |e| typed_value(e) } } }
-      else              { stringValue: v.to_s }
+      when true, false then { boolValue: v }
+      when Integer     then { intValue: v }
+      when Float       then { doubleValue: v }
+      when Array       then { arrayValue: { values: v.map { |e| typed_value(e) } } }
+      else                  { stringValue: v.to_s }
       end
     end
 

data/lib/dash/output/otel_logger.rb

@@ -43,9 +43,61 @@ class Dash::Output::OtelLogger < Dash::Output::BaseLogger
           "kamal.command": full_command(payload), "kamal.runtime": runtime,
           **deployment_attrs(payload, status: "succeeded"))
       end
+      ship_report(payload)
       puts "Logs sent to #{@endpoint}"
     end
 
+    # The same numbers the table printed, as events a backend can chart across deploys.
+    # Shipping them must never be the thing that fails a deploy that already succeeded,
+    # so anything that goes wrong here costs one line on stderr.
+    def ship_report(payload)
+      report = payload[:report] or return
+      attrs = deployment_attrs(payload)
+
+      report.timings.to_h.each { |phase| @shipper.event("dash.phase", **phase_attrs(phase), **attrs) }
+      ship_build(report.build, attrs) if report.build&.any?
+      report.advice.each { |finding| @shipper.event("dash.advice", **advice_attrs(finding), **attrs) }
+    rescue StandardError => e
+      $stderr.puts "OTel report events failed: #{e.class}: #{e.message}"
+      $stderr.puts e.backtrace.join("\n") if ENV["VERBOSE"]
+    end
+
+    def ship_build(build, attrs)
+      @shipper.event("dash.build", **build_attrs(build), **attrs)
+      build.dockerfile_steps.each { |step| @shipper.event("dash.build.step", **step_attrs(step), **attrs) }
+    end
+
+    def phase_attrs(phase)
+      {
+        "dash.phase.name": phase[:name], "dash.phase.depth": phase[:depth], "dash.phase.seconds": phase[:seconds],
+        "dash.phase.detail": phase[:detail], "dash.phase.commands": phase[:commands],
+        "dash.phase.command_seconds": phase[:command_seconds], "dash.phase.connect_seconds": phase[:connect_seconds]
+      }.compact
+    end
+
+    def build_attrs(build)
+      {
+        "dash.build.context_bytes": build.context_bytes, "dash.build.context_seconds": build.context_seconds,
+        "dash.build.cached_steps": build.cached_steps.size, "dash.build.total_steps": build.dockerfile_steps.size,
+        "dash.build.export_seconds": build.export_seconds, "dash.build.cache_export_seconds": build.cache_export_seconds,
+        "dash.build.push_seconds": build.push_seconds
+      }.compact
+    end
+
+    def step_attrs(step)
+      {
+        "dash.build.stage": step.stage, "dash.build.ordinal": step.ordinal, "dash.build.instruction": step.instruction,
+        "dash.build.seconds": step.seconds, "dash.build.cached": step.cached
+      }.compact
+    end
+
+    def advice_attrs(finding)
+      {
+        "dash.advice.rule": finding.rule, "dash.advice.severity": finding.severity.to_s,
+        "dash.advice.location": finding.location, "dash.advice.message": finding.message
+      }.compact
+    end
+
     def on_close
       @shipper.shutdown
     end

data/lib/dash/report/history.rb

@@ -0,0 +1,94 @@
+require "json"
+
+# The saved reports for one destination, newest first.
+#
+# Everything here is best-effort by design: the directory is the operator's, a report may
+# be half-written by a deploy that was killed mid-flush, and a future dash may write a
+# schema this one does not know. None of that is worth a word on a deploy, so an
+# unreadable file is skipped rather than reported.
+class Dash::Report::History
+  attr_reader :directory, :destination
+
+  def initialize(directory, destination: nil)
+    @directory = directory
+    @destination = destination
+  end
+
+  def recent(count)
+    documents.first(count)
+  end
+
+  def any?
+    documents.any?
+  end
+
+  # Keeps the newest `count` reports of this destination and deletes the rest. Other
+  # destinations are left alone: they have their own budget, and a staging deploy must
+  # not age out production's history.
+  def prune(count)
+    entries.drop(count).each do |path, _document|
+      File.delete(path)
+    rescue SystemCallError
+      nil
+    end
+  end
+
+  private
+    def documents
+      entries.map(&:last)
+    end
+
+    # Sorted by filename rather than by the timestamp inside, because the name is what an
+    # operator sorts by too — and a report whose body we could not read is not one we can
+    # order by its contents. Newest first.
+    def entries
+      @entries ||= Dir.glob(File.join(directory, "*.json")).sort_by { |path| order_key(path) }.reverse.filter_map { |path| entry_for(path) }
+    end
+
+    # Two runs in the same second are `X.json` and `X-2.json`, and byte for byte the
+    # unsuffixed one sorts last — which would make the older run the newest. The suffix
+    # is the run order; the command a name ends in is never a number, so a trailing
+    # `-N` is only ever ours.
+    def order_key(path)
+      name = File.basename(path, ".json")
+      base, suffix = name.match(/\A(.*)-(\d+)\z/)&.captures
+
+      base ? [ base, suffix.to_i ] : [ name, 1 ]
+    end
+
+    def entry_for(path)
+      document = JSON.parse(File.read(path), symbolize_names: true)
+      return unless document.is_a?(Hash) && document[:schema] == Dash::Report::SCHEMA
+      return unless document[:destination] == destination
+      return unless well_formed?(document)
+
+      [ path, document ]
+    rescue StandardError
+      nil
+    end
+
+    # Claiming schema 1 is not the same as being one. A hand-edited file that parses but
+    # holds the wrong shapes would crash `dash report` somewhere far from the mistake, or
+    # — worse — render as an empty table. The top-level fields the writer always sets are
+    # checked by shape; then the test that matters for everything nested: render it.
+    # Anything that cannot be is skipped with the unreadable files, the raise landing in
+    # #entry_for's rescue.
+    def well_formed?(document)
+      return false unless list_of_hashes?(document[:phases])
+      return false unless optional?(document[:advice]) { |advice| list_of_hashes?(advice) }
+      return false unless optional?(document[:build]) { |build| build.is_a?(Hash) }
+      return false unless optional?(document[:error]) { |error| error.is_a?(Hash) }
+      return false unless optional?(document[:runtime]) { |runtime| runtime.is_a?(Numeric) }
+
+      Dash::Report.from_h(document).lines
+      true
+    end
+
+    def optional?(value)
+      value.nil? || yield(value)
+    end
+
+    def list_of_hashes?(value)
+      value.is_a?(Array) && value.all?(Hash)
+    end
+end