dash 4.0.8 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dash/build/progress_parser.rb +136 -0
- data/lib/dash/build/report.rb +104 -0
- data/lib/dash/build/step.rb +49 -0
- data/lib/dash/cli/app/boot.rb +46 -24
- data/lib/dash/cli/app.rb +4 -4
- data/lib/dash/cli/base.rb +117 -2
- data/lib/dash/cli/build.rb +83 -20
- data/lib/dash/cli/doctor/config_checks.rb +36 -1
- data/lib/dash/cli/doctor.rb +2 -1
- data/lib/dash/cli/healthcheck/poller.rb +15 -3
- data/lib/dash/cli/healthcheck/progress_reporter.rb +39 -0
- data/lib/dash/cli/main.rb +24 -7
- data/lib/dash/cli/proxy/drift.rb +17 -2
- data/lib/dash/cli/proxy/legacy_rename.rb +8 -21
- data/lib/dash/cli/proxy/loadbalancer_reboot.rb +8 -1
- data/lib/dash/cli/proxy/reboot.rb +6 -1
- data/lib/dash/cli/proxy.rb +19 -12
- data/lib/dash/cli/prune.rb +5 -8
- data/lib/dash/cli/report.rb +97 -0
- data/lib/dash/cli/templates/sample_hooks/post-deploy.sample +5 -0
- data/lib/dash/commander.rb +10 -2
- data/lib/dash/commands/app.rb +81 -0
- data/lib/dash/commands/auditor.rb +10 -0
- data/lib/dash/commands/base.rb +42 -1
- data/lib/dash/commands/builder/base.rb +18 -0
- data/lib/dash/commands/builder.rb +1 -1
- data/lib/dash/commands/loadbalancer.rb +54 -0
- data/lib/dash/commands/proxy/state.rb +31 -0
- data/lib/dash/commands/proxy.rb +74 -2
- data/lib/dash/commands/registry.rb +14 -0
- data/lib/dash/configuration/docs/configuration.yml +6 -0
- data/lib/dash/configuration/docs/report.yml +39 -0
- data/lib/dash/configuration/docs/role.yml +7 -7
- data/lib/dash/configuration/proxy.rb +3 -0
- data/lib/dash/configuration/report.rb +65 -0
- data/lib/dash/configuration.rb +2 -1
- data/lib/dash/dockerfile/analyzer.rb +66 -0
- data/lib/dash/dockerfile/context.rb +147 -0
- data/lib/dash/dockerfile/dockerignore.rb +29 -0
- data/lib/dash/dockerfile/document.rb +28 -0
- data/lib/dash/dockerfile/finding.rb +20 -0
- data/lib/dash/dockerfile/hadolint.rb +75 -0
- data/lib/dash/dockerfile/instruction.rb +58 -0
- data/lib/dash/dockerfile/parser.rb +199 -0
- data/lib/dash/dockerfile/rules/apt_hygiene.rb +35 -0
- data/lib/dash/dockerfile/rules/base.rb +44 -0
- data/lib/dash/dockerfile/rules/cache_busting_arg.rb +40 -0
- data/lib/dash/dockerfile/rules/cache_export_cost.rb +20 -0
- data/lib/dash/dockerfile/rules/context_size.rb +20 -0
- data/lib/dash/dockerfile/rules/copy_before_install.rb +34 -0
- data/lib/dash/dockerfile/rules/curl_pipe_shell.rb +16 -0
- data/lib/dash/dockerfile/rules/dockerignore_gaps.rb +36 -0
- data/lib/dash/dockerfile/rules/inline_env_blob.rb +23 -0
- data/lib/dash/dockerfile/rules/latest_base.rb +24 -0
- data/lib/dash/dockerfile/rules/missing_dockerignore.rb +11 -0
- data/lib/dash/dockerfile/rules/no_cache_mount.rb +26 -0
- data/lib/dash/dockerfile/rules/root_user.rb +12 -0
- data/lib/dash/dockerfile/rules/secret_in_build_arg.rb +30 -0
- data/lib/dash/dockerfile/rules/single_stage_build_deps.rb +19 -0
- data/lib/dash/dockerfile/rules/uncached_install.rb +20 -0
- data/lib/dash/dockerfile/stage.rb +65 -0
- data/lib/dash/otel_shipper.rb +5 -4
- data/lib/dash/output/otel_logger.rb +52 -0
- data/lib/dash/report/history.rb +94 -0
- data/lib/dash/report/trends.rb +129 -0
- data/lib/dash/report/writer.rb +142 -0
- data/lib/dash/report.rb +170 -0
- data/lib/dash/sshkit_with_ext.rb +62 -0
- data/lib/dash/timings.rb +163 -10
- data/lib/dash/utils.rb +7 -0
- data/lib/dash/version.rb +1 -1
- data/lib/dash.rb +4 -0
- metadata +38 -1
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# One build stage: everything from a FROM up to the next one.
|
|
2
|
+
#
|
|
3
|
+
# `shipped?` is the distinction that matters for advice. A stage only reachable through
|
|
4
|
+
# `COPY --from=` contributes files, not layers, so apt hygiene and a missing USER in it
|
|
5
|
+
# say nothing about the image that ends up on the server. The last stage is shipped, and
|
|
6
|
+
# so is anything it (transitively) builds FROM.
|
|
7
|
+
class Dash::Dockerfile::Stage
|
|
8
|
+
# A registry may carry a port (`localhost:5000/app`), so the tag is only what follows a
|
|
9
|
+
# colon in the last path segment.
|
|
10
|
+
IMAGE_REF = %r{\A(?<image>(?:[^/@\s]+/)*[^:/@\s]+)(?::(?<tag>[^@\s]+))?(?:@(?<digest>\S+))?\z}
|
|
11
|
+
INTERPOLATION = /\$\{?\w+\}?/
|
|
12
|
+
# Docker's reserved empty base: nothing to pin, nothing to resolve.
|
|
13
|
+
SCRATCH = "scratch".freeze
|
|
14
|
+
|
|
15
|
+
attr_reader :name, :index, :base, :instructions
|
|
16
|
+
attr_writer :shipped
|
|
17
|
+
|
|
18
|
+
def initialize(name:, index:, base:, from:, named: true)
|
|
19
|
+
@name = name
|
|
20
|
+
@named = named
|
|
21
|
+
@index = index
|
|
22
|
+
@base = base
|
|
23
|
+
@from = from
|
|
24
|
+
@instructions = [ from ]
|
|
25
|
+
@shipped = false
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def shipped?
|
|
29
|
+
@shipped
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# An explicit `AS` name, as opposed to the `stage-N` label dash generates.
|
|
33
|
+
def named?
|
|
34
|
+
@named
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def scratch?
|
|
38
|
+
base == SCRATCH
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def image
|
|
42
|
+
ref[:image]
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def tag
|
|
46
|
+
ref[:tag]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def digest
|
|
50
|
+
ref[:digest]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# `FROM ruby:$RUBY_VERSION` pins a version through an ARG, so it is not the unpinned
|
|
54
|
+
# base the latest-base rule is looking for. With no tag at all, `FROM $IMAGE` might be
|
|
55
|
+
# carrying one inside the variable — unknowable, so it passes too. An explicit `:latest`
|
|
56
|
+
# is explicit whatever the registry in front of it was.
|
|
57
|
+
def interpolated_tag?
|
|
58
|
+
(tag || image).to_s.match?(INTERPOLATION)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
def ref
|
|
63
|
+
@ref ||= base.match(IMAGE_REF) || {}
|
|
64
|
+
end
|
|
65
|
+
end
|
data/lib/dash/otel_shipper.rb
CHANGED
|
@@ -131,10 +131,11 @@ class Dash::OtelShipper
|
|
|
131
131
|
|
|
132
132
|
def typed_value(v)
|
|
133
133
|
case v
|
|
134
|
-
when
|
|
135
|
-
when
|
|
136
|
-
when
|
|
137
|
-
|
|
134
|
+
when true, false then { boolValue: v }
|
|
135
|
+
when Integer then { intValue: v }
|
|
136
|
+
when Float then { doubleValue: v }
|
|
137
|
+
when Array then { arrayValue: { values: v.map { |e| typed_value(e) } } }
|
|
138
|
+
else { stringValue: v.to_s }
|
|
138
139
|
end
|
|
139
140
|
end
|
|
140
141
|
|
|
@@ -43,9 +43,61 @@ class Dash::Output::OtelLogger < Dash::Output::BaseLogger
|
|
|
43
43
|
"kamal.command": full_command(payload), "kamal.runtime": runtime,
|
|
44
44
|
**deployment_attrs(payload, status: "succeeded"))
|
|
45
45
|
end
|
|
46
|
+
ship_report(payload)
|
|
46
47
|
puts "Logs sent to #{@endpoint}"
|
|
47
48
|
end
|
|
48
49
|
|
|
50
|
+
# The same numbers the table printed, as events a backend can chart across deploys.
|
|
51
|
+
# Shipping them must never be the thing that fails a deploy that already succeeded,
|
|
52
|
+
# so anything that goes wrong here costs one line on stderr.
|
|
53
|
+
def ship_report(payload)
|
|
54
|
+
report = payload[:report] or return
|
|
55
|
+
attrs = deployment_attrs(payload)
|
|
56
|
+
|
|
57
|
+
report.timings.to_h.each { |phase| @shipper.event("dash.phase", **phase_attrs(phase), **attrs) }
|
|
58
|
+
ship_build(report.build, attrs) if report.build&.any?
|
|
59
|
+
report.advice.each { |finding| @shipper.event("dash.advice", **advice_attrs(finding), **attrs) }
|
|
60
|
+
rescue StandardError => e
|
|
61
|
+
$stderr.puts "OTel report events failed: #{e.class}: #{e.message}"
|
|
62
|
+
$stderr.puts e.backtrace.join("\n") if ENV["VERBOSE"]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def ship_build(build, attrs)
|
|
66
|
+
@shipper.event("dash.build", **build_attrs(build), **attrs)
|
|
67
|
+
build.dockerfile_steps.each { |step| @shipper.event("dash.build.step", **step_attrs(step), **attrs) }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def phase_attrs(phase)
|
|
71
|
+
{
|
|
72
|
+
"dash.phase.name": phase[:name], "dash.phase.depth": phase[:depth], "dash.phase.seconds": phase[:seconds],
|
|
73
|
+
"dash.phase.detail": phase[:detail], "dash.phase.commands": phase[:commands],
|
|
74
|
+
"dash.phase.command_seconds": phase[:command_seconds], "dash.phase.connect_seconds": phase[:connect_seconds]
|
|
75
|
+
}.compact
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def build_attrs(build)
|
|
79
|
+
{
|
|
80
|
+
"dash.build.context_bytes": build.context_bytes, "dash.build.context_seconds": build.context_seconds,
|
|
81
|
+
"dash.build.cached_steps": build.cached_steps.size, "dash.build.total_steps": build.dockerfile_steps.size,
|
|
82
|
+
"dash.build.export_seconds": build.export_seconds, "dash.build.cache_export_seconds": build.cache_export_seconds,
|
|
83
|
+
"dash.build.push_seconds": build.push_seconds
|
|
84
|
+
}.compact
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def step_attrs(step)
|
|
88
|
+
{
|
|
89
|
+
"dash.build.stage": step.stage, "dash.build.ordinal": step.ordinal, "dash.build.instruction": step.instruction,
|
|
90
|
+
"dash.build.seconds": step.seconds, "dash.build.cached": step.cached
|
|
91
|
+
}.compact
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def advice_attrs(finding)
|
|
95
|
+
{
|
|
96
|
+
"dash.advice.rule": finding.rule, "dash.advice.severity": finding.severity.to_s,
|
|
97
|
+
"dash.advice.location": finding.location, "dash.advice.message": finding.message
|
|
98
|
+
}.compact
|
|
99
|
+
end
|
|
100
|
+
|
|
49
101
|
def on_close
|
|
50
102
|
@shipper.shutdown
|
|
51
103
|
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
# The saved reports for one destination, newest first.
|
|
4
|
+
#
|
|
5
|
+
# Everything here is best-effort by design: the directory is the operator's, a report may
|
|
6
|
+
# be half-written by a deploy that was killed mid-flush, and a future dash may write a
|
|
7
|
+
# schema this one does not know. None of that is worth a word on a deploy, so an
|
|
8
|
+
# unreadable file is skipped rather than reported.
|
|
9
|
+
class Dash::Report::History
|
|
10
|
+
attr_reader :directory, :destination
|
|
11
|
+
|
|
12
|
+
def initialize(directory, destination: nil)
|
|
13
|
+
@directory = directory
|
|
14
|
+
@destination = destination
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def recent(count)
|
|
18
|
+
documents.first(count)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def any?
|
|
22
|
+
documents.any?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Keeps the newest `count` reports of this destination and deletes the rest. Other
|
|
26
|
+
# destinations are left alone: they have their own budget, and a staging deploy must
|
|
27
|
+
# not age out production's history.
|
|
28
|
+
def prune(count)
|
|
29
|
+
entries.drop(count).each do |path, _document|
|
|
30
|
+
File.delete(path)
|
|
31
|
+
rescue SystemCallError
|
|
32
|
+
nil
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
def documents
|
|
38
|
+
entries.map(&:last)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Sorted by filename rather than by the timestamp inside, because the name is what an
|
|
42
|
+
# operator sorts by too — and a report whose body we could not read is not one we can
|
|
43
|
+
# order by its contents. Newest first.
|
|
44
|
+
def entries
|
|
45
|
+
@entries ||= Dir.glob(File.join(directory, "*.json")).sort_by { |path| order_key(path) }.reverse.filter_map { |path| entry_for(path) }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Two runs in the same second are `X.json` and `X-2.json`, and byte for byte the
|
|
49
|
+
# unsuffixed one sorts last — which would make the older run the newest. The suffix
|
|
50
|
+
# is the run order; the command a name ends in is never a number, so a trailing
|
|
51
|
+
# `-N` is only ever ours.
|
|
52
|
+
def order_key(path)
|
|
53
|
+
name = File.basename(path, ".json")
|
|
54
|
+
base, suffix = name.match(/\A(.*)-(\d+)\z/)&.captures
|
|
55
|
+
|
|
56
|
+
base ? [ base, suffix.to_i ] : [ name, 1 ]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def entry_for(path)
|
|
60
|
+
document = JSON.parse(File.read(path), symbolize_names: true)
|
|
61
|
+
return unless document.is_a?(Hash) && document[:schema] == Dash::Report::SCHEMA
|
|
62
|
+
return unless document[:destination] == destination
|
|
63
|
+
return unless well_formed?(document)
|
|
64
|
+
|
|
65
|
+
[ path, document ]
|
|
66
|
+
rescue StandardError
|
|
67
|
+
nil
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Claiming schema 1 is not the same as being one. A hand-edited file that parses but
|
|
71
|
+
# holds the wrong shapes would crash `dash report` somewhere far from the mistake, or
|
|
72
|
+
# — worse — render as an empty table. The top-level fields the writer always sets are
|
|
73
|
+
# checked by shape; then the test that matters for everything nested: render it.
|
|
74
|
+
# Anything that cannot be is skipped with the unreadable files, the raise landing in
|
|
75
|
+
# #entry_for's rescue.
|
|
76
|
+
def well_formed?(document)
|
|
77
|
+
return false unless list_of_hashes?(document[:phases])
|
|
78
|
+
return false unless optional?(document[:advice]) { |advice| list_of_hashes?(advice) }
|
|
79
|
+
return false unless optional?(document[:build]) { |build| build.is_a?(Hash) }
|
|
80
|
+
return false unless optional?(document[:error]) { |error| error.is_a?(Hash) }
|
|
81
|
+
return false unless optional?(document[:runtime]) { |runtime| runtime.is_a?(Numeric) }
|
|
82
|
+
|
|
83
|
+
Dash::Report.from_h(document).lines
|
|
84
|
+
true
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def optional?(value)
|
|
88
|
+
value.nil? || yield(value)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def list_of_hashes?(value)
|
|
92
|
+
value.is_a?(Array) && value.all?(Hash)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# Compares this run with the last few saved reports of the same command and destination.
|
|
2
|
+
#
|
|
3
|
+
# The Dockerfile rules can only see what one build looked like; these see what a project's
|
|
4
|
+
# deploys usually look like, which is the only way to tell "the build takes 84 seconds"
|
|
5
|
+
# from "the build suddenly takes twice what it used to". Everything here is informational:
|
|
6
|
+
# a slow deploy is a fact about today, not a defect to fix.
|
|
7
|
+
#
|
|
8
|
+
# The phase names are the ones a deploy records in Dash::Cli::Main and Dash::Cli::Base.
|
|
9
|
+
# They are pinned from the other side by test/cli/main_test.rb, which asserts the table a
|
|
10
|
+
# real deploy prints, so a rename cannot quietly turn these rules off.
|
|
11
|
+
class Dash::Report::Trends
|
|
12
|
+
BUILD_PHASE = "Build and push app image".freeze
|
|
13
|
+
BOOT_PHASE = "Boot".freeze
|
|
14
|
+
# Everything a deploy spends before and around the work itself: loading the gem, parsing
|
|
15
|
+
# the config, resolving secrets, and taking the locks.
|
|
16
|
+
OVERHEAD_PHASES = [ "Startup (load, config)", "Validate config and secrets",
|
|
17
|
+
"Acquire deploy lock", "Acquire server lock" ].freeze
|
|
18
|
+
|
|
19
|
+
# Two deploys are an anecdote. Three are the fewest that can have a median worth
|
|
20
|
+
# comparing against.
|
|
21
|
+
MINIMUM_HISTORY = 3
|
|
22
|
+
WINDOW = 5
|
|
23
|
+
FACTOR = 1.5
|
|
24
|
+
# Overhead this large is worth naming even to a project whose deploys have always
|
|
25
|
+
# carried it — "normal" is not the same as "cheap".
|
|
26
|
+
OVERHEAD_FLOOR = 10.0
|
|
27
|
+
|
|
28
|
+
LOCATION = "deploy history".freeze
|
|
29
|
+
|
|
30
|
+
def initialize(document, history: [], ignore: [])
|
|
31
|
+
@document = document
|
|
32
|
+
@history = comparable(history)
|
|
33
|
+
@ignore = Array(ignore).map(&:to_s)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def findings
|
|
37
|
+
return [] if @history.size < MINIMUM_HISTORY
|
|
38
|
+
|
|
39
|
+
[ build_finding, boot_finding, overhead_finding, total_finding ].compact.reject { |finding| @ignore.include?(finding.rule) }
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
def comparable(history)
|
|
44
|
+
Array(history)
|
|
45
|
+
.map { |document| document.transform_keys(&:to_sym) }
|
|
46
|
+
.select { |document| document[:command] == @document[:command] && document[:status] == "succeeded" }
|
|
47
|
+
.first(WINDOW)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def build_finding
|
|
51
|
+
phase_finding "trend-build", "build", BUILD_PHASE
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def boot_finding
|
|
55
|
+
phase_finding "trend-boot", "boot", BOOT_PHASE
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def phase_finding(rule, label, name)
|
|
59
|
+
current = phase_seconds(@document, name) or return
|
|
60
|
+
past = @history.filter_map { |document| phase_seconds(document, name) }
|
|
61
|
+
return if past.size < MINIMUM_HISTORY || current <= median(past) * FACTOR
|
|
62
|
+
|
|
63
|
+
note rule, "#{label} #{seconds(current)} vs median #{seconds(median(past))} over the last #{past.size} deploys"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def total_finding
|
|
67
|
+
current = @document[:runtime].to_f
|
|
68
|
+
past = @history.map { |document| document[:runtime].to_f }
|
|
69
|
+
return if current <= median(past) * FACTOR
|
|
70
|
+
|
|
71
|
+
note "trend-total", "total #{seconds(current)} vs median #{seconds(median(past))} over the last #{past.size} deploys"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Fires on the absolute number as well as the trend, and names the row responsible —
|
|
75
|
+
# ten seconds before the first docker command is nearly always one slow thing (a
|
|
76
|
+
# secrets adapter shelling out, a lock waiting on a sweep) rather than a diffuse cost.
|
|
77
|
+
def overhead_finding
|
|
78
|
+
current = overhead_seconds(@document)
|
|
79
|
+
past = @history.map { |document| overhead_seconds(document) }
|
|
80
|
+
return if current < OVERHEAD_FLOOR && current <= median(past) * FACTOR
|
|
81
|
+
|
|
82
|
+
message = "dash overhead #{seconds(current)} vs median #{seconds(median(past))} over the last #{past.size} deploys"
|
|
83
|
+
if (slowest = slowest_overhead_phase)
|
|
84
|
+
message += "; #{slowest[:name]} was #{seconds(slowest[:seconds])} of it"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
note "trend-overhead", message, "a secrets adapter or a lock sweep is the usual cause"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def overhead_seconds(document)
|
|
91
|
+
overhead_phases(document).sum { |phase| phase[:seconds].to_f }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def slowest_overhead_phase
|
|
95
|
+
overhead_phases(@document).max_by { |phase| phase[:seconds].to_f }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def overhead_phases(document)
|
|
99
|
+
Array(document[:phases])
|
|
100
|
+
.map { |phase| phase.transform_keys(&:to_sym) }
|
|
101
|
+
.select { |phase| OVERHEAD_PHASES.include?(phase[:name]) }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Only depth-0 rows: a per-host row inside Boot is named after the host, but a role
|
|
105
|
+
# could be named "Boot" and would otherwise be counted as the phase.
|
|
106
|
+
def phase_seconds(document, name)
|
|
107
|
+
phase = Array(document[:phases])
|
|
108
|
+
.map { |candidate| candidate.transform_keys(&:to_sym) }
|
|
109
|
+
.find { |candidate| candidate[:name] == name && candidate[:depth].to_i.zero? }
|
|
110
|
+
|
|
111
|
+
phase[:seconds].to_f if phase
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def median(values)
|
|
115
|
+
sorted = values.sort
|
|
116
|
+
middle = sorted.size / 2
|
|
117
|
+
|
|
118
|
+
sorted.size.odd? ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2.0
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def seconds(value)
|
|
122
|
+
format("%.1fs", value.to_f)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def note(rule, message, suggestion = nil)
|
|
126
|
+
Dash::Dockerfile::Finding.new \
|
|
127
|
+
rule: rule, severity: :info, location: LOCATION, message: message, suggestion: suggestion
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
require "fileutils"
|
|
2
|
+
require "json"
|
|
3
|
+
|
|
4
|
+
# Writes one JSON report per run under `.dash/reports`, so the next deploy has something
|
|
5
|
+
# to compare itself against and CI has something to archive.
|
|
6
|
+
#
|
|
7
|
+
# A failed deploy is written too, with whatever was measured before it broke: the run an
|
|
8
|
+
# operator most wants to look at afterwards is the one that went wrong.
|
|
9
|
+
class Dash::Report::Writer
|
|
10
|
+
GITIGNORE = "*\n!.gitignore\n".freeze
|
|
11
|
+
|
|
12
|
+
# A destination is an operator-supplied string and a command can carry a subcommand, so
|
|
13
|
+
# neither is trusted to be a filename.
|
|
14
|
+
UNSAFE = /[^\w.-]+/
|
|
15
|
+
|
|
16
|
+
attr_reader :report, :run, :keep, :directory
|
|
17
|
+
|
|
18
|
+
# `run` is the metadata Dash::Cli::Base assembled for this invocation — the same hash
|
|
19
|
+
# the trend rules compared against, so the file and the advice cannot disagree.
|
|
20
|
+
def initialize(report, run:, keep:, directory: Dash::ProjectDirectory.join("reports"))
|
|
21
|
+
@report = report
|
|
22
|
+
@run = run
|
|
23
|
+
@keep = keep
|
|
24
|
+
@directory = directory
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Returns the path it wrote, or nil when the operator turned history off.
|
|
28
|
+
def write
|
|
29
|
+
return if keep.zero?
|
|
30
|
+
|
|
31
|
+
prepare_directory
|
|
32
|
+
@path = publish JSON.pretty_generate(report.to_h(**run))
|
|
33
|
+
Dash::Report::History.new(directory, destination: run[:destination]).prune(keep)
|
|
34
|
+
|
|
35
|
+
@path
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
# The name and the content arrive together, always. A hard link publishes a file
|
|
40
|
+
# that is already complete, atomically, and only if the name is free — so two runs
|
|
41
|
+
# racing for one name (started_at is recorded to the second, and `safe` maps
|
|
42
|
+
# `eu/west` and `eu-west` onto the same string) each keep their own report, and
|
|
43
|
+
# there is never a moment where a report exists empty or half-written. That last
|
|
44
|
+
# part matters more than it sounds: every reader skips a file it cannot parse, and
|
|
45
|
+
# the prune only counts the files it could read, so anything left behind here would
|
|
46
|
+
# stay in the directory forever.
|
|
47
|
+
def publish(content)
|
|
48
|
+
scratch = File.join(directory, ".#{base_name}.#{Process.pid}.tmp")
|
|
49
|
+
File.write(scratch, content)
|
|
50
|
+
|
|
51
|
+
begin
|
|
52
|
+
linked(scratch)
|
|
53
|
+
rescue SystemCallError
|
|
54
|
+
created(scratch)
|
|
55
|
+
end
|
|
56
|
+
ensure
|
|
57
|
+
File.delete(scratch) if scratch && File.exist?(scratch)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def linked(scratch)
|
|
61
|
+
claim do |candidate|
|
|
62
|
+
File.link(scratch, candidate)
|
|
63
|
+
candidate
|
|
64
|
+
rescue Errno::EEXIST
|
|
65
|
+
nil
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# For a filesystem with no hard links. The name is claimed with an exclusive create —
|
|
70
|
+
# which never replaces, so another run's report is safe — and the completed scratch
|
|
71
|
+
# file is then renamed onto that placeholder, atomically, so no reader ever sees a
|
|
72
|
+
# half-written report. The placeholder is the one thing that can be left behind
|
|
73
|
+
# here, and it is the one thing nothing would ever clean up, so it comes down in an
|
|
74
|
+
# `ensure`: a full disk and the operator's Ctrl-C (an Interrupt, not a
|
|
75
|
+
# StandardError) both take the claimed name with them.
|
|
76
|
+
def created(scratch)
|
|
77
|
+
claim do |candidate|
|
|
78
|
+
next unless placeholder?(candidate)
|
|
79
|
+
|
|
80
|
+
filled(candidate, scratch)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def placeholder?(candidate)
|
|
85
|
+
File.open(candidate, File::WRONLY | File::CREAT | File::EXCL) { }
|
|
86
|
+
true
|
|
87
|
+
rescue Errno::EEXIST
|
|
88
|
+
false
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The cleanup asks the filesystem what happened rather than trusting a flag set after
|
|
92
|
+
# the fact: the rename consumes the scratch file, so a scratch that is still there is
|
|
93
|
+
# a rename that did not happen. A flag would have its own window — an interrupt
|
|
94
|
+
# between a successful rename and the assignment would delete a published report.
|
|
95
|
+
def filled(candidate, scratch)
|
|
96
|
+
File.rename(scratch, candidate)
|
|
97
|
+
|
|
98
|
+
candidate
|
|
99
|
+
ensure
|
|
100
|
+
File.delete(candidate) if File.exist?(scratch) && File.exist?(candidate)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Walks the candidate names until the block takes one, yielding nil for a name that
|
|
104
|
+
# was already gone. Suffixed names sort after the one they collided with, which
|
|
105
|
+
# Dash::Report::History#order_key relies on to read them back in run order.
|
|
106
|
+
def claim
|
|
107
|
+
suffix = 1
|
|
108
|
+
candidate = File.join(directory, "#{base_name}.json")
|
|
109
|
+
|
|
110
|
+
until (claimed = yield candidate)
|
|
111
|
+
candidate = File.join(directory, "#{base_name}-#{suffix += 1}.json")
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
claimed
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def base_name
|
|
118
|
+
"#{timestamp}-#{safe(destination)}-#{safe(run[:command])}"
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The started_at the report already carries, with the colons a filename cannot have.
|
|
122
|
+
def timestamp
|
|
123
|
+
safe(run[:started_at].to_s.tr(":", "-"))
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def destination
|
|
127
|
+
run[:destination].presence || "default"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def safe(part)
|
|
131
|
+
part.to_s.gsub(UNSAFE, "-")
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# The .gitignore is written next to the reports rather than left to the operator: a
|
|
135
|
+
# project that commits `.dash/` would otherwise start committing a deploy report on
|
|
136
|
+
# every deploy, and nobody asked for that in their diff.
|
|
137
|
+
def prepare_directory
|
|
138
|
+
FileUtils.mkdir_p(directory)
|
|
139
|
+
gitignore = File.join(directory, ".gitignore")
|
|
140
|
+
File.write(gitignore, GITIGNORE) unless File.exist?(gitignore)
|
|
141
|
+
end
|
|
142
|
+
end
|
data/lib/dash/report.rb
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
require "active_support/core_ext/string/filters"
|
|
2
|
+
require "active_support/core_ext/module/delegation"
|
|
3
|
+
|
|
4
|
+
# The deploy report: the phase table plus everything measured inside a phase that the
|
|
5
|
+
# table itself has no column for. Today that is the build; the Dockerfile advice and the
|
|
6
|
+
# JSON export hang off the same object.
|
|
7
|
+
#
|
|
8
|
+
# It owns the rendering rather than Timings because the build rows have to be spliced
|
|
9
|
+
# into the middle of the table, under the phase they belong to, and Timings has no
|
|
10
|
+
# business knowing what a buildx vertex is.
|
|
11
|
+
class Dash::Report
|
|
12
|
+
delegate :human_bytes, to: Dash::Utils
|
|
13
|
+
|
|
14
|
+
# Depth-1 rows, so build steps line up with the per-host rows a Boot phase prints.
|
|
15
|
+
INDENT = " "
|
|
16
|
+
# Wider than the phase table's 36-column name, because a Dockerfile instruction is the
|
|
17
|
+
# whole point of the row and most of them are longer than that. The build rows line up
|
|
18
|
+
# with each other as their own block under the phase; 4 + 60 + 8 still fits 80 columns.
|
|
19
|
+
NAME_WIDTH = 60
|
|
20
|
+
VALUE_WIDTH = 7
|
|
21
|
+
SLOWEST_STEPS = 5
|
|
22
|
+
|
|
23
|
+
# Severity, then the file or key to open, then the sentence. The suggestion hangs under
|
|
24
|
+
# the sentence so the eye can skip the whole block or read one finding in full.
|
|
25
|
+
SEVERITY_WIDTH = 4
|
|
26
|
+
LOCATION_WIDTH = 15
|
|
27
|
+
SEVERITY_COLORS = { warn: "\e[33m" }.freeze
|
|
28
|
+
|
|
29
|
+
# The version of the JSON documents #to_h writes and #from_h reads. Bumping it is a
|
|
30
|
+
# promise to whatever reads .dash/reports, so a reader that does not recognise the
|
|
31
|
+
# number skips the file rather than guessing.
|
|
32
|
+
SCHEMA = 1
|
|
33
|
+
|
|
34
|
+
attr_reader :timings
|
|
35
|
+
attr_accessor :build, :build_entry, :advice
|
|
36
|
+
|
|
37
|
+
# Rebuilds a saved report so `dash report` prints it the way the deploy printed it —
|
|
38
|
+
# same table, same build rows under the same phase, same advice.
|
|
39
|
+
def self.from_h(document)
|
|
40
|
+
document = document.transform_keys(&:to_sym)
|
|
41
|
+
timings = Dash::Timings.from_h(document[:phases])
|
|
42
|
+
|
|
43
|
+
new(timings: timings).tap do |report|
|
|
44
|
+
report.build_entry = timings.entry_at(document[:build_phase])
|
|
45
|
+
report.build = Dash::Build::Report.from_h(document[:build]) if document[:build]
|
|
46
|
+
report.advice = Array(document[:advice]).map { |finding| Dash::Dockerfile::Finding.from_h(finding) }
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def initialize(timings:)
|
|
51
|
+
@timings = timings
|
|
52
|
+
@advice = []
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The run's own facts (command, service, destination, version, timings of the whole
|
|
56
|
+
# thing) belong to the caller that knows them; the report contributes what it measured.
|
|
57
|
+
# `build_phase` is the row the build rows hang under, by position, so a reader can put
|
|
58
|
+
# them back without matching on a phase name dash is free to reword.
|
|
59
|
+
def to_h(**run)
|
|
60
|
+
{
|
|
61
|
+
schema: SCHEMA, dash_version: Dash::VERSION, **run,
|
|
62
|
+
phases: timings.to_h, build_phase: build_entry && timings.index_of(build_entry),
|
|
63
|
+
build: build&.to_h, advice: advice.map(&:to_h)
|
|
64
|
+
}.compact
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def lines
|
|
68
|
+
lines = timings.lines
|
|
69
|
+
rows = build_lines
|
|
70
|
+
|
|
71
|
+
unless rows.empty?
|
|
72
|
+
index = build_entry && timings.index_of(build_entry)
|
|
73
|
+
lines = index ? lines.insert(index + 1, *rows) : lines + rows
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
lines + advice_lines
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Runs the Dockerfile rules against the file this deploy would build, upgraded with what
|
|
80
|
+
# the build measured when there was one. Silent about a Dockerfile that is not there:
|
|
81
|
+
# a --skip-push deploy never looks at one, and a missing file is `dash doctor`'s finding
|
|
82
|
+
# to report, not a deploy's.
|
|
83
|
+
#
|
|
84
|
+
# `build_directory` is where the Dockerfile and context are read from: the git clone for
|
|
85
|
+
# a `push`, the working directory for a `dev` build that never clones.
|
|
86
|
+
def analyze!(config, build: @build, build_directory: config.builder.build_directory)
|
|
87
|
+
@advice = []
|
|
88
|
+
return unless config.report.advice?
|
|
89
|
+
|
|
90
|
+
dockerfile = File.expand_path(config.builder.dockerfile, build_directory)
|
|
91
|
+
return unless File.exist?(dockerfile)
|
|
92
|
+
|
|
93
|
+
@advice = Dash::Dockerfile::Analyzer.new(
|
|
94
|
+
document: Dash::Dockerfile::Parser.parse(File.read(dockerfile)),
|
|
95
|
+
path: config.builder.dockerfile,
|
|
96
|
+
file: dockerfile,
|
|
97
|
+
context_dir: File.expand_path(config.builder.context, build_directory),
|
|
98
|
+
build: build,
|
|
99
|
+
builder: config.builder,
|
|
100
|
+
ignore: config.report.ignore,
|
|
101
|
+
hadolint: config.report.hadolint?
|
|
102
|
+
).findings
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Also printed on their own by `dash build push`, for the same reason the build rows are.
|
|
106
|
+
def advice_lines
|
|
107
|
+
return [] if advice.blank?
|
|
108
|
+
|
|
109
|
+
[ " Advice", *advice.flat_map { |finding| advice_rows(finding) } ]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Also printed on their own by `dash build push`, which has no phase table to sit under.
|
|
113
|
+
def build_lines
|
|
114
|
+
return [] unless build&.any?
|
|
115
|
+
|
|
116
|
+
rows = []
|
|
117
|
+
rows << context_row if build.context_bytes
|
|
118
|
+
build.slowest(SLOWEST_STEPS).select(&:seconds).each { |step| rows << row(step.label, seconds(step.seconds)) }
|
|
119
|
+
rows << row("cached steps", "#{build.cached_steps.size} of #{build.dockerfile_steps.size}") if build.dockerfile_steps.any?
|
|
120
|
+
rows << export_row if export_and_push_seconds > 0
|
|
121
|
+
# A multi-platform build runs the same instruction once per platform, so one broken
|
|
122
|
+
# step fails once per platform with the same message. Print that once.
|
|
123
|
+
build.failed_steps.uniq { |step| [ step.label, step.error ] }.each { |step| rows << row(step.label, "error", step.error) }
|
|
124
|
+
rows
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
private
|
|
128
|
+
# The suggestion hangs under the message rather than under the row, so a location
|
|
129
|
+
# longer than its column (a Dockerfile somewhere deep in the tree) shifts both.
|
|
130
|
+
def advice_rows(finding)
|
|
131
|
+
prefix = format("%s%-#{SEVERITY_WIDTH}s %-#{LOCATION_WIDTH}s ", INDENT, finding.severity, finding.location)
|
|
132
|
+
|
|
133
|
+
rows = [ colorize(finding.severity, "#{prefix}#{finding.message}") ]
|
|
134
|
+
rows << "#{" " * prefix.length}→ #{finding.suggestion}" if finding.suggestion.present?
|
|
135
|
+
rows
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Colour is for the terminal only: a report piped to a file or asserted in a test
|
|
139
|
+
# should be the same text without the escape codes.
|
|
140
|
+
def colorize(severity, line)
|
|
141
|
+
color = SEVERITY_COLORS[severity] if $stdout.tty?
|
|
142
|
+
color ? "#{color}#{line}\e[0m" : line
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# A build killed between the context transfer and that vertex's DONE has a size but
|
|
146
|
+
# no duration. "0.0s" would be a measurement nobody took.
|
|
147
|
+
def context_row
|
|
148
|
+
row "build context", build.context_seconds ? seconds(build.context_seconds) : "n/a", human_bytes(build.context_bytes)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def export_row
|
|
152
|
+
note = "cache export #{seconds(build.cache_export_seconds)}" if build.cache_export_seconds > 0
|
|
153
|
+
row "export + push", seconds(export_and_push_seconds), note
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def export_and_push_seconds
|
|
157
|
+
build.export_seconds + build.push_seconds
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Indented like a Dash::Timings depth-1 row, with a wider name column of its own; a
|
|
161
|
+
# step longer than that is cut rather than allowed to run off the line.
|
|
162
|
+
def row(name, value, detail = nil)
|
|
163
|
+
line = format("%s%-#{NAME_WIDTH}s %#{VALUE_WIDTH}s", INDENT, name.truncate(NAME_WIDTH), value)
|
|
164
|
+
detail ? "#{line} (#{detail})" : line
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def seconds(value)
|
|
168
|
+
format("%.1fs", value.to_f)
|
|
169
|
+
end
|
|
170
|
+
end
|