dash 4.0.7 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dash/build/progress_parser.rb +136 -0
- data/lib/dash/build/report.rb +104 -0
- data/lib/dash/build/step.rb +49 -0
- data/lib/dash/cli/app/boot.rb +20 -10
- data/lib/dash/cli/app.rb +2 -2
- data/lib/dash/cli/base.rb +117 -2
- data/lib/dash/cli/build.rb +61 -5
- data/lib/dash/cli/doctor/config_checks.rb +36 -1
- data/lib/dash/cli/doctor.rb +2 -1
- data/lib/dash/cli/main.rb +24 -7
- data/lib/dash/cli/prune.rb +5 -8
- data/lib/dash/cli/report.rb +97 -0
- data/lib/dash/cli/templates/sample_hooks/post-deploy.sample +5 -0
- data/lib/dash/commander.rb +10 -2
- data/lib/dash/commands/app.rb +18 -0
- data/lib/dash/commands/auditor.rb +10 -0
- data/lib/dash/commands/builder/base.rb +18 -0
- data/lib/dash/commands/builder.rb +1 -1
- data/lib/dash/configuration/docs/configuration.yml +6 -0
- data/lib/dash/configuration/docs/report.yml +39 -0
- data/lib/dash/configuration/docs/ssh.yml +8 -0
- data/lib/dash/configuration/docs/sshkit.yml +2 -1
- data/lib/dash/configuration/report.rb +65 -0
- data/lib/dash/configuration/ssh.rb +8 -1
- data/lib/dash/configuration.rb +2 -1
- data/lib/dash/dockerfile/analyzer.rb +66 -0
- data/lib/dash/dockerfile/context.rb +147 -0
- data/lib/dash/dockerfile/dockerignore.rb +29 -0
- data/lib/dash/dockerfile/document.rb +28 -0
- data/lib/dash/dockerfile/finding.rb +20 -0
- data/lib/dash/dockerfile/hadolint.rb +75 -0
- data/lib/dash/dockerfile/instruction.rb +58 -0
- data/lib/dash/dockerfile/parser.rb +199 -0
- data/lib/dash/dockerfile/rules/apt_hygiene.rb +35 -0
- data/lib/dash/dockerfile/rules/base.rb +44 -0
- data/lib/dash/dockerfile/rules/cache_busting_arg.rb +40 -0
- data/lib/dash/dockerfile/rules/cache_export_cost.rb +20 -0
- data/lib/dash/dockerfile/rules/context_size.rb +20 -0
- data/lib/dash/dockerfile/rules/copy_before_install.rb +34 -0
- data/lib/dash/dockerfile/rules/curl_pipe_shell.rb +16 -0
- data/lib/dash/dockerfile/rules/dockerignore_gaps.rb +36 -0
- data/lib/dash/dockerfile/rules/inline_env_blob.rb +23 -0
- data/lib/dash/dockerfile/rules/latest_base.rb +24 -0
- data/lib/dash/dockerfile/rules/missing_dockerignore.rb +11 -0
- data/lib/dash/dockerfile/rules/no_cache_mount.rb +26 -0
- data/lib/dash/dockerfile/rules/root_user.rb +12 -0
- data/lib/dash/dockerfile/rules/secret_in_build_arg.rb +30 -0
- data/lib/dash/dockerfile/rules/single_stage_build_deps.rb +19 -0
- data/lib/dash/dockerfile/rules/uncached_install.rb +20 -0
- data/lib/dash/dockerfile/stage.rb +65 -0
- data/lib/dash/otel_shipper.rb +5 -4
- data/lib/dash/output/otel_logger.rb +52 -0
- data/lib/dash/report/history.rb +94 -0
- data/lib/dash/report/trends.rb +129 -0
- data/lib/dash/report/writer.rb +142 -0
- data/lib/dash/report.rb +170 -0
- data/lib/dash/sshkit_with_ext.rb +77 -4
- data/lib/dash/timings.rb +163 -10
- data/lib/dash/utils.rb +7 -0
- data/lib/dash/version.rb +1 -1
- data/lib/dash.rb +4 -0
- metadata +36 -1
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# Everything a rule is allowed to look at: the parsed Dockerfile, the build context on
|
|
2
|
+
# disk, the builder's configuration, and — when the advice is printed next to a build
|
|
3
|
+
# that actually ran — what that build measured.
|
|
4
|
+
#
|
|
5
|
+
# The shared predicates live here rather than in each rule so that "what counts as a
|
|
6
|
+
# dependency install" has one answer, and so the measured half of a rule can find the
|
|
7
|
+
# buildx vertex that corresponds to a Dockerfile line.
|
|
8
|
+
class Dash::Dockerfile::Context
|
|
9
|
+
# The package managers whose install step is worth protecting from cache busting, and
|
|
10
|
+
# the cache directory each conventionally wants mounted.
|
|
11
|
+
DEPENDENCY_INSTALLS = [
|
|
12
|
+
[ /\bbundle\s+(?:_[\d._]+_\s+)?install\b/, "/usr/local/bundle/cache" ],
|
|
13
|
+
[ /\bnpm\s+(?:ci|install)\b/, "/root/.npm" ],
|
|
14
|
+
[ /\byarn\s+install\b/, "/usr/local/share/.cache/yarn" ],
|
|
15
|
+
[ /\bpnpm\s+install\b/, "/root/.local/share/pnpm/store" ],
|
|
16
|
+
[ /\bbun\s+install\b/, "/root/.bun/install/cache" ],
|
|
17
|
+
[ /\bpip3?\s+install\b/, "/root/.cache/pip" ],
|
|
18
|
+
[ /\bpoetry\s+install\b/, "/root/.cache/pypoetry" ],
|
|
19
|
+
[ /\bgo\s+mod\s+download\b/, "/go/pkg/mod" ],
|
|
20
|
+
[ /\bcargo\s+(?:build|fetch)\b/, "/usr/local/cargo/registry" ],
|
|
21
|
+
[ /\bcomposer\s+install\b/, "/root/.composer/cache" ],
|
|
22
|
+
[ /\bmix\s+deps\.get\b/, "/root/.hex" ],
|
|
23
|
+
[ /\bdotnet\s+restore\b/, "/root/.nuget/packages" ]
|
|
24
|
+
].freeze
|
|
25
|
+
|
|
26
|
+
# apt takes its options before or after the verb (`apt-get -y install`,
|
|
27
|
+
# `apt-get -t bookworm-backports install`), so the verb is found past any of them. A
|
|
28
|
+
# shell separator is never an option or its value, so `apt-get -y && install` is not
|
|
29
|
+
# an apt install.
|
|
30
|
+
APT_OPTIONS = /(?:-[^\s;&|]+(?:\s+[^-\s;&|][^\s;&|]*)?\s+)*/
|
|
31
|
+
APT_INSTALL = [ /\bapt-get\s+#{APT_OPTIONS.source}install\b/, "/var/cache/apt" ].freeze
|
|
32
|
+
|
|
33
|
+
# A copy that ships the whole tree, so every commit invalidates it and everything
|
|
34
|
+
# layered on top of it.
|
|
35
|
+
BROAD_SOURCES = [ ".", "./", "*", "/" ].freeze
|
|
36
|
+
|
|
37
|
+
# buildx expands ARG and ENV references in the vertex name it prints, so a step's text
|
|
38
|
+
# and the Dockerfile line it came from stop agreeing at the first ${…}. Match on the
|
|
39
|
+
# longest shared prefix instead, and require enough of it that two unrelated RUNs
|
|
40
|
+
# cannot be confused for each other.
|
|
41
|
+
MINIMUM_STEP_MATCH = 12
|
|
42
|
+
|
|
43
|
+
attr_reader :document, :context_dir, :dockerignore, :build, :builder, :path
|
|
44
|
+
|
|
45
|
+
def initialize(document:, context_dir: nil, dockerignore: nil, build: nil, builder: nil, path: "Dockerfile")
|
|
46
|
+
@document = document
|
|
47
|
+
@context_dir = context_dir
|
|
48
|
+
@dockerignore = dockerignore
|
|
49
|
+
@build = build
|
|
50
|
+
@builder = builder
|
|
51
|
+
@path = path
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def location_for(instruction)
|
|
55
|
+
"#{path}:#{instruction.line}"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def dependency_install?(instruction)
|
|
59
|
+
!install_match(instruction).nil?
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The command that made it a dependency install ("bundle install"), for advice that
|
|
63
|
+
# names what the operator wrote rather than the whole RUN.
|
|
64
|
+
def install_command(instruction)
|
|
65
|
+
install_match(instruction)&.first
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def install_cache_target(instruction)
|
|
69
|
+
install_match(instruction)&.last
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def apt_install?(instruction)
|
|
73
|
+
instruction.name == "RUN" && instruction.shell_command.match?(APT_INSTALL.first)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def broad_copy?(instruction)
|
|
77
|
+
return false unless %w[ COPY ADD ].include?(instruction.name)
|
|
78
|
+
return false if instruction.flag?("from")
|
|
79
|
+
|
|
80
|
+
sources(instruction).any? { |source| BROAD_SOURCES.include?(source) }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# A dependency install directly after a broad copy is invalidated by every commit,
|
|
84
|
+
# which is a finding of its own — rules that would otherwise report the same slow step
|
|
85
|
+
# twice defer to it.
|
|
86
|
+
def busted_by_broad_copy?(instruction)
|
|
87
|
+
stage = instruction.stage or return false
|
|
88
|
+
|
|
89
|
+
stage.instructions.any? { |other| other.line < instruction.line && broad_copy?(other) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# buildx labels a step with its stage name except in a single-stage build, where it
|
|
93
|
+
# prints none. A multi-platform build reports the same step once per platform; the
|
|
94
|
+
# slowest one is the number worth quoting.
|
|
95
|
+
def build_step_for(instruction)
|
|
96
|
+
return unless build
|
|
97
|
+
|
|
98
|
+
text = normalize(instruction.to_s)
|
|
99
|
+
candidates = build.instruction_steps.select { |step| same_stage?(step, instruction) }
|
|
100
|
+
|
|
101
|
+
best = candidates.max_by do |step|
|
|
102
|
+
matched = normalize(step.instruction)
|
|
103
|
+
[ matched == text ? 1 : 0, shared_prefix(text, matched), step.seconds.to_f ]
|
|
104
|
+
end
|
|
105
|
+
return unless best
|
|
106
|
+
|
|
107
|
+
matched = normalize(best.instruction)
|
|
108
|
+
best if matched == text || shared_prefix(text, matched) >= MINIMUM_STEP_MATCH
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def context_entries(name)
|
|
112
|
+
return [] unless context_dir
|
|
113
|
+
|
|
114
|
+
Dir.glob(name, base: context_dir, flags: ::File::FNM_DOTMATCH)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
private
|
|
118
|
+
def install_match(instruction)
|
|
119
|
+
return unless instruction.name == "RUN"
|
|
120
|
+
|
|
121
|
+
command = instruction.shell_command
|
|
122
|
+
DEPENDENCY_INSTALLS.each do |pattern, target|
|
|
123
|
+
matched = command[pattern]
|
|
124
|
+
return [ matched, target ] if matched
|
|
125
|
+
end
|
|
126
|
+
nil
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def same_stage?(step, instruction)
|
|
130
|
+
step.stage.nil? ? document.stages.one? : step.stage == instruction.stage&.name
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def sources(instruction)
|
|
134
|
+
words = instruction.json? ? instruction.argv : instruction.args.split(/\s+/)
|
|
135
|
+
words.size > 1 ? words[0..-2] : words
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def normalize(text)
|
|
139
|
+
text.to_s.squeeze(" ").strip
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def shared_prefix(one, other)
|
|
143
|
+
length = 0
|
|
144
|
+
length += 1 while length < one.length && length < other.length && one[length] == other[length]
|
|
145
|
+
length
|
|
146
|
+
end
|
|
147
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Just enough .dockerignore to answer "does this file ship in the build context?".
|
|
2
|
+
#
|
|
3
|
+
# Not a reimplementation of BuildKit's matcher: negations are skipped rather than
|
|
4
|
+
# applied, so a path this says is covered might still ship. That direction is the safe
|
|
5
|
+
# one — it costs a piece of advice, never a false accusation.
|
|
6
|
+
class Dash::Dockerfile::Dockerignore
|
|
7
|
+
FILENAME = ".dockerignore"
|
|
8
|
+
|
|
9
|
+
attr_reader :patterns
|
|
10
|
+
|
|
11
|
+
def self.in(directory)
|
|
12
|
+
path = ::File.join(directory.to_s, FILENAME)
|
|
13
|
+
new(::File.read(path).lines) if ::File.file?(path)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def initialize(lines)
|
|
17
|
+
@patterns = lines
|
|
18
|
+
.map(&:strip)
|
|
19
|
+
.reject { |line| line.empty? || line.start_with?("#", "!") }
|
|
20
|
+
.map { |line| line.delete_prefix("**/").delete_prefix("./").delete_prefix("/").delete_suffix("/") }
|
|
21
|
+
.reject(&:empty?)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def covers?(path)
|
|
25
|
+
patterns.any? do |pattern|
|
|
26
|
+
::File.fnmatch?(pattern, path, ::File::FNM_DOTMATCH) || path.start_with?("#{pattern}/")
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# A parsed Dockerfile: its instructions in file order, grouped into stages.
|
|
2
|
+
#
|
|
3
|
+
# Named Document rather than File so that `File.fnmatch` inside Dash::Dockerfile still
|
|
4
|
+
# means the one in Ruby's core library.
|
|
5
|
+
class Dash::Dockerfile::Document
|
|
6
|
+
attr_reader :instructions, :stages, :directives
|
|
7
|
+
|
|
8
|
+
def initialize(instructions:, stages:, directives: {})
|
|
9
|
+
@instructions = instructions
|
|
10
|
+
@stages = stages
|
|
11
|
+
@directives = directives
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def shipped_stages
|
|
15
|
+
stages.select(&:shipped?)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def final_stage
|
|
19
|
+
stages.last
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def each_instruction(name)
|
|
23
|
+
return to_enum(:each_instruction, name) unless block_given?
|
|
24
|
+
|
|
25
|
+
name = name.to_s.upcase
|
|
26
|
+
instructions.each { |instruction| yield instruction if instruction.name == name }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# One piece of advice about the Dockerfile or the build context.
|
|
2
|
+
#
|
|
3
|
+
# `rule` is a stable public string an operator can put in `report: ignore:`, so renaming
|
|
4
|
+
# one is a breaking change to their deploy.yml. `location` is whatever they should open:
|
|
5
|
+
# a Dockerfile line, `.dockerignore`, `build context`, or a deploy.yml key.
|
|
6
|
+
Dash::Dockerfile::Finding = Struct.new(:rule, :severity, :location, :message, :suggestion, keyword_init: true) do
|
|
7
|
+
def warn?
|
|
8
|
+
severity == :warn
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.from_h(hash)
|
|
12
|
+
hash = hash.transform_keys(&:to_sym)
|
|
13
|
+
|
|
14
|
+
new(**hash.slice(:rule, :location, :message, :suggestion), severity: hash[:severity].to_sym)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def to_h
|
|
18
|
+
{ rule: rule, severity: severity.to_s, location: location, message: message, suggestion: suggestion }
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "open3"
|
|
3
|
+
require "active_support/core_ext/string/filters"
|
|
4
|
+
|
|
5
|
+
# Optional supplement to the built-in rules: whatever `hadolint` has to say about the same
|
|
6
|
+
# file, when the operator already has it installed.
|
|
7
|
+
#
|
|
8
|
+
# Run as a plain local process rather than through SSHKit, so the command sequence a deploy
|
|
9
|
+
# prints — and the cost-guard test that pins it — is unchanged. `--no-fail` keeps its exit
|
|
10
|
+
# status out of the deploy, and anything that goes wrong becomes one informational finding
|
|
11
|
+
# rather than an exception.
|
|
12
|
+
class Dash::Dockerfile::Hadolint
|
|
13
|
+
EXECUTABLE = "hadolint".freeze
|
|
14
|
+
# hadolint's own levels. `error` is the only one worth a warning next to a deploy; the
|
|
15
|
+
# rest are style notes that should not compete with a measured finding.
|
|
16
|
+
SEVERITIES = { "error" => :warn }.freeze
|
|
17
|
+
|
|
18
|
+
class << self
|
|
19
|
+
# No shell: PATH is walked directly, so a directory with a space or a semicolon in it
|
|
20
|
+
# cannot turn a lookup into a command.
|
|
21
|
+
def available?
|
|
22
|
+
ENV["PATH"].to_s.split(::File::PATH_SEPARATOR).any? do |directory|
|
|
23
|
+
path = ::File.join(directory, EXECUTABLE)
|
|
24
|
+
::File.executable?(path) && !::File.directory?(path)
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def initialize(path:, file: path)
|
|
30
|
+
@path = path
|
|
31
|
+
@file = file
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def findings
|
|
35
|
+
return [] unless self.class.available?
|
|
36
|
+
|
|
37
|
+
output, status = Open3.capture2(EXECUTABLE, "--format", "json", "--no-fail", @file)
|
|
38
|
+
return unavailable("exited #{status.exitstatus}") unless status.success?
|
|
39
|
+
return [] if output.strip.empty?
|
|
40
|
+
|
|
41
|
+
issues = JSON.parse(output)
|
|
42
|
+
return unparsable("expected a JSON array, got #{issues.class.name.downcase}") unless issues.is_a?(Array)
|
|
43
|
+
|
|
44
|
+
issues.map { |issue| finding_for(issue) }
|
|
45
|
+
# Only JSON.parse raises this, so it is the one error that means "it ran fine, dash
|
|
46
|
+
# could not read what it printed". Anything else that raises in here is dash's own.
|
|
47
|
+
rescue JSON::ParserError => e
|
|
48
|
+
unparsable(e.message.truncate(80))
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
unavailable(e.message)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
def finding_for(issue)
|
|
55
|
+
Dash::Dockerfile::Finding.new \
|
|
56
|
+
rule: issue["code"],
|
|
57
|
+
severity: SEVERITIES.fetch(issue["level"], :info),
|
|
58
|
+
location: "#{@path}:#{issue["line"]}",
|
|
59
|
+
message: "#{issue["code"]}: #{issue["message"]}",
|
|
60
|
+
suggestion: nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def unparsable(reason)
|
|
64
|
+
note "hadolint output could not be parsed (#{reason})"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def unavailable(reason)
|
|
68
|
+
note "hadolint could not run (#{reason})"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def note(message)
|
|
72
|
+
[ Dash::Dockerfile::Finding.new(rule: "hadolint", severity: :info, location: EXECUTABLE,
|
|
73
|
+
message: message, suggestion: "silence this with report: hadolint: false") ]
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
# One logical Dockerfile instruction: the keyword, its flags, and everything else on the
|
|
4
|
+
# line — continuations joined, heredoc bodies appended, comments dropped.
|
|
5
|
+
#
|
|
6
|
+
# `line` is the first physical line the instruction started on, because that is the line
|
|
7
|
+
# an operator opens their editor at when advice names it.
|
|
8
|
+
class Dash::Dockerfile::Instruction
|
|
9
|
+
attr_reader :name, :args, :flags, :line
|
|
10
|
+
attr_accessor :stage
|
|
11
|
+
|
|
12
|
+
def initialize(name:, args:, flags: {}, line: 1)
|
|
13
|
+
@name = name
|
|
14
|
+
@args = args
|
|
15
|
+
@flags = flags
|
|
16
|
+
@line = line
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The first value of a flag. Repeated flags (several `--mount`s on one RUN) keep every
|
|
20
|
+
# value in `flags`; callers that only care whether one is present ask for the first.
|
|
21
|
+
def flag(key)
|
|
22
|
+
Array(flags[key]).first
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def flag?(key)
|
|
26
|
+
flags.key?(key)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def json?
|
|
30
|
+
args.start_with?("[") && !argv.nil?
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# The exec form's arguments, or nil when this is the shell form (or malformed JSON,
|
|
34
|
+
# which BuildKit would reject but which must not take the analyzer down with it).
|
|
35
|
+
def argv
|
|
36
|
+
return @argv if defined?(@argv)
|
|
37
|
+
|
|
38
|
+
@argv = begin
|
|
39
|
+
parsed = JSON.parse(args) if args.start_with?("[")
|
|
40
|
+
parsed if parsed.is_a?(Array) && parsed.all?(String)
|
|
41
|
+
rescue JSON::ParserError
|
|
42
|
+
nil
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# What the instruction runs, as one string, whichever form it was written in — the
|
|
47
|
+
# rules match against shell text and should not have to care.
|
|
48
|
+
def shell_command
|
|
49
|
+
json? ? argv.join(" ") : args
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Collapsed to a single line so it can be compared with what buildx printed for the
|
|
53
|
+
# matching vertex, and so advice can name it without wrapping the terminal.
|
|
54
|
+
def to_s
|
|
55
|
+
[ name, *flags.flat_map { |key, values| values.map { |value| "--#{key}=#{value}" } }, args ]
|
|
56
|
+
.reject(&:blank?).join(" ").gsub(/\s+/, " ")
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# Turns Dockerfile text into instructions and stages.
|
|
2
|
+
#
|
|
3
|
+
# Line-oriented, because that is how BuildKit reads it: a physical line is joined to the
|
|
4
|
+
# next while it ends in the escape character, comment lines in between are dropped, and a
|
|
5
|
+
# heredoc redirection pulls the following lines in verbatim until its delimiter.
|
|
6
|
+
#
|
|
7
|
+
# It is deliberately forgiving. Advice is a courtesy printed next to a deploy, so a file
|
|
8
|
+
# this parser cannot make sense of must produce fewer findings, never an exception — the
|
|
9
|
+
# authority on whether a Dockerfile builds is BuildKit, not this.
|
|
10
|
+
class Dash::Dockerfile::Parser
|
|
11
|
+
DIRECTIVE = /\A#\s*(?<name>syntax|escape)\s*=\s*(?<value>\S+)\s*\z/i
|
|
12
|
+
COMMENT = /\A\s*#/
|
|
13
|
+
BLANK = /\A\s*\z/
|
|
14
|
+
INSTRUCTION = /\A\s*(?<name>[A-Za-z]+)(?:\s+(?<rest>.*))?\z/m
|
|
15
|
+
FLAG = /\A--(?<key>[a-zA-Z][\w-]*)=(?<value>(?:"[^"]*"|'[^']*'|\S)*)\s*/
|
|
16
|
+
# `<<EOF`, `<<-EOF`, `<<"EOF"`, `<<'EOF'` — the quoted forms only change how BuildKit
|
|
17
|
+
# expands the body, not where it ends.
|
|
18
|
+
HEREDOC = /<<-?\s*(?<quote>["']?)(?<delimiter>[A-Za-z_]\w*)\k<quote>/
|
|
19
|
+
STAGE_NAME = /\A(?<base>\S+)(?:\s+AS\s+(?<name>\S+))?\z/i
|
|
20
|
+
|
|
21
|
+
def self.parse(text)
|
|
22
|
+
new(text).parse
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def initialize(text)
|
|
26
|
+
@lines = text.to_s.lines.map(&:chomp)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def parse
|
|
30
|
+
directives = parse_directives
|
|
31
|
+
escape = directives["escape"] == "`" ? "`" : "\\"
|
|
32
|
+
|
|
33
|
+
instructions = parse_instructions(escape)
|
|
34
|
+
stages = build_stages(instructions)
|
|
35
|
+
|
|
36
|
+
Dash::Dockerfile::Document.new(instructions: instructions, stages: stages, directives: directives)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
attr_reader :lines
|
|
41
|
+
|
|
42
|
+
# Only the comment block at the very top can carry directives; after the first
|
|
43
|
+
# instruction a `# syntax=` line is an ordinary comment.
|
|
44
|
+
def parse_directives
|
|
45
|
+
directives = {}
|
|
46
|
+
|
|
47
|
+
lines.each do |line|
|
|
48
|
+
break unless line.match?(COMMENT) || line.match?(BLANK)
|
|
49
|
+
next unless (match = line.match(DIRECTIVE))
|
|
50
|
+
|
|
51
|
+
directives[match[:name].downcase] = match[:value]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
directives
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def parse_instructions(escape)
|
|
58
|
+
instructions = []
|
|
59
|
+
index = 0
|
|
60
|
+
|
|
61
|
+
while index < lines.size
|
|
62
|
+
line = lines[index]
|
|
63
|
+
|
|
64
|
+
if line.match?(BLANK) || line.match?(COMMENT)
|
|
65
|
+
index += 1
|
|
66
|
+
next
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
started_at = index
|
|
70
|
+
text, index = join_continuations(index, escape)
|
|
71
|
+
text, index = append_heredocs(text, index)
|
|
72
|
+
|
|
73
|
+
instruction = build_instruction(text, started_at + 1)
|
|
74
|
+
instructions << instruction if instruction
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
instructions
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Joins physical lines while each ends in the escape character. Comment lines between
|
|
81
|
+
# them are BuildKit's own convention for annotating a long RUN, and are not part of
|
|
82
|
+
# the command.
|
|
83
|
+
def join_continuations(index, escape)
|
|
84
|
+
parts = []
|
|
85
|
+
|
|
86
|
+
while index < lines.size
|
|
87
|
+
line = lines[index]
|
|
88
|
+
index += 1
|
|
89
|
+
|
|
90
|
+
next if line.match?(COMMENT) && parts.any?
|
|
91
|
+
|
|
92
|
+
continues = line.rstrip.end_with?(escape)
|
|
93
|
+
parts << (continues ? line.rstrip.delete_suffix(escape) : line)
|
|
94
|
+
break unless continues
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
[ parts.map(&:strip).reject(&:empty?).join(" "), index ]
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Every heredoc opened on the instruction line consumes lines until its delimiter, in
|
|
101
|
+
# the order they were opened. The bodies are appended to the arguments so rules can
|
|
102
|
+
# match what the instruction actually runs.
|
|
103
|
+
#
|
|
104
|
+
# A delimiter that never arrives means this was not a heredoc after all (`'<<EOF'` as
|
|
105
|
+
# a quoted shell word, or a typo): nothing is consumed, so the rest of the file is
|
|
106
|
+
# still parsed rather than folded into this one instruction.
|
|
107
|
+
def append_heredocs(text, index)
|
|
108
|
+
delimiters = text.scan(HEREDOC).map(&:last)
|
|
109
|
+
return [ text, index ] if delimiters.empty?
|
|
110
|
+
|
|
111
|
+
body = []
|
|
112
|
+
at = index
|
|
113
|
+
|
|
114
|
+
delimiters.each do |delimiter|
|
|
115
|
+
terminator = (at...lines.size).find { |line| lines[line].strip == delimiter }
|
|
116
|
+
return [ text, index ] unless terminator
|
|
117
|
+
|
|
118
|
+
body.concat heredoc_commands(lines[at...terminator])
|
|
119
|
+
at = terminator + 1
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Line breaks are kept: in a RUN heredoc each line is its own shell command, and the
|
|
123
|
+
# apt rules need to know where one ends.
|
|
124
|
+
[ [ text, *body ].join("\n"), at ]
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Inside the body the shell's own continuation applies: a line ending in `\` is the
|
|
128
|
+
# same command as the next one.
|
|
129
|
+
def heredoc_commands(body)
|
|
130
|
+
body.map(&:strip).each_with_object([]) do |line, commands|
|
|
131
|
+
if commands.last&.end_with?("\\")
|
|
132
|
+
commands[-1] = "#{commands.last.delete_suffix("\\").rstrip} #{line}"
|
|
133
|
+
else
|
|
134
|
+
commands << line
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def build_instruction(text, line)
|
|
140
|
+
match = text.match(INSTRUCTION)
|
|
141
|
+
return unless match
|
|
142
|
+
|
|
143
|
+
flags, args = extract_flags(match[:rest].to_s.strip)
|
|
144
|
+
|
|
145
|
+
Dash::Dockerfile::Instruction.new(name: match[:name].upcase, args: args, flags: flags, line: line)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def extract_flags(rest)
|
|
149
|
+
flags = {}
|
|
150
|
+
|
|
151
|
+
while (match = rest.match(FLAG))
|
|
152
|
+
(flags[match[:key]] ||= []) << match[:value].delete_prefix('"').delete_suffix('"')
|
|
153
|
+
rest = match.post_match.lstrip
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
[ flags, rest ]
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def build_stages(instructions)
|
|
160
|
+
stages = []
|
|
161
|
+
|
|
162
|
+
instructions.each do |instruction|
|
|
163
|
+
if instruction.name == "FROM"
|
|
164
|
+
stages << new_stage(instruction, stages.size)
|
|
165
|
+
elsif (stage = stages.last)
|
|
166
|
+
stage.instructions << instruction
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
instruction.stage = stages.last unless stages.empty? && instruction.name != "FROM"
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
mark_shipped stages
|
|
173
|
+
stages
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def new_stage(instruction, index)
|
|
177
|
+
match = instruction.args.match(STAGE_NAME)
|
|
178
|
+
|
|
179
|
+
Dash::Dockerfile::Stage.new \
|
|
180
|
+
name: match && match[:name] || "stage-#{index}",
|
|
181
|
+
named: !(match && match[:name]).nil?,
|
|
182
|
+
index: index,
|
|
183
|
+
base: (match ? match[:base] : instruction.args),
|
|
184
|
+
from: instruction
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Walk back from the final stage through the bases it inherits. A stage reached only
|
|
188
|
+
# by `COPY --from=` is not on that chain, which is the point. Only an explicit `AS`
|
|
189
|
+
# name can be inherited from; the generated `stage-N` labels are dash's, not BuildKit's.
|
|
190
|
+
def mark_shipped(stages)
|
|
191
|
+
by_name = stages.select(&:named?).to_h { |stage| [ stage.name, stage ] }
|
|
192
|
+
stage = stages.last
|
|
193
|
+
|
|
194
|
+
while stage && !stage.shipped?
|
|
195
|
+
stage.shipped = true
|
|
196
|
+
stage = by_name[stage.base]
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# apt's recommended packages and its package lists both ship in the layer unless the same
|
|
2
|
+
# RUN gets rid of them. Only worth saying about a stage that ends up in the image.
|
|
3
|
+
class Dash::Dockerfile::Rules::AptHygiene < Dash::Dockerfile::Rules::Base
|
|
4
|
+
NO_RECOMMENDS = /--no-install-recommends/
|
|
5
|
+
LIST_CLEANUP = %r{rm\s+-rf\s+/var/lib/apt/lists}
|
|
6
|
+
APT_OPERATION = /\bapt-get\s+#{Dash::Dockerfile::Context::APT_OPTIONS.source}(?:update|install|upgrade)\b/
|
|
7
|
+
# One shell command at a time: a later, compliant install must not vouch for an
|
|
8
|
+
# earlier one, and a cleanup only counts after the last thing that refilled the lists.
|
|
9
|
+
# A heredoc body keeps its line breaks, so each of its lines is a command too.
|
|
10
|
+
SEGMENT = /&&|\|\||;|\n/
|
|
11
|
+
SUGGESTION = "add --no-install-recommends to every install and rm -rf /var/lib/apt/lists/* at the end of the same RUN"
|
|
12
|
+
|
|
13
|
+
def findings
|
|
14
|
+
document.shipped_stages.flat_map { |stage| stage.instructions }.filter_map do |instruction|
|
|
15
|
+
next unless context.apt_install?(instruction)
|
|
16
|
+
|
|
17
|
+
problems = problems_in(instruction.shell_command.split(SEGMENT))
|
|
18
|
+
next if problems.empty?
|
|
19
|
+
|
|
20
|
+
note at(instruction), "apt-get install #{problems.join(" and ")}", SUGGESTION
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
def problems_in(segments)
|
|
26
|
+
installs = segments.select { |segment| segment.match?(Dash::Dockerfile::Context::APT_INSTALL.first) }
|
|
27
|
+
last_apt = segments.rindex { |segment| segment.match?(APT_OPERATION) } or return []
|
|
28
|
+
cleaned = segments.drop(last_apt + 1).any? { |segment| segment.match?(LIST_CLEANUP) }
|
|
29
|
+
|
|
30
|
+
problems = []
|
|
31
|
+
problems << "installs recommended packages" if installs.any? { |segment| !segment.match?(NO_RECOMMENDS) }
|
|
32
|
+
problems << "leaves /var/lib/apt/lists in the layer" unless cleaned
|
|
33
|
+
problems
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
require "active_support/core_ext/module/delegation"
|
|
2
|
+
require "active_support/core_ext/string/inflections"
|
|
3
|
+
|
|
4
|
+
# A rule looks at the analysis context and returns findings. Nothing else: no IO, no
|
|
5
|
+
# state, no ordering assumptions about the other rules.
|
|
6
|
+
#
|
|
7
|
+
# The id is derived from the class name and is a public string — operators put it in
|
|
8
|
+
# `report: ignore:`, so renaming a rule class renames a config value.
|
|
9
|
+
class Dash::Dockerfile::Rules::Base
|
|
10
|
+
attr_reader :context
|
|
11
|
+
delegate :document, :build, :builder, :context_dir, :dockerignore, to: :context
|
|
12
|
+
|
|
13
|
+
class << self
|
|
14
|
+
def id
|
|
15
|
+
@id ||= name.demodulize.underscore.dasherize
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def initialize(context)
|
|
20
|
+
@context = context
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def findings
|
|
24
|
+
[]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
def warning(location, message, suggestion = nil)
|
|
29
|
+
finding :warn, location, message, suggestion
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def note(location, message, suggestion = nil)
|
|
33
|
+
finding :info, location, message, suggestion
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def finding(severity, location, message, suggestion)
|
|
37
|
+
Dash::Dockerfile::Finding.new \
|
|
38
|
+
rule: self.class.id, severity: severity, location: location, message: message, suggestion: suggestion
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def at(instruction)
|
|
42
|
+
context.location_for(instruction)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# An ARG that changes on every commit — a git SHA, a build timestamp — invalidates every
|
|
2
|
+
# layer from its first use onwards. Referenced after the dependency install it costs
|
|
3
|
+
# nothing; referenced before it, it costs the whole install.
|
|
4
|
+
class Dash::Dockerfile::Rules::CacheBustingArg < Dash::Dockerfile::Rules::Base
|
|
5
|
+
CACHE_BUSTING = /\A(?:.*_)?(?:GIT_SHA|COMMIT|SHA|BUILD_DATE|BUILD_TIME|BUILDTIME|VERSION)\z/i
|
|
6
|
+
# RUBY_VERSION, NODE_VERSION and friends name a toolchain and change once a quarter.
|
|
7
|
+
TOOLCHAIN = /_VERSION\z/i
|
|
8
|
+
SUGGESTION = "reference it after the dependency install, so a new commit does not invalidate it"
|
|
9
|
+
|
|
10
|
+
def findings
|
|
11
|
+
document.each_instruction("ARG").filter_map { |arg| finding_for(arg) }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
def finding_for(arg)
|
|
16
|
+
name = arg.args.split("=").first.to_s
|
|
17
|
+
return unless name.match?(CACHE_BUSTING) && !name.match?(TOOLCHAIN)
|
|
18
|
+
|
|
19
|
+
stage = arg.stage || document.stages.first or return
|
|
20
|
+
barrier = last_install_line(stage) or return
|
|
21
|
+
reference = reference_before(name, arg.line, barrier)
|
|
22
|
+
return unless reference
|
|
23
|
+
|
|
24
|
+
note at(arg), "ARG #{name} changes on every commit and is referenced on line #{reference.line}, before the dependency install on line #{barrier}", SUGGESTION
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def last_install_line(stage)
|
|
28
|
+
stage.instructions.select { |instruction| context.dependency_install?(instruction) }.last&.line
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# A FROM that uses the ARG is pinning its base with it, which is a different thing
|
|
32
|
+
# and not something "reference it later" could ever fix.
|
|
33
|
+
def reference_before(name, from_line, barrier)
|
|
34
|
+
pattern = /\$(?:\{#{Regexp.escape(name)}\}|#{Regexp.escape(name)}(?!\w))/
|
|
35
|
+
|
|
36
|
+
document.instructions.find do |instruction|
|
|
37
|
+
instruction.name != "FROM" && instruction.line > from_line && instruction.line < barrier && instruction.to_s.match?(pattern)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|