mutineer 1.8.2 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +611 -0
- data/README.md +84 -49
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/child_cause.rb +149 -0
- data/lib/mutineer/child_stdout.rb +12 -1
- data/lib/mutineer/cli.rb +536 -58
- data/lib/mutineer/command_redact.rb +38 -0
- data/lib/mutineer/config.rb +112 -6
- data/lib/mutineer/coverage_map.rb +1345 -156
- data/lib/mutineer/daemon_backend.rb +40 -17
- data/lib/mutineer/daemon_client.rb +63 -15
- data/lib/mutineer/daemon_server.rb +75 -5
- data/lib/mutineer/database_urls.rb +189 -0
- data/lib/mutineer/external_backend.rb +47 -1
- data/lib/mutineer/fs_path.rb +125 -0
- data/lib/mutineer/isolation.rb +374 -20
- data/lib/mutineer/job_plan.rb +349 -39
- data/lib/mutineer/minitest_integration/file_order.rb +56 -0
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +243 -19
- data/lib/mutineer/minitest_integration.rb +26 -58
- data/lib/mutineer/mutant_id.rb +4 -1
- data/lib/mutineer/mutation.rb +6 -1
- data/lib/mutineer/mutator_registry.rb +6 -3
- data/lib/mutineer/mutators/collection_method.rb +195 -6
- data/lib/mutineer/mutators/literal_mutation.rb +33 -3
- data/lib/mutineer/mutators/range_literal.rb +78 -29
- data/lib/mutineer/mutators/rescue_body.rb +126 -0
- data/lib/mutineer/mutators/return_nil.rb +23 -6
- data/lib/mutineer/output_filter.rb +451 -0
- data/lib/mutineer/pairing.rb +463 -31
- data/lib/mutineer/parser.rb +14 -0
- data/lib/mutineer/project_path.rb +15 -8
- data/lib/mutineer/rails_worker_db.rb +356 -6
- data/lib/mutineer/reach_probe.rb +229 -0
- data/lib/mutineer/redaction.rb +264 -32
- data/lib/mutineer/reload_guards.rb +460 -39
- data/lib/mutineer/reporter.rb +465 -57
- data/lib/mutineer/result.rb +6 -7
- data/lib/mutineer/run_seed.rb +16 -0
- data/lib/mutineer/runner.rb +175 -51
- data/lib/mutineer/source_bytes.rb +187 -0
- data/lib/mutineer/statement_lines.rb +108 -14
- data/lib/mutineer/test_runners/rspec.rb +118 -10
- data/lib/mutineer/test_runners/rspec_seed.rb +151 -0
- data/lib/mutineer/test_runners.rb +3 -2
- data/lib/mutineer/triage.rb +55 -10
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer.rb +1 -0
- data/skills/mutineer/SKILL.md +12 -4
- metadata +12 -1
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
require "tempfile"
|
|
5
|
+
require_relative "parser"
|
|
6
|
+
require_relative "statement_lines"
|
|
7
|
+
|
|
8
|
+
module Mutineer
|
|
9
|
+
# Records whether the expression that holds a mutation ran in the child.
|
|
10
|
+
#
|
|
11
|
+
# Ruby counts one line of a statement. A mutant on another line is selected
|
|
12
|
+
# as `:maybe` (#254, #289): the tests of the statement may not have reached
|
|
13
|
+
# it. A kill is still a kill. A survivor used to stay `no_coverage`, because
|
|
14
|
+
# the line count cannot tell "ran" from "did not run". The wrapper below
|
|
15
|
+
# writes a sentinel when that expression runs, so a survivor that ran counts
|
|
16
|
+
# as `survived` and one that did not stays `no_coverage` (#345).
|
|
17
|
+
#
|
|
18
|
+
# The inserted text is stdlib only (`File`). The daemon loads it as source,
|
|
19
|
+
# so the app process needs no Prism and no Mutineer constant.
|
|
20
|
+
module ReachProbe
|
|
21
|
+
# A fiber-local key for one sentinel: `mutineer_r` plus 16 hex digits.
|
|
22
|
+
# `Thread.current[]` is fiber-local, not one value for the whole thread.
|
|
23
|
+
# The generated source and {forget_load} use this key, not a fixed global,
|
|
24
|
+
# so an application variable of the same shape is not read or overwritten.
|
|
25
|
+
# The daemon checks the same pattern; it cannot load this file.
|
|
26
|
+
KEY_FORMAT = /\Amutineer_r[0-9a-f]{16}\z/
|
|
27
|
+
|
|
28
|
+
# A new key for one mutant's sentinel.
|
|
29
|
+
#
|
|
30
|
+
# @return [String]
|
|
31
|
+
def self.reach_key
|
|
32
|
+
"mutineer_r#{SecureRandom.hex(8)}"
|
|
33
|
+
end
|
|
34
|
+
# An empty sentinel file. The child writes `"1"` when the expression runs.
|
|
35
|
+
# The caller checks {reached?} and then {close_sentinel}.
|
|
36
|
+
#
|
|
37
|
+
# @return [Tempfile]
|
|
38
|
+
def self.sentinel
|
|
39
|
+
file = Tempfile.new(["mutineer-reach-", ".touch"])
|
|
40
|
+
file.close
|
|
41
|
+
file
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Whether a test wrote the sentinel after {forget_load}. False when there
|
|
45
|
+
# is no sentinel, or the expression did not return during a test.
|
|
46
|
+
#
|
|
47
|
+
# @param sentinel [Tempfile, nil]
|
|
48
|
+
# @return [Boolean]
|
|
49
|
+
def self.reached?(sentinel)
|
|
50
|
+
!sentinel.nil? && File.size(sentinel.path).positive?
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Empties the sentinel after the mutant has loaded and before its tests
|
|
54
|
+
# run. A reload runs top-level calls, and those can reach the expression
|
|
55
|
+
# before any test does. That load is not a test reaching it (#345). The
|
|
56
|
+
# daemon does the same clear itself, because this file pulls in Prism and
|
|
57
|
+
# the daemon cannot.
|
|
58
|
+
#
|
|
59
|
+
# @param path [String, nil] sentinel path.
|
|
60
|
+
# @param key [String, nil] fiber-local key from {reach_key}.
|
|
61
|
+
# @return [void]
|
|
62
|
+
def self.forget_load(path, key)
|
|
63
|
+
return unless path
|
|
64
|
+
|
|
65
|
+
Thread.current[key.to_sym] = nil if key.to_s.match?(KEY_FORMAT)
|
|
66
|
+
File.write(path, "")
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Removes the sentinel file. A missing file is already gone.
|
|
70
|
+
#
|
|
71
|
+
# @param sentinel [Tempfile, nil]
|
|
72
|
+
# @return [void]
|
|
73
|
+
def self.close_sentinel(sentinel)
|
|
74
|
+
return unless sentinel
|
|
75
|
+
|
|
76
|
+
sentinel.close unless sentinel.closed?
|
|
77
|
+
sentinel.unlink
|
|
78
|
+
rescue Errno::ENOENT
|
|
79
|
+
nil
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# The mutated source with a probe around the expression that holds the
|
|
83
|
+
# replacement, or `source` when `path` is nil. The original source when
|
|
84
|
+
# the expression cannot be wrapped without changing how it parses: the
|
|
85
|
+
# caller then has no evidence the expression ran, and a survivor stays
|
|
86
|
+
# `no_coverage`.
|
|
87
|
+
#
|
|
88
|
+
# @param source [String] mutated source.
|
|
89
|
+
# @param mutation [Mutineer::Mutation] the edit, already applied to `source`.
|
|
90
|
+
# @param path [String, nil] sentinel path, or nil to leave `source` alone.
|
|
91
|
+
# @param key [String, nil] fiber-local key from {reach_key}.
|
|
92
|
+
# @return [String]
|
|
93
|
+
def self.wrap_mutation(source, mutation, path, key)
|
|
94
|
+
return source if path.nil? || !key.to_s.match?(KEY_FORMAT)
|
|
95
|
+
|
|
96
|
+
start = mutation.start_offset
|
|
97
|
+
wrap(source, start, start + mutation.replacement.bytesize, path, key) || source
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Wraps the smallest expression that holds the byte range `[start_offset,
|
|
101
|
+
# end_offset)`. An empty range is the point where a deletion used to be.
|
|
102
|
+
# Returns nil when no expression holds it, or the wrapped text does not
|
|
103
|
+
# parse: a wrap that does not parse is not used.
|
|
104
|
+
#
|
|
105
|
+
# The wrapper returns the expression's value. It writes the sentinel only
|
|
106
|
+
# after that value has come back, so a raise inside the expression — an
|
|
107
|
+
# operand that fails before a mutated operator runs — does not count as a
|
|
108
|
+
# run. Later returns on the same thread do not touch the file.
|
|
109
|
+
#
|
|
110
|
+
# @param source [String]
|
|
111
|
+
# @param start_offset [Integer] byte offset of the replacement.
|
|
112
|
+
# @param end_offset [Integer] byte offset just after the replacement.
|
|
113
|
+
# @param path [String] absolute sentinel path.
|
|
114
|
+
# @param key [String] fiber-local key from {reach_key}.
|
|
115
|
+
# @return [String, nil]
|
|
116
|
+
def self.wrap(source, start_offset, end_offset, path, key)
|
|
117
|
+
return nil unless key.to_s.match?(KEY_FORMAT)
|
|
118
|
+
root = Parser.parse_string(source).value
|
|
119
|
+
node = smallest(root, start_offset, end_offset)
|
|
120
|
+
return nil unless node
|
|
121
|
+
|
|
122
|
+
parents = parent_map(root)
|
|
123
|
+
while node
|
|
124
|
+
wrapped = splice(source, node, path, key)
|
|
125
|
+
return wrapped unless Parser.invalid?(wrapped)
|
|
126
|
+
|
|
127
|
+
parent = parents[node]
|
|
128
|
+
break if parent.nil? || climb_stops?(parent, node)
|
|
129
|
+
|
|
130
|
+
node = parent
|
|
131
|
+
end
|
|
132
|
+
nil
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# True when wrapping `parent` would record a run of code that can run
|
|
136
|
+
# without `child`: a conditional child, or a node that is a whole def,
|
|
137
|
+
# class, or file.
|
|
138
|
+
#
|
|
139
|
+
# @param parent [Prism::Node]
|
|
140
|
+
# @param child [Prism::Node]
|
|
141
|
+
# @return [Boolean]
|
|
142
|
+
def self.climb_stops?(parent, child)
|
|
143
|
+
return true if CLIMB_STOP.any? { |klass| parent.is_a?(klass) }
|
|
144
|
+
|
|
145
|
+
StatementLines.sometimes?(parent, child)
|
|
146
|
+
end
|
|
147
|
+
private_class_method :climb_stops?
|
|
148
|
+
|
|
149
|
+
# Nodes that must not be wrapped. Wrapping one records every run of the
|
|
150
|
+
# method or file, not the mutated expression.
|
|
151
|
+
CLIMB_STOP = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::ProgramNode,
|
|
152
|
+
Prism::SingletonClassNode].freeze
|
|
153
|
+
|
|
154
|
+
# Parent of each node, by identity. A heredoc body is a child that lies
|
|
155
|
+
# outside its opener, so every child is recorded.
|
|
156
|
+
#
|
|
157
|
+
# @param root [Prism::Node]
|
|
158
|
+
# @return [Hash{Prism::Node => Prism::Node}]
|
|
159
|
+
def self.parent_map(root)
|
|
160
|
+
map = {}.compare_by_identity
|
|
161
|
+
walk = lambda do |node|
|
|
162
|
+
return unless node.is_a?(Prism::Node)
|
|
163
|
+
|
|
164
|
+
node.compact_child_nodes.each do |child|
|
|
165
|
+
map[child] = node
|
|
166
|
+
walk.call(child)
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
walk.call(root)
|
|
170
|
+
map
|
|
171
|
+
end
|
|
172
|
+
private_class_method :parent_map
|
|
173
|
+
|
|
174
|
+
# The deepest node whose location holds the range. A child replaces a
|
|
175
|
+
# parent of the same span, so a one-statement body wraps the statement
|
|
176
|
+
# and not the statement list around it. A heredoc body is a child that
|
|
177
|
+
# lies outside its opener, so every child is searched.
|
|
178
|
+
#
|
|
179
|
+
# @param node [Prism::Node, Object]
|
|
180
|
+
# @param start_offset [Integer]
|
|
181
|
+
# @param end_offset [Integer]
|
|
182
|
+
# @param best [Prism::Node, nil]
|
|
183
|
+
# @return [Prism::Node, nil]
|
|
184
|
+
def self.smallest(node, start_offset, end_offset, best = nil)
|
|
185
|
+
return best unless node.is_a?(Prism::Node)
|
|
186
|
+
|
|
187
|
+
loc = node.location
|
|
188
|
+
span = loc.end_offset - loc.start_offset
|
|
189
|
+
covers = if start_offset == end_offset
|
|
190
|
+
loc.start_offset <= start_offset && start_offset < loc.end_offset
|
|
191
|
+
else
|
|
192
|
+
span.positive? && loc.start_offset <= start_offset && end_offset <= loc.end_offset
|
|
193
|
+
end
|
|
194
|
+
best = node if covers && (best.nil? || span <= (best.location.end_offset - best.location.start_offset))
|
|
195
|
+
node.compact_child_nodes.each do |child|
|
|
196
|
+
best = smallest(child, start_offset, end_offset, best)
|
|
197
|
+
end
|
|
198
|
+
best
|
|
199
|
+
end
|
|
200
|
+
private_class_method :smallest
|
|
201
|
+
|
|
202
|
+
# Replaces `node` with an expression that writes `path` only when `node`
|
|
203
|
+
# returns. The mark is the second element of an array, so it does not run
|
|
204
|
+
# when `node` raises. A one-line node stays one line. A node that spans
|
|
205
|
+
# lines keeps its text and puts the array on the following lines, so the
|
|
206
|
+
# indent warning stays quiet.
|
|
207
|
+
#
|
|
208
|
+
# @param source [String]
|
|
209
|
+
# @param node [Prism::Node]
|
|
210
|
+
# @param path [String]
|
|
211
|
+
# @param key [String] fiber-local key from {reach_key}.
|
|
212
|
+
# @return [String]
|
|
213
|
+
def self.splice(source, node, path, key)
|
|
214
|
+
loc = node.location
|
|
215
|
+
inner = source.byteslice(loc.start_offset, loc.end_offset - loc.start_offset)
|
|
216
|
+
quoted = path.dump
|
|
217
|
+
# The fiber-local skips the write after the first return on this
|
|
218
|
+
# fiber. A stat on every later pass would hit a loop.
|
|
219
|
+
mark = "(Thread.current[:#{key}] ||= (::File.write(#{quoted}, \"1\"); true))"
|
|
220
|
+
replacement = if inner.include?("\n")
|
|
221
|
+
"\n[(#{inner}), #{mark}][0]"
|
|
222
|
+
else
|
|
223
|
+
"[(#{inner}), #{mark}][0]"
|
|
224
|
+
end
|
|
225
|
+
source.byteslice(0, loc.start_offset) + replacement + source.byteslice(loc.end_offset..)
|
|
226
|
+
end
|
|
227
|
+
private_class_method :splice
|
|
228
|
+
end
|
|
229
|
+
end
|
data/lib/mutineer/redaction.rb
CHANGED
|
@@ -8,7 +8,12 @@ module Mutineer
|
|
|
8
8
|
# a malformed one can quote the whole URL. Outside a URL a password form
|
|
9
9
|
# made only of word characters is hidden only as a whole token, so it cannot
|
|
10
10
|
# blank letters inside other words (#341). A password that exactly equals the
|
|
11
|
-
# username is left visible outside a URL,
|
|
11
|
+
# username, the host, or the database name is left visible outside a URL,
|
|
12
|
+
# because the redacted URL already shows that word (`role "postgres"`,
|
|
13
|
+
# `localhost`, `database "app"`). So is a one-character password, and a
|
|
14
|
+
# password that is one of {COMMON_WORDS}: those words are ordinary text in
|
|
15
|
+
# errors and paths. The password inside a URL, and the value of a
|
|
16
|
+
# `password=` query parameter, stays hidden in every one of those cases.
|
|
12
17
|
#
|
|
13
18
|
# Stdlib only: the app-side daemon loads this file without the rest of
|
|
14
19
|
# mutineer.
|
|
@@ -20,10 +25,12 @@ module Mutineer
|
|
|
20
25
|
# such `@`, it ends at the last `@`, as before. A malformed password with
|
|
21
26
|
# an `@`, a `:`, a space, or a quote `inspect` escaped (`\"`) is hidden
|
|
22
27
|
# too, so when the two goals conflict, hiding wins: in
|
|
23
|
-
# `u:pw@h/x and y@h2/db` the text up to `h2` is hidden.
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
#
|
|
28
|
+
# `u:pw@h/x and y@h2/db` the text up to `h2` is hidden. The matcher
|
|
29
|
+
# contract, pinned by the redaction test, still shows the tail of one
|
|
30
|
+
# malformed password: it holds an `@`, a host, a `/`, `:`, `?` or `#`,
|
|
31
|
+
# and a space (`a@b/c d`), in a URL whose own host has no port, path or
|
|
32
|
+
# query, or has an `@` in them (`?n=ci@corp`). A DATABASE_URL password
|
|
33
|
+
# is still hidden by the prefix and blanket passes.
|
|
27
34
|
URL_PASSWORD = %r{
|
|
28
35
|
([a-z][a-z0-9+.\-]*://[^\s:/@"'\\]*):
|
|
29
36
|
(?:
|
|
@@ -50,47 +57,102 @@ module Mutineer
|
|
|
50
57
|
# (`mysql -ps3cret`, `["-ps3cret"]`). A form right after one is still a
|
|
51
58
|
# whole token.
|
|
52
59
|
ESCAPE = /%\h{2}|\\x\h{2}|\\u(?:\h{4}|\{\h+\})|\\[0-7]{1,3}|(?:\e|\\(?:e|033|x1[bB]|u001[bB]))?\[[0-9;]*[A-Za-z]|\\[a-z]|(?<![^\s"'(\[,=])-p/.source
|
|
60
|
+
# Passwords left visible as bare words (#341). Each one is also ordinary
|
|
61
|
+
# text in a backtrace, a log line, or a database error (`password
|
|
62
|
+
# authentication failed`, `/app/test/`). Hiding the word makes that line
|
|
63
|
+
# unreadable. Local and CI URLs often use one of these; a production
|
|
64
|
+
# password is usually not a dictionary word. The URL and `password=`
|
|
65
|
+
# passes still hide the value. Matched exactly, in this case: `Password`
|
|
66
|
+
# is still hidden.
|
|
67
|
+
COMMON_WORDS = %w[
|
|
68
|
+
password passwd secret test admin root guest user username
|
|
69
|
+
database localhost host port name token key login credential
|
|
70
|
+
postgres mysql redis default welcome
|
|
71
|
+
].freeze
|
|
72
|
+
# Where a `password=` query value ends in a message: the next query
|
|
73
|
+
# delimiter, or the quote or bracket that closed the URL.
|
|
74
|
+
QUERY_VALUE_END = /(?=&|#|\s|"|'|`|\)|\]|\}|,|\\|\.|\?|!|;|\z)/
|
|
53
75
|
|
|
54
76
|
module_function
|
|
55
77
|
|
|
56
|
-
#
|
|
78
|
+
# Extra DATABASE_URL values, besides `env` in {text}. The CLI and the
|
|
79
|
+
# daemon set this from dotenv files so a password the app loads itself
|
|
80
|
+
# is still hidden (#341). Empty unless a caller sets it.
|
|
81
|
+
#
|
|
82
|
+
# @return [Array<String>]
|
|
83
|
+
def extra_urls
|
|
84
|
+
@extra_urls || []
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Replace {extra_urls}. Blank values are dropped. Frozen, so a caller
|
|
88
|
+
# cannot change the list the redaction thread is reading.
|
|
89
|
+
#
|
|
90
|
+
# @param urls [Array<String>]
|
|
91
|
+
# @return [Array<String>]
|
|
92
|
+
def extra_urls=(urls)
|
|
93
|
+
@extra_urls = Array(urls).map { |url| utf8(url.to_s) }.uniq.reject(&:empty?).freeze
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Hide the password in any URL, and each DATABASE_URL password (from its
|
|
57
97
|
# `user:password@` pair and from its `password=` query parameter) wherever
|
|
58
98
|
# it appears: whole, percent-decoded, `inspect` escaped, or cut short after
|
|
59
|
-
# its `scheme://user:` prefix.
|
|
60
|
-
#
|
|
61
|
-
#
|
|
62
|
-
#
|
|
99
|
+
# its `scheme://user:` prefix. `env` supplies one URL. {extra_urls} and
|
|
100
|
+
# `urls` supply the rest (a dotenv file, or a URL the app set itself).
|
|
101
|
+
# Outside a URL a form of word characters, or of {SHORT_FORM} characters or
|
|
102
|
+
# fewer, is hidden only as a whole token; a longer form with another
|
|
103
|
+
# character is hidden anywhere. A raw password that equals the username,
|
|
104
|
+
# the host, or the database name, a form of one character, and a form in
|
|
105
|
+
# {COMMON_WORDS} are left visible outside a URL. The `password=` value
|
|
106
|
+
# itself is still replaced. Never raises: text in any encoding comes back
|
|
107
|
+
# as UTF-8, and an internal error returns {FAILED}.
|
|
63
108
|
#
|
|
64
109
|
# @param text [#to_s]
|
|
65
110
|
# @param env [Hash{String => String}] where DATABASE_URL is read.
|
|
111
|
+
# @param urls [Array<String>] more credential URLs, besides {extra_urls}.
|
|
66
112
|
# @return [String]
|
|
67
|
-
def text(text, env: ENV)
|
|
113
|
+
def text(text, env: ENV, urls: [])
|
|
68
114
|
out = utf8(text.to_s)
|
|
69
|
-
|
|
70
|
-
# Two readings of the userinfo: up to the last `@` (a malformed password
|
|
71
|
-
# may hold any character) and up to the last `@` before the path (a
|
|
72
|
-
# query value may hold an `@`). Both are hidden: hiding too much is safe.
|
|
73
|
-
parts = [database_url_parts(url), database_url_parts(authority(url))].compact.uniq
|
|
115
|
+
sources = source_urls(env, urls)
|
|
74
116
|
# Prefixes first, on the caller's text, so no replacement made here can
|
|
75
|
-
# be mistaken for part of a password.
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
117
|
+
# be mistaken for part of a password. Two readings of each userinfo: up
|
|
118
|
+
# to the last `@` (a malformed password may hold any character) and up
|
|
119
|
+
# to the last `@` before the path (a query value may hold an `@`). Both
|
|
120
|
+
# are hidden: hiding too much is safe.
|
|
121
|
+
sources.each do |url|
|
|
122
|
+
credential_parts(url).each do |head, raw|
|
|
123
|
+
forms = password_forms(raw)
|
|
124
|
+
[head, head.inspect[1...-1]].uniq.each do |prefix|
|
|
125
|
+
out = forms.reduce(out) { |acc, secret| redact_prefixes(acc, prefix, secret) }
|
|
126
|
+
end
|
|
80
127
|
end
|
|
81
128
|
end
|
|
82
129
|
out = out.gsub(URL_PASSWORD) { "#{Regexp.last_match(1)}:#{REDACTED}@" }
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
130
|
+
secrets = []
|
|
131
|
+
sources.each do |url|
|
|
132
|
+
host, database = url_labels(url).values_at(1, 2)
|
|
133
|
+
credential_parts(url).each do |head, raw|
|
|
134
|
+
# The username is not a secret: the redacted URL prints it. The skip
|
|
135
|
+
# needs an exact raw match. A password that only shares a form or a
|
|
136
|
+
# prefix with the username is a separate secret and stays hidden.
|
|
137
|
+
user = head[(head.index("://") + 3)...-1]
|
|
138
|
+
password_forms(raw).each { |form| out = redact_query_value(out, form) }
|
|
139
|
+
next if outside_url_visible?(raw, user: user, host: host, database: database)
|
|
140
|
+
|
|
141
|
+
secrets.concat(password_forms(raw))
|
|
142
|
+
end
|
|
143
|
+
# A `password=` value is always replaced in the parameter. Rails
|
|
144
|
+
# prefers it over the userinfo password. The bare word follows the
|
|
145
|
+
# same outside-URL rules as a userinfo password.
|
|
146
|
+
query_passwords(url).each do |raw|
|
|
147
|
+
password_forms(raw).each { |form| out = redact_query_value(out, form) }
|
|
148
|
+
next if outside_url_visible?(raw, user: nil, host: host, database: database)
|
|
149
|
+
|
|
150
|
+
secrets.concat(password_forms(raw))
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
secrets.uniq.sort_by { |form| -form.length }.reduce(out) do |acc, secret|
|
|
154
|
+
next acc unless blanket_form?(secret)
|
|
155
|
+
|
|
94
156
|
whole_token = secret.length <= SHORT_FORM || secret.match?(/\A#{WORD_CHAR}+\z/)
|
|
95
157
|
whole_token ? redact_word(acc, secret) : acc.gsub(secret, REDACTED)
|
|
96
158
|
end
|
|
@@ -98,6 +160,176 @@ module Mutineer
|
|
|
98
160
|
FAILED
|
|
99
161
|
end
|
|
100
162
|
|
|
163
|
+
# Every DATABASE_URL this call should treat as a credential. Blank values
|
|
164
|
+
# are dropped. The order does not matter: each password is applied.
|
|
165
|
+
#
|
|
166
|
+
# @param env [Hash{String => String}]
|
|
167
|
+
# @param urls [Array<String>]
|
|
168
|
+
# @return [Array<String>]
|
|
169
|
+
def source_urls(env, urls)
|
|
170
|
+
([env["DATABASE_URL"], *extra_urls, *urls]).map { |url| utf8(url.to_s) }.uniq.reject(&:empty?)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# The userinfo readings of `url`. See {text}.
|
|
174
|
+
#
|
|
175
|
+
# @param url [String]
|
|
176
|
+
# @return [Array<Array(String, String)>]
|
|
177
|
+
def credential_parts(url)
|
|
178
|
+
[database_url_parts(url), database_url_parts(authority(url))].compact.uniq
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Username, host, and database name of `url`, or nils when that part is
|
|
182
|
+
# missing. The host drops a trailing `:port`. The database name is the
|
|
183
|
+
# path before `?` or `#`. An `@` in the query or the path is not the
|
|
184
|
+
# userinfo separator: the host is read from the authority, the substring
|
|
185
|
+
# before the first `/`, `?`, or `#`.
|
|
186
|
+
#
|
|
187
|
+
# @param url [String]
|
|
188
|
+
# @return [Array(String, String, String)] username, host, and database name; a missing part is nil.
|
|
189
|
+
def url_labels(url)
|
|
190
|
+
start = url.index("://")
|
|
191
|
+
return [nil, nil, nil] unless start
|
|
192
|
+
|
|
193
|
+
body = url[(start + 3)..]
|
|
194
|
+
cut = body.index(%r{[/?#]})
|
|
195
|
+
authority = cut ? body[0...cut] : body
|
|
196
|
+
path = cut && body[cut] == "/" ? body[(cut + 1)..].split(%r{[?#]}, 2).first : nil
|
|
197
|
+
database = path unless path.nil? || path.empty?
|
|
198
|
+
at = authority.rindex("@")
|
|
199
|
+
if at
|
|
200
|
+
userinfo = authority[0...at]
|
|
201
|
+
hostport = authority[(at + 1)..]
|
|
202
|
+
colon = userinfo.index(":")
|
|
203
|
+
user = colon ? userinfo[0...colon] : userinfo
|
|
204
|
+
else
|
|
205
|
+
user = nil
|
|
206
|
+
hostport = authority
|
|
207
|
+
end
|
|
208
|
+
host = hostport.sub(/:\d+\z/, "")
|
|
209
|
+
host = nil if host.empty?
|
|
210
|
+
[user, host, database]
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Byte index where a password form that contains a newline is not safe to
|
|
214
|
+
# emit yet. That is a proper prefix of the form at the end of `buffer`, or
|
|
215
|
+
# a complete form with no newline after it. The index is the buffer length
|
|
216
|
+
# when nothing must be held. Empty when no form contains a newline.
|
|
217
|
+
#
|
|
218
|
+
# @param buffer [String]
|
|
219
|
+
# @return [Integer]
|
|
220
|
+
def newline_pending_index(buffer)
|
|
221
|
+
forms = newline_forms
|
|
222
|
+
return buffer.bytesize if forms.empty?
|
|
223
|
+
|
|
224
|
+
earliest = buffer.bytesize
|
|
225
|
+
forms.each do |form|
|
|
226
|
+
raw = form.b
|
|
227
|
+
limit = raw.bytesize - 1
|
|
228
|
+
size = [limit, buffer.bytesize].min
|
|
229
|
+
size.downto(1) do |n|
|
|
230
|
+
next unless buffer.byteslice(-n, n) == raw.byteslice(0, n)
|
|
231
|
+
|
|
232
|
+
start = buffer.bytesize - n
|
|
233
|
+
earliest = start if start < earliest
|
|
234
|
+
break
|
|
235
|
+
end
|
|
236
|
+
from = 0
|
|
237
|
+
while (idx = buffer.index(raw, from))
|
|
238
|
+
after = buffer.byteslice(idx + raw.bytesize, buffer.bytesize - idx - raw.bytesize)
|
|
239
|
+
earliest = idx if idx < earliest && !after&.include?("\n".b)
|
|
240
|
+
from = idx + 1
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
earliest
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# The prefix of `buffer` that can be redacted now, and the rest to keep.
|
|
247
|
+
# Unless `flush`, the prefix ends on a newline and does not cut a password
|
|
248
|
+
# that itself contains a newline. A complete form stays buffered until a
|
|
249
|
+
# newline follows it, or until `flush`.
|
|
250
|
+
#
|
|
251
|
+
# @param buffer [String]
|
|
252
|
+
# @param flush [Boolean] emit everything, including a partial tail.
|
|
253
|
+
# @return [Array(String, String)]
|
|
254
|
+
def take_redactable(buffer, flush:)
|
|
255
|
+
buffer = buffer.b
|
|
256
|
+
return [buffer, +"".b] if flush
|
|
257
|
+
|
|
258
|
+
stop = newline_pending_index(buffer)
|
|
259
|
+
usable = buffer.byteslice(0, stop) || +"".b
|
|
260
|
+
cut = usable.rindex("\n".b)
|
|
261
|
+
return [+"".b, buffer] unless cut
|
|
262
|
+
|
|
263
|
+
ready_end = cut + 1
|
|
264
|
+
rest = buffer.byteslice(ready_end, buffer.bytesize - ready_end) || +"".b
|
|
265
|
+
[buffer.byteslice(0, ready_end), rest]
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# Password forms that contain a newline. Cached for the current
|
|
269
|
+
# DATABASE_URL and {extra_urls}. Empty for an ordinary URL.
|
|
270
|
+
#
|
|
271
|
+
# @return [Array<String>]
|
|
272
|
+
def newline_forms
|
|
273
|
+
marker = ENV["DATABASE_URL"].to_s
|
|
274
|
+
urls = extra_urls
|
|
275
|
+
return @newline_forms if @newline_marker == marker && @newline_urls.equal?(urls)
|
|
276
|
+
|
|
277
|
+
@newline_marker = marker.dup.freeze
|
|
278
|
+
@newline_urls = urls
|
|
279
|
+
forms = []
|
|
280
|
+
source_urls(ENV, []).each do |url|
|
|
281
|
+
credential_parts(url).each do |_head, raw|
|
|
282
|
+
password_forms(raw).each { |form| forms << form if form.include?("\n") }
|
|
283
|
+
end
|
|
284
|
+
query_passwords(url).each do |raw|
|
|
285
|
+
password_forms(raw).each { |form| forms << form if form.include?("\n") }
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
@newline_forms = forms.uniq.freeze
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# True when `raw` must not be blanked outside a URL or a `password=`
|
|
292
|
+
# parameter (#341). Exact match only, same as the username rule.
|
|
293
|
+
#
|
|
294
|
+
# @param raw [String]
|
|
295
|
+
# @param user [String, nil]
|
|
296
|
+
# @param host [String, nil]
|
|
297
|
+
# @param database [String, nil]
|
|
298
|
+
# @return [Boolean]
|
|
299
|
+
def outside_url_visible?(raw, user:, host:, database:)
|
|
300
|
+
return true if raw.nil? || raw.empty? || raw.length <= 1
|
|
301
|
+
return true if COMMON_WORDS.include?(raw)
|
|
302
|
+
|
|
303
|
+
raw == user || raw == host || raw == database
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# True when this password form is hidden outside a URL. A one-character
|
|
307
|
+
# form and a {COMMON_WORDS} form are not: they are ordinary text.
|
|
308
|
+
#
|
|
309
|
+
# @param form [String]
|
|
310
|
+
# @return [Boolean]
|
|
311
|
+
def blanket_form?(form)
|
|
312
|
+
return false if form.nil? || form.empty? || form.length <= 1
|
|
313
|
+
return false if COMMON_WORDS.include?(form)
|
|
314
|
+
|
|
315
|
+
true
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
# Replace one `password=` query value and keep the parameter name. `form`
|
|
319
|
+
# must be the whole value, so a one-character password does not eat the
|
|
320
|
+
# start of a longer value.
|
|
321
|
+
#
|
|
322
|
+
# @param text [String]
|
|
323
|
+
# @param form [String]
|
|
324
|
+
# @return [String]
|
|
325
|
+
def redact_query_value(text, form)
|
|
326
|
+
return text if form.nil? || form.empty?
|
|
327
|
+
|
|
328
|
+
text.gsub(/([?&]password=)#{Regexp.escape(form)}#{QUERY_VALUE_END}/) do
|
|
329
|
+
"#{Regexp.last_match(1)}#{REDACTED}"
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
|
|
101
333
|
# A UTF-8 copy of `str` that every pattern here can match. Valid UTF-8 and
|
|
102
334
|
# ASCII-only text is returned as is. Binary text, or ASCII-compatible text
|
|
103
335
|
# whose bytes are invalid in its own encoding, is read as UTF-8 with bad
|