mutineer 1.8.1 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +733 -0
  3. data/README.md +85 -50
  4. data/lib/mutineer/changed_lines.rb +2 -1
  5. data/lib/mutineer/child_cause.rb +149 -0
  6. data/lib/mutineer/child_stdout.rb +12 -1
  7. data/lib/mutineer/cli.rb +540 -59
  8. data/lib/mutineer/command_redact.rb +38 -0
  9. data/lib/mutineer/concern_blocks.rb +95 -24
  10. data/lib/mutineer/config.rb +112 -6
  11. data/lib/mutineer/coverage_map.rb +1345 -156
  12. data/lib/mutineer/daemon_backend.rb +41 -18
  13. data/lib/mutineer/daemon_client.rb +104 -21
  14. data/lib/mutineer/daemon_server.rb +85 -14
  15. data/lib/mutineer/database_urls.rb +189 -0
  16. data/lib/mutineer/external_backend.rb +93 -6
  17. data/lib/mutineer/fs_path.rb +125 -0
  18. data/lib/mutineer/isolation.rb +380 -25
  19. data/lib/mutineer/job_plan.rb +352 -41
  20. data/lib/mutineer/minitest_integration/file_order.rb +56 -0
  21. data/lib/mutineer/minitest_integration/in_test_database_error.rb +243 -19
  22. data/lib/mutineer/minitest_integration.rb +26 -58
  23. data/lib/mutineer/mutant_id.rb +4 -1
  24. data/lib/mutineer/mutation.rb +10 -4
  25. data/lib/mutineer/mutator_registry.rb +6 -3
  26. data/lib/mutineer/mutators/collection_method.rb +195 -6
  27. data/lib/mutineer/mutators/literal_mutation.rb +33 -3
  28. data/lib/mutineer/mutators/range_literal.rb +78 -29
  29. data/lib/mutineer/mutators/rescue_body.rb +126 -0
  30. data/lib/mutineer/mutators/return_nil.rb +23 -6
  31. data/lib/mutineer/output_filter.rb +451 -0
  32. data/lib/mutineer/pairing.rb +463 -31
  33. data/lib/mutineer/parser.rb +53 -0
  34. data/lib/mutineer/project_path.rb +15 -8
  35. data/lib/mutineer/rails_worker_db.rb +366 -6
  36. data/lib/mutineer/reach_probe.rb +229 -0
  37. data/lib/mutineer/redaction.rb +301 -41
  38. data/lib/mutineer/reload_guards.rb +985 -0
  39. data/lib/mutineer/reporter.rb +467 -58
  40. data/lib/mutineer/result.rb +6 -7
  41. data/lib/mutineer/run_seed.rb +16 -0
  42. data/lib/mutineer/runner.rb +194 -55
  43. data/lib/mutineer/source_bytes.rb +187 -0
  44. data/lib/mutineer/statement_lines.rb +108 -14
  45. data/lib/mutineer/test_runners/rspec.rb +118 -10
  46. data/lib/mutineer/test_runners/rspec_seed.rb +151 -0
  47. data/lib/mutineer/test_runners.rb +3 -2
  48. data/lib/mutineer/triage.rb +55 -10
  49. data/lib/mutineer/version.rb +1 -1
  50. data/lib/mutineer.rb +1 -0
  51. data/skills/mutineer/SKILL.md +12 -4
  52. metadata +13 -1
@@ -0,0 +1,229 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+ require "tempfile"
5
+ require_relative "parser"
6
+ require_relative "statement_lines"
7
+
8
+ module Mutineer
9
+ # Records whether the expression that holds a mutation ran in the child.
10
+ #
11
+ # Ruby counts one line of a statement. A mutant on another line is selected
12
+ # as `:maybe` (#254, #289): the tests of the statement may not have reached
13
+ # it. A kill is still a kill. A survivor used to stay `no_coverage`, because
14
+ # the line count cannot tell "ran" from "did not run". The wrapper below
15
+ # writes a sentinel when that expression runs, so a survivor that ran counts
16
+ # as `survived` and one that did not stays `no_coverage` (#345).
17
+ #
18
+ # The inserted text is stdlib only (`File`). The daemon loads it as source,
19
+ # so the app process needs no Prism and no Mutineer constant.
20
+ module ReachProbe
21
+ # A fiber-local key for one sentinel: `mutineer_r` plus 16 hex digits.
22
+ # `Thread.current[]` is fiber-local, not one value for the whole thread.
23
+ # The generated source and {forget_load} use this key, not a fixed global,
24
+ # so an application variable of the same shape is not read or overwritten.
25
+ # The daemon checks the same pattern; it cannot load this file.
26
+ KEY_FORMAT = /\Amutineer_r[0-9a-f]{16}\z/
27
+
28
+ # A new key for one mutant's sentinel.
29
+ #
30
+ # @return [String]
31
+ def self.reach_key
32
+ "mutineer_r#{SecureRandom.hex(8)}"
33
+ end
34
+ # An empty sentinel file. The child writes `"1"` when the expression runs.
35
+ # The caller checks {reached?} and then {close_sentinel}.
36
+ #
37
+ # @return [Tempfile]
38
+ def self.sentinel
39
+ file = Tempfile.new(["mutineer-reach-", ".touch"])
40
+ file.close
41
+ file
42
+ end
43
+
44
+ # Whether a test wrote the sentinel after {forget_load}. False when there
45
+ # is no sentinel, or the expression did not return during a test.
46
+ #
47
+ # @param sentinel [Tempfile, nil]
48
+ # @return [Boolean]
49
+ def self.reached?(sentinel)
50
+ !sentinel.nil? && File.size(sentinel.path).positive?
51
+ end
52
+
53
+ # Empties the sentinel after the mutant has loaded and before its tests
54
+ # run. A reload runs top-level calls, and those can reach the expression
55
+ # before any test does. That load is not a test reaching it (#345). The
56
+ # daemon does the same clear itself, because this file pulls in Prism and
57
+ # the daemon cannot.
58
+ #
59
+ # @param path [String, nil] sentinel path.
60
+ # @param key [String, nil] fiber-local key from {reach_key}.
61
+ # @return [void]
62
+ def self.forget_load(path, key)
63
+ return unless path
64
+
65
+ Thread.current[key.to_sym] = nil if key.to_s.match?(KEY_FORMAT)
66
+ File.write(path, "")
67
+ end
68
+
69
+ # Removes the sentinel file. A missing file is already gone.
70
+ #
71
+ # @param sentinel [Tempfile, nil]
72
+ # @return [void]
73
+ def self.close_sentinel(sentinel)
74
+ return unless sentinel
75
+
76
+ sentinel.close unless sentinel.closed?
77
+ sentinel.unlink
78
+ rescue Errno::ENOENT
79
+ nil
80
+ end
81
+
82
+ # The mutated source with a probe around the expression that holds the
83
+ # replacement, or `source` when `path` is nil. The original source when
84
+ # the expression cannot be wrapped without changing how it parses: the
85
+ # caller then has no evidence the expression ran, and a survivor stays
86
+ # `no_coverage`.
87
+ #
88
+ # @param source [String] mutated source.
89
+ # @param mutation [Mutineer::Mutation] the edit, already applied to `source`.
90
+ # @param path [String, nil] sentinel path, or nil to leave `source` alone.
91
+ # @param key [String, nil] fiber-local key from {reach_key}.
92
+ # @return [String]
93
+ def self.wrap_mutation(source, mutation, path, key)
94
+ return source if path.nil? || !key.to_s.match?(KEY_FORMAT)
95
+
96
+ start = mutation.start_offset
97
+ wrap(source, start, start + mutation.replacement.bytesize, path, key) || source
98
+ end
99
+
100
+ # Wraps the smallest expression that holds the byte range `[start_offset,
101
+ # end_offset)`. An empty range is the point where a deletion used to be.
102
+ # Returns nil when no expression holds it, or the wrapped text does not
103
+ # parse: a wrap that does not parse is not used.
104
+ #
105
+ # The wrapper returns the expression's value. It writes the sentinel only
106
+ # after that value has come back, so a raise inside the expression — an
107
+ # operand that fails before a mutated operator runs — does not count as a
108
+ # run. Later returns on the same thread do not touch the file.
109
+ #
110
+ # @param source [String]
111
+ # @param start_offset [Integer] byte offset of the replacement.
112
+ # @param end_offset [Integer] byte offset just after the replacement.
113
+ # @param path [String] absolute sentinel path.
114
+ # @param key [String] fiber-local key from {reach_key}.
115
+ # @return [String, nil]
116
+ def self.wrap(source, start_offset, end_offset, path, key)
117
+ return nil unless key.to_s.match?(KEY_FORMAT)
118
+ root = Parser.parse_string(source).value
119
+ node = smallest(root, start_offset, end_offset)
120
+ return nil unless node
121
+
122
+ parents = parent_map(root)
123
+ while node
124
+ wrapped = splice(source, node, path, key)
125
+ return wrapped unless Parser.invalid?(wrapped)
126
+
127
+ parent = parents[node]
128
+ break if parent.nil? || climb_stops?(parent, node)
129
+
130
+ node = parent
131
+ end
132
+ nil
133
+ end
134
+
135
+ # True when wrapping `parent` would record a run of code that can run
136
+ # without `child`: a conditional child, or a node that is a whole def,
137
+ # class, or file.
138
+ #
139
+ # @param parent [Prism::Node]
140
+ # @param child [Prism::Node]
141
+ # @return [Boolean]
142
+ def self.climb_stops?(parent, child)
143
+ return true if CLIMB_STOP.any? { |klass| parent.is_a?(klass) }
144
+
145
+ StatementLines.sometimes?(parent, child)
146
+ end
147
+ private_class_method :climb_stops?
148
+
149
+ # Nodes that must not be wrapped. Wrapping one records every run of the
150
+ # method or file, not the mutated expression.
151
+ CLIMB_STOP = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::ProgramNode,
152
+ Prism::SingletonClassNode].freeze
153
+
154
+ # Parent of each node, by identity. A heredoc body is a child that lies
155
+ # outside its opener, so every child is recorded.
156
+ #
157
+ # @param root [Prism::Node]
158
+ # @return [Hash{Prism::Node => Prism::Node}]
159
+ def self.parent_map(root)
160
+ map = {}.compare_by_identity
161
+ walk = lambda do |node|
162
+ return unless node.is_a?(Prism::Node)
163
+
164
+ node.compact_child_nodes.each do |child|
165
+ map[child] = node
166
+ walk.call(child)
167
+ end
168
+ end
169
+ walk.call(root)
170
+ map
171
+ end
172
+ private_class_method :parent_map
173
+
174
+ # The deepest node whose location holds the range. A child replaces a
175
+ # parent of the same span, so a one-statement body wraps the statement
176
+ # and not the statement list around it. A heredoc body is a child that
177
+ # lies outside its opener, so every child is searched.
178
+ #
179
+ # @param node [Prism::Node, Object]
180
+ # @param start_offset [Integer]
181
+ # @param end_offset [Integer]
182
+ # @param best [Prism::Node, nil]
183
+ # @return [Prism::Node, nil]
184
+ def self.smallest(node, start_offset, end_offset, best = nil)
185
+ return best unless node.is_a?(Prism::Node)
186
+
187
+ loc = node.location
188
+ span = loc.end_offset - loc.start_offset
189
+ covers = if start_offset == end_offset
190
+ loc.start_offset <= start_offset && start_offset < loc.end_offset
191
+ else
192
+ span.positive? && loc.start_offset <= start_offset && end_offset <= loc.end_offset
193
+ end
194
+ best = node if covers && (best.nil? || span <= (best.location.end_offset - best.location.start_offset))
195
+ node.compact_child_nodes.each do |child|
196
+ best = smallest(child, start_offset, end_offset, best)
197
+ end
198
+ best
199
+ end
200
+ private_class_method :smallest
201
+
202
+ # Replaces `node` with an expression that writes `path` only when `node`
203
+ # returns. The mark is the second element of an array, so it does not run
204
+ # when `node` raises. A one-line node stays one line. A node that spans
205
+ # lines keeps its text and puts the array on the following lines, so the
206
+ # indent warning stays quiet.
207
+ #
208
+ # @param source [String]
209
+ # @param node [Prism::Node]
210
+ # @param path [String]
211
+ # @param key [String] fiber-local key from {reach_key}.
212
+ # @return [String]
213
+ def self.splice(source, node, path, key)
214
+ loc = node.location
215
+ inner = source.byteslice(loc.start_offset, loc.end_offset - loc.start_offset)
216
+ quoted = path.dump
217
+ # The fiber-local skips the write after the first return on this
218
+ # fiber. A stat on every later pass would hit a loop.
219
+ mark = "(Thread.current[:#{key}] ||= (::File.write(#{quoted}, \"1\"); true))"
220
+ replacement = if inner.include?("\n")
221
+ "\n[(#{inner}), #{mark}][0]"
222
+ else
223
+ "[(#{inner}), #{mark}][0]"
224
+ end
225
+ source.byteslice(0, loc.start_offset) + replacement + source.byteslice(loc.end_offset..)
226
+ end
227
+ private_class_method :splice
228
+ end
229
+ end
@@ -5,18 +5,39 @@ require "uri"
5
5
  module Mutineer
6
6
  # Hides database credentials in text bound for a terminal: an error reply,
7
7
  # a log line, a warning. Rails parses DATABASE_URL itself, so an error about
8
- # a malformed one can quote the whole URL. Outside a URL a password of 3
9
- # characters or fewer is hidden only as a whole token, so it cannot blank
10
- # letters inside other words. A password that exactly equals the username is
11
- # left visible outside a URL, so `role "postgres"` stays readable.
8
+ # a malformed one can quote the whole URL. Outside a URL a password form
9
+ # made only of word characters is hidden only as a whole token, so it cannot
10
+ # blank letters inside other words (#341). A password that exactly equals the
11
+ # username, the host, or the database name is left visible outside a URL,
12
+ # because the redacted URL already shows that word (`role "postgres"`,
13
+ # `localhost`, `database "app"`). So is a one-character password, and a
14
+ # password that is one of {COMMON_WORDS}: those words are ordinary text in
15
+ # errors and paths. The password inside a URL, and the value of a
16
+ # `password=` query parameter, stays hidden in every one of those cases.
12
17
  #
13
18
  # Stdlib only: the app-side daemon loads this file without the rest of
14
19
  # mutineer.
15
20
  module Redaction
16
- # The password in a `scheme://user:password@host` URL. Greedy up to the last
17
- # `@` before an unescaped quote or line end, so a malformed password with a
18
- # space, an unescaped `@`, or a quote `inspect` escaped (`\"`) is hidden too.
19
- URL_PASSWORD = %r{([a-z][a-z0-9+.\-]*://[^\s:/@"'\\]*):(?:\\.|[^"'\r\n\\])*@}i
21
+ # The password in a `scheme://user:password@host` URL. It ends at the last
22
+ # `@` before an unescaped quote or line end that starts a host: a run with
23
+ # no `@`, then a `/`, `:`, `?` or `#`, and no other `@` before the next
24
+ # space. So in `u:p@ss@h/db and p@ss` only `p@ss` is hidden (#341). With no
25
+ # such `@`, it ends at the last `@`, as before. A malformed password with
26
+ # an `@`, a `:`, a space, or a quote `inspect` escaped (`\"`) is hidden
27
+ # too, so when the two goals conflict, hiding wins: in
28
+ # `u:pw@h/x and y@h2/db` the text up to `h2` is hidden. The matcher
29
+ # contract, pinned by the redaction test, still shows the tail of one
30
+ # malformed password: it holds an `@`, a host, a `/`, `:`, `?` or `#`,
31
+ # and a space (`a@b/c d`), in a URL whose own host has no port, path or
32
+ # query, or has an `@` in them (`?n=ci@corp`). A DATABASE_URL password
33
+ # is still hidden by the prefix and blanket passes.
34
+ URL_PASSWORD = %r{
35
+ ([a-z][a-z0-9+.\-]*://[^\s:/@"'\\]*):
36
+ (?:
37
+ (?:\\.|[^"'\r\n\\])*@(?=[^@\s"'\\/:?\#]+[/:?\#][^@\s]*(?:\s|\z))
38
+ | (?:\\.|[^"'\r\n\\])*@
39
+ )
40
+ }xi
20
41
  # What replaces a hidden password.
21
42
  REDACTED = "[REDACTED]"
22
43
  # What {text} returns when it fails: never the input, which may hold the
@@ -24,56 +45,291 @@ module Mutineer
24
45
  FAILED = "[message hidden: redaction failed]"
25
46
  # An ASCII word character, the edge a whole-token match must not touch.
26
47
  WORD_CHAR = "[A-Za-z0-9_]"
27
- # The longest password form hidden only as a whole token. A longer form is
28
- # hidden wherever it appears, even glued to an escape such as `%40` or
29
- # `\x41`: failing toward hiding too much is safer for a long secret.
48
+ # The longest password form hidden only as a whole token whatever its
49
+ # characters. A longer form with a non-word character is hidden wherever
50
+ # it appears, because its edges are no reliable token boundary.
30
51
  SHORT_FORM = 3
52
+ # An escape or prefix that glues a word character to the left of a
53
+ # password form in a message: `%40`, `\x41`, `\u0041`, `\u{41}`, `\101`,
54
+ # `\n`, an ANSI code such as the colour Rails logs put on a span (raw,
55
+ # escaped as `\e`, `\033`, `\x1b` or `\u001b`, or with its ESC byte
56
+ # stripped), or a `-p` flag after a space, quote, bracket, comma or `=`
57
+ # (`mysql -ps3cret`, `["-ps3cret"]`). A form right after one is still a
58
+ # whole token.
59
+ ESCAPE = /%\h{2}|\\x\h{2}|\\u(?:\h{4}|\{\h+\})|\\[0-7]{1,3}|(?:\e|\\(?:e|033|x1[bB]|u001[bB]))?\[[0-9;]*[A-Za-z]|\\[a-z]|(?<![^\s"'(\[,=])-p/.source
60
+ # Passwords left visible as bare words (#341). Each one is also ordinary
61
+ # text in a backtrace, a log line, or a database error (`password
62
+ # authentication failed`, `/app/test/`). Hiding the word makes that line
63
+ # unreadable. Local and CI URLs often use one of these; a production
64
+ # password is usually not a dictionary word. The URL and `password=`
65
+ # passes still hide the value. Matched exactly, in this case: `Password`
66
+ # is still hidden.
67
+ COMMON_WORDS = %w[
68
+ password passwd secret test admin root guest user username
69
+ database localhost host port name token key login credential
70
+ postgres mysql redis default welcome
71
+ ].freeze
72
+ # Where a `password=` query value ends in a message: the next query
73
+ # delimiter, or the quote or bracket that closed the URL.
74
+ QUERY_VALUE_END = /(?=&|#|\s|"|'|`|\)|\]|\}|,|\\|\.|\?|!|;|\z)/
31
75
 
32
76
  module_function
33
77
 
34
- # Hide the password in any URL, and the DATABASE_URL password (from its
78
+ # Extra DATABASE_URL values, besides `env` in {text}. The CLI and the
79
+ # daemon set this from dotenv files so a password the app loads itself
80
+ # is still hidden (#341). Empty unless a caller sets it.
81
+ #
82
+ # @return [Array<String>]
83
+ def extra_urls
84
+ @extra_urls || []
85
+ end
86
+
87
+ # Replace {extra_urls}. Blank values are dropped. Frozen, so a caller
88
+ # cannot change the list the redaction thread is reading.
89
+ #
90
+ # @param urls [Array<String>]
91
+ # @return [Array<String>]
92
+ def extra_urls=(urls)
93
+ @extra_urls = Array(urls).map { |url| utf8(url.to_s) }.uniq.reject(&:empty?).freeze
94
+ end
95
+
96
+ # Hide the password in any URL, and each DATABASE_URL password (from its
35
97
  # `user:password@` pair and from its `password=` query parameter) wherever
36
98
  # it appears: whole, percent-decoded, `inspect` escaped, or cut short after
37
- # its `scheme://user:` prefix. Outside a URL a form of {SHORT_FORM}
38
- # characters or fewer is hidden only as a whole token. Never raises: text in
39
- # any encoding comes back as UTF-8, and an internal error returns {FAILED}.
99
+ # its `scheme://user:` prefix. `env` supplies one URL. {extra_urls} and
100
+ # `urls` supply the rest (a dotenv file, or a URL the app set itself).
101
+ # Outside a URL a form of word characters, or of {SHORT_FORM} characters or
102
+ # fewer, is hidden only as a whole token; a longer form with another
103
+ # character is hidden anywhere. A raw password that equals the username,
104
+ # the host, or the database name, a form of one character, and a form in
105
+ # {COMMON_WORDS} are left visible outside a URL. The `password=` value
106
+ # itself is still replaced. Never raises: text in any encoding comes back
107
+ # as UTF-8, and an internal error returns {FAILED}.
40
108
  #
41
109
  # @param text [#to_s]
42
110
  # @param env [Hash{String => String}] where DATABASE_URL is read.
111
+ # @param urls [Array<String>] more credential URLs, besides {extra_urls}.
43
112
  # @return [String]
44
- def text(text, env: ENV)
113
+ def text(text, env: ENV, urls: [])
45
114
  out = utf8(text.to_s)
46
- url = utf8(env["DATABASE_URL"].to_s)
47
- # Two readings of the userinfo: up to the last `@` (a malformed password
48
- # may hold any character) and up to the last `@` before the path (a
49
- # query value may hold an `@`). Both are hidden: hiding too much is safe.
50
- parts = [database_url_parts(url), database_url_parts(authority(url))].compact.uniq
115
+ sources = source_urls(env, urls)
51
116
  # Prefixes first, on the caller's text, so no replacement made here can
52
- # be mistaken for part of a password.
53
- parts.each do |head, raw|
54
- forms = password_forms(raw)
55
- [head, head.inspect[1...-1]].uniq.each do |prefix|
56
- out = forms.reduce(out) { |acc, secret| redact_prefixes(acc, prefix, secret) }
117
+ # be mistaken for part of a password. Two readings of each userinfo: up
118
+ # to the last `@` (a malformed password may hold any character) and up
119
+ # to the last `@` before the path (a query value may hold an `@`). Both
120
+ # are hidden: hiding too much is safe.
121
+ sources.each do |url|
122
+ credential_parts(url).each do |head, raw|
123
+ forms = password_forms(raw)
124
+ [head, head.inspect[1...-1]].uniq.each do |prefix|
125
+ out = forms.reduce(out) { |acc, secret| redact_prefixes(acc, prefix, secret) }
126
+ end
57
127
  end
58
128
  end
59
129
  out = out.gsub(URL_PASSWORD) { "#{Regexp.last_match(1)}:#{REDACTED}@" }
60
- # A password that exactly equals the username is left visible outside a
61
- # URL. The username is not a secret: mutineer prints it in every
62
- # redacted URL, so the value is already shown, and blanking it would
63
- # hide every role and database name in a `postgres:postgres` setup. The
64
- # skip needs an exact raw match: a password that only shares a form or a
65
- # prefix with the username is a separate secret and stays hidden. A
66
- # `password=` query parameter, which Rails prefers, is always hidden.
67
- raws = parts.reject { |head, raw| raw == head[(head.index("://") + 3)...-1] }.map(&:last)
68
- raws.concat(query_passwords(url))
69
- secrets = raws.compact.uniq.flat_map { |raw| password_forms(raw) }.uniq.sort_by { |form| -form.length }
70
- secrets.reduce(out) do |acc, secret|
71
- secret.length > SHORT_FORM ? acc.gsub(secret, REDACTED) : redact_word(acc, secret)
130
+ secrets = []
131
+ sources.each do |url|
132
+ host, database = url_labels(url).values_at(1, 2)
133
+ credential_parts(url).each do |head, raw|
134
+ # The username is not a secret: the redacted URL prints it. The skip
135
+ # needs an exact raw match. A password that only shares a form or a
136
+ # prefix with the username is a separate secret and stays hidden.
137
+ user = head[(head.index("://") + 3)...-1]
138
+ password_forms(raw).each { |form| out = redact_query_value(out, form) }
139
+ next if outside_url_visible?(raw, user: user, host: host, database: database)
140
+
141
+ secrets.concat(password_forms(raw))
142
+ end
143
+ # A `password=` value is always replaced in the parameter. Rails
144
+ # prefers it over the userinfo password. The bare word follows the
145
+ # same outside-URL rules as a userinfo password.
146
+ query_passwords(url).each do |raw|
147
+ password_forms(raw).each { |form| out = redact_query_value(out, form) }
148
+ next if outside_url_visible?(raw, user: nil, host: host, database: database)
149
+
150
+ secrets.concat(password_forms(raw))
151
+ end
152
+ end
153
+ secrets.uniq.sort_by { |form| -form.length }.reduce(out) do |acc, secret|
154
+ next acc unless blanket_form?(secret)
155
+
156
+ whole_token = secret.length <= SHORT_FORM || secret.match?(/\A#{WORD_CHAR}+\z/)
157
+ whole_token ? redact_word(acc, secret) : acc.gsub(secret, REDACTED)
72
158
  end
73
159
  rescue StandardError
74
160
  FAILED
75
161
  end
76
162
 
163
+ # Every DATABASE_URL this call should treat as a credential. Blank values
164
+ # are dropped. The order does not matter: each password is applied.
165
+ #
166
+ # @param env [Hash{String => String}]
167
+ # @param urls [Array<String>]
168
+ # @return [Array<String>]
169
+ def source_urls(env, urls)
170
+ ([env["DATABASE_URL"], *extra_urls, *urls]).map { |url| utf8(url.to_s) }.uniq.reject(&:empty?)
171
+ end
172
+
173
+ # The userinfo readings of `url`. See {text}.
174
+ #
175
+ # @param url [String]
176
+ # @return [Array<Array(String, String)>]
177
+ def credential_parts(url)
178
+ [database_url_parts(url), database_url_parts(authority(url))].compact.uniq
179
+ end
180
+
181
+ # Username, host, and database name of `url`, or nils when that part is
182
+ # missing. The host drops a trailing `:port`. The database name is the
183
+ # path before `?` or `#`. An `@` in the query or the path is not the
184
+ # userinfo separator: the host is read from the authority, the substring
185
+ # before the first `/`, `?`, or `#`.
186
+ #
187
+ # @param url [String]
188
+ # @return [Array(String, String, String)] username, host, and database name; a missing part is nil.
189
+ def url_labels(url)
190
+ start = url.index("://")
191
+ return [nil, nil, nil] unless start
192
+
193
+ body = url[(start + 3)..]
194
+ cut = body.index(%r{[/?#]})
195
+ authority = cut ? body[0...cut] : body
196
+ path = cut && body[cut] == "/" ? body[(cut + 1)..].split(%r{[?#]}, 2).first : nil
197
+ database = path unless path.nil? || path.empty?
198
+ at = authority.rindex("@")
199
+ if at
200
+ userinfo = authority[0...at]
201
+ hostport = authority[(at + 1)..]
202
+ colon = userinfo.index(":")
203
+ user = colon ? userinfo[0...colon] : userinfo
204
+ else
205
+ user = nil
206
+ hostport = authority
207
+ end
208
+ host = hostport.sub(/:\d+\z/, "")
209
+ host = nil if host.empty?
210
+ [user, host, database]
211
+ end
212
+
213
+ # Byte index where a password form that contains a newline is not safe to
214
+ # emit yet. That is a proper prefix of the form at the end of `buffer`, or
215
+ # a complete form with no newline after it. The index is the buffer length
216
+ # when nothing must be held. Empty when no form contains a newline.
217
+ #
218
+ # @param buffer [String]
219
+ # @return [Integer]
220
+ def newline_pending_index(buffer)
221
+ forms = newline_forms
222
+ return buffer.bytesize if forms.empty?
223
+
224
+ earliest = buffer.bytesize
225
+ forms.each do |form|
226
+ raw = form.b
227
+ limit = raw.bytesize - 1
228
+ size = [limit, buffer.bytesize].min
229
+ size.downto(1) do |n|
230
+ next unless buffer.byteslice(-n, n) == raw.byteslice(0, n)
231
+
232
+ start = buffer.bytesize - n
233
+ earliest = start if start < earliest
234
+ break
235
+ end
236
+ from = 0
237
+ while (idx = buffer.index(raw, from))
238
+ after = buffer.byteslice(idx + raw.bytesize, buffer.bytesize - idx - raw.bytesize)
239
+ earliest = idx if idx < earliest && !after&.include?("\n".b)
240
+ from = idx + 1
241
+ end
242
+ end
243
+ earliest
244
+ end
245
+
246
+ # The prefix of `buffer` that can be redacted now, and the rest to keep.
247
+ # Unless `flush`, the prefix ends on a newline and does not cut a password
248
+ # that itself contains a newline. A complete form stays buffered until a
249
+ # newline follows it, or until `flush`.
250
+ #
251
+ # @param buffer [String]
252
+ # @param flush [Boolean] emit everything, including a partial tail.
253
+ # @return [Array(String, String)]
254
+ def take_redactable(buffer, flush:)
255
+ buffer = buffer.b
256
+ return [buffer, +"".b] if flush
257
+
258
+ stop = newline_pending_index(buffer)
259
+ usable = buffer.byteslice(0, stop) || +"".b
260
+ cut = usable.rindex("\n".b)
261
+ return [+"".b, buffer] unless cut
262
+
263
+ ready_end = cut + 1
264
+ rest = buffer.byteslice(ready_end, buffer.bytesize - ready_end) || +"".b
265
+ [buffer.byteslice(0, ready_end), rest]
266
+ end
267
+
268
+ # Password forms that contain a newline. Cached for the current
269
+ # DATABASE_URL and {extra_urls}. Empty for an ordinary URL.
270
+ #
271
+ # @return [Array<String>]
272
+ def newline_forms
273
+ marker = ENV["DATABASE_URL"].to_s
274
+ urls = extra_urls
275
+ return @newline_forms if @newline_marker == marker && @newline_urls.equal?(urls)
276
+
277
+ @newline_marker = marker.dup.freeze
278
+ @newline_urls = urls
279
+ forms = []
280
+ source_urls(ENV, []).each do |url|
281
+ credential_parts(url).each do |_head, raw|
282
+ password_forms(raw).each { |form| forms << form if form.include?("\n") }
283
+ end
284
+ query_passwords(url).each do |raw|
285
+ password_forms(raw).each { |form| forms << form if form.include?("\n") }
286
+ end
287
+ end
288
+ @newline_forms = forms.uniq.freeze
289
+ end
290
+
291
+ # True when `raw` must not be blanked outside a URL or a `password=`
292
+ # parameter (#341). Exact match only, same as the username rule.
293
+ #
294
+ # @param raw [String]
295
+ # @param user [String, nil]
296
+ # @param host [String, nil]
297
+ # @param database [String, nil]
298
+ # @return [Boolean]
299
+ def outside_url_visible?(raw, user:, host:, database:)
300
+ return true if raw.nil? || raw.empty? || raw.length <= 1
301
+ return true if COMMON_WORDS.include?(raw)
302
+
303
+ raw == user || raw == host || raw == database
304
+ end
305
+
306
+ # True when this password form is hidden outside a URL. A one-character
307
+ # form and a {COMMON_WORDS} form are not: they are ordinary text.
308
+ #
309
+ # @param form [String]
310
+ # @return [Boolean]
311
+ def blanket_form?(form)
312
+ return false if form.nil? || form.empty? || form.length <= 1
313
+ return false if COMMON_WORDS.include?(form)
314
+
315
+ true
316
+ end
317
+
318
+ # Replace one `password=` query value and keep the parameter name. `form`
319
+ # must be the whole value, so a one-character password does not eat the
320
+ # start of a longer value.
321
+ #
322
+ # @param text [String]
323
+ # @param form [String]
324
+ # @return [String]
325
+ def redact_query_value(text, form)
326
+ return text if form.nil? || form.empty?
327
+
328
+ text.gsub(/([?&]password=)#{Regexp.escape(form)}#{QUERY_VALUE_END}/) do
329
+ "#{Regexp.last_match(1)}#{REDACTED}"
330
+ end
331
+ end
332
+
77
333
  # A UTF-8 copy of `str` that every pattern here can match. Valid UTF-8 and
78
334
  # ASCII-only text is returned as is. Binary text, or ASCII-compatible text
79
335
  # whose bytes are invalid in its own encoding, is read as UTF-8 with bad
@@ -93,14 +349,18 @@ module Mutineer
93
349
  end
94
350
 
95
351
  # Replace `form` in `text` only where it is a whole token: no ASCII word
96
- # character touches either edge, including a punctuation edge.
352
+ # character touches either edge, including a punctuation edge. A form
353
+ # right after an {ESCAPE} also counts as a whole token, and the escape is
354
+ # kept. An escape may not start where a copy of the form starts, so it
355
+ # cannot swallow that copy (`%41%41` with password `%41`).
97
356
  #
98
357
  # @param text [String]
99
358
  # @param form [String] one form of the password.
100
359
  # @return [String]
101
360
  def redact_word(text, form)
102
- pattern = "(?<!#{WORD_CHAR})#{Regexp.escape(form)}(?!#{WORD_CHAR})"
103
- text.gsub(Regexp.new(pattern), REDACTED)
361
+ quoted = Regexp.escape(form)
362
+ pattern = "((?!#{quoted})(?:#{ESCAPE})|(?<!#{WORD_CHAR}))#{quoted}(?!#{WORD_CHAR})"
363
+ text.gsub(Regexp.new(pattern)) { "#{Regexp.last_match(1)}#{REDACTED}" }
104
364
  end
105
365
 
106
366
  # An exception as `Class: message`, redacted. Each part is redacted on its