mutineer 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +680 -0
- data/README.md +210 -499
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/child_coverage.rb +53 -0
- data/lib/mutineer/cli.rb +293 -42
- data/lib/mutineer/concern_blocks.rb +243 -0
- data/lib/mutineer/config.rb +154 -8
- data/lib/mutineer/coverage_map.rb +265 -27
- data/lib/mutineer/daemon_backend.rb +244 -75
- data/lib/mutineer/daemon_client.rb +73 -10
- data/lib/mutineer/daemon_server.rb +226 -42
- data/lib/mutineer/external_backend.rb +159 -15
- data/lib/mutineer/init.rb +124 -0
- data/lib/mutineer/isolation.rb +78 -24
- data/lib/mutineer/job_plan.rb +234 -44
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +707 -0
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutator_registry.rb +6 -2
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +36 -7
- data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +84 -0
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +302 -14
- data/lib/mutineer/rails_run_env.rb +73 -0
- data/lib/mutineer/rails_worker_db.rb +1596 -57
- data/lib/mutineer/redaction.rb +199 -0
- data/lib/mutineer/reporter.rb +173 -31
- data/lib/mutineer/result.rb +52 -15
- data/lib/mutineer/runner.rb +148 -28
- data/lib/mutineer/statement_lines.rb +24 -11
- data/lib/mutineer/subject.rb +30 -4
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners/rspec.rb +15 -3
- data/lib/mutineer/test_runners.rb +5 -2
- data/lib/mutineer/triage.rb +111 -0
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/lib/mutineer.rb +4 -0
- data/skills/mutineer/SKILL.md +84 -0
- metadata +13 -1
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
5
|
+
module Mutineer
|
|
6
|
+
# Hides database credentials in text bound for a terminal: an error reply,
|
|
7
|
+
# a log line, a warning. Rails parses DATABASE_URL itself, so an error about
|
|
8
|
+
# a malformed one can quote the whole URL. Outside a URL a password of 3
|
|
9
|
+
# characters or fewer is hidden only as a whole token, so it cannot blank
|
|
10
|
+
# letters inside other words. A password that exactly equals the username is
|
|
11
|
+
# left visible outside a URL, so `role "postgres"` stays readable.
|
|
12
|
+
#
|
|
13
|
+
# Stdlib only: the app-side daemon loads this file without the rest of
|
|
14
|
+
# mutineer.
|
|
15
|
+
module Redaction
|
|
16
|
+
# The password in a `scheme://user:password@host` URL. Greedy up to the last
|
|
17
|
+
# `@` before an unescaped quote or line end, so a malformed password with a
|
|
18
|
+
# space, an unescaped `@`, or a quote `inspect` escaped (`\"`) is hidden too.
|
|
19
|
+
URL_PASSWORD = %r{([a-z][a-z0-9+.\-]*://[^\s:/@"'\\]*):(?:\\.|[^"'\r\n\\])*@}i
|
|
20
|
+
# What replaces a hidden password.
|
|
21
|
+
REDACTED = "[REDACTED]"
|
|
22
|
+
# What {text} returns when it fails: never the input, which may hold the
|
|
23
|
+
# password.
|
|
24
|
+
FAILED = "[message hidden: redaction failed]"
|
|
25
|
+
# An ASCII word character, the edge a whole-token match must not touch.
|
|
26
|
+
WORD_CHAR = "[A-Za-z0-9_]"
|
|
27
|
+
# The longest password form hidden only as a whole token. A longer form is
|
|
28
|
+
# hidden wherever it appears, even glued to an escape such as `%40` or
|
|
29
|
+
# `\x41`: failing toward hiding too much is safer for a long secret.
|
|
30
|
+
SHORT_FORM = 3
|
|
31
|
+
|
|
32
|
+
module_function
|
|
33
|
+
|
|
34
|
+
# Hide the password in any URL, and the DATABASE_URL password (from its
|
|
35
|
+
# `user:password@` pair and from its `password=` query parameter) wherever
|
|
36
|
+
# it appears: whole, percent-decoded, `inspect` escaped, or cut short after
|
|
37
|
+
# its `scheme://user:` prefix. Outside a URL a form of {SHORT_FORM}
|
|
38
|
+
# characters or fewer is hidden only as a whole token. Never raises: text in
|
|
39
|
+
# any encoding comes back as UTF-8, and an internal error returns {FAILED}.
|
|
40
|
+
#
|
|
41
|
+
# @param text [#to_s]
|
|
42
|
+
# @param env [Hash{String => String}] where DATABASE_URL is read.
|
|
43
|
+
# @return [String]
|
|
44
|
+
def text(text, env: ENV)
|
|
45
|
+
out = utf8(text.to_s)
|
|
46
|
+
url = utf8(env["DATABASE_URL"].to_s)
|
|
47
|
+
# Two readings of the userinfo: up to the last `@` (a malformed password
|
|
48
|
+
# may hold any character) and up to the last `@` before the path (a
|
|
49
|
+
# query value may hold an `@`). Both are hidden: hiding too much is safe.
|
|
50
|
+
parts = [database_url_parts(url), database_url_parts(authority(url))].compact.uniq
|
|
51
|
+
# Prefixes first, on the caller's text, so no replacement made here can
|
|
52
|
+
# be mistaken for part of a password.
|
|
53
|
+
parts.each do |head, raw|
|
|
54
|
+
forms = password_forms(raw)
|
|
55
|
+
[head, head.inspect[1...-1]].uniq.each do |prefix|
|
|
56
|
+
out = forms.reduce(out) { |acc, secret| redact_prefixes(acc, prefix, secret) }
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
out = out.gsub(URL_PASSWORD) { "#{Regexp.last_match(1)}:#{REDACTED}@" }
|
|
60
|
+
# A password that exactly equals the username is left visible outside a
|
|
61
|
+
# URL. The username is not a secret: mutineer prints it in every
|
|
62
|
+
# redacted URL, so the value is already shown, and blanking it would
|
|
63
|
+
# hide every role and database name in a `postgres:postgres` setup. The
|
|
64
|
+
# skip needs an exact raw match: a password that only shares a form or a
|
|
65
|
+
# prefix with the username is a separate secret and stays hidden. A
|
|
66
|
+
# `password=` query parameter, which Rails prefers, is always hidden.
|
|
67
|
+
raws = parts.reject { |head, raw| raw == head[(head.index("://") + 3)...-1] }.map(&:last)
|
|
68
|
+
raws.concat(query_passwords(url))
|
|
69
|
+
secrets = raws.compact.uniq.flat_map { |raw| password_forms(raw) }.uniq.sort_by { |form| -form.length }
|
|
70
|
+
secrets.reduce(out) do |acc, secret|
|
|
71
|
+
secret.length > SHORT_FORM ? acc.gsub(secret, REDACTED) : redact_word(acc, secret)
|
|
72
|
+
end
|
|
73
|
+
rescue StandardError
|
|
74
|
+
FAILED
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# A UTF-8 copy of `str` that every pattern here can match. Valid UTF-8 and
|
|
78
|
+
# ASCII-only text is returned as is. Binary text, or ASCII-compatible text
|
|
79
|
+
# whose bytes are invalid in its own encoding, is read as UTF-8 with bad
|
|
80
|
+
# bytes replaced. Other text (including UTF-16 and UTF-32, valid or not)
|
|
81
|
+
# is converted, with bad bytes and characters UTF-8 lacks replaced:
|
|
82
|
+
# reading UTF-16 bytes as UTF-8 would hide the password behind NULs.
|
|
83
|
+
#
|
|
84
|
+
# @param str [String]
|
|
85
|
+
# @return [String]
|
|
86
|
+
def utf8(str)
|
|
87
|
+
return str if str.ascii_only? || (str.encoding == Encoding::UTF_8 && str.valid_encoding?)
|
|
88
|
+
|
|
89
|
+
reread = str.encoding == Encoding::BINARY || (!str.valid_encoding? && str.encoding.ascii_compatible?)
|
|
90
|
+
return String.new(str, encoding: Encoding::UTF_8).scrub if reread
|
|
91
|
+
|
|
92
|
+
str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Replace `form` in `text` only where it is a whole token: no ASCII word
|
|
96
|
+
# character touches either edge, including a punctuation edge.
|
|
97
|
+
#
|
|
98
|
+
# @param text [String]
|
|
99
|
+
# @param form [String] one form of the password.
|
|
100
|
+
# @return [String]
|
|
101
|
+
def redact_word(text, form)
|
|
102
|
+
pattern = "(?<!#{WORD_CHAR})#{Regexp.escape(form)}(?!#{WORD_CHAR})"
|
|
103
|
+
text.gsub(Regexp.new(pattern), REDACTED)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# An exception as `Class: message`, redacted. Each part is redacted on its
|
|
107
|
+
# own, so the join cannot raise on mixed encodings (a non-ASCII class name
|
|
108
|
+
# next to, say, a Latin-1 message): a raise inside a rescue would make the
|
|
109
|
+
# raw error the cause of the new one. For the same reason an exception
|
|
110
|
+
# whose own `message` raises gives {FAILED}.
|
|
111
|
+
#
|
|
112
|
+
# @param error [Exception]
|
|
113
|
+
# @return [String]
|
|
114
|
+
def describe(error)
|
|
115
|
+
"#{text(error.class)}: #{text(error.message)}"
|
|
116
|
+
rescue StandardError
|
|
117
|
+
FAILED
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Split DATABASE_URL into its `scheme://user:` head and its raw password.
|
|
121
|
+
# Read by position, not URI or a pattern, because the URL may be the
|
|
122
|
+
# malformed one and its password may hold any character: the password runs
|
|
123
|
+
# from the first `:` after `scheme://` to the last `@`.
|
|
124
|
+
#
|
|
125
|
+
# @param url [String]
|
|
126
|
+
# @return [Array(String, String), nil]
|
|
127
|
+
def database_url_parts(url)
|
|
128
|
+
start = url.index("://")
|
|
129
|
+
at = url.rindex("@")
|
|
130
|
+
return unless start && at
|
|
131
|
+
|
|
132
|
+
colon = url.index(":", start + 3)
|
|
133
|
+
return unless colon && colon < at
|
|
134
|
+
|
|
135
|
+
raw = url[(colon + 1)...at]
|
|
136
|
+
[url[0..colon], raw] unless raw.empty?
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# `url` cut before the first `/`, `?` or `#` after `scheme://`: the part
|
|
140
|
+
# that holds the userinfo of a well-formed URL.
|
|
141
|
+
#
|
|
142
|
+
# @param url [String]
|
|
143
|
+
# @return [String]
|
|
144
|
+
def authority(url)
|
|
145
|
+
start = url.index("://")
|
|
146
|
+
return url unless start
|
|
147
|
+
|
|
148
|
+
url[0...(url.index(%r{[/?#]}, start + 3) || url.length)]
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# The raw value of each `password=` query parameter, which Rails and libpq
|
|
152
|
+
# both accept in place of a `user:password@` pair. Every one, because
|
|
153
|
+
# Rails keeps the last duplicate.
|
|
154
|
+
#
|
|
155
|
+
# @param url [String]
|
|
156
|
+
# @return [Array<String>]
|
|
157
|
+
def query_passwords(url)
|
|
158
|
+
url.scan(/[?&]password=([^&#]+)/).flatten
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# The password as it may appear in a message: raw, percent-decoded, and
|
|
162
|
+
# each `inspect` escaped. Longest first, so a longer form is replaced
|
|
163
|
+
# before a form it contains.
|
|
164
|
+
#
|
|
165
|
+
# @param raw [String]
|
|
166
|
+
# @return [Array<String>]
|
|
167
|
+
def password_forms(raw)
|
|
168
|
+
decoded = begin
|
|
169
|
+
# Decoding can yield invalid UTF-8 (`%E4`); scrub it like the text.
|
|
170
|
+
utf8(URI.decode_www_form_component(raw.gsub("+", "%2B")))
|
|
171
|
+
rescue ArgumentError
|
|
172
|
+
raw
|
|
173
|
+
end
|
|
174
|
+
[raw, decoded].flat_map { |form| [form, form.inspect[1...-1]] }
|
|
175
|
+
.uniq.reject(&:empty?).sort_by { |form| -form.length }
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Hide what follows each `head` while it matches the start of `secret`,
|
|
179
|
+
# for a message that cut the URL short inside the password.
|
|
180
|
+
#
|
|
181
|
+
# @param text [String]
|
|
182
|
+
# @param head [String] the DATABASE_URL `scheme://user:` prefix, raw or
|
|
183
|
+
# `inspect` escaped.
|
|
184
|
+
# @param secret [String]
|
|
185
|
+
# @return [String]
|
|
186
|
+
def redact_prefixes(text, head, secret)
|
|
187
|
+
out = +""
|
|
188
|
+
pos = 0
|
|
189
|
+
while (found = text.index(head, pos))
|
|
190
|
+
start = found + head.length
|
|
191
|
+
length = 0
|
|
192
|
+
length += 1 while length < secret.length && text[start + length] == secret[length]
|
|
193
|
+
out << text[pos...start] << (length.positive? ? REDACTED : "")
|
|
194
|
+
pos = start + length
|
|
195
|
+
end
|
|
196
|
+
out << text[pos..]
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
data/lib/mutineer/reporter.rb
CHANGED
|
@@ -25,8 +25,13 @@ module Mutineer
|
|
|
25
25
|
# workflow, so one bad mutant never fails the gate on its own, at any size.
|
|
26
26
|
BROKEN_FLOOR = 1
|
|
27
27
|
|
|
28
|
+
# Share of the mutants a run tried to score that may go unrun (unplaceable or
|
|
29
|
+
# skipped invalid) before stderr warns. A warning only: it never changes the
|
|
30
|
+
# exit code. {BROKEN_FLOOR} also applies, so one unrun mutant never warns.
|
|
31
|
+
UNRUN_SHARE_LIMIT = 0.10
|
|
32
|
+
|
|
28
33
|
# The JSON report's `schema_version` (see docs/json-schema.md).
|
|
29
|
-
SCHEMA_VERSION = "1.
|
|
34
|
+
SCHEMA_VERSION = "1.9"
|
|
30
35
|
|
|
31
36
|
# The warning both matrix renderers give under the redundant tests.
|
|
32
37
|
MATRIX_REDUNDANT_NOTE = "Delete redundant tests one at a time: two of them can be the only killers of one mutant."
|
|
@@ -52,16 +57,18 @@ module Mutineer
|
|
|
52
57
|
# `--baseline` load) knows the score covers only the changed-line mutants.
|
|
53
58
|
# `legacy_id_matches` (`{ignore:, baseline:}`) counts stored ids still in the
|
|
54
59
|
# old format (#126); only the JSON report records it (`summary.legacy_id_matches`).
|
|
60
|
+
# `declined` (operator name => count, from {JobPlan.collect_jobs}) counts the
|
|
61
|
+
# candidates the operators did not mutate; the JSON and human reports show it.
|
|
55
62
|
def report(out: $stdout, err: $stderr, threshold: 0.0, format: "human", output: nil,
|
|
56
|
-
baseline: nil, scoped: false, legacy_id_matches: { ignore: 0, baseline: 0 })
|
|
63
|
+
baseline: nil, scoped: false, legacy_id_matches: { ignore: 0, baseline: 0 }, declined: {})
|
|
57
64
|
rendered =
|
|
58
65
|
if format == "json"
|
|
59
|
-
json_report(baseline, scoped: scoped, legacy_id_matches: legacy_id_matches)
|
|
66
|
+
json_report(baseline, scoped: scoped, legacy_id_matches: legacy_id_matches, declined: declined)
|
|
60
67
|
elsif format == "html"
|
|
61
68
|
html_report
|
|
62
69
|
else
|
|
63
70
|
sio = StringIO.new
|
|
64
|
-
human_report(sio, err, threshold)
|
|
71
|
+
human_report(sio, err, threshold, declined: declined, scoped: scoped)
|
|
65
72
|
baseline_section(sio, baseline) if baseline
|
|
66
73
|
sio.string
|
|
67
74
|
end
|
|
@@ -74,11 +81,12 @@ module Mutineer
|
|
|
74
81
|
out.print rendered
|
|
75
82
|
end
|
|
76
83
|
|
|
84
|
+
warn_unrun_share(err)
|
|
85
|
+
return warn_errors_without_gate(err) unless threshold&.positive?
|
|
86
|
+
|
|
77
87
|
# Both ways a run can fail the gate on completeness, said here rather than in
|
|
78
88
|
# the human renderer: --format json is the documented CI path, and a run that
|
|
79
89
|
# exits 1 must say why on every format, not only the one a person reads.
|
|
80
|
-
return unless threshold&.positive?
|
|
81
|
-
|
|
82
90
|
if @agg.mutation_score.nil? && broken_nil_score?
|
|
83
91
|
err.puts "[mutineer] nothing could be scored (#{broken_counts_detail}), so the " \
|
|
84
92
|
"--threshold gate fails. See no_verdict[] in --format json for the cause of each."
|
|
@@ -89,23 +97,69 @@ module Mutineer
|
|
|
89
97
|
end
|
|
90
98
|
end
|
|
91
99
|
|
|
100
|
+
# Errors are out of the score, so a run with no --threshold passes however
|
|
101
|
+
# many mutants errored. Say so, and say what --threshold would enforce.
|
|
102
|
+
#
|
|
103
|
+
# @api private
|
|
104
|
+
# @param err [IO] error stream.
|
|
105
|
+
# @return [void]
|
|
106
|
+
def warn_errors_without_gate(err)
|
|
107
|
+
count = @agg.errored_count
|
|
108
|
+
return unless count.positive?
|
|
109
|
+
|
|
110
|
+
err.puts "[mutineer] #{count} #{count == 1 ? 'mutant' : 'mutants'} errored. Errors are not in the " \
|
|
111
|
+
"score, so they do not fail a run without a positive --threshold. With one, the run fails " \
|
|
112
|
+
"when nothing can be scored, or when more than one mutant has no verdict and they exceed " \
|
|
113
|
+
"#{(BROKEN_SHARE_LIMIT * 100).round}% of those attempted (no_verdict[] in --format json)."
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Warns when the unrun mutants (unplaceable plus skipped invalid) are more
|
|
117
|
+
# than {UNRUN_SHARE_LIMIT} of the mutants the run tried to score, and more
|
|
118
|
+
# than {BROKEN_FLOOR}. Every format, with or without --threshold; the exit
|
|
119
|
+
# code does not change.
|
|
120
|
+
#
|
|
121
|
+
# @api private
|
|
122
|
+
# @param err [IO] error stream.
|
|
123
|
+
# @return [void]
|
|
124
|
+
def warn_unrun_share(err)
|
|
125
|
+
return unless unrun_share_exceeded?
|
|
126
|
+
|
|
127
|
+
unrun = @agg.unrun_count
|
|
128
|
+
tried = attempted_count + unrun
|
|
129
|
+
parts = []
|
|
130
|
+
parts << "#{@agg.unplaceable_count} unplaceable" if @agg.unplaceable_count.positive?
|
|
131
|
+
parts << "#{@agg.skipped_invalid_count} skipped (invalid)" if @agg.skipped_invalid_count.positive?
|
|
132
|
+
pct = (unrun * 100.0 / tried).round(1)
|
|
133
|
+
message = +"[mutineer] #{unrun} of the #{tried} mutants mutineer tried to score were not run " \
|
|
134
|
+
"(#{parts.join(', ')}; #{pct}%, limit #{(UNRUN_SHARE_LIMIT * 100).round}%). " \
|
|
135
|
+
"The score leaves them out. This warning does not change the exit code."
|
|
136
|
+
if @agg.unplaceable_count.positive?
|
|
137
|
+
message << " See unplaceable[] in --format json; --test-command runs them."
|
|
138
|
+
end
|
|
139
|
+
if @agg.skipped_invalid_count.positive?
|
|
140
|
+
message << " A mutant skipped (invalid) did not re-parse after mutation. If its source file " \
|
|
141
|
+
"parses on its own, that is a mutineer bug: please report it at " \
|
|
142
|
+
"https://github.com/davidteren/mutineer/issues."
|
|
143
|
+
end
|
|
144
|
+
err.puts message
|
|
145
|
+
end
|
|
146
|
+
|
|
92
147
|
# Renders the human report.
|
|
93
148
|
#
|
|
94
149
|
# @param out [IO] output stream.
|
|
95
150
|
# @param err [IO] error stream.
|
|
96
151
|
# @param threshold [Float] score threshold.
|
|
152
|
+
# @param declined [Hash{String => Integer}] operator name => declined count.
|
|
153
|
+
# @param scoped [Boolean] the run was diff-scoped (`--since`).
|
|
97
154
|
# @return [void]
|
|
98
|
-
def human_report(out, err, threshold)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
"operators and are reached by the suite."
|
|
102
|
-
return
|
|
103
|
-
end
|
|
155
|
+
def human_report(out, err, threshold, declined: {}, scoped: false)
|
|
156
|
+
# The CLI explains an empty run for every format (CLI.warn_empty_run).
|
|
157
|
+
return if @agg.total.zero?
|
|
104
158
|
|
|
105
159
|
out.puts "Mutineer — Mutation Results"
|
|
106
160
|
out.puts "========================="
|
|
107
161
|
out.puts
|
|
108
|
-
summary(out)
|
|
162
|
+
summary(out, declined: declined, scoped: scoped)
|
|
109
163
|
out.puts
|
|
110
164
|
score_line(out, err)
|
|
111
165
|
per_source(out)
|
|
@@ -136,7 +190,7 @@ module Mutineer
|
|
|
136
190
|
# reports 90% and exits 0, so CI cannot tell a complete run from a broken one.
|
|
137
191
|
return 1 if broken_share_exceeded?
|
|
138
192
|
|
|
139
|
-
|
|
193
|
+
@agg.meets_threshold?(threshold) ? 0 : 1
|
|
140
194
|
end
|
|
141
195
|
|
|
142
196
|
private
|
|
@@ -153,7 +207,10 @@ module Mutineer
|
|
|
153
207
|
# old-format ignore entries that matched, and survivors matched in an
|
|
154
208
|
# old-format baseline only through their old id (#126).
|
|
155
209
|
# @return [String] JSON text.
|
|
156
|
-
|
|
210
|
+
# @param declined [Hash{String => Integer}] operator name => count of
|
|
211
|
+
# candidates that operator did not mutate (additive `summary.declined`
|
|
212
|
+
# and `summary.declined_by_operator` keys).
|
|
213
|
+
def json_report(baseline = nil, scoped: false, legacy_id_matches: { ignore: 0, baseline: 0 }, declined: {})
|
|
157
214
|
killed = @agg.killed_count
|
|
158
215
|
survived = @agg.survived_count
|
|
159
216
|
# null (not 0.0) on an empty denominator, matching the nil-vs-0.0
|
|
@@ -186,7 +243,11 @@ module Mutineer
|
|
|
186
243
|
# Additive (1.4, #126): stored ids still in the old format. `ignore` is
|
|
187
244
|
# the number of old-format ignore entries that matched; `baseline` the
|
|
188
245
|
# survivors matched in the baseline only through their old id.
|
|
189
|
-
legacy_id_matches: legacy_id_matches
|
|
246
|
+
legacy_id_matches: legacy_id_matches,
|
|
247
|
+
# Additive (1.9, #291): candidates the operators found and did not
|
|
248
|
+
# mutate (a heredoc split or an equivalent skip), so not in `total`.
|
|
249
|
+
declined: declined.values.sum,
|
|
250
|
+
declined_by_operator: declined
|
|
190
251
|
},
|
|
191
252
|
survivors: @agg.surviving_mutants.map { |r| survivor_json(r) }
|
|
192
253
|
.sort_by { |h| [h[:file], h[:line], h[:operator]] },
|
|
@@ -220,7 +281,7 @@ module Mutineer
|
|
|
220
281
|
# Equivalent mutants the user suppressed: emitted with their stable id so
|
|
221
282
|
# the user can audit what is silenced (and copy ids for survivors they
|
|
222
283
|
# want to add). Excluded from the score; never in `survivors`.
|
|
223
|
-
ignored: @agg.results.select(&:ignored?).map { |r|
|
|
284
|
+
ignored: @agg.results.select(&:ignored?).map { |r| ignored_json(r) }
|
|
224
285
|
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] },
|
|
225
286
|
# Per-source breakdown (additive; baseline consumes it). Sorted by file so
|
|
226
287
|
# output is byte-stable. Reuses AggregateResult via by_source.
|
|
@@ -284,6 +345,7 @@ module Mutineer
|
|
|
284
345
|
#{summary_html}
|
|
285
346
|
#{per_source_html(per_source)}
|
|
286
347
|
#{survivors_html(survivors)}
|
|
348
|
+
#{ignored_html}
|
|
287
349
|
#{matrix_html}
|
|
288
350
|
</body>
|
|
289
351
|
</html>
|
|
@@ -345,6 +407,28 @@ module Mutineer
|
|
|
345
407
|
"<h2>Surviving Mutants</h2>\n#{cards}"
|
|
346
408
|
end
|
|
347
409
|
|
|
410
|
+
# Ignored mutants, each with its id and its reason when the user gave one.
|
|
411
|
+
#
|
|
412
|
+
# @api private
|
|
413
|
+
# @return [String] HTML, or "" when none were ignored.
|
|
414
|
+
def ignored_html
|
|
415
|
+
rows = @agg.results.select(&:ignored?).map { |r| ignored_json(r) }
|
|
416
|
+
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] }
|
|
417
|
+
return "" if rows.empty?
|
|
418
|
+
|
|
419
|
+
cards = rows.map do |row|
|
|
420
|
+
reason = row[:reason] ? "<div class=\"meta\">#{esc(row[:reason])}</div>" : ""
|
|
421
|
+
<<~CARD.chomp
|
|
422
|
+
<div class="survivor">
|
|
423
|
+
<h3>#{esc(row[:subject])}</h3>
|
|
424
|
+
<div class="meta">#{esc(row[:file])}:#{row[:line]} · #{esc(row[:operator])} · <span class="id">#{esc(row[:id])}</span></div>
|
|
425
|
+
#{reason}
|
|
426
|
+
</div>
|
|
427
|
+
CARD
|
|
428
|
+
end.join("\n")
|
|
429
|
+
"<h2>Ignored mutants</h2>\n#{cards}"
|
|
430
|
+
end
|
|
431
|
+
|
|
348
432
|
# The kill-matrix section of the HTML report: the counts, then the blind and
|
|
349
433
|
# redundant tests. Empty without --matrix.
|
|
350
434
|
#
|
|
@@ -485,7 +569,7 @@ module Mutineer
|
|
|
485
569
|
file = result.subject&.file
|
|
486
570
|
return unless result.mutation && file
|
|
487
571
|
|
|
488
|
-
source =
|
|
572
|
+
source = source_for(file)
|
|
489
573
|
source.byteslice(0, result.mutation.start_offset).count("\n") + 1
|
|
490
574
|
end
|
|
491
575
|
|
|
@@ -572,7 +656,7 @@ module Mutineer
|
|
|
572
656
|
def survivor_json(result)
|
|
573
657
|
m = result.mutation
|
|
574
658
|
file = result.subject.file
|
|
575
|
-
source =
|
|
659
|
+
source = source_for(file)
|
|
576
660
|
start_line, original_block, mutated_block, token = diff_for(m, source)
|
|
577
661
|
# A block at the end of the file has no newline after it; elsewhere the
|
|
578
662
|
# file's next byte is one. Each side's own text decides its marker.
|
|
@@ -631,13 +715,29 @@ module Mutineer
|
|
|
631
715
|
lines.size == 1 ? start_line.to_s : "#{start_line},#{lines.size}"
|
|
632
716
|
end
|
|
633
717
|
|
|
718
|
+
# One ignored mutant. `reason` is present only when the user wrote one.
|
|
719
|
+
#
|
|
720
|
+
# @api private
|
|
721
|
+
# @param result [Mutineer::Result] an ignored result.
|
|
722
|
+
# @return [Hash]
|
|
723
|
+
def ignored_json(result)
|
|
724
|
+
row = mutant_json(result)
|
|
725
|
+
text = result.reason
|
|
726
|
+
row[:reason] = text if text.is_a?(String) && !text.strip.empty?
|
|
727
|
+
row
|
|
728
|
+
end
|
|
729
|
+
|
|
634
730
|
# The fields that name one mutant, for the lists that point at mutants:
|
|
635
731
|
# `no_coverage`, `uncapturable`, `unplaceable`, `ran_at_load`, `ignored` and
|
|
636
732
|
# `baseline.new_survivors`.
|
|
733
|
+
#
|
|
734
|
+
# @api private
|
|
735
|
+
# @param result [Mutineer::Result] the mutant result.
|
|
736
|
+
# @return [Hash] subject, file, line, operator, token, and id.
|
|
637
737
|
def mutant_json(result)
|
|
638
738
|
m = result.mutation
|
|
639
739
|
file = result.subject.file
|
|
640
|
-
source =
|
|
740
|
+
source = source_for(file)
|
|
641
741
|
start_line, _orig, _mut, token = diff_for(m, source)
|
|
642
742
|
{
|
|
643
743
|
subject: result.subject.qualified_name,
|
|
@@ -690,8 +790,11 @@ module Mutineer
|
|
|
690
790
|
# Writes the summary block.
|
|
691
791
|
#
|
|
692
792
|
# @param out [IO] output stream.
|
|
793
|
+
# @param declined [Hash{String => Integer}] operator name => declined count.
|
|
794
|
+
# @param scoped [Boolean] the run was diff-scoped (`--since`), which does not
|
|
795
|
+
# narrow the declined count.
|
|
693
796
|
# @return [void]
|
|
694
|
-
def summary(out)
|
|
797
|
+
def summary(out, declined: {}, scoped: false)
|
|
695
798
|
out.puts "Summary"
|
|
696
799
|
out.puts "-------"
|
|
697
800
|
out.puts format("Total: %-6d Killed: %d", @agg.total, @agg.killed_count)
|
|
@@ -700,12 +803,17 @@ module Mutineer
|
|
|
700
803
|
out.puts format("Timeout: %-6d (over the per-mutant time limit)", @agg.timeout_count)
|
|
701
804
|
# A broken harness, not a coverage gap: report it distinctly from No coverage.
|
|
702
805
|
out.puts format("Uncapturable: %-6d (tests failed to run)", @agg.uncapturable_count)
|
|
703
|
-
# Not broken:
|
|
704
|
-
|
|
806
|
+
# Not broken: the strategy cannot load these where the tests call them (no
|
|
807
|
+
# named class under redefine, or some methods in a concern's `included` or
|
|
808
|
+
# `prepended` block; see docs/json-schema.md).
|
|
809
|
+
out.puts format("Unplaceable: %-6d (not loadable where the tests call it; verify with --test-command)",
|
|
705
810
|
@agg.unplaceable_count)
|
|
706
811
|
# Not a verdict: the line ran before the mutant was applied (#187).
|
|
707
812
|
out.puts format("Ran at load: %-6d (ran while the app or class loaded; mutineer cannot re-run that; " \
|
|
708
813
|
"verify with --test-command)", @agg.ran_at_load_count)
|
|
814
|
+
# Not mutants: candidates an operator chose not to mutate (#291).
|
|
815
|
+
out.puts format("Declined: %-6d (candidates an operator did not mutate; --dry-run shows the count " \
|
|
816
|
+
"per operator%s)", declined.values.sum, scoped ? "; not narrowed by --since" : "")
|
|
709
817
|
# Equivalent mutants the user suppressed; excluded from the denominator.
|
|
710
818
|
out.puts format("Ignored: %-6d (equivalent, suppressed)", @agg.ignored_count)
|
|
711
819
|
end
|
|
@@ -765,13 +873,14 @@ module Mutineer
|
|
|
765
873
|
end
|
|
766
874
|
|
|
767
875
|
# Mutants that were actually run. Deliberately not `total`: no_coverage,
|
|
768
|
-
# skipped-invalid and ignored mutants
|
|
876
|
+
# skipped-invalid and ignored mutants are not counted (a #254 no_coverage
|
|
877
|
+
# mutant ran its statement's tests but is not a verdict), so counting them
|
|
769
878
|
# would dilute the share and let a broken run slip under the limit.
|
|
770
879
|
#
|
|
771
880
|
# skipped-invalid is excluded from both sides: it means a mutant did not
|
|
772
881
|
# re-parse and was correctly never run, which is a validity outcome rather
|
|
773
|
-
# than a broken harness.
|
|
774
|
-
#
|
|
882
|
+
# than a broken harness. Its own signal is the unrun-share warning
|
|
883
|
+
# ({#warn_unrun_share}), together with unplaceable.
|
|
775
884
|
#
|
|
776
885
|
# @api private
|
|
777
886
|
# @return [Integer] killed + survived + no-verdict.
|
|
@@ -779,6 +888,16 @@ module Mutineer
|
|
|
779
888
|
@agg.killed_count + @agg.survived_count + no_verdict_count
|
|
780
889
|
end
|
|
781
890
|
|
|
891
|
+
# Whether the unrun mutants (unplaceable plus skipped invalid) are more than
|
|
892
|
+
# {BROKEN_FLOOR} and more than {UNRUN_SHARE_LIMIT} of attempted plus unrun.
|
|
893
|
+
#
|
|
894
|
+
# @api private
|
|
895
|
+
# @return [Boolean] true when the unrun-share warning applies.
|
|
896
|
+
def unrun_share_exceeded?
|
|
897
|
+
unrun = @agg.unrun_count
|
|
898
|
+
unrun > BROKEN_FLOOR && unrun > UNRUN_SHARE_LIMIT * (attempted_count + unrun)
|
|
899
|
+
end
|
|
900
|
+
|
|
782
901
|
# The sentence both the verdict line and the stderr note are built from, so a
|
|
783
902
|
# user cannot read one number in the report and a different one beside it.
|
|
784
903
|
#
|
|
@@ -833,10 +952,10 @@ module Mutineer
|
|
|
833
952
|
out.puts "-------------------"
|
|
834
953
|
out.puts "killed #{@agg.killed_count}, #{delta.new_survivors.size} new survivors vs baseline"
|
|
835
954
|
delta.new_survivors
|
|
836
|
-
.sort_by { |r| [r.subject.file, r
|
|
955
|
+
.sort_by { |r| [r.subject.file, *tie_key(r)] }
|
|
837
956
|
.each do |r|
|
|
838
957
|
file = r.subject.file
|
|
839
|
-
source =
|
|
958
|
+
source = source_for(file)
|
|
840
959
|
line, = diff_for(r.mutation, source)
|
|
841
960
|
out.puts " + #{printable(r.subject.qualified_name)} (#{printable(file)}:#{line}) #{r.mutation.operator}"
|
|
842
961
|
end
|
|
@@ -847,6 +966,16 @@ module Mutineer
|
|
|
847
966
|
out.puts(delta.regressed ? "REGRESSION vs baseline" : "OK: no regression vs baseline")
|
|
848
967
|
end
|
|
849
968
|
|
|
969
|
+
# Orders mutants in one file: by offset, then operator and replacement, so
|
|
970
|
+
# mutants at the same offset list the same way whatever order they ran in.
|
|
971
|
+
#
|
|
972
|
+
# @param result [Mutineer::Result] a result with a mutation.
|
|
973
|
+
# @return [Array] the sort key.
|
|
974
|
+
def tie_key(result)
|
|
975
|
+
m = result.mutation
|
|
976
|
+
[m.start_offset, m.operator.to_s, m.replacement.to_s]
|
|
977
|
+
end
|
|
978
|
+
|
|
850
979
|
# Writes the survivors block.
|
|
851
980
|
#
|
|
852
981
|
# @param out [IO] output stream.
|
|
@@ -861,7 +990,7 @@ module Mutineer
|
|
|
861
990
|
mutants.group_by { |r| r.subject.file }.sort.each do |file, group|
|
|
862
991
|
out.puts
|
|
863
992
|
out.puts printable(file)
|
|
864
|
-
group.sort_by { |r| r
|
|
993
|
+
group.sort_by { |r| tie_key(r) }.each { |r| survivor(out, file, r) }
|
|
865
994
|
end
|
|
866
995
|
end
|
|
867
996
|
|
|
@@ -873,7 +1002,7 @@ module Mutineer
|
|
|
873
1002
|
# @return [void]
|
|
874
1003
|
def survivor(out, file, result)
|
|
875
1004
|
m = result.mutation
|
|
876
|
-
source =
|
|
1005
|
+
source = source_for(file)
|
|
877
1006
|
start_line, original_block, mutated_block, token = diff_for(m, source)
|
|
878
1007
|
|
|
879
1008
|
out.puts " #{printable(result.subject.qualified_name)} (#{printable(File.basename(file))}:#{start_line})"
|
|
@@ -882,6 +1011,14 @@ module Mutineer
|
|
|
882
1011
|
mutated_block.each_line { |l| out.puts " + #{printable(l.chomp)}" }
|
|
883
1012
|
end
|
|
884
1013
|
|
|
1014
|
+
# The source text of `file`: the run's copy when it has one, else the file
|
|
1015
|
+
# read as UTF-8 whatever the locale (#303).
|
|
1016
|
+
#
|
|
1017
|
+
# @api private
|
|
1018
|
+
# @param file [String] source file path.
|
|
1019
|
+
# @return [String] source text.
|
|
1020
|
+
def source_for(file) = @source_map[file] || File.read(file, encoding: "UTF-8")
|
|
1021
|
+
|
|
885
1022
|
# Source text, a method name or a file path made safe for a terminal (#163):
|
|
886
1023
|
# each control character except tab becomes its Ruby escape (ESC prints as
|
|
887
1024
|
# `\e`). The text is read as UTF-8 whatever the locale, and each byte that is
|
|
@@ -914,9 +1051,14 @@ module Mutineer
|
|
|
914
1051
|
|
|
915
1052
|
# Same rule as exit_code, or the report says PASSED on a run that exits 1 —
|
|
916
1053
|
# and with --output that wrong verdict is what gets archived.
|
|
1054
|
+
passed = @agg.meets_threshold?(threshold)
|
|
917
1055
|
if broken_share_exceeded?
|
|
918
1056
|
out.puts "FAILED: #{no_verdict_ratio}; #{score}% covers only part of the run"
|
|
919
|
-
elsif score >= threshold
|
|
1057
|
+
elsif passed != (score >= threshold)
|
|
1058
|
+
# The rounded score points the other way, so give the exact counts.
|
|
1059
|
+
out.puts "#{passed ? 'PASSED' : 'FAILED'}: #{@agg.killed_count} of #{@agg.covered_count} killed " \
|
|
1060
|
+
"#{passed ? 'meets' : 'is below'} threshold #{threshold}% (the #{score}% score is rounded)"
|
|
1061
|
+
elsif passed
|
|
920
1062
|
out.puts "PASSED: #{score}% >= threshold #{threshold}%"
|
|
921
1063
|
else
|
|
922
1064
|
out.puts "FAILED: #{score}% < threshold #{threshold}%"
|