mutineer 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +475 -0
- data/README.md +224 -28
- data/lib/mutineer/changed_lines.rb +24 -1
- data/lib/mutineer/cli.rb +117 -20
- data/lib/mutineer/config.rb +64 -13
- data/lib/mutineer/coverage_map.rb +533 -100
- data/lib/mutineer/daemon_backend.rb +51 -33
- data/lib/mutineer/daemon_client.rb +212 -22
- data/lib/mutineer/daemon_server.rb +74 -27
- data/lib/mutineer/external_backend.rb +2 -2
- data/lib/mutineer/file_swap.rb +1 -1
- data/lib/mutineer/isolation.rb +92 -6
- data/lib/mutineer/job_plan.rb +352 -0
- data/lib/mutineer/kill_channel.rb +284 -0
- data/lib/mutineer/kill_matrix.rb +88 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +301 -0
- data/lib/mutineer/minitest_integration/outer_reporter.rb +126 -0
- data/lib/mutineer/minitest_integration/stop_at_first_failure.rb +14 -68
- data/lib/mutineer/minitest_integration.rb +75 -11
- data/lib/mutineer/mutant_id.rb +5 -4
- data/lib/mutineer/mutators/array_literal.rb +0 -7
- data/lib/mutineer/mutators/base.rb +63 -4
- data/lib/mutineer/mutators/chain_link.rb +0 -7
- data/lib/mutineer/mutators/collection_method.rb +1 -1
- data/lib/mutineer/mutators/condition_forcing.rb +0 -7
- data/lib/mutineer/mutators/operand_removal.rb +0 -7
- data/lib/mutineer/mutators/operator_assignment.rb +0 -7
- data/lib/mutineer/mutators/return_nil.rb +5 -11
- data/lib/mutineer/mutators/string_literal.rb +1 -1
- data/lib/mutineer/orphan_guard.rb +28 -0
- data/lib/mutineer/pairing.rb +222 -15
- data/lib/mutineer/parser.rb +12 -3
- data/lib/mutineer/project.rb +344 -21
- data/lib/mutineer/rails_worker_db.rb +83 -9
- data/lib/mutineer/reporter.rb +306 -53
- data/lib/mutineer/result.rb +73 -5
- data/lib/mutineer/runner.rb +123 -242
- data/lib/mutineer/statement_lines.rb +183 -0
- data/lib/mutineer/subject.rb +6 -2
- data/lib/mutineer/test_runners/minitest.rb +4 -2
- data/lib/mutineer/test_runners/rspec.rb +282 -3
- data/lib/mutineer/test_runners.rb +5 -3
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +4 -2
- data/lib/mutineer.rb +2 -0
- metadata +10 -3
data/lib/mutineer/reporter.rb
CHANGED
|
@@ -25,9 +25,24 @@ module Mutineer
|
|
|
25
25
|
# workflow, so one bad mutant never fails the gate on its own, at any size.
|
|
26
26
|
BROKEN_FLOOR = 1
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
# The JSON report's `schema_version` (see docs/json-schema.md).
|
|
29
|
+
SCHEMA_VERSION = "1.7"
|
|
30
|
+
|
|
31
|
+
# The warning both matrix renderers give under the redundant tests.
|
|
32
|
+
MATRIX_REDUNDANT_NOTE = "Delete redundant tests one at a time: two of them can be the only killers of one mutant."
|
|
33
|
+
|
|
34
|
+
# How many blind or redundant tests the human report lists before it points
|
|
35
|
+
# to `--format json` for the rest.
|
|
36
|
+
MATRIX_LIST_LIMIT = 20
|
|
37
|
+
|
|
38
|
+
# @param aggregate [Mutineer::AggregateResult] the run's results.
|
|
39
|
+
# @param source_map [Hash{String => String}] source file path => source text.
|
|
40
|
+
# @param matrix [Mutineer::KillMatrix, nil] the kill matrix of a `--matrix`
|
|
41
|
+
# run; nil leaves every format exactly as it is without the flag.
|
|
42
|
+
def initialize(aggregate, source_map, matrix: nil)
|
|
29
43
|
@agg = aggregate
|
|
30
44
|
@source_map = source_map
|
|
45
|
+
@matrix = matrix
|
|
31
46
|
end
|
|
32
47
|
|
|
33
48
|
# Single entry point. Branches on `format` ("human" | "json" | "html") and
|
|
@@ -96,6 +111,7 @@ module Mutineer
|
|
|
96
111
|
per_source(out)
|
|
97
112
|
|
|
98
113
|
survivors(out)
|
|
114
|
+
matrix_section(out) if @matrix
|
|
99
115
|
verdict(out, threshold) if threshold && threshold.positive?
|
|
100
116
|
end
|
|
101
117
|
|
|
@@ -146,11 +162,13 @@ module Mutineer
|
|
|
146
162
|
score = @agg.mutation_score
|
|
147
163
|
|
|
148
164
|
doc = {
|
|
149
|
-
schema_version:
|
|
165
|
+
schema_version: SCHEMA_VERSION,
|
|
150
166
|
summary: {
|
|
151
167
|
total: @agg.total, killed: killed, survived: survived,
|
|
152
168
|
no_coverage: @agg.no_coverage_count,
|
|
153
169
|
uncapturable: @agg.uncapturable_count,
|
|
170
|
+
unplaceable: @agg.unplaceable_count,
|
|
171
|
+
ran_at_load: @agg.ran_at_load_count,
|
|
154
172
|
skipped_invalid: @agg.skipped_invalid_count,
|
|
155
173
|
errored: @agg.errored_count, timeout: @agg.timeout_count,
|
|
156
174
|
ignored: @agg.ignored_count,
|
|
@@ -172,17 +190,24 @@ module Mutineer
|
|
|
172
190
|
},
|
|
173
191
|
survivors: @agg.surviving_mutants.map { |r| survivor_json(r) }
|
|
174
192
|
.sort_by { |h| [h[:file], h[:line], h[:operator]] },
|
|
175
|
-
|
|
176
|
-
|
|
193
|
+
# id extends the key to a total order: several mutants share a line.
|
|
194
|
+
no_coverage: @agg.results.select(&:no_coverage?).map { |r| mutant_json(r) }
|
|
195
|
+
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] },
|
|
177
196
|
# Same shape as no_coverage; additive key.
|
|
178
|
-
uncapturable: @agg.results.select(&:uncapturable?).map { |r|
|
|
179
|
-
.sort_by { |h| [h[:file], h[:line]] },
|
|
197
|
+
uncapturable: @agg.results.select(&:uncapturable?).map { |r| mutant_json(r) }
|
|
198
|
+
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] },
|
|
199
|
+
# Additive (1.7): owner-unknown mutants redefine did not run. Not in no_verdict.
|
|
200
|
+
unplaceable: @agg.results.select(&:unplaceable?).map { |r| mutant_json(r) }
|
|
201
|
+
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] },
|
|
202
|
+
# Additive (1.7, #187): mutants on a line that ran at load. Not in no_verdict.
|
|
203
|
+
ran_at_load: @agg.results.select(&:ran_at_load?).map { |r| mutant_json(r) }
|
|
204
|
+
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] },
|
|
180
205
|
# Every mutant that was attempted and produced no verdict, whatever the
|
|
181
206
|
# reason — the set the --threshold completeness gate counts. Named for the
|
|
182
207
|
# condition rather than one status, because summary.errored means :error
|
|
183
208
|
# alone and a key that reconciled with neither would be worse. `details`
|
|
184
209
|
# carries the cause where there is one. Uncapturable mutants also appear in
|
|
185
|
-
# uncapturable[]
|
|
210
|
+
# uncapturable[], which stays for consumers that already read it.
|
|
186
211
|
# to_s/to_i because a pre-fork failure has no subject, so its file and line
|
|
187
212
|
# are null and would not compare against a real entry. id and status extend
|
|
188
213
|
# the key to a total order: these entries collide on (file, line) far more
|
|
@@ -195,13 +220,15 @@ module Mutineer
|
|
|
195
220
|
# Equivalent mutants the user suppressed: emitted with their stable id so
|
|
196
221
|
# the user can audit what is silenced (and copy ids for survivors they
|
|
197
222
|
# want to add). Excluded from the score; never in `survivors`.
|
|
198
|
-
ignored: @agg.results.select(&:ignored?).map { |r|
|
|
199
|
-
.sort_by { |h| [h[:file], h[:line], h[:operator]] },
|
|
223
|
+
ignored: @agg.results.select(&:ignored?).map { |r| mutant_json(r) }
|
|
224
|
+
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] },
|
|
200
225
|
# Per-source breakdown (additive; baseline consumes it). Sorted by file so
|
|
201
226
|
# output is byte-stable. Reuses AggregateResult via by_source.
|
|
202
227
|
per_source: @agg.by_source.map { |file, agg| per_source_json(file, agg) }
|
|
203
228
|
.sort_by { |h| h[:file] }
|
|
204
229
|
}
|
|
230
|
+
# Additive (1.5): the kill matrix, present only with --matrix.
|
|
231
|
+
doc[:matrix] = matrix_json if @matrix
|
|
205
232
|
# Additive baseline-delta block, present only with --baseline. Existing
|
|
206
233
|
# consumers ignore the extra key; it does not move schema_version on its own.
|
|
207
234
|
doc[:baseline] = baseline_json(baseline) if baseline
|
|
@@ -257,6 +284,7 @@ module Mutineer
|
|
|
257
284
|
#{summary_html}
|
|
258
285
|
#{per_source_html(per_source)}
|
|
259
286
|
#{survivors_html(survivors)}
|
|
287
|
+
#{matrix_html}
|
|
260
288
|
</body>
|
|
261
289
|
</html>
|
|
262
290
|
HTML
|
|
@@ -267,9 +295,10 @@ module Mutineer
|
|
|
267
295
|
counts = {
|
|
268
296
|
"total" => @agg.total, "killed" => @agg.killed_count,
|
|
269
297
|
"survived" => @agg.survived_count, "no_coverage" => @agg.no_coverage_count,
|
|
270
|
-
"uncapturable" => @agg.uncapturable_count, "
|
|
298
|
+
"uncapturable" => @agg.uncapturable_count, "unplaceable" => @agg.unplaceable_count,
|
|
299
|
+
"ran_at_load" => @agg.ran_at_load_count, "ignored" => @agg.ignored_count,
|
|
271
300
|
"skipped" => @agg.skipped_invalid_count,
|
|
272
|
-
"errored" => @agg.errored_count
|
|
301
|
+
"errored" => @agg.errored_count, "timeout" => @agg.timeout_count
|
|
273
302
|
}
|
|
274
303
|
spans = counts.map { |k, v| "<span><strong>#{v}</strong> #{esc(k)}</span>" }.join("\n ")
|
|
275
304
|
"<div class=\"counts\">\n #{spans}\n</div>"
|
|
@@ -316,11 +345,191 @@ module Mutineer
|
|
|
316
345
|
"<h2>Surviving Mutants</h2>\n#{cards}"
|
|
317
346
|
end
|
|
318
347
|
|
|
348
|
+
# The kill-matrix section of the HTML report: the counts, then the blind and
|
|
349
|
+
# redundant tests. Empty without --matrix.
|
|
350
|
+
#
|
|
351
|
+
# @api private
|
|
352
|
+
# @return [String] HTML.
|
|
353
|
+
def matrix_html
|
|
354
|
+
return "" unless @matrix
|
|
355
|
+
|
|
356
|
+
lists = { "Blind tests" => @matrix.blind, "Redundant tests" => @matrix.redundant }.map do |title, tests|
|
|
357
|
+
items = tests.map { |test| "<li><span class=\"id\">#{esc(test[0])}</span> #{esc(test_label(test))}</li>" }
|
|
358
|
+
body = items.empty? ? "<p>None.</p>" : "<ul>\n#{items.join("\n")}\n</ul>"
|
|
359
|
+
"<h3>#{esc(title)} (#{tests.size})</h3>\n#{body}"
|
|
360
|
+
end
|
|
361
|
+
note = @matrix.complete? ? "" : "\n<p>#{esc(matrix_incomplete_note)}</p>\n#{matrix_incomplete_html}"
|
|
362
|
+
delete = @matrix.redundant.empty? ? "" : "\n<p>#{esc(MATRIX_REDUNDANT_NOTE)}</p>"
|
|
363
|
+
"<h2>Kill matrix</h2>\n<p>#{esc(matrix_counts_line)}</p>#{note}\n#{lists.join("\n")}#{delete}"
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
# Every incomplete row of the HTML kill-matrix section.
|
|
367
|
+
#
|
|
368
|
+
# @api private
|
|
369
|
+
# @return [String] HTML.
|
|
370
|
+
def matrix_incomplete_html
|
|
371
|
+
items = @matrix.incomplete_rows.map { |r| "<li>#{esc(matrix_row_label(r))}</li>" }
|
|
372
|
+
"<h3>Incomplete rows (#{items.size})</h3>\n<ul>\n#{items.join("\n")}\n</ul>"
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# The JSON `matrix` block: every test with its kill count, one row per
|
|
376
|
+
# mutant that ran (its killers as indexes into `tests`), and the blind and
|
|
377
|
+
# redundant tests. Rows sort like survivors, then by id, so output is
|
|
378
|
+
# byte-stable regardless of --jobs.
|
|
379
|
+
#
|
|
380
|
+
# @api private
|
|
381
|
+
# @return [Hash] the matrix JSON object.
|
|
382
|
+
def matrix_json
|
|
383
|
+
tests = @matrix.tests
|
|
384
|
+
index = tests.each_with_index.to_h
|
|
385
|
+
{
|
|
386
|
+
complete: @matrix.complete?,
|
|
387
|
+
tests: tests.map { |test| test_json(test).merge(kills: @matrix.kill_count(test)) },
|
|
388
|
+
mutants: @matrix.rows.map { |r| matrix_row_json(r, index) }
|
|
389
|
+
.sort_by { |h| [h[:file].to_s, h[:line].to_i, h[:operator].to_s, h[:id].to_s] },
|
|
390
|
+
blind: @matrix.blind.map { |test| test_json(test) },
|
|
391
|
+
redundant: @matrix.redundant.map { |test| test_json(test) }
|
|
392
|
+
}
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
# One test as JSON: its file, display name and id.
|
|
396
|
+
#
|
|
397
|
+
# @api private
|
|
398
|
+
# @param test [Array(String, String, String)] a `[file, name, id]` test.
|
|
399
|
+
# @return [Hash]
|
|
400
|
+
def test_json(test)
|
|
401
|
+
file, name, id = test
|
|
402
|
+
{ file: file, name: name, id: id }
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
# A test's name for people: the display name, plus the id when it differs
|
|
406
|
+
# (an RSpec example id, which tells apart examples that share a name and
|
|
407
|
+
# runs exactly that example with `rspec`).
|
|
408
|
+
#
|
|
409
|
+
# @api private
|
|
410
|
+
# @param test [Array(String, String, String)] a `[file, name, id]` test.
|
|
411
|
+
# @return [String]
|
|
412
|
+
def test_label(test)
|
|
413
|
+
_file, name, id = test
|
|
414
|
+
id == name ? name : "#{name} (#{id})"
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
# One mutant's row under `matrix.mutants`.
|
|
418
|
+
#
|
|
419
|
+
# @api private
|
|
420
|
+
# @param result [Mutineer::Result] a result with a {Kills} row.
|
|
421
|
+
# @param index [Hash{Array => Integer}] test => its index in `matrix.tests`.
|
|
422
|
+
# @return [Hash] the row JSON object.
|
|
423
|
+
def matrix_row_json(result, index)
|
|
424
|
+
file = result.subject&.file
|
|
425
|
+
kills = result.kills
|
|
426
|
+
{
|
|
427
|
+
subject: result.subject&.qualified_name, file: file, line: result_line(result),
|
|
428
|
+
operator: result.mutation&.operator&.to_s, id: result.id, status: result.status.to_s,
|
|
429
|
+
killed_by: kills.killed_by.map { |test| index.fetch(test) }.sort,
|
|
430
|
+
ran: kills.ran.size, complete: kills.complete
|
|
431
|
+
}
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
# The kill-matrix section of the human report.
|
|
435
|
+
#
|
|
436
|
+
# @api private
|
|
437
|
+
# @param out [IO] output stream.
|
|
438
|
+
# @return [void]
|
|
439
|
+
def matrix_section(out)
|
|
440
|
+
out.puts
|
|
441
|
+
out.puts "Kill matrix"
|
|
442
|
+
out.puts "-----------"
|
|
443
|
+
out.puts matrix_counts_line
|
|
444
|
+
matrix_incomplete_section(out) unless @matrix.complete?
|
|
445
|
+
matrix_list(out, "Blind tests (ran, killed no mutant)", @matrix.blind)
|
|
446
|
+
matrix_list(out, "Redundant tests (each mutant they kill has another killer)", @matrix.redundant)
|
|
447
|
+
return if @matrix.redundant.empty?
|
|
448
|
+
|
|
449
|
+
out.puts MATRIX_REDUNDANT_NOTE
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# The human report's warning about incomplete rows, with the first
|
|
453
|
+
# {MATRIX_LIST_LIMIT} of them named.
|
|
454
|
+
#
|
|
455
|
+
# @api private
|
|
456
|
+
# @param out [IO] output stream.
|
|
457
|
+
# @return [void]
|
|
458
|
+
def matrix_incomplete_section(out)
|
|
459
|
+
rows = @matrix.incomplete_rows
|
|
460
|
+
out.puts matrix_incomplete_note
|
|
461
|
+
rows.first(MATRIX_LIST_LIMIT).each { |r| out.puts " #{printable(matrix_row_label(r))}" }
|
|
462
|
+
rest = rows.size - MATRIX_LIST_LIMIT
|
|
463
|
+
out.puts " and #{rest} more" if rest.positive?
|
|
464
|
+
out.puts "The HTML and JSON reports list every incomplete row (JSON: matrix.mutants with complete: false)."
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
# A row's label: subject, place, operator, status and id.
|
|
468
|
+
#
|
|
469
|
+
# @api private
|
|
470
|
+
# @param result [Mutineer::Result] a result with a {Kills} row.
|
|
471
|
+
# @return [String]
|
|
472
|
+
def matrix_row_label(result)
|
|
473
|
+
place = [result.subject&.file, result_line(result)].compact.join(":")
|
|
474
|
+
[result.subject&.qualified_name, ("(#{place})" unless place.empty?), result.mutation&.operator,
|
|
475
|
+
result.status, result.id].compact.join(" ")
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
# The 1-based line of a result's mutation, or nil without one (a pre-fork
|
|
479
|
+
# failure has no subject or mutation).
|
|
480
|
+
#
|
|
481
|
+
# @api private
|
|
482
|
+
# @param result [Mutineer::Result] any result.
|
|
483
|
+
# @return [Integer, nil]
|
|
484
|
+
def result_line(result)
|
|
485
|
+
file = result.subject&.file
|
|
486
|
+
return unless result.mutation && file
|
|
487
|
+
|
|
488
|
+
source = @source_map[file] || File.read(file)
|
|
489
|
+
source.byteslice(0, result.mutation.start_offset).count("\n") + 1
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
# One titled list of tests in the human kill-matrix section, cut at
|
|
493
|
+
# {MATRIX_LIST_LIMIT}; the JSON report has the whole list.
|
|
494
|
+
#
|
|
495
|
+
# @api private
|
|
496
|
+
# @param out [IO] output stream.
|
|
497
|
+
# @param title [String] the list's title.
|
|
498
|
+
# @param tests [Array<Array(String, String, String)>] `[file, name, id]` tests.
|
|
499
|
+
# @return [void]
|
|
500
|
+
def matrix_list(out, title, tests)
|
|
501
|
+
out.puts
|
|
502
|
+
out.puts "#{title}: #{tests.size}"
|
|
503
|
+
tests.first(MATRIX_LIST_LIMIT).each { |test| out.puts " #{printable(test[0])} #{printable(test_label(test))}" }
|
|
504
|
+
rest = tests.size - MATRIX_LIST_LIMIT
|
|
505
|
+
out.puts " and #{rest} more; see --format json" if rest.positive?
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
# The counts sentence both matrix renderers open with.
|
|
509
|
+
#
|
|
510
|
+
# @api private
|
|
511
|
+
# @return [String]
|
|
512
|
+
def matrix_counts_line
|
|
513
|
+
"#{@matrix.tests.size} tests ran against #{@matrix.rows.size} mutants " \
|
|
514
|
+
"(every test in each mutant's covering files): " \
|
|
515
|
+
"#{@matrix.blind.size} blind, #{@matrix.redundant.size} redundant."
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
# The warning both matrix renderers give when a row is incomplete.
|
|
519
|
+
#
|
|
520
|
+
# @api private
|
|
521
|
+
# @return [String]
|
|
522
|
+
def matrix_incomplete_note
|
|
523
|
+
"#{@matrix.incomplete_rows.size} mutants could not be verified as complete (a timeout, an error, " \
|
|
524
|
+
"an exit, an interrupt or a broken stream), so a blind test may have killed one of them. " \
|
|
525
|
+
"Do not delete a blind test until these rows are complete:"
|
|
526
|
+
end
|
|
527
|
+
|
|
319
528
|
# HTML-escapes any text destined for the document (stdlib CGI).
|
|
320
529
|
def esc(text) = CGI.escapeHTML(text.to_s)
|
|
321
530
|
|
|
322
531
|
# The same delta facts the human report prints, for dashboards. new_survivors
|
|
323
|
-
# reuse the
|
|
532
|
+
# reuse the mutant_json shape (subject/file/line/operator/token/id) and sort
|
|
324
533
|
# byte-stably so output does not depend on --jobs finish order.
|
|
325
534
|
def baseline_json(delta)
|
|
326
535
|
{
|
|
@@ -332,8 +541,8 @@ module Mutineer
|
|
|
332
541
|
# a diff-scoped side), so a consumer knows not to render the two scores
|
|
333
542
|
# as a comparison.
|
|
334
543
|
score_comparable: delta.score_comparable,
|
|
335
|
-
new_survivors: delta.new_survivors.map { |r|
|
|
336
|
-
.sort_by { |h| [h[:file], h[:line], h[:operator]] },
|
|
544
|
+
new_survivors: delta.new_survivors.map { |r| mutant_json(r) }
|
|
545
|
+
.sort_by { |h| [h[:file], h[:line], h[:operator], h[:id].to_s] },
|
|
337
546
|
fixed_survivors: delta.fixed_survivors.map do |h|
|
|
338
547
|
{ subject: h["subject"], file: h["file"], line: h["line"],
|
|
339
548
|
operator: h["operator"], id: h["id"] }
|
|
@@ -365,8 +574,11 @@ module Mutineer
|
|
|
365
574
|
file = result.subject.file
|
|
366
575
|
source = @source_map[file] || File.read(file)
|
|
367
576
|
start_line, original_block, mutated_block, token = diff_for(m, source)
|
|
368
|
-
|
|
369
|
-
|
|
577
|
+
# A block at the end of the file has no newline after it; elsewhere the
|
|
578
|
+
# file's next byte is one. Each side's own text decides its marker.
|
|
579
|
+
at_eof = source.byteindex("\n", m.end_offset).nil?
|
|
580
|
+
old_lines = side_lines(original_block, at_eof)
|
|
581
|
+
new_lines = side_lines(mutated_block, at_eof)
|
|
370
582
|
{
|
|
371
583
|
subject: result.subject.qualified_name,
|
|
372
584
|
file: file,
|
|
@@ -376,12 +588,53 @@ module Mutineer
|
|
|
376
588
|
# user can paste it straight into .mutineer.yml `ignore:`.
|
|
377
589
|
id: result.id,
|
|
378
590
|
token: token,
|
|
379
|
-
diff: "--- a/#{file}\n+++ b/#{file}\n
|
|
591
|
+
diff: "--- a/#{file}\n+++ b/#{file}\n" \
|
|
592
|
+
"@@ -#{hunk_range(start_line, old_lines)} +#{hunk_range(start_line, new_lines)} @@\n" \
|
|
593
|
+
"#{diff_body("-", old_lines)}#{diff_body("+", new_lines)}"
|
|
380
594
|
}
|
|
381
595
|
end
|
|
382
596
|
|
|
383
|
-
#
|
|
384
|
-
|
|
597
|
+
# One side of the diff as the file holds it: the {#diff_for} block plus the
|
|
598
|
+
# newline that follows it, unless the block ends the file. Each line keeps
|
|
599
|
+
# its own ending, so a CRLF line keeps its "\r" and a last line with no
|
|
600
|
+
# newline is told apart from one with a newline.
|
|
601
|
+
#
|
|
602
|
+
# @api private
|
|
603
|
+
# @param block [String] the block's text.
|
|
604
|
+
# @param at_eof [Boolean] no newline follows the block in the file.
|
|
605
|
+
# @return [Array<String>] its lines, with their endings.
|
|
606
|
+
def side_lines(block, at_eof)
|
|
607
|
+
(at_eof ? block : "#{block}\n").lines
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
# The `-` or `+` lines of one side. A line with no newline is the file's
|
|
611
|
+
# last, so it gets the unified diff's no-newline marker (#106).
|
|
612
|
+
#
|
|
613
|
+
# @api private
|
|
614
|
+
# @param sign [String] "-" or "+".
|
|
615
|
+
# @param lines [Array<String>] the side's lines, with their endings.
|
|
616
|
+
# @return [String]
|
|
617
|
+
def diff_body(sign, lines)
|
|
618
|
+
lines.map { |l| l.end_with?("\n") ? "#{sign}#{l}" : "#{sign}#{l}\n\\n" }.join
|
|
619
|
+
end
|
|
620
|
+
|
|
621
|
+
# One side of a unified diff hunk header: `start` for one line, else
|
|
622
|
+
# `start,count`; an empty side is `start-1,0` (#106).
|
|
623
|
+
#
|
|
624
|
+
# @api private
|
|
625
|
+
# @param start_line [Integer] 1-based first line of the block.
|
|
626
|
+
# @param lines [Array<String>] the block's lines.
|
|
627
|
+
# @return [String] the range, e.g. "3", "3,2" or "2,0".
|
|
628
|
+
def hunk_range(start_line, lines)
|
|
629
|
+
return "#{start_line - 1},0" if lines.empty? # an empty side names the line before it
|
|
630
|
+
|
|
631
|
+
lines.size == 1 ? start_line.to_s : "#{start_line},#{lines.size}"
|
|
632
|
+
end
|
|
633
|
+
|
|
634
|
+
# The fields that name one mutant, for the lists that point at mutants:
|
|
635
|
+
# `no_coverage`, `uncapturable`, `unplaceable`, `ran_at_load`, `ignored` and
|
|
636
|
+
# `baseline.new_survivors`.
|
|
637
|
+
def mutant_json(result)
|
|
385
638
|
m = result.mutation
|
|
386
639
|
file = result.subject.file
|
|
387
640
|
source = @source_map[file] || File.read(file)
|
|
@@ -415,22 +668,6 @@ module Mutineer
|
|
|
415
668
|
[start_line, original_block, mutated_block, token]
|
|
416
669
|
end
|
|
417
670
|
|
|
418
|
-
# Builds no-coverage JSON.
|
|
419
|
-
#
|
|
420
|
-
# @api private
|
|
421
|
-
# @param result [Mutineer::Result] result object.
|
|
422
|
-
# @return [Hash] no-coverage JSON object.
|
|
423
|
-
def no_coverage_json(result)
|
|
424
|
-
m = result.mutation
|
|
425
|
-
file = result.subject.file
|
|
426
|
-
source = @source_map[file] || File.read(file)
|
|
427
|
-
{
|
|
428
|
-
subject: result.subject.qualified_name,
|
|
429
|
-
file: file,
|
|
430
|
-
line: source.byteslice(0, m.start_offset).count("\n") + 1
|
|
431
|
-
}
|
|
432
|
-
end
|
|
433
|
-
|
|
434
671
|
# An entry under the JSON `no_verdict:` key: an attempted mutant with no verdict.
|
|
435
672
|
# A pre-fork failure has no subject or mutation attached, so those degrade to
|
|
436
673
|
# nulls rather than dropping the entry — the count must still reconcile with
|
|
@@ -440,16 +677,10 @@ module Mutineer
|
|
|
440
677
|
# @param result [Mutineer::Result] an errored or timed-out result.
|
|
441
678
|
# @return [Hash] no-verdict JSON object.
|
|
442
679
|
def no_verdict_json(result)
|
|
443
|
-
file = result.subject&.file
|
|
444
|
-
line =
|
|
445
|
-
if result.mutation && file
|
|
446
|
-
source = @source_map[file] || File.read(file)
|
|
447
|
-
source.byteslice(0, result.mutation.start_offset).count("\n") + 1
|
|
448
|
-
end
|
|
449
680
|
{
|
|
450
681
|
subject: result.subject&.qualified_name,
|
|
451
|
-
file: file,
|
|
452
|
-
line:
|
|
682
|
+
file: result.subject&.file,
|
|
683
|
+
line: result_line(result),
|
|
453
684
|
id: result.id,
|
|
454
685
|
status: result.status.to_s,
|
|
455
686
|
details: result.details
|
|
@@ -465,10 +696,16 @@ module Mutineer
|
|
|
465
696
|
out.puts "-------"
|
|
466
697
|
out.puts format("Total: %-6d Killed: %d", @agg.total, @agg.killed_count)
|
|
467
698
|
out.puts format("Survived: %-6d No coverage: %d", @agg.survived_count, @agg.no_coverage_count)
|
|
468
|
-
out.puts format("Skipped: %-6d Errored: %d", @agg.skipped_invalid_count,
|
|
469
|
-
|
|
699
|
+
out.puts format("Skipped: %-6d Errored: %d", @agg.skipped_invalid_count, @agg.errored_count)
|
|
700
|
+
out.puts format("Timeout: %-6d (over the per-mutant time limit)", @agg.timeout_count)
|
|
470
701
|
# A broken harness, not a coverage gap: report it distinctly from No coverage.
|
|
471
702
|
out.puts format("Uncapturable: %-6d (tests failed to run)", @agg.uncapturable_count)
|
|
703
|
+
# Not broken: redefine has no named class to load these onto; reload runs them.
|
|
704
|
+
out.puts format("Unplaceable: %-6d (class cannot be named statically; --strategy reload runs these)",
|
|
705
|
+
@agg.unplaceable_count)
|
|
706
|
+
# Not a verdict: the line ran before the mutant was applied (#187).
|
|
707
|
+
out.puts format("Ran at load: %-6d (ran while the app or class loaded; mutineer cannot re-run that; " \
|
|
708
|
+
"verify with --test-command)", @agg.ran_at_load_count)
|
|
472
709
|
# Equivalent mutants the user suppressed; excluded from the denominator.
|
|
473
710
|
out.puts format("Ignored: %-6d (equivalent, suppressed)", @agg.ignored_count)
|
|
474
711
|
end
|
|
@@ -481,8 +718,9 @@ module Mutineer
|
|
|
481
718
|
def score_line(out, err)
|
|
482
719
|
score = @agg.mutation_score
|
|
483
720
|
excluded = "#{@agg.no_coverage_count} no-coverage, #{@agg.uncapturable_count} uncapturable, " \
|
|
721
|
+
"#{@agg.unplaceable_count} unplaceable, #{@agg.ran_at_load_count} ran at load, " \
|
|
484
722
|
"#{@agg.skipped_invalid_count} skipped, " \
|
|
485
|
-
"#{@agg.errored_count
|
|
723
|
+
"#{@agg.errored_count} errored, #{@agg.timeout_count} timeout, " \
|
|
486
724
|
"#{@agg.ignored_count} ignored excluded"
|
|
487
725
|
if score.nil?
|
|
488
726
|
out.puts "Mutation score: N/A (no covered mutants)"
|
|
@@ -581,7 +819,7 @@ module Mutineer
|
|
|
581
819
|
sources.sort.each do |file, agg|
|
|
582
820
|
score = agg.mutation_score
|
|
583
821
|
out.puts format("%s %s (%d killed / %d survived / %d no-cov)",
|
|
584
|
-
file, score.nil? ? "N/A" : "#{score}%",
|
|
822
|
+
printable(file), score.nil? ? "N/A" : "#{score}%",
|
|
585
823
|
agg.killed_count, agg.survived_count, agg.no_coverage_count)
|
|
586
824
|
end
|
|
587
825
|
end
|
|
@@ -600,7 +838,7 @@ module Mutineer
|
|
|
600
838
|
file = r.subject.file
|
|
601
839
|
source = @source_map[file] || File.read(file)
|
|
602
840
|
line, = diff_for(r.mutation, source)
|
|
603
|
-
out.puts " + #{r.subject.qualified_name} (#{file}:#{line}) #{r.mutation.operator}"
|
|
841
|
+
out.puts " + #{printable(r.subject.qualified_name)} (#{printable(file)}:#{line}) #{r.mutation.operator}"
|
|
604
842
|
end
|
|
605
843
|
out.puts "score dropped #{delta.score_before}% -> #{delta.score_after}%" if delta.score_drop
|
|
606
844
|
# An OK verdict must not imply a check that never ran: say when the score
|
|
@@ -622,7 +860,7 @@ module Mutineer
|
|
|
622
860
|
out.puts "-----------------"
|
|
623
861
|
mutants.group_by { |r| r.subject.file }.sort.each do |file, group|
|
|
624
862
|
out.puts
|
|
625
|
-
out.puts file
|
|
863
|
+
out.puts printable(file)
|
|
626
864
|
group.sort_by { |r| r.mutation.start_offset }.each { |r| survivor(out, file, r) }
|
|
627
865
|
end
|
|
628
866
|
end
|
|
@@ -638,10 +876,25 @@ module Mutineer
|
|
|
638
876
|
source = @source_map[file] || File.read(file)
|
|
639
877
|
start_line, original_block, mutated_block, token = diff_for(m, source)
|
|
640
878
|
|
|
641
|
-
out.puts " #{result.subject.qualified_name} (#{File.basename(file)}:#{start_line})"
|
|
642
|
-
out.puts " Operator: #{m.operator} (#{token} -> #{m.replacement})"
|
|
643
|
-
original_block.each_line { |l| out.puts " - #{l.chomp}" }
|
|
644
|
-
mutated_block.each_line { |l| out.puts " + #{l.chomp}" }
|
|
879
|
+
out.puts " #{printable(result.subject.qualified_name)} (#{printable(File.basename(file))}:#{start_line})"
|
|
880
|
+
out.puts " Operator: #{m.operator} (#{printable(token)} -> #{printable(m.replacement)})"
|
|
881
|
+
original_block.each_line { |l| out.puts " - #{printable(l.chomp)}" }
|
|
882
|
+
mutated_block.each_line { |l| out.puts " + #{printable(l.chomp)}" }
|
|
883
|
+
end
|
|
884
|
+
|
|
885
|
+
# Source text, a method name or a file path made safe for a terminal (#163):
|
|
886
|
+
# each control character except tab becomes its Ruby escape (ESC prints as
|
|
887
|
+
# `\e`). The text is read as UTF-8 whatever the locale, and each byte that is
|
|
888
|
+
# not valid UTF-8 (a Latin-1 source, say) prints as `\xNN`, so nothing is
|
|
889
|
+
# lost and the regex never raises. JSON and HTML escape on their own.
|
|
890
|
+
#
|
|
891
|
+
# @api private
|
|
892
|
+
# @param text [String] source text.
|
|
893
|
+
# @return [String] the text with control characters and invalid bytes escaped.
|
|
894
|
+
def printable(text)
|
|
895
|
+
text.dup.force_encoding(Encoding::UTF_8)
|
|
896
|
+
.scrub { |bytes| bytes.unpack("C*").map { |b| format("\\x%02X", b) }.join }
|
|
897
|
+
.gsub(/[[:cntrl:]&&[^\t]]/) { |c| c.dump[1..-2] }
|
|
645
898
|
end
|
|
646
899
|
|
|
647
900
|
# Writes the final verdict line.
|
data/lib/mutineer/result.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Mutineer
|
|
4
|
-
# Immutable outcome of running one mutant.
|
|
4
|
+
# Immutable outcome of running one mutant. Ten distinct states:
|
|
5
5
|
# killed - a test failed/errored, so the mutation was caught.
|
|
6
6
|
# survived - every test passed, so the mutation went undetected.
|
|
7
7
|
# error - the child crashed (unhandled exception): exit status 2.
|
|
@@ -13,6 +13,19 @@ module Mutineer
|
|
|
13
13
|
# like no_coverage, but reported separately: it signals a
|
|
14
14
|
# broken harness (a test that failed to run), not a genuine
|
|
15
15
|
# coverage gap.
|
|
16
|
+
# unplaceable - under `--strategy redefine`, the method belongs to a class
|
|
17
|
+
# or module that cannot be named statically, so it has no
|
|
18
|
+
# owner to be loaded onto and is not run. Excluded from the
|
|
19
|
+
# denominator and, unlike uncapturable, from the
|
|
20
|
+
# no-verdict gate: nothing is broken. `--strategy reload`
|
|
21
|
+
# runs these mutants.
|
|
22
|
+
# ran_at_load - the mutated line ran while the app booted or its class
|
|
23
|
+
# loaded. That run happened before the mutant was applied,
|
|
24
|
+
# so a test can check a value the original code computed.
|
|
25
|
+
# A survivor on such a line, and a line that ran only at
|
|
26
|
+
# load, get this status; a kill stays killed. Excluded from
|
|
27
|
+
# the denominator and from the no-verdict gate, like
|
|
28
|
+
# unplaceable. `--test-command` verifies these mutants.
|
|
16
29
|
# ignored - a known-equivalent mutant the user suppressed, via an
|
|
17
30
|
# inline `# mutineer:disable-line` comment or a
|
|
18
31
|
# `.mutineer.yml` `ignore:` id. A pre-fork classification
|
|
@@ -21,8 +34,9 @@ module Mutineer
|
|
|
21
34
|
#
|
|
22
35
|
# `error` and `skipped` are deliberately distinct: skipped is a pre-fork
|
|
23
36
|
# validity failure (counted separately by the reporter), error is a runtime
|
|
24
|
-
# crash. Never conflate them via `details` string parsing. `no_coverage
|
|
25
|
-
# `uncapturable` are pre-fork
|
|
37
|
+
# crash. Never conflate them via `details` string parsing. `no_coverage`,
|
|
38
|
+
# `uncapturable` and `unplaceable` are pre-fork results, and `ran_at_load` is
|
|
39
|
+
# pre-fork or a reclassified survivor: all excluded from the score
|
|
26
40
|
# denominator.
|
|
27
41
|
#
|
|
28
42
|
# `subject`, `mutation`, and `id` are nil when the Result is built by
|
|
@@ -30,7 +44,28 @@ module Mutineer
|
|
|
30
44
|
# them afterwards via `result.with(subject:, mutation:, id:)` so the Reporter
|
|
31
45
|
# can render survivor diffs and emit the id. `id` is the content-based
|
|
32
46
|
# MutantId (it includes the project-relative file path).
|
|
33
|
-
|
|
47
|
+
#
|
|
48
|
+
# `kills` is nil except in a `--matrix` run, where a mutant that was forked
|
|
49
|
+
# carries the {Kills} its child reported. It annotates the verdict and never
|
|
50
|
+
# decides it.
|
|
51
|
+
#
|
|
52
|
+
# A Result crosses a process boundary only through WorkerPool's Marshal pipe,
|
|
53
|
+
# from a fork back to the process that forked it, so both ends always run the
|
|
54
|
+
# same code. Nothing persists a marshaled Result (reports, baselines and the
|
|
55
|
+
# coverage cache are JSON), so a new field needs no Marshal compatibility.
|
|
56
|
+
Result = Data.define(:status, :details, :subject, :mutation, :id, :kills) do
|
|
57
|
+
# Every field but `status` defaults to nil.
|
|
58
|
+
#
|
|
59
|
+
# @param status [Symbol] the outcome.
|
|
60
|
+
# @param details [String, nil] error or skip details.
|
|
61
|
+
# @param subject [Mutineer::Subject, nil] the mutated subject.
|
|
62
|
+
# @param mutation [Mutineer::Mutation, nil] the mutation.
|
|
63
|
+
# @param id [String, nil] the stable mutant id.
|
|
64
|
+
# @param kills [Mutineer::Kills, nil] the matrix row of a `--matrix` run.
|
|
65
|
+
def initialize(status:, details: nil, subject: nil, mutation: nil, id: nil, kills: nil)
|
|
66
|
+
super
|
|
67
|
+
end
|
|
68
|
+
|
|
34
69
|
# Builds a killed result.
|
|
35
70
|
#
|
|
36
71
|
# @return [Mutineer::Result] killed result.
|
|
@@ -68,6 +103,16 @@ module Mutineer
|
|
|
68
103
|
# @return [Mutineer::Result] uncapturable result.
|
|
69
104
|
def self.uncapturable = new(status: :uncapturable, details: nil, subject: nil, mutation: nil, id: nil)
|
|
70
105
|
|
|
106
|
+
# Builds an unplaceable result.
|
|
107
|
+
#
|
|
108
|
+
# @return [Mutineer::Result] unplaceable result.
|
|
109
|
+
def self.unplaceable = new(status: :unplaceable, details: nil, subject: nil, mutation: nil, id: nil)
|
|
110
|
+
|
|
111
|
+
# Builds a ran_at_load result.
|
|
112
|
+
#
|
|
113
|
+
# @return [Mutineer::Result] ran-at-load result.
|
|
114
|
+
def self.ran_at_load = new(status: :ran_at_load, details: nil, subject: nil, mutation: nil, id: nil)
|
|
115
|
+
|
|
71
116
|
# Builds an ignored result.
|
|
72
117
|
#
|
|
73
118
|
# @return [Mutineer::Result] ignored result.
|
|
@@ -87,14 +132,33 @@ module Mutineer
|
|
|
87
132
|
def no_coverage? = status == :no_coverage
|
|
88
133
|
# @return [Boolean] true when the status is uncapturable.
|
|
89
134
|
def uncapturable? = status == :uncapturable
|
|
135
|
+
# @return [Boolean] true when the status is unplaceable.
|
|
136
|
+
def unplaceable? = status == :unplaceable
|
|
137
|
+
# @return [Boolean] true when the status is ran_at_load.
|
|
138
|
+
def ran_at_load? = status == :ran_at_load
|
|
90
139
|
# @return [Boolean] true when the status is ignored.
|
|
91
140
|
def ignored? = status == :ignored
|
|
92
141
|
|
|
93
142
|
end
|
|
94
143
|
|
|
144
|
+
# One mutant's row of the kill matrix (`--matrix`): which tests killed it and
|
|
145
|
+
# which ran against it. A test is a `[file, name, id]` triple: the
|
|
146
|
+
# project-relative file that defines it, its name (`CalculatorTest#test_add`,
|
|
147
|
+
# or an RSpec example's full description), and an id that tells apart tests
|
|
148
|
+
# sharing a file and name (an RSpec example id; the name again for Minitest).
|
|
149
|
+
# File and id identify a test; the name is display data that a mutant can
|
|
150
|
+
# change. Both lists are sorted and unique, and `ran` includes the killers.
|
|
151
|
+
#
|
|
152
|
+
# `complete` is true when the child ran its whole suite and reported it
|
|
153
|
+
# cleanly (see Isolation.finish). It is false when the child timed out,
|
|
154
|
+
# errored or exited the process early, when the recorder never armed, when
|
|
155
|
+
# its lines arrived out of order, when Minitest returned after an Interrupt
|
|
156
|
+
# with tests unseen, or when the row does not agree with the verdict.
|
|
157
|
+
Kills = Data.define(:killed_by, :ran, :complete)
|
|
158
|
+
|
|
95
159
|
# Aggregates a flat list of Results into counts, the mutation score, and the
|
|
96
160
|
# surviving-mutant list. The score denominator is killed + survived ONLY:
|
|
97
|
-
# no-coverage, uncapturable, skipped (invalid), errored, timeout, and ignored
|
|
161
|
+
# no-coverage, uncapturable, unplaceable, ran-at-load, skipped (invalid), errored, timeout, and ignored
|
|
98
162
|
# (equivalent-mutant suppression) are each excluded and surfaced separately,
|
|
99
163
|
# so suppressing every survivor reaches 100%. An empty denominator yields a
|
|
100
164
|
# nil score (rendered "N/A"), never 0.0, distinguishing "no testable mutants"
|
|
@@ -118,6 +182,10 @@ module Mutineer
|
|
|
118
182
|
def no_coverage_count = count(:no_coverage)
|
|
119
183
|
# @return [Integer] uncapturable count.
|
|
120
184
|
def uncapturable_count = count(:uncapturable)
|
|
185
|
+
# @return [Integer] unplaceable count.
|
|
186
|
+
def unplaceable_count = count(:unplaceable)
|
|
187
|
+
# @return [Integer] ran-at-load count.
|
|
188
|
+
def ran_at_load_count = count(:ran_at_load)
|
|
121
189
|
# @return [Integer] skipped-invalid count.
|
|
122
190
|
def skipped_invalid_count = count(:skipped)
|
|
123
191
|
# @return [Integer] errored count.
|