kimera 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +86 -0
- data/README.md +59 -1
- data/lib/kimera/cli/baseline/entry.rb +56 -0
- data/lib/kimera/cli/baseline/ledger.rb +32 -0
- data/lib/kimera/cli/baseline/prune.rb +68 -0
- data/lib/kimera/cli/baseline/review.rb +56 -0
- data/lib/kimera/cli/baseline.rb +49 -9
- data/lib/kimera/cli/help_text.rb +2 -2
- data/lib/kimera/cli/mutant.rb +17 -11
- data/lib/kimera/cli/report_file.rb +4 -0
- data/lib/kimera/cli/run/arguments.rb +7 -3
- data/lib/kimera/cli/run/cycle.rb +14 -9
- data/lib/kimera/cli/run/digest.rb +10 -2
- data/lib/kimera/cli/run/emission.rb +5 -4
- data/lib/kimera/cli/run/narrowing.rb +48 -0
- data/lib/kimera/cli/run/options.rb +8 -2
- data/lib/kimera/cli/run.rb +15 -1
- data/lib/kimera/cli/survivors/panel.rb +2 -2
- data/lib/kimera/cli/survivors.rb +2 -2
- data/lib/kimera/cli/test_command.rb +22 -3
- data/lib/kimera/execution/isolated_child.rb +5 -0
- data/lib/kimera/execution/isolated_child_minitest.rb +2 -1
- data/lib/kimera/execution/parallel_test_databases.rb +16 -4
- data/lib/kimera/execution/shift.rb +4 -2
- data/lib/kimera/execution/worker_pool/worker.rb +2 -2
- data/lib/kimera/execution/worker_pool.rb +1 -1
- data/lib/kimera/frameworks/minitest_adapter.rb +2 -1
- data/lib/kimera/frameworks/rspec_adapter.rb +4 -1
- data/lib/kimera/incremental/session.rb +5 -1
- data/lib/kimera/registry/mutant_keys.rb +45 -0
- data/lib/kimera/registry/registry.rb +5 -0
- data/lib/kimera/report/actions.rb +5 -1
- data/lib/kimera/report/sections.rb +15 -0
- data/lib/kimera/report/text.rb +1 -0
- data/lib/kimera/results/result.rb +17 -2
- data/lib/kimera/results/run_report.rb +3 -2
- data/lib/kimera/scope/config.rb +6 -1
- data/lib/kimera/scope/ignore_drift.rb +13 -0
- data/lib/kimera/scope/ignore_list.rb +21 -14
- data/lib/kimera/scope/ignore_placement.rb +30 -0
- data/lib/kimera/self_protection.rb +1 -0
- data/lib/kimera/support/test_exit.rb +40 -0
- data/lib/kimera/support/version.rb +1 -1
- data/skills/kimera/SKILL.md +66 -4
- metadata +10 -1
data/lib/kimera/cli/run/cycle.rb
CHANGED
|
@@ -4,6 +4,7 @@ require_relative "../../execution/suite_env"
|
|
|
4
4
|
require_relative "../../incremental/selection"
|
|
5
5
|
require_relative "../../incremental/session"
|
|
6
6
|
require_relative "digest"
|
|
7
|
+
require_relative "narrowing"
|
|
7
8
|
require_relative "pass"
|
|
8
9
|
require_relative "sources"
|
|
9
10
|
|
|
@@ -24,19 +25,21 @@ class Kimera::CLI::Run::Cycle
|
|
|
24
25
|
|
|
25
26
|
def scope(selected)
|
|
26
27
|
eligible = focus(selected || @registry.each.map { |mutant, _| mutant.id })
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
resolution = Kimera::IgnoreList.resolve(@registry, @options[:ignore])
|
|
29
|
+
ignored = resolution.ids & eligible
|
|
30
|
+
notify(selected, ignored, resolution)
|
|
29
31
|
session(eligible - ignored, ignored)
|
|
30
32
|
end
|
|
31
33
|
|
|
32
|
-
def notify(selected, ignored)
|
|
33
|
-
@digest.
|
|
34
|
+
def notify(selected, ignored, resolution)
|
|
35
|
+
@digest.anchors(resolution)
|
|
34
36
|
@digest.announce(selected, ignored, files: @registry.files.size, since: @options[:since])
|
|
37
|
+
@digest.evaluating(ignored) if @options[:evaluate_ignored]
|
|
35
38
|
end
|
|
36
39
|
|
|
37
40
|
def session(todo, ignored)
|
|
38
41
|
loaded = Kimera::Incremental::Session.load(@options[:session], registry: @registry)
|
|
39
|
-
perform(todo, loaded)
|
|
42
|
+
perform(@options[:evaluate_ignored] ? todo + ignored : todo, loaded)
|
|
40
43
|
conclude(todo, ignored, loaded)
|
|
41
44
|
end
|
|
42
45
|
|
|
@@ -55,21 +58,23 @@ class Kimera::CLI::Run::Cycle
|
|
|
55
58
|
|
|
56
59
|
def emission
|
|
57
60
|
{ path: @options[:report], format: @options.fetch(:format, "text"), metadata: provenance }
|
|
58
|
-
.merge(coverage: coverage, log: @options[:log])
|
|
61
|
+
.merge(coverage: coverage, log: @options[:log], scope: narrowing.note)
|
|
59
62
|
end
|
|
60
63
|
|
|
64
|
+
def narrowing = @_narrowing ||= Kimera::CLI::Run::Narrowing.new(@options)
|
|
65
|
+
|
|
61
66
|
def coverage = @options[:fail_on_no_coverage] ? :list : :hint
|
|
62
67
|
|
|
63
68
|
def focus(eligible)
|
|
64
|
-
ids = Array(@options[:focus])
|
|
69
|
+
ids = Array(@options[:focus]).map { |token| @registry.keys.id(token) }
|
|
65
70
|
return eligible if ids.empty?
|
|
66
71
|
missing = ids - eligible
|
|
67
72
|
missing.empty? ? ids : raise(Kimera::UsageError, "focused mutant(s) not in scope: #{missing.join(", ")}")
|
|
68
73
|
end
|
|
69
74
|
|
|
70
75
|
def provenance
|
|
71
|
-
@options.slice(:framework, :source_root, :tests, :exclude_tests, :operators, :coverage, :isolated, :jobs)
|
|
72
|
-
.transform_keys(&:to_s)
|
|
76
|
+
@options.slice(:framework, :source_root, :tests, :exclude_tests, :operators, :coverage, :isolated, :jobs, :since)
|
|
77
|
+
.transform_keys(&:to_s).merge(narrowing.provenance)
|
|
73
78
|
end
|
|
74
79
|
|
|
75
80
|
def harness(remaining, loaded, env: ENV)
|
|
@@ -21,8 +21,14 @@ class Kimera::CLI::Run::Digest
|
|
|
21
21
|
@io.puts("Ignoring #{ignored.size} mutant(s) marked equivalent.") unless ignored.empty?
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
def
|
|
25
|
-
|
|
24
|
+
def evaluating(ignored)
|
|
25
|
+
return if ignored.empty?
|
|
26
|
+
@io.puts("Evaluating them anyway (--evaluate-ignored): each keeps status ignored and records its verdict.")
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def anchors(resolution)
|
|
30
|
+
resolution.stale.each { |rule| @errors.puts(anchor(rule)) }
|
|
31
|
+
resolution.moved.each { |shift| @errors.puts(moved(shift)) }
|
|
26
32
|
end
|
|
27
33
|
|
|
28
34
|
def verbose(options)
|
|
@@ -43,6 +49,8 @@ class Kimera::CLI::Run::Digest
|
|
|
43
49
|
"kimera: warning: ignore entry matches no mutant (stale anchor?): #{spot(rule)} #{rule[:label]}".rstrip
|
|
44
50
|
end
|
|
45
51
|
|
|
52
|
+
def moved(shift) = "kimera: warning: ignore entry re-anchored: #{shift} (update the entry to silence this)"
|
|
53
|
+
|
|
46
54
|
def spot(rule)
|
|
47
55
|
[rule[:file], rule[:line]].compact.join(":")
|
|
48
56
|
end
|
|
@@ -17,8 +17,8 @@ class Kimera::CLI::Run::Emission
|
|
|
17
17
|
@quiet = quiet
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
def emit(report, registry, path: nil, format: "text", metadata: nil, coverage: :hint, log: nil)
|
|
21
|
-
text = rendered(report, registry, coverage, path)
|
|
20
|
+
def emit(report, registry, path: nil, format: "text", metadata: nil, coverage: :hint, log: nil, scope: nil)
|
|
21
|
+
text = rendered(report, registry, coverage, path, scope)
|
|
22
22
|
document = report.document(metadata)
|
|
23
23
|
present(text, format, document)
|
|
24
24
|
save(log, text) if log
|
|
@@ -27,9 +27,10 @@ class Kimera::CLI::Run::Emission
|
|
|
27
27
|
|
|
28
28
|
private
|
|
29
29
|
|
|
30
|
-
def rendered(report, registry, coverage, path)
|
|
30
|
+
def rendered(report, registry, coverage, path, scope)
|
|
31
31
|
io = StringIO.new
|
|
32
|
-
Kimera::Report::Text.new(registry, io: io, color: color?)
|
|
32
|
+
Kimera::Report::Text.new(registry, io: io, color: color?)
|
|
33
|
+
.report(report, scope: scope, coverage: coverage, path: path)
|
|
33
34
|
io.string
|
|
34
35
|
end
|
|
35
36
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../scope/file_set"
|
|
4
|
+
|
|
5
|
+
class Kimera::CLI
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
class Kimera::CLI::Run
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
class Kimera::CLI::Run::Narrowing
|
|
12
|
+
def initialize(options)
|
|
13
|
+
@options = options
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def narrowed?
|
|
17
|
+
kept, total = counts
|
|
18
|
+
kept < total
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def provenance
|
|
22
|
+
narrowed? ? { "narrowed" => true, "configured_tests" => @options[:configured_tests] } : { "narrowed" => false }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def note
|
|
26
|
+
return unless narrowed?
|
|
27
|
+
kept, total = counts
|
|
28
|
+
"narrowed run: --tests matched #{count(kept)} of #{count(total)} test files from the configured tests: " \
|
|
29
|
+
"glob; survivors may be killed by tests outside it"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def counts = @_counts ||= measure
|
|
35
|
+
|
|
36
|
+
def measure
|
|
37
|
+
tests, configured = @options.values_at(:tests, :configured_tests)
|
|
38
|
+
return [0, 0] if tests == configured
|
|
39
|
+
all = files(configured)
|
|
40
|
+
[(all & files(tests)).size, all.size]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def files(globs)
|
|
44
|
+
Kimera::FileSet.expand(globs, exclude: @options[:exclude_tests]).map { |file| File.expand_path(file) }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def count(number) = number.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
|
|
48
|
+
end
|
|
@@ -35,8 +35,8 @@ module Kimera::CLI::RunOptions
|
|
|
35
35
|
Kimera::Flag.build("--since REF", :since, "Incremental: only mutate lines changed vs REF (git diff)"),
|
|
36
36
|
Kimera::Flag.build("--session FILE", :session, "Persist/resume per-mutant results in FILE"),
|
|
37
37
|
Kimera::Flag.build(
|
|
38
|
-
"--focus ID", :focus, "Evaluate only
|
|
39
|
-
|
|
38
|
+
"--focus ID|KEY", :focus, "Evaluate only these mutants, by ID or report key (repeatable)",
|
|
39
|
+
collect: true
|
|
40
40
|
),
|
|
41
41
|
Kimera::Flag.build("--[no-]gate", :gate, "Return gate status (default: on; use --no-gate for exploration)"),
|
|
42
42
|
Kimera::Flag.build("--max-survivors N", :max_survivors, "Gate: fail only if survivors exceed N", type: Integer),
|
|
@@ -44,6 +44,11 @@ module Kimera::CLI::RunOptions
|
|
|
44
44
|
"--max-ignored N", :max_ignored,
|
|
45
45
|
"Gate: fail if more than N mutants are ignore-listed", type: Integer
|
|
46
46
|
),
|
|
47
|
+
Kimera::Flag.build(
|
|
48
|
+
"--evaluate-ignored", :evaluate_ignored,
|
|
49
|
+
"Also run ignored mutants; they stay ignored but record their verdict (killed, survived, ...)"
|
|
50
|
+
),
|
|
51
|
+
Kimera::Flag.build("--no-baseline", :baseline, "Skip the config's baseline: file (its mutants are judged)"),
|
|
47
52
|
Kimera::Flag.build(
|
|
48
53
|
"--max-errors N", :max_errors,
|
|
49
54
|
"Gate: fail if more than N mutants could not be judged (default 0)", type: Integer
|
|
@@ -63,6 +68,7 @@ module Kimera::CLI::RunOptions
|
|
|
63
68
|
Kimera::Flag.build(["-q", "--quiet"], :quiet, "Suppress the final human-readable report"),
|
|
64
69
|
Kimera::Flag.build(["-v", "--verbose"], :verbose, "Print resolved scope and execution settings"),
|
|
65
70
|
Kimera::Flag.build("--log FILE", :log, "Write the final human-readable report to FILE"),
|
|
71
|
+
Kimera::Flag.build("--pidfile FILE", :pidfile, "Write kimera's process id to FILE, removed when the run ends"),
|
|
66
72
|
Kimera::Flag.build(
|
|
67
73
|
"--[no-]isolate-db", :isolate_db, "Wrap each mutant in a rolled-back ActiveRecord transaction"
|
|
68
74
|
),
|
data/lib/kimera/cli/run.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "fileutils"
|
|
3
4
|
require_relative "../error"
|
|
4
5
|
require_relative "../plugins"
|
|
5
6
|
|
|
@@ -27,7 +28,8 @@ class Kimera::CLI::Run
|
|
|
27
28
|
end
|
|
28
29
|
|
|
29
30
|
def run(argv)
|
|
30
|
-
|
|
31
|
+
options = parse(argv)
|
|
32
|
+
pid_file(options[:pidfile]) { cycle(options).call }
|
|
31
33
|
rescue Kimera::Error => error
|
|
32
34
|
@errors.puts("kimera: #{error.message}")
|
|
33
35
|
1
|
|
@@ -35,6 +37,18 @@ class Kimera::CLI::Run
|
|
|
35
37
|
|
|
36
38
|
private
|
|
37
39
|
|
|
40
|
+
def pid_file(path, &)
|
|
41
|
+
return yield unless path
|
|
42
|
+
File.write(path, "#{Process.pid}\n")
|
|
43
|
+
held(path, &)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def held(path)
|
|
47
|
+
yield
|
|
48
|
+
ensure
|
|
49
|
+
FileUtils.rm_f(path)
|
|
50
|
+
end
|
|
51
|
+
|
|
38
52
|
def parse(argv) = Kimera::CLI::Run::Arguments.new.parse(argv)
|
|
39
53
|
|
|
40
54
|
def cycle(options)
|
|
@@ -11,7 +11,7 @@ class Kimera::CLI::Survivors::Panel
|
|
|
11
11
|
@io.puts("#{matching.size} #{options[:status]} mutant(s):")
|
|
12
12
|
sorted(matching).each { |row| entry(row) }
|
|
13
13
|
@io.puts
|
|
14
|
-
@io.puts("detail: kimera mutant <
|
|
14
|
+
@io.puts("detail: kimera mutant <ID|KEY> --report #{options[:report]}")
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def describe(row)
|
|
@@ -76,6 +76,6 @@ class Kimera::CLI::Survivors::Panel
|
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
def location(row)
|
|
79
|
-
[row["file"], row["line"]].compact.join(":")
|
|
79
|
+
row["key"] || [row["file"], row["line"]].compact.join(":")
|
|
80
80
|
end
|
|
81
81
|
end
|
data/lib/kimera/cli/survivors.rb
CHANGED
|
@@ -11,7 +11,7 @@ class Kimera::CLI::Survivors
|
|
|
11
11
|
"Lists a report's surviving mutants; FILE_SUBSTRING keeps only " \
|
|
12
12
|
"mutants whose file path contains it (e.g. models/discount).",
|
|
13
13
|
flags: [
|
|
14
|
-
Kimera::Flag.build("--id
|
|
14
|
+
Kimera::Flag.build("--id ID|KEY", :id, "Show one mutant, by ID or key, in full detail"),
|
|
15
15
|
Kimera::Flag.build(
|
|
16
16
|
"--status NAME", :status,
|
|
17
17
|
"List a different status (survived, no_coverage, " \
|
|
@@ -74,7 +74,7 @@ class Kimera::CLI::Survivors
|
|
|
74
74
|
end
|
|
75
75
|
|
|
76
76
|
def show(results, id)
|
|
77
|
-
row = results
|
|
77
|
+
row = Kimera::CLI::ReportFile.row(results, id)
|
|
78
78
|
return missing(id) unless row
|
|
79
79
|
Panel.new(io: @io).describe(row)
|
|
80
80
|
0
|
|
@@ -5,6 +5,12 @@ require_relative "../execution/suite_env"
|
|
|
5
5
|
|
|
6
6
|
class Kimera::CLI::TestCommand
|
|
7
7
|
LOADER = "ARGV.map { |f| File.expand_path(f) }.tap { ARGV.clear }.each { |f| require(f) }"
|
|
8
|
+
MAX_LISTED = 10
|
|
9
|
+
COUNT_LINE = /\d+ (?:failures?|examples?)\b/i
|
|
10
|
+
SUMMARIES = [COUNT_LINE, /failed|Error\b/i].freeze
|
|
11
|
+
RSPEC_RERUN = /^rspec (\S+) #/
|
|
12
|
+
MINITEST_HEADER = /^\s*\d+\) (?:Failure|Error):\n(.+?)(?: \[[^\]]*\])?:$/
|
|
13
|
+
CULPRITS = [RSPEC_RERUN, MINITEST_HEADER].freeze
|
|
8
14
|
DRY_RUN = "require 'minitest'; Minitest.class_variable_set(:@@installed_at_exit, true); #{LOADER}; exit!(0)".freeze
|
|
9
15
|
|
|
10
16
|
def initialize(framework, files, root:)
|
|
@@ -14,7 +20,9 @@ class Kimera::CLI::TestCommand
|
|
|
14
20
|
end
|
|
15
21
|
|
|
16
22
|
def baseline
|
|
17
|
-
check(run, "Baseline: configured test suite is green")
|
|
23
|
+
check(run, "Baseline: configured test suite is green") do |failure, output|
|
|
24
|
+
["✗", "Baseline: #{advice(failure)}#{failing(output)}"]
|
|
25
|
+
end
|
|
18
26
|
end
|
|
19
27
|
|
|
20
28
|
def loading
|
|
@@ -32,7 +40,7 @@ class Kimera::CLI::TestCommand
|
|
|
32
40
|
|
|
33
41
|
def check(command, passed)
|
|
34
42
|
output, status = Open3.capture2e(Kimera::Execution::SUITE_ENV, *command, chdir: @root)
|
|
35
|
-
status.success? ? ["✓", passed] : yield(summary(output))
|
|
43
|
+
status.success? ? ["✓", passed] : yield(summary(output), output)
|
|
36
44
|
rescue Errno::ENOENT
|
|
37
45
|
yield(nil)
|
|
38
46
|
end
|
|
@@ -41,8 +49,19 @@ class Kimera::CLI::TestCommand
|
|
|
41
49
|
|
|
42
50
|
def dry_run = minitest? ? ruby(DRY_RUN) : rspec("--dry-run")
|
|
43
51
|
|
|
52
|
+
def failing(output)
|
|
53
|
+
ids = culprits(output.to_s)
|
|
54
|
+
return "" if ids.empty?
|
|
55
|
+
"\n failing tests:#{ids.first(MAX_LISTED).map { |id| "\n #{id}" }.join}#{overflow(ids)}"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def overflow(ids) = (ids.size - MAX_LISTED).then { |more| more.positive? ? "\n … and #{more} more" : "" }
|
|
59
|
+
|
|
60
|
+
def culprits(output) = CULPRITS.flat_map { |pattern| output.scan(pattern).flatten }.uniq
|
|
61
|
+
|
|
44
62
|
def summary(output)
|
|
45
|
-
output.lines.reverse
|
|
63
|
+
lines = output.lines.reverse
|
|
64
|
+
SUMMARIES.lazy.filter_map { |pattern| lines.find { |line| line.match?(pattern) } }.first&.strip ||
|
|
46
65
|
"test command failed"
|
|
47
66
|
end
|
|
48
67
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "rspec/core"
|
|
5
5
|
require "stringio"
|
|
6
|
+
require_relative "../support/test_exit"
|
|
6
7
|
|
|
7
8
|
begin
|
|
8
9
|
RSpec::Core::ConfigurationOptions.new([]).configure(RSpec.configuration)
|
|
@@ -14,6 +15,10 @@ RSpec.configuration.output_stream = output
|
|
|
14
15
|
RSpec.configuration.error_stream = StringIO.new
|
|
15
16
|
RSpec.configuration.deprecation_stream = StringIO.new
|
|
16
17
|
|
|
18
|
+
RSpec.configuration.before(:suite) do
|
|
19
|
+
RSpec.world.all_examples.each { |example| example.extend(Kimera::TestExit::Example) }
|
|
20
|
+
end
|
|
21
|
+
|
|
17
22
|
ledger, request = ARGV
|
|
18
23
|
locations = JSON.parse(File.read(request, encoding: Encoding::UTF_8)).fetch("tests")
|
|
19
24
|
code = RSpec::Core::Runner.run(locations)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "minitest"
|
|
5
|
+
require_relative "../support/test_exit"
|
|
5
6
|
|
|
6
7
|
Minitest.seed ||= 1
|
|
7
8
|
begin
|
|
@@ -36,7 +37,7 @@ killer =
|
|
|
36
37
|
ids.find do |id|
|
|
37
38
|
klass, name = methods[id]
|
|
38
39
|
next false unless klass
|
|
39
|
-
result = klass.new(name).run
|
|
40
|
+
result = klass.new(name).extend(Kimera::TestExit::Test).run
|
|
40
41
|
!result.passed? && !result.skipped?
|
|
41
42
|
end
|
|
42
43
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class Kimera::Execution::ParallelTestDatabases
|
|
4
|
+
LOCK_WAITS = { /postg/i => "SET lock_timeout = '5s'", /mysql|trilogy/i => "SET SESSION lock_wait_timeout = 5" }.freeze
|
|
5
|
+
|
|
4
6
|
def initialize(adapter:, jobs:, errors: nil)
|
|
5
7
|
@adapter = adapter
|
|
6
8
|
@jobs = jobs
|
|
@@ -26,16 +28,14 @@ class Kimera::Execution::ParallelTestDatabases
|
|
|
26
28
|
end
|
|
27
29
|
|
|
28
30
|
def before_exit(index)
|
|
29
|
-
Array(index).each { |worker| cleanup(worker) }
|
|
30
|
-
rescue StandardError => error
|
|
31
|
-
errors.puts("kimera: parallelize_teardown failed (#{error.class}: #{error.message})")
|
|
31
|
+
attempt("parallelize_teardown failed") { Array(index).each { |worker| cleanup(worker) } }
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
private
|
|
35
35
|
|
|
36
36
|
def cleanup(index)
|
|
37
37
|
return unless active?
|
|
38
|
-
scrub
|
|
38
|
+
attempt("could not empty the worker's test database") { scrub }
|
|
39
39
|
ActiveSupport::Testing::Parallelization.run_cleanup_hooks.each { |hook| hook.call(index) }
|
|
40
40
|
end
|
|
41
41
|
|
|
@@ -46,12 +46,24 @@ class Kimera::Execution::ParallelTestDatabases
|
|
|
46
46
|
nil
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
def attempt(failure)
|
|
50
|
+
yield
|
|
51
|
+
rescue StandardError => error
|
|
52
|
+
errors.puts("kimera: #{failure} (#{error.class}: #{error.message})")
|
|
53
|
+
end
|
|
54
|
+
|
|
49
55
|
def scrub
|
|
50
56
|
return unless defined?(ActiveRecord::Base)
|
|
51
57
|
connection = ActiveRecord::Base.connection
|
|
58
|
+
bound(connection)
|
|
52
59
|
connection.truncate_tables(*connection.tables)
|
|
53
60
|
end
|
|
54
61
|
|
|
62
|
+
def bound(connection)
|
|
63
|
+
statement = LOCK_WAITS.find { |pattern, _| connection.adapter_name.match?(pattern) }&.last
|
|
64
|
+
connection.execute(statement) if statement
|
|
65
|
+
end
|
|
66
|
+
|
|
55
67
|
def errors = @errors || $stderr
|
|
56
68
|
|
|
57
69
|
def eligible?
|
|
@@ -73,11 +73,13 @@ class Kimera::Execution::Shift
|
|
|
73
73
|
|
|
74
74
|
def step(line, responses, history, index)
|
|
75
75
|
request = JSON.parse(line)
|
|
76
|
-
return if process(request["id"], responses, history, index, MODES.fetch(request["recheck"], :warm))
|
|
76
|
+
return if tainted?(process(request["id"], responses, history, index, MODES.fetch(request["recheck"], :warm)))
|
|
77
77
|
emit(responses, t: "ready")
|
|
78
78
|
index + 1
|
|
79
79
|
end
|
|
80
80
|
|
|
81
|
+
def tainted?(result) = result.is_a?(Suspect) || result.status == :timeout
|
|
82
|
+
|
|
81
83
|
def safely(subject, mode)
|
|
82
84
|
tests = available(subject.id)
|
|
83
85
|
return subject.verdict(:no_coverage) if tests.empty?
|
|
@@ -103,7 +105,7 @@ class Kimera::Execution::Shift
|
|
|
103
105
|
result = evaluate(id, mode)
|
|
104
106
|
emit(io, **result.message)
|
|
105
107
|
history << id if result.killed?
|
|
106
|
-
leaks.check(io, history, index) unless
|
|
108
|
+
leaks.check(io, history, index) unless tainted?(result)
|
|
107
109
|
result
|
|
108
110
|
end
|
|
109
111
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "adapter"
|
|
4
|
+
require_relative "../support/test_exit"
|
|
4
5
|
|
|
5
6
|
module Kimera
|
|
6
7
|
module Frameworks
|
|
@@ -75,7 +76,7 @@ class Kimera::Frameworks::MinitestAdapter < Kimera::Frameworks::Adapter
|
|
|
75
76
|
def record(id, failed, failures)
|
|
76
77
|
klass, name = @methods[id]
|
|
77
78
|
return unless klass
|
|
78
|
-
result = klass.new(name).run
|
|
79
|
+
result = klass.new(name).extend(Kimera::TestExit::Test).run
|
|
79
80
|
return unless killing?(result)
|
|
80
81
|
failure(id, result, failed, failures)
|
|
81
82
|
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "rspec/core"
|
|
4
4
|
require_relative "adapter"
|
|
5
5
|
require_relative "rspec_group_index"
|
|
6
|
+
require_relative "../support/test_exit"
|
|
6
7
|
|
|
7
8
|
module Kimera
|
|
8
9
|
module Frameworks
|
|
@@ -105,7 +106,9 @@ class Kimera::Frameworks::RSpecAdapter < Kimera::Frameworks::Adapter
|
|
|
105
106
|
end
|
|
106
107
|
|
|
107
108
|
def index(top)
|
|
108
|
-
Kimera::Frameworks::RSpecGroupIndex.examples(top).each
|
|
109
|
+
Kimera::Frameworks::RSpecGroupIndex.examples(top).each do |example|
|
|
110
|
+
@examples[example.id] = example.extend(Kimera::TestExit::Example)
|
|
111
|
+
end
|
|
109
112
|
end
|
|
110
113
|
|
|
111
114
|
def clear(examples)
|
|
@@ -68,10 +68,14 @@ class Kimera::Incremental::Session
|
|
|
68
68
|
|
|
69
69
|
def report(evaluated, waived, registry)
|
|
70
70
|
results = evaluated.filter_map { |id| @results[id] }
|
|
71
|
-
results += waived.map { |id|
|
|
71
|
+
results += waived.map { |id| waiver(id, registry) }
|
|
72
72
|
Kimera::RunReport.new(results: results, leaks: @leaks, registry: registry)
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
+
def waiver(id, registry)
|
|
76
|
+
@results[id]&.waive || Kimera::MutantResult.waived(id, registry.index[id]&.file)
|
|
77
|
+
end
|
|
78
|
+
|
|
75
79
|
def merge!(report)
|
|
76
80
|
report.results.each { |r| @results[r.mutant_id] = r }
|
|
77
81
|
@leaks.concat(report.leaks)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require_relative "mutation_point"
|
|
5
|
+
|
|
6
|
+
class Kimera::MutantKeys
|
|
7
|
+
ORDINAL = /\A\d+\z/
|
|
8
|
+
LINE = /:\d+:(?=\h{8}\z)/
|
|
9
|
+
|
|
10
|
+
Fingerprint =
|
|
11
|
+
Data.define(:point, :mutant) do
|
|
12
|
+
def key(repeats)
|
|
13
|
+
repeats[facts] += 1
|
|
14
|
+
"#{point.file}:#{point.location.start_line}:#{digest(repeats[facts])}"
|
|
15
|
+
end
|
|
16
|
+
private
|
|
17
|
+
def facts = [point.file, point.method_name, point.operator, point.original_source.split.join(" "), mutant.label]
|
|
18
|
+
def digest(repeat) = Digest::SHA256.hexdigest([*facts, repeat].join("\n"))[0, 8]
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.of(registry)
|
|
22
|
+
repeats = Hash.new(0)
|
|
23
|
+
new(registry.each.to_h { |mutant, point| [mutant.id, Fingerprint.new(point, mutant).key(repeats)] })
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(keys)
|
|
27
|
+
@keys = keys
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def [](id) = @keys[id]
|
|
31
|
+
|
|
32
|
+
def id(token)
|
|
33
|
+
text = token.to_s
|
|
34
|
+
text.match?(ORDINAL) ? text.to_i : @keys.key(text) || moved(text) || text
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def moved(text)
|
|
40
|
+
found = @keys.select { |_id, key| unlined(key) == unlined(text) }.keys
|
|
41
|
+
found.first if found.one?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def unlined(key) = key.sub(LINE, ":")
|
|
45
|
+
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
4
|
require_relative "mutation_point"
|
|
5
|
+
require_relative "mutant_keys"
|
|
5
6
|
|
|
6
7
|
class Kimera::Registry
|
|
7
8
|
FORMAT_VERSION = 1
|
|
@@ -53,6 +54,10 @@ class Kimera::Registry
|
|
|
53
54
|
paired[id]
|
|
54
55
|
end
|
|
55
56
|
|
|
57
|
+
def keys
|
|
58
|
+
@_keys ||= Kimera::MutantKeys.of(self)
|
|
59
|
+
end
|
|
60
|
+
|
|
56
61
|
def each
|
|
57
62
|
return enum_for(:each) unless block_given?
|
|
58
63
|
@points.each do |point|
|
|
@@ -14,7 +14,7 @@ class Kimera::Report::Actions
|
|
|
14
14
|
hints = path ? SavedHints.new(path) : UnsavedHints.new
|
|
15
15
|
lines = [
|
|
16
16
|
advice(report.survived, hints, :survivors), advice(report.uncovered, hints, :uncovered),
|
|
17
|
-
advice(report.errors, hints, :unjudged)
|
|
17
|
+
advice(report.errors, hints, :unjudged), advice(report.statuses(:ignored).select(&:lapsed?), hints, :lapsed)
|
|
18
18
|
].compact
|
|
19
19
|
@io.puts("", "Next actions:", *lines) unless lines.empty?
|
|
20
20
|
end
|
|
@@ -37,6 +37,8 @@ class Kimera::Report::Actions
|
|
|
37
37
|
|
|
38
38
|
def unjudged(id) = "unjudged mutant(s): see why with `#{mutant(id)}`"
|
|
39
39
|
|
|
40
|
+
def lapsed(_id) = "ignored mutant(s) now killed: `kimera baseline prune BASELINE.yml --report #{@path}`"
|
|
41
|
+
|
|
40
42
|
private
|
|
41
43
|
|
|
42
44
|
def mutant(id) = "kimera mutant #{id} --report #{@path}"
|
|
@@ -48,5 +50,7 @@ class Kimera::Report::Actions
|
|
|
48
50
|
def uncovered(_id) = "uncovered mutant(s): rerun with --report tmp/kimera.json"
|
|
49
51
|
|
|
50
52
|
def unjudged(_id) = "unjudged mutant(s): retry with `kimera run --isolated`"
|
|
53
|
+
|
|
54
|
+
def lapsed(_id) = "ignored mutant(s) now killed: rerun with --report FILE, then `kimera baseline prune`"
|
|
51
55
|
end
|
|
52
56
|
end
|
|
@@ -10,6 +10,21 @@ module Kimera::Report::Sections
|
|
|
10
10
|
|
|
11
11
|
def unjudged(report) = section("Mutants Kimera could not judge", report.errors) { |r| unjudge(r) }
|
|
12
12
|
|
|
13
|
+
def waivers(report)
|
|
14
|
+
judged = report.statuses(:ignored).select(&:verdict)
|
|
15
|
+
return if judged.empty?
|
|
16
|
+
@io.puts
|
|
17
|
+
@io.puts(verdicts(judged))
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def verdicts(judged)
|
|
21
|
+
lapsed = judged.count(&:lapsed?)
|
|
22
|
+
surviving = judged.count { |result| result.verdict == :survived }
|
|
23
|
+
other = judged.size - lapsed - surviving
|
|
24
|
+
line = "#{lapsed} ignored mutant(s) are now killed (their entries can be pruned); #{surviving} still survive"
|
|
25
|
+
other.zero? ? "#{line}." : "#{line}; #{other} could not be judged."
|
|
26
|
+
end
|
|
27
|
+
|
|
13
28
|
def unjudge(result)
|
|
14
29
|
id = result.mutant_id
|
|
15
30
|
@io.puts(" #{red("unjudged")} ##{id} #{where(@registry.index[id], result)} #{result.detail}")
|