kimera 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +86 -0
- data/README.md +59 -1
- data/lib/kimera/cli/baseline/entry.rb +56 -0
- data/lib/kimera/cli/baseline/ledger.rb +32 -0
- data/lib/kimera/cli/baseline/prune.rb +68 -0
- data/lib/kimera/cli/baseline/review.rb +56 -0
- data/lib/kimera/cli/baseline.rb +49 -9
- data/lib/kimera/cli/help_text.rb +2 -2
- data/lib/kimera/cli/mutant.rb +17 -11
- data/lib/kimera/cli/report_file.rb +4 -0
- data/lib/kimera/cli/run/arguments.rb +7 -3
- data/lib/kimera/cli/run/cycle.rb +14 -9
- data/lib/kimera/cli/run/digest.rb +10 -2
- data/lib/kimera/cli/run/emission.rb +5 -4
- data/lib/kimera/cli/run/narrowing.rb +48 -0
- data/lib/kimera/cli/run/options.rb +8 -2
- data/lib/kimera/cli/run.rb +15 -1
- data/lib/kimera/cli/survivors/panel.rb +2 -2
- data/lib/kimera/cli/survivors.rb +2 -2
- data/lib/kimera/cli/test_command.rb +22 -3
- data/lib/kimera/execution/isolated_child.rb +5 -0
- data/lib/kimera/execution/isolated_child_minitest.rb +2 -1
- data/lib/kimera/execution/parallel_test_databases.rb +16 -4
- data/lib/kimera/execution/shift.rb +4 -2
- data/lib/kimera/execution/worker_pool/worker.rb +2 -2
- data/lib/kimera/execution/worker_pool.rb +1 -1
- data/lib/kimera/frameworks/minitest_adapter.rb +2 -1
- data/lib/kimera/frameworks/rspec_adapter.rb +4 -1
- data/lib/kimera/incremental/session.rb +5 -1
- data/lib/kimera/registry/mutant_keys.rb +45 -0
- data/lib/kimera/registry/registry.rb +5 -0
- data/lib/kimera/report/actions.rb +5 -1
- data/lib/kimera/report/sections.rb +15 -0
- data/lib/kimera/report/text.rb +1 -0
- data/lib/kimera/results/result.rb +17 -2
- data/lib/kimera/results/run_report.rb +3 -2
- data/lib/kimera/scope/config.rb +6 -1
- data/lib/kimera/scope/ignore_drift.rb +13 -0
- data/lib/kimera/scope/ignore_list.rb +21 -14
- data/lib/kimera/scope/ignore_placement.rb +30 -0
- data/lib/kimera/self_protection.rb +1 -0
- data/lib/kimera/support/test_exit.rb +40 -0
- data/lib/kimera/support/version.rb +1 -1
- data/skills/kimera/SKILL.md +66 -4
- metadata +10 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 95ba2c00cdac5d85b206e1e82ec76af626763f010f88206c756e8b91b0fb617a
|
|
4
|
+
data.tar.gz: 1212b25f14d752f02ae0144a41f09cf3cf0be48c8ffc1bfbed97ddbf765219f8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cbfd38f27c4f1a230a7d3919209e6e4c40e9255d1791f8e0c1676269a2d8e7496008ee3e435a131aefa7d8717c0bed405cf05459d4946e936ed282bbfb24dffd
|
|
7
|
+
data.tar.gz: 6893b2f9a4ad91dbd0d0f3acdb1922315b584efc4c018226f2ea39d88dd6f3dc2e242beef1cca9b3e6e619bd2dacbf37dd1a09010dd9768f5b5f5af740542bfe
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,91 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.1.6 (2026-09-29)
|
|
4
|
+
|
|
5
|
+
- A warm worker that hits the soft timeout is replaced before it takes
|
|
6
|
+
another mutant. The timeout interrupts only the test's own thread, so
|
|
7
|
+
threads the test started (a lock holder in a concurrency test, say) kept
|
|
8
|
+
their connections, transactions and locks. Later mutants on that worker
|
|
9
|
+
blocked on them, timed out, and could be scored as kills.
|
|
10
|
+
- A retiring worker's shutdown (`parallelize_teardown` hooks, emptying its
|
|
11
|
+
test database) is bounded by the hard timeout. Before, the watchdog stopped
|
|
12
|
+
watching a worker once the queue drained, so a teardown blocked on a leaked
|
|
13
|
+
lock hung `kimera run` forever.
|
|
14
|
+
- Emptying a worker's test database at shutdown waits at most 5 seconds for
|
|
15
|
+
table locks on PostgreSQL and MySQL. If it gives up, kimera warns and still
|
|
16
|
+
runs the `parallelize_teardown` hooks.
|
|
17
|
+
- `kimera doctor --check-baseline` lists the failing tests (up to ten) of a
|
|
18
|
+
red baseline, for RSpec and Minitest.
|
|
19
|
+
- Every report result carries a `key`, such as
|
|
20
|
+
`app/models/discount.rb:44:8557dadd`, that names the mutant in any run. A
|
|
21
|
+
mutant ID numbers every mutant the run scanned, so `--focus 3740` from a
|
|
22
|
+
full report meant nothing, or a different mutant, in a one-file run. The
|
|
23
|
+
key's digest covers the mutant's own code (file, method, operator,
|
|
24
|
+
whitespace-squeezed source, label, and which repeat it is), not its line or
|
|
25
|
+
ID, so it is the same whether the run scanned one file or five hundred.
|
|
26
|
+
`kimera report` and `kimera mutant` show it next to the ID.
|
|
27
|
+
- `--focus` (on `run`, `changed`, and `ci`) and `kimera mutant` accept a key
|
|
28
|
+
as well as an ID. A key whose line moved still resolves when its path and
|
|
29
|
+
digest match exactly one mutant; an unknown key is "not in scope".
|
|
30
|
+
- `kimera mutant ID --report R --rerun` re-evaluates the right mutant. It
|
|
31
|
+
scanned only the mutant's file, where IDs restart at 1, then focused the
|
|
32
|
+
report's ID, so a mutant outside the report's first file was "not in scope"
|
|
33
|
+
or silently swapped for another. It now focuses the key. A report written
|
|
34
|
+
before keys existed can't be rerun; regenerate it.
|
|
35
|
+
- A test that makes code call `exit` or `abort` now fails like any other
|
|
36
|
+
test. RSpec and Minitest let the `SystemExit` through, so it ended the warm
|
|
37
|
+
worker and the mutant was `harness_error` ("worker crashed before result");
|
|
38
|
+
a serial baseline exited kimera silently, and an isolated child died before
|
|
39
|
+
reporting. The test now fails after its teardown, with the exit status,
|
|
40
|
+
where it was called and `abort`'s message in the failure. A mutant that
|
|
41
|
+
makes a test abort is `killed`, and a baseline test that aborts turns the
|
|
42
|
+
baseline red with that message. `exit!` and interrupts are not caught.
|
|
43
|
+
- A run whose `--tests` leaves out test files the configured `tests:` glob
|
|
44
|
+
(or the default) would run now says so above the summary: `narrowed run:
|
|
45
|
+
--tests matched 3 of 1,120 test files from the configured tests: glob;
|
|
46
|
+
survivors may be killed by tests outside it`. The JSON report's `run`
|
|
47
|
+
section records `"narrowed": true` and the configured globs. Narrowed runs
|
|
48
|
+
are faster, but their survivors (and kills that hold only within the narrow
|
|
49
|
+
set) used to read like a full run's. Verdicts are unchanged.
|
|
50
|
+
- `kimera run --pidfile FILE` writes the run's process id to FILE and removes
|
|
51
|
+
it when the run ends, including on errors. Scripts waiting with `pgrep -f
|
|
52
|
+
"kimera run ..."` matched their own shell and waited forever; they can wait
|
|
53
|
+
on that pid instead.
|
|
54
|
+
- `kimera run --evaluate-ignored` (also `changed` and `ci`) runs ignored
|
|
55
|
+
mutants, from `ignore:` and from the baseline, like any other. They keep
|
|
56
|
+
status `ignored`, so they never gate as survivors and still count toward
|
|
57
|
+
`max_ignored`, but each report row gains a `verdict` (`killed`,
|
|
58
|
+
`survived`, ...) and a `detail` such as `ignored (killed): ...`. The text
|
|
59
|
+
report sums them up: how many are now killed (their entries can be
|
|
60
|
+
pruned) and how many still survive. Until now a baselined mutant was never
|
|
61
|
+
evaluated, so finding out which entries were still alive meant copying
|
|
62
|
+
`.kimera.yml` without its `baseline:` line. A `--session` file keeps these
|
|
63
|
+
verdicts across resumes.
|
|
64
|
+
- `kimera run --no-baseline` leaves out the `baseline:` file's entries, so
|
|
65
|
+
those mutants are judged and gated like any other; `.kimera.yml`'s own
|
|
66
|
+
`ignore:` entries still apply.
|
|
67
|
+
- `kimera baseline review BASELINE.yml --report REPORT.json` judges every
|
|
68
|
+
entry against a report (best one from `--evaluate-ignored` or
|
|
69
|
+
`--no-baseline`): killed and safe to prune, still surviving, unjudged,
|
|
70
|
+
stale (the report covers the file but no mutant matches), or out of the
|
|
71
|
+
report's scope. Without `--report` it still just lists the entries.
|
|
72
|
+
- `kimera baseline prune BASELINE.yml --report REPORT.json [--dry-run]`
|
|
73
|
+
rewrites the baseline without the killed and stale entries, keeping the
|
|
74
|
+
order of the rest, prints what it removed, and says by how much
|
|
75
|
+
`max_ignored` can drop. An incremental (`--since`) report holds only the
|
|
76
|
+
mutants on changed lines, so prune never treats an entry missing from it
|
|
77
|
+
as stale. Reports now record `since` in their `run` provenance for this.
|
|
78
|
+
- A line-anchored ignore entry that no longer matches at its line (a line
|
|
79
|
+
was added above it) still applies when its other anchors (label, plus
|
|
80
|
+
`original` and `method` when given) single out exactly one mutant in the
|
|
81
|
+
file. Kimera warns `ignore entry re-anchored: path:12 → 13 [label]` so the
|
|
82
|
+
entry can be updated, instead of reporting it stale and bringing the same
|
|
83
|
+
mutant back as a new survivor. `baseline review` and `prune` follow the
|
|
84
|
+
same rule, and `prune` moves such entries to their current line.
|
|
85
|
+
- `kimera baseline create` records each survivor's `original` snippet next
|
|
86
|
+
to its file, line, and label, so a re-anchored entry matches only the
|
|
87
|
+
mutant it was written for. Baselines without it keep working.
|
|
88
|
+
|
|
3
89
|
## 0.1.5 (2026-09-27)
|
|
4
90
|
|
|
5
91
|
- Re-running a mutated concern's `included do` block no longer clobbers a
|
data/README.md
CHANGED
|
@@ -101,8 +101,20 @@ suite.
|
|
|
101
101
|
bundle exec kimera run --report tmp/kimera/report.json
|
|
102
102
|
bundle exec kimera report tmp/kimera/report.json --status survived
|
|
103
103
|
bundle exec kimera mutant 42 --report tmp/kimera/report.json
|
|
104
|
+
bundle exec kimera mutant app/models/discount.rb:44:8557dadd --report tmp/kimera/report.json
|
|
105
|
+
bundle exec kimera mutant 42 --report tmp/kimera/report.json --rerun
|
|
104
106
|
```
|
|
105
107
|
|
|
108
|
+
A mutant has two names. Its ID (`42`) numbers every mutant the run scanned,
|
|
109
|
+
so the same mutant gets a different ID when a run covers a different set of
|
|
110
|
+
files. Its key (`app/models/discount.rb:44:8557dadd`, the `key` field of each
|
|
111
|
+
report result) is `path:line:digest`. The digest is computed from the mutant's
|
|
112
|
+
own code (its file, method, operator, source, and label), so the key names
|
|
113
|
+
the same mutant in a one-file run and in a full one. `kimera mutant` and
|
|
114
|
+
`--focus` take either. A key whose line moved still resolves when the path
|
|
115
|
+
and digest match exactly one mutant. `--rerun` re-evaluates only that mutant,
|
|
116
|
+
focused by its key.
|
|
117
|
+
|
|
106
118
|
For each survivor, write the test that kills it. If a mutant is truly
|
|
107
119
|
equivalent, add it to `ignore:` in `.kimera.yml` with a reason (see
|
|
108
120
|
[Configuration](#configuration)).
|
|
@@ -158,6 +170,9 @@ bundle exec kimera run app --since origin/main \
|
|
|
158
170
|
# (a no_coverage mutant is untested new logic, not a pass)
|
|
159
171
|
bundle exec kimera run app --since origin/main --fail-on-no-coverage
|
|
160
172
|
|
|
173
|
+
# Evaluate only the given mutants, by report key or by ID (repeatable)
|
|
174
|
+
bundle exec kimera run app/models/discount.rb --focus app/models/discount.rb:44:8557dadd
|
|
175
|
+
|
|
161
176
|
# N warm workers pull from one shared queue, so this scales with cores
|
|
162
177
|
# even on a single big file
|
|
163
178
|
bundle exec kimera run app --jobs 4
|
|
@@ -168,6 +183,12 @@ bundle exec kimera run app --jobs 4
|
|
|
168
183
|
# only slow under parallel load
|
|
169
184
|
bundle exec kimera run app --jobs 8 --hard-timeout 60
|
|
170
185
|
|
|
186
|
+
# Run only some test files, for speed. --tests replaces the configured tests:
|
|
187
|
+
# glob, so verdicts hold only for those files: a survivor may be killed by a
|
|
188
|
+
# test left out. The report says so ("narrowed run: --tests matched 1 of 1,120
|
|
189
|
+
# test files ...", and "narrowed": true in the JSON report's run section)
|
|
190
|
+
bundle exec kimera run app/models/order.rb --tests 'spec/models/order_spec.rb'
|
|
191
|
+
|
|
171
192
|
# Drop spec files that can't run this way (order-dependent, need a browser)
|
|
172
193
|
# without rewriting the whole --tests glob
|
|
173
194
|
bundle exec kimera run app --exclude-test 'spec/system/**/*_spec.rb'
|
|
@@ -196,8 +217,26 @@ bundle exec kimera mutant 42 --report tmp/kimera/report.json
|
|
|
196
217
|
bundle exec kimera baseline create tmp/kimera/report.json --reason "adopting Kimera"
|
|
197
218
|
# Add the printed `baseline:` entry to .kimera.yml, then review it in code review
|
|
198
219
|
bundle exec kimera baseline review .kimera-baseline.yml
|
|
220
|
+
|
|
221
|
+
# Burn the baseline down: evaluate the ignored mutants too, see which entries
|
|
222
|
+
# are now killed or stale, and drop them
|
|
223
|
+
bundle exec kimera run --evaluate-ignored --report tmp/kimera/report.json
|
|
224
|
+
bundle exec kimera baseline review .kimera-baseline.yml --report tmp/kimera/report.json
|
|
225
|
+
bundle exec kimera baseline prune .kimera-baseline.yml --report tmp/kimera/report.json
|
|
199
226
|
```
|
|
200
227
|
|
|
228
|
+
Ignored mutants, from `ignore:` or the baseline, are skipped by default.
|
|
229
|
+
`--evaluate-ignored` runs them anyway: each keeps status `ignored` (it never
|
|
230
|
+
gates as a survivor and still counts toward `max_ignored`) and its report row
|
|
231
|
+
gains a `verdict` (`killed`, `survived`, ...). `--no-baseline` leaves the
|
|
232
|
+
baseline out entirely, so its mutants are judged and gated like any other.
|
|
233
|
+
|
|
234
|
+
`baseline review --report` sorts the entries into killed (safe to prune),
|
|
235
|
+
still surviving, unjudged, stale (the report covers the file but no mutant
|
|
236
|
+
matches), and out of the report's scope. `baseline prune` drops the killed and
|
|
237
|
+
stale ones (`--dry-run` only prints them) and says how far `max_ignored` can
|
|
238
|
+
drop. After an incremental `--since` run it never prunes an entry as stale.
|
|
239
|
+
|
|
201
240
|
### Output formats and exit codes
|
|
202
241
|
|
|
203
242
|
- `--format json`, `ndjson`, `github`, and `sarif` write *only* that format to
|
|
@@ -206,6 +245,10 @@ bundle exec kimera baseline review .kimera-baseline.yml
|
|
|
206
245
|
- Text output honors `NO_COLOR`. `--no-color` forces it off.
|
|
207
246
|
- `--quiet` suits scripts that only need an artifact. `--verbose` prints the
|
|
208
247
|
resolved scope. `--log FILE` keeps the final text report.
|
|
248
|
+
- `--pidfile FILE` writes kimera's process id to FILE when the run starts and
|
|
249
|
+
removes the file when the run ends, whether it passes, fails or errors
|
|
250
|
+
(short of `kill -9`). A script that waits on a background run can watch
|
|
251
|
+
that pid; `pgrep -f "kimera run"` also matches the shell that started it.
|
|
209
252
|
|
|
210
253
|
Exit codes:
|
|
211
254
|
|
|
@@ -299,7 +342,10 @@ max_ignored: 1
|
|
|
299
342
|
|
|
300
343
|
# Known-equivalent mutants. Equivalence is undecidable, so Kimera doesn't
|
|
301
344
|
# guess: you mark a mutant and it stops being a survivor. An entry without
|
|
302
|
-
# a reason: is rejected at startup.
|
|
345
|
+
# a reason: is rejected at startup. Anchors: file (a glob, required), line,
|
|
346
|
+
# column, label, method, original. If a line anchor drifts (a line was added
|
|
347
|
+
# above), the entry still applies when its label (plus original/method, if
|
|
348
|
+
# given) singles out one mutant in the file; Kimera warns so you can update it.
|
|
303
349
|
ignore:
|
|
304
350
|
- file: app/models/discount.rb
|
|
305
351
|
line: 33
|
|
@@ -519,6 +565,18 @@ source ──Prism──▶ registry (mutation points, JSON) ──┬─▶ syn
|
|
|
519
565
|
Before the kill it asks the worker for every thread's backtrace (SIGQUIT)
|
|
520
566
|
and puts them in the verdict's `detail`, or in the baseline error. A test
|
|
521
567
|
interrupted by the soft timeout is a `timeout`, not the mutant's killer.
|
|
568
|
+
The interrupt reaches only the test's own thread, so threads it started
|
|
569
|
+
may still hold connections and locks: after a soft timeout the worker
|
|
570
|
+
takes no more mutants and is replaced. A retiring worker's teardown stays
|
|
571
|
+
under the hard timeout too.
|
|
572
|
+
- Code under test that calls `exit` or `abort` (a rake task, a CLI entry
|
|
573
|
+
point): RSpec and Minitest let the `SystemExit` through, so it would end
|
|
574
|
+
the worker. Kimera records it as that test's failure instead, after the
|
|
575
|
+
test's teardown, with the status, where it was called, and `abort`'s
|
|
576
|
+
message (`SystemExit: exit(1) called from lib/tasks/import.rb:12:in
|
|
577
|
+
'Kernel#abort': no such file`). A mutant that makes a test exit is killed;
|
|
578
|
+
a baseline test that exits turns the baseline red. `exit!` still ends the
|
|
579
|
+
process, and interrupts still stop the run.
|
|
522
580
|
- Slow tests under parallel load: a baseline test whose worker hits the hard
|
|
523
581
|
timeout reruns once, alone. If it passes, the run goes on with a notice and
|
|
524
582
|
the stacks. If it times out again, the baseline is red.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../scope/ignore_list"
|
|
4
|
+
|
|
5
|
+
class Kimera::CLI::Baseline::Entry
|
|
6
|
+
include Kimera::IgnoreList::Drift
|
|
7
|
+
|
|
8
|
+
KILLING = %w[killed timeout error].freeze
|
|
9
|
+
PRUNABLE = %i[killed stale].freeze
|
|
10
|
+
|
|
11
|
+
attr_reader :fields
|
|
12
|
+
|
|
13
|
+
def initialize(fields, rows, missing: :stale)
|
|
14
|
+
@fields = fields
|
|
15
|
+
@rows = rows
|
|
16
|
+
@missing = missing
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def kind
|
|
20
|
+
return :out_of_scope if @rows.empty?
|
|
21
|
+
hits.empty? ? @missing : verdict
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def prunable? = PRUNABLE.include?(kind)
|
|
25
|
+
|
|
26
|
+
def moved = exact.empty? ? hits.first&.fetch("line") : nil
|
|
27
|
+
|
|
28
|
+
def statuses = hits.map { |row| row["verdict"] || row["status"] }.uniq
|
|
29
|
+
|
|
30
|
+
def kept = moved ? @fields.merge("line" => moved) : @fields
|
|
31
|
+
|
|
32
|
+
def place
|
|
33
|
+
file, line, label = @fields.values_at("file", "line", "label")
|
|
34
|
+
"#{[file, line].compact.join(":")}#{" → #{moved}" if moved} [#{label}]"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def verdict
|
|
40
|
+
return :surviving if statuses.include?("survived")
|
|
41
|
+
(statuses - KILLING).empty? ? :killed : :unjudged
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def anchors = @_anchors ||= @fields.except("file").transform_keys(&:to_sym)
|
|
45
|
+
|
|
46
|
+
def matching(anchors) = @rows.select { |row| fits?(anchors, row) }
|
|
47
|
+
|
|
48
|
+
def fits?(anchors, row)
|
|
49
|
+
anchors.all? do |key, value|
|
|
50
|
+
field = key.to_s
|
|
51
|
+
!row.key?(field) || squeeze(value) == squeeze(row[field])
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def squeeze(value) = value.to_s.split.join(" ")
|
|
56
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "yaml"
|
|
4
|
+
require_relative "../report_file"
|
|
5
|
+
require_relative "entry"
|
|
6
|
+
|
|
7
|
+
class Kimera::CLI::Baseline::Ledger
|
|
8
|
+
attr_reader :path, :report
|
|
9
|
+
|
|
10
|
+
def initialize(path, report)
|
|
11
|
+
@path = path
|
|
12
|
+
@report = report
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def document = @_document ||= YAML.safe_load_file(@path) || {}
|
|
16
|
+
|
|
17
|
+
def entries = @_entries ||= Array(document["ignore"]).map { |fields| judged(fields) }
|
|
18
|
+
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def judged(fields)
|
|
22
|
+
Kimera::CLI::Baseline::Entry.new(fields, scoped(fields["file"].to_s), missing: missing)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def scoped(glob) = rows.select { |row| File.fnmatch?(glob, row["file"].to_s, File::FNM_PATHNAME) }
|
|
26
|
+
|
|
27
|
+
def rows = @_rows ||= parsed.fetch("results", [])
|
|
28
|
+
|
|
29
|
+
def missing = parsed.dig("run", "since") ? :out_of_scope : :stale
|
|
30
|
+
|
|
31
|
+
def parsed = @_parsed ||= Kimera::CLI::ReportFile.parse(@report)
|
|
32
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "yaml"
|
|
4
|
+
|
|
5
|
+
class Kimera::CLI::Baseline::Prune
|
|
6
|
+
Tense = Data.define(:removed, :holds)
|
|
7
|
+
DONE = Tense.new("Removed", "now holds")
|
|
8
|
+
PREVIEW = Tense.new("Would remove", "would hold")
|
|
9
|
+
|
|
10
|
+
def initialize(ledger, io: $stdout)
|
|
11
|
+
@ledger = ledger
|
|
12
|
+
@io = io
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def preview
|
|
16
|
+
summarize(PREVIEW)
|
|
17
|
+
@io.puts("(dry run: #{@ledger.path} is unchanged)")
|
|
18
|
+
0
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def apply
|
|
22
|
+
summarize(DONE)
|
|
23
|
+
File.write(@ledger.path, YAML.dump(pruned)) if changed?
|
|
24
|
+
0
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def entries = @ledger.entries
|
|
30
|
+
|
|
31
|
+
def dropped = entries.select(&:prunable?)
|
|
32
|
+
|
|
33
|
+
def kept = entries.reject(&:prunable?)
|
|
34
|
+
|
|
35
|
+
def moved = kept.select(&:moved)
|
|
36
|
+
|
|
37
|
+
def changed? = !(dropped.empty? && moved.empty?)
|
|
38
|
+
|
|
39
|
+
def pruned = @ledger.document.merge("ignore" => kept.map(&:kept))
|
|
40
|
+
|
|
41
|
+
def summarize(tense)
|
|
42
|
+
return unchanged unless changed?
|
|
43
|
+
removed(tense)
|
|
44
|
+
anchored
|
|
45
|
+
total(tense)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def unchanged = @io.puts("Nothing to prune: #{@ledger.path} keeps its #{entries.size} accepted mutant(s).")
|
|
49
|
+
|
|
50
|
+
def removed(tense)
|
|
51
|
+
return if dropped.empty?
|
|
52
|
+
@io.puts("#{tense.removed} #{dropped.size} entr(ies) from #{@ledger.path}:")
|
|
53
|
+
dropped.each { |entry| @io.puts(" - #{entry.place} (#{entry.kind})") }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def anchored
|
|
57
|
+
return if moved.empty?
|
|
58
|
+
@io.puts("Re-anchored #{moved.size} entr(ies) to their mutant's current line:")
|
|
59
|
+
moved.each { |entry| @io.puts(" #{entry.place}") }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def total(tense)
|
|
63
|
+
@io.puts(
|
|
64
|
+
"#{@ledger.path} #{tense.holds} #{kept.size} accepted mutant(s) " \
|
|
65
|
+
"(was #{entries.size}); max_ignored can be lowered by #{dropped.size}."
|
|
66
|
+
)
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class Kimera::CLI::Baseline::Review
|
|
4
|
+
Group =
|
|
5
|
+
Data.define(:kind, :title) do
|
|
6
|
+
def entries(grouped) = grouped.fetch(kind, [])
|
|
7
|
+
|
|
8
|
+
def heading(count) = "#{title} (#{count}):"
|
|
9
|
+
end
|
|
10
|
+
GROUPS = [
|
|
11
|
+
Group.new(:killed, "Killed, safe to prune"),
|
|
12
|
+
Group.new(:surviving, "Still surviving"),
|
|
13
|
+
Group.new(:unjudged, "Unjudged (no verdict in this report)"),
|
|
14
|
+
Group.new(:stale, "Stale: the report covers the file, but no mutant matches"),
|
|
15
|
+
Group.new(:out_of_scope, "Not in the report's scope")
|
|
16
|
+
].freeze
|
|
17
|
+
|
|
18
|
+
def initialize(io: $stdout)
|
|
19
|
+
@io = io
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def show(ledger)
|
|
23
|
+
entries = ledger.entries
|
|
24
|
+
@io.puts("#{entries.size} accepted mutant(s) in #{ledger.path}, judged against #{ledger.report}:")
|
|
25
|
+
groups(entries.group_by(&:kind))
|
|
26
|
+
advise(ledger, entries)
|
|
27
|
+
0
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def groups(grouped) = GROUPS.each { |group| list(group, group.entries(grouped)) }
|
|
33
|
+
|
|
34
|
+
def list(group, entries)
|
|
35
|
+
return if entries.empty?
|
|
36
|
+
@io.puts("", group.heading(entries.size))
|
|
37
|
+
entries.each { |entry| @io.puts(" #{entry.place}#{verdicts(entry)} — #{entry.fields["reason"]}") }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def verdicts(entry)
|
|
41
|
+
statuses = entry.statuses
|
|
42
|
+
statuses.empty? ? "" : " (#{statuses.join(", ")})"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def advise(ledger, entries)
|
|
46
|
+
unevaluated(entries.count { |entry| entry.statuses.include?("ignored") })
|
|
47
|
+
prunable = entries.count(&:prunable?)
|
|
48
|
+
return if prunable.zero?
|
|
49
|
+
@io.puts("", "Prune #{prunable} entr(ies): kimera baseline prune #{ledger.path} --report #{ledger.report}")
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def unevaluated(count)
|
|
53
|
+
return if count.zero?
|
|
54
|
+
@io.puts("", "#{count} entr(ies) were not evaluated: rerun with `kimera run --evaluate-ignored --report FILE`")
|
|
55
|
+
end
|
|
56
|
+
end
|
data/lib/kimera/cli/baseline.rb
CHANGED
|
@@ -6,8 +6,9 @@ require_relative "flag"
|
|
|
6
6
|
require_relative "report_file"
|
|
7
7
|
|
|
8
8
|
class Kimera::CLI::Baseline
|
|
9
|
-
COMMANDS = { "create" => :create, "review" => :review }.freeze
|
|
9
|
+
COMMANDS = { "create" => :create, "review" => :review, "prune" => :prune }.freeze
|
|
10
10
|
ENTRY_KEYS = %w[file line label].freeze
|
|
11
|
+
REPORT_FLAG = Kimera::Flag.build("--report FILE", :report, "Judge each entry against this report")
|
|
11
12
|
|
|
12
13
|
CREATE = Kimera::FlagTable.new(
|
|
13
14
|
banner: "Usage: kimera baseline create REPORT.json --reason TEXT [--output FILE]",
|
|
@@ -18,6 +19,16 @@ class Kimera::CLI::Baseline
|
|
|
18
19
|
]
|
|
19
20
|
)
|
|
20
21
|
|
|
22
|
+
REVIEW = Kimera::FlagTable.new(
|
|
23
|
+
banner: "Usage: kimera baseline review BASELINE.yml [--report REPORT.json]",
|
|
24
|
+
flags: [REPORT_FLAG]
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
PRUNE = Kimera::FlagTable.new(
|
|
28
|
+
banner: "Usage: kimera baseline prune BASELINE.yml --report REPORT.json [--dry-run]",
|
|
29
|
+
flags: [REPORT_FLAG, Kimera::Flag.build("--dry-run", :dry_run, "Print what would change; leave the file as is")]
|
|
30
|
+
)
|
|
31
|
+
|
|
21
32
|
def initialize(io: $stdout, errors: $stderr)
|
|
22
33
|
@io = io
|
|
23
34
|
@errors = errors
|
|
@@ -32,7 +43,9 @@ class Kimera::CLI::Baseline
|
|
|
32
43
|
|
|
33
44
|
private
|
|
34
45
|
|
|
35
|
-
def command(name)
|
|
46
|
+
def command(name)
|
|
47
|
+
COMMANDS.fetch(name) { raise(Kimera::UsageError, "usage: kimera baseline <create|review|prune> ...") }
|
|
48
|
+
end
|
|
36
49
|
|
|
37
50
|
def create(argv)
|
|
38
51
|
options = { reason: nil, output: ".kimera-baseline.yml", force: false }
|
|
@@ -60,13 +73,35 @@ class Kimera::CLI::Baseline
|
|
|
60
73
|
end
|
|
61
74
|
|
|
62
75
|
def review(argv)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
76
|
+
options = { report: nil }
|
|
77
|
+
file = baseline(REVIEW, argv, options)
|
|
78
|
+
report = options[:report]
|
|
79
|
+
report ? Review.new(io: @io).show(ledger(file, report)) : list(file)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def prune(argv)
|
|
83
|
+
options = { dry_run: false }
|
|
84
|
+
file = baseline(PRUNE, argv, options)
|
|
85
|
+
report = options.fetch(:report) { raise(Kimera::UsageError, "baseline prune needs --report FILE") }
|
|
86
|
+
pruning = Prune.new(ledger(file, report), io: @io)
|
|
87
|
+
options[:dry_run] ? pruning.preview : pruning.apply
|
|
67
88
|
end
|
|
68
89
|
|
|
69
|
-
def
|
|
90
|
+
def baseline(table, argv, options)
|
|
91
|
+
file, *rest = table.parse(argv, options)
|
|
92
|
+
raise(Kimera::UsageError, table.banner) unless file && rest.empty?
|
|
93
|
+
existing(file)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def existing(file) = File.file?(file) ? file : raise(Kimera::UsageError, "no such baseline: #{file}")
|
|
97
|
+
|
|
98
|
+
def ledger(file, report)
|
|
99
|
+
raise(Kimera::UsageError, "no such report: #{report}") unless File.file?(report)
|
|
100
|
+
Ledger.new(file, report)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def list(file)
|
|
104
|
+
entries = Array((YAML.safe_load_file(file) || {})["ignore"])
|
|
70
105
|
@io.puts("#{entries.size} accepted mutant(s) in #{file}:")
|
|
71
106
|
entries.sort_by { |entry| [entry["file"], entry["line"], entry["label"]] }.each { |entry| @io.puts(line(entry)) }
|
|
72
107
|
0
|
|
@@ -79,9 +114,14 @@ class Kimera::CLI::Baseline
|
|
|
79
114
|
end
|
|
80
115
|
|
|
81
116
|
def entry(row, reason)
|
|
82
|
-
id, *fields = row.values_at("mutant_id", *ENTRY_KEYS)
|
|
83
|
-
ENTRY_KEYS.zip(fields).to_h { |key, value| [key, value || missing(key, id)] }
|
|
117
|
+
id, *fields, original = row.values_at("mutant_id", *ENTRY_KEYS, "original")
|
|
118
|
+
ENTRY_KEYS.zip(fields).to_h { |key, value| [key, value || missing(key, id)] }
|
|
119
|
+
.merge({ "original" => original }.compact, "reason" => reason)
|
|
84
120
|
end
|
|
85
121
|
|
|
86
122
|
def missing(key, id) = raise(Kimera::UsageError, "report is missing #{key} for mutant ##{id}")
|
|
87
123
|
end
|
|
124
|
+
|
|
125
|
+
require_relative "baseline/ledger"
|
|
126
|
+
require_relative "baseline/prune"
|
|
127
|
+
require_relative "baseline/review"
|
data/lib/kimera/cli/help_text.rb
CHANGED
|
@@ -21,12 +21,12 @@ module Kimera
|
|
|
21
21
|
|
|
22
22
|
Investigate:
|
|
23
23
|
report REPORT.json Filter a saved report (survivors, uncovered, errors)
|
|
24
|
-
mutant ID --report R Show one mutant in full detail
|
|
24
|
+
mutant ID --report R Show one mutant, by ID or key, in full detail
|
|
25
25
|
registry [paths...] Inspect what Kimera would mutate
|
|
26
26
|
synthesize [paths...] Write schemata sources for inspection
|
|
27
27
|
|
|
28
28
|
Setup:
|
|
29
|
-
baseline ... Create or
|
|
29
|
+
baseline ... Create, review, or prune a baseline of accepted survivors
|
|
30
30
|
completion SHELL Print Bash, Zsh, or Fish completion setup
|
|
31
31
|
skill Print the guide for AI agents running Kimera
|
|
32
32
|
version Print version
|
data/lib/kimera/cli/mutant.rb
CHANGED
|
@@ -8,7 +8,7 @@ require_relative "survivors"
|
|
|
8
8
|
|
|
9
9
|
class Kimera::CLI::Mutant
|
|
10
10
|
OPTIONS = Kimera::FlagTable.new(
|
|
11
|
-
banner: "Usage: kimera mutant ID --report REPORT.json",
|
|
11
|
+
banner: "Usage: kimera mutant ID|KEY --report REPORT.json",
|
|
12
12
|
flags: [
|
|
13
13
|
Kimera::Flag.build("--report FILE", :report, "Report generated by kimera run"),
|
|
14
14
|
Kimera::Flag.build("--rerun", :rerun, "Re-evaluate only this mutant in the current project")
|
|
@@ -38,26 +38,25 @@ class Kimera::CLI::Mutant
|
|
|
38
38
|
def identify(id, options)
|
|
39
39
|
raise(Kimera::UsageError, "mutant ID is required") unless id
|
|
40
40
|
raise(Kimera::UsageError, "--report REPORT.json is required") unless options[:report]
|
|
41
|
-
|
|
42
|
-
rescue ArgumentError
|
|
43
|
-
raise(Kimera::UsageError, "mutant ID must be an integer")
|
|
41
|
+
id
|
|
44
42
|
end
|
|
45
43
|
|
|
46
44
|
def show(id, options)
|
|
47
45
|
report = options[:report]
|
|
48
46
|
return rerun(id, report) if options[:rerun]
|
|
49
|
-
Kimera::CLI::Survivors.new(io: @io, errors: @errors).run([report, "--id", id
|
|
47
|
+
Kimera::CLI::Survivors.new(io: @io, errors: @errors).run([report, "--id", id])
|
|
50
48
|
end
|
|
51
49
|
|
|
52
50
|
def rerun(id, report)
|
|
53
51
|
document = Kimera::CLI::ReportFile.parse(report)
|
|
54
52
|
row = locate(document, id)
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
settings = provenance(document)
|
|
54
|
+
@io.puts("Re-running mutant ##{row["mutant_id"]}: #{keyed(row)}")
|
|
55
|
+
Kimera::CLI::Run.new(io: @io, errors: @errors).run(arguments_for(row, settings))
|
|
57
56
|
end
|
|
58
57
|
|
|
59
58
|
def locate(document, id)
|
|
60
|
-
row = document.fetch("results", [])
|
|
59
|
+
row = Kimera::CLI::ReportFile.row(document.fetch("results", []), id)
|
|
61
60
|
raise(Kimera::UsageError, "no mutant ##{id} in this report") unless row
|
|
62
61
|
raise(Kimera::UsageError, "mutant ##{id} has no source file") unless row["file"]
|
|
63
62
|
row
|
|
@@ -68,14 +67,21 @@ class Kimera::CLI::Mutant
|
|
|
68
67
|
raise(Kimera::UsageError, "report has no run provenance; rerun with a report created by this Kimera version")
|
|
69
68
|
end
|
|
70
69
|
|
|
71
|
-
def
|
|
72
|
-
[
|
|
70
|
+
def keyed(row)
|
|
71
|
+
row["key"] ||
|
|
72
|
+
raise(Kimera::UsageError, "report has no mutant keys; regenerate it with kimera run --report, then rerun")
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def arguments_for(row, settings)
|
|
76
|
+
[row.fetch("file"), "--focus", keyed(row), "--framework", settings.fetch("framework", "rspec")]
|
|
73
77
|
.push("--source-root", settings["source_root"])
|
|
74
78
|
.concat(Array(settings["tests"]).flat_map { |glob| ["--tests", glob] })
|
|
75
79
|
.push("--operators", Array(settings["operators"]).join(","))
|
|
76
|
-
.concat(modes(settings))
|
|
80
|
+
.concat(modes(settings)).concat(waived(row))
|
|
77
81
|
end
|
|
78
82
|
|
|
83
|
+
def waived(row) = row["status"] == "ignored" ? ["--evaluate-ignored"] : []
|
|
84
|
+
|
|
79
85
|
def modes(settings)
|
|
80
86
|
[settings["coverage"] == false ? "--no-coverage" : nil, settings["isolated"] ? "--isolated" : nil].compact
|
|
81
87
|
end
|
|
@@ -12,6 +12,10 @@ end
|
|
|
12
12
|
module Kimera::CLI::ReportFile
|
|
13
13
|
module_function
|
|
14
14
|
|
|
15
|
+
def row(results, token)
|
|
16
|
+
results.find { |result| [result["mutant_id"].to_s, result["key"]].include?(token.to_s) }
|
|
17
|
+
end
|
|
18
|
+
|
|
15
19
|
def parse(path)
|
|
16
20
|
JSON.parse(File.read(path, encoding: Encoding::UTF_8))
|
|
17
21
|
rescue JSON::ParserError => error
|
|
@@ -16,20 +16,22 @@ class Kimera::CLI::Run::Arguments
|
|
|
16
16
|
|
|
17
17
|
def finish(options, rest)
|
|
18
18
|
validate(options)
|
|
19
|
-
options.except(:cli_tests).merge(scoped(options, rest))
|
|
19
|
+
options.except(:cli_tests, :baseline_ignore).merge(scoped(options, rest))
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
def validate(options) = Kimera::Report::Formats.new.normalize(options[:format])
|
|
23
23
|
|
|
24
24
|
def scoped(options, rest)
|
|
25
25
|
{
|
|
26
|
-
tests:
|
|
26
|
+
tests: tests(options[:cli_tests], options), configured_tests: tests([], options),
|
|
27
27
|
paths: Kimera::Config.prefer(rest, options[:paths], Kimera::FileSet::DEFAULT_GLOBS)
|
|
28
28
|
}.merge(arrays(options))
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
+
def tests(cli, options) = Kimera::Config.prefer(cli, options[:tests], Kimera::CLI::RunOptions::DEFAULT_TESTS)
|
|
32
|
+
|
|
31
33
|
def arrays(options)
|
|
32
|
-
{ exclude: Array(options[:exclude]), ignore:
|
|
34
|
+
{ exclude: Array(options[:exclude]), ignore: Kimera::Config.ignores(options) }
|
|
33
35
|
.merge(isolate_when_covered_by: Array(options[:isolate_when_covered_by]), focus: Array(options[:focus]))
|
|
34
36
|
end
|
|
35
37
|
|
|
@@ -46,12 +48,14 @@ class Kimera::CLI::Run::Arguments
|
|
|
46
48
|
|
|
47
49
|
def gates
|
|
48
50
|
{ since: nil, session: nil, max_survivors: nil }.merge(max_ignored: nil, max_errors: 0, jobs: 1)
|
|
51
|
+
.merge(evaluate_ignored: false, baseline: nil)
|
|
49
52
|
end
|
|
50
53
|
|
|
51
54
|
def scope
|
|
52
55
|
{ exclude: [], exclude_tests: [], config: nil }
|
|
53
56
|
.merge(ignore: [], isolate_db: false, isolated: false)
|
|
54
57
|
.merge(fail_on_no_coverage: false, progress: nil, color: nil, quiet: false, verbose: false, log: nil)
|
|
58
|
+
.merge(pidfile: nil)
|
|
55
59
|
.merge(isolate_when_covered_by: [])
|
|
56
60
|
.merge(cli_tests: [])
|
|
57
61
|
end
|