@jesdi/skills 0.0.50 → 0.0.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/skills/implement-spec/ARCHITECTURE.md +3 -0
- package/skills/implement-spec/README.md +2 -2
- package/skills/implement-spec/driver_fixloop.py +27 -6
- package/skills/implement-spec/driver_report.py +3 -2
- package/skills/implement-spec/prompts/correction.md +1 -0
- package/skills/implement-spec/prompts/test-writer.md +3 -2
- package/skills/implement-spec/tests/test_fixloop.py +50 -0
- package/skills/test-prune/SKILL.md +156 -0
- package/skills/test-prune/prune.py +202 -0
- package/skills/test-prune/prune_repo.py +319 -0
- package/skills/test-prune/prune_review.py +363 -0
- package/skills/test-prune/prune_runs.py +240 -0
- package/skills/test-prune/prune_source.py +237 -0
- package/skills/test-prune/prune_strengthen.py +301 -0
- package/skills/test-prune/tests/conftest.py +15 -0
- package/skills/test-prune/tests/prune_harness.py +205 -0
- package/skills/test-prune/tests/test_prune_guards.py +871 -0
- package/skills/test-prune/tests/test_prune_inputs.py +97 -0
- package/skills/test-prune/tests/test_prune_review.py +113 -0
- package/skills/test-prune/tests/test_prune_source.py +84 -0
- package/skills/test-prune/tests/test_prune_strengthen.py +183 -0
- package/skills/test-prune/tests/test_prune_unattended.py +225 -0
- package/skills/test-prune/tests/testdata/gh +16 -0
- package/skills/test-prune/tests/testdata/repo/Makefile +25 -0
- package/skills/test-prune/tests/testdata/repo/backend/mutation_baseline.json +16 -0
- package/skills/test-prune/tests/testdata/repo/backend/services/concentration.py +2 -0
- package/skills/test-prune/tests/testdata/repo/backend/services/rebalancer.py +11 -0
- package/skills/test-prune/tests/testdata/repo/backend/tests/conftest.py +6 -0
- package/skills/test-prune/tests/testdata/repo/backend/tests/test_concentration.py +73 -0
- package/skills/test-prune/tests/testdata/repo/backend/tests/test_rebalancer.py +77 -0
- package/skills/test-prune/tests/testdata/repo/fake_engine.py +289 -0
- package/skills/test-prune/tests/testdata/repo/kill_map.json +131 -0
- package/skills-manifest.json +9 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jesdi/skills",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.52",
|
|
4
4
|
"description": "jesdi's agent skills (content package consumed by @jesdi/skills-cli)",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"scripts": {
|
|
30
30
|
"test": "vitest run",
|
|
31
31
|
"test:coverage": "vitest run --coverage --coverage.reportOnFailure",
|
|
32
|
-
"test:python": "mkdir -p .crap && python3 -m coverage run -m pytest skills/backlog skills/crap-gate -q && python3 -m coverage run -a -m pytest skills/implement-spec -q && python3 -m coverage json -o .crap/python.json",
|
|
32
|
+
"test:python": "mkdir -p .crap && python3 -m coverage erase && python3 -m coverage run -m pytest skills/backlog skills/crap-gate -q && python3 -m coverage run -a -m pytest skills/implement-spec -q && COVERAGE_SUBPROCESS_PATCH=subprocess python3 -m coverage run -m pytest skills/test-prune -q && python3 -m coverage combine --append -q && python3 -m coverage json -o .crap/python.json",
|
|
33
33
|
"crap-gate": "sh skills/crap-gate/run.sh",
|
|
34
34
|
"check": "sh scripts/check.sh",
|
|
35
35
|
"generate-manifest": "tsx scripts/generate-manifest.ts"
|
|
@@ -132,6 +132,9 @@ and do not know about the run, except `driver_run.py`, which the three deciding
|
|
|
132
132
|
it blocks only after round 4. A red gate on a test-writer correction blocks.
|
|
133
133
|
- The implementer and each fixer get the `--gate` command and run it before they report. The gate of the driver
|
|
134
134
|
stays the source of truth: an agent's run of the gate counts for nothing.
|
|
135
|
+
- A finding that a correction reports in `not_addressed` goes to the fixer of the same round as kind `code`
|
|
136
|
+
(`driver_fixloop._correct` rewrites it in the findings file): the test needs a production change first, so a
|
|
137
|
+
second correction could not close it.
|
|
135
138
|
- No test-writer correction runs after round 4 (`driver_fixloop._act_on`): its findings become `finding` questions
|
|
136
139
|
with the others. A correction there would add red locked tests that no fix round makes green, and `accept` could
|
|
137
140
|
not merge the ticket.
|
|
@@ -117,7 +117,7 @@ Rules for each agent:
|
|
|
117
117
|
|
|
118
118
|
| Finding | What the driver does |
|
|
119
119
|
|---|---|
|
|
120
|
-
| kind `fixture`, any severity, or kind `test`, `critical` or `important`, on a locked file of the ticket | One test-writer correction, outside the fix rounds. The fixer may not edit a locked file, so a fix round cannot close a `test` finding there. The corrected files are the new locked versions, and the next review is a full review. A finding of this kind that is open after the correction blocks the ticket. No correction runs after round 4. |
|
|
120
|
+
| kind `fixture`, any severity, or kind `test`, `critical` or `important`, on a locked file of the ticket | One test-writer correction, outside the fix rounds. The fixer may not edit a locked file, so a fix round cannot close a `test` finding there. The corrected files are the new locked versions, and the next review is a full review. A finding that the correction reports in `not_addressed` (it needs a production change first) becomes a finding of kind `code`, with the reason as its evidence, and goes to a fix round at once; the ledger says it was not corrected. A finding of this kind that is open after the correction blocks the ticket. No correction runs after round 4. |
|
|
121
121
|
| `critical` or `important` | One fix round. |
|
|
122
122
|
| `minor` | It stays in the ledger and in the findings file. The final reviewer gets the list. |
|
|
123
123
|
|
|
@@ -176,7 +176,7 @@ Every report has `status` (`DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, `BLOCK
|
|
|
176
176
|
|
|
177
177
|
| Role | Template | More fields | Checks after the schema |
|
|
178
178
|
|---|---|---|---|
|
|
179
|
-
| test-writer | `prompts/test-writer.md` | `locked` (test file paths), `red` (`test`, `line`) | The driver reads a test node id `<file>::<test>` in `locked` as its file. With DONE: one commit, one locked file and one red line or more. Each locked path is in a commit that the role made. Each `line` is in the log. A locked file of a merged ticket that the role changed is in `locked`. A correction (after `overrule-test`, or for a review finding) changes only the files that it lists in `locked` and the files that the driver restored before it. When the gate is red on the restore commit, a DONE correction lists a restored file in `locked`. |
|
|
179
|
+
| test-writer | `prompts/test-writer.md` | `locked` (test file paths), `red` (`test`, `line`) | The driver reads a test node id `<file>::<test>` in `locked` as its file. With DONE: one commit, one locked file and one red line or more. Each locked path is in a commit that the role made. Each `line` is in the log. A locked file of a merged ticket that the role changed is in `locked`. A correction (after `overrule-test`, or for a review finding) changes only the files that it lists in `locked` and the files that the driver restored before it. When the gate is red on the restore commit, a DONE correction lists a restored file in `locked`. Each `not_addressed[].finding` is the title of a finding of the correction; a DONE correction with each of its findings there needs no commit. |
|
|
180
180
|
| implementer | `prompts/implementer.md` | `green` | With DONE: one commit or more. |
|
|
181
181
|
| reviewer | `prompts/reviewer.md` | `findings` (`severity`, `file`, `line`, `title`, `evidence`, `kind`), `mutations` (`test`, `file`, `line`, `red`); `commits` is empty | In a full review (one with no fix range: round 1, and the review after a test-writer correction): `mutations` is required, with a proof for each locked test file (`test` is `<locked file>::<test name>`). Each `red` is in the log. |
|
|
182
182
|
| correctness | `prompts/correctness.md` | the same as the reviewer; `commits` is empty | `mutations` is optional. |
|
|
@@ -104,8 +104,9 @@ def _review(job: Job) -> list[dict]:
|
|
|
104
104
|
return found
|
|
105
105
|
|
|
106
106
|
|
|
107
|
-
def _correct(job: Job, findings: list[dict]) ->
|
|
108
|
-
"""One test-writer correction for the findings on the locked tests. It is outside the fix rounds.
|
|
107
|
+
def _correct(job: Job, findings: list[dict]) -> list[dict]:
|
|
108
|
+
"""One test-writer correction for the findings on the locked tests. It is outside the fix rounds. Returns the
|
|
109
|
+
findings that it did not correct, as findings of kind code for the fixer (issue #111)."""
|
|
109
110
|
run, number = job.run, job.ticket.number
|
|
110
111
|
if job.corrected:
|
|
111
112
|
titles = "; ".join(finding["title"] for finding in findings)
|
|
@@ -120,19 +121,39 @@ def _correct(job: Job, findings: list[dict]) -> None:
|
|
|
120
121
|
job.previous = "" # no review has seen the restore commit
|
|
121
122
|
ledger(f"Ticket {number}: the code does not need the change to {', '.join(restored)}: the restore closes it, "
|
|
122
123
|
"with no correction.")
|
|
123
|
-
return
|
|
124
|
+
return []
|
|
124
125
|
values = {**job.pointers(job.round), "restored": " ".join(restored),
|
|
125
126
|
"record": record and os.path.relpath(record, job.slot)}
|
|
126
127
|
notes = ["correction"] + ["restored"] * bool(restored) + ["restored-red"] * bool(record)
|
|
127
128
|
with during("the test-writer correction"):
|
|
128
129
|
report = run.agent("test-writer", number, job.slot, values, notes=notes, fix_round=job.round + 1,
|
|
129
|
-
check={"correction": True, "restored": restored, "restore_red": bool(record)
|
|
130
|
+
check={"correction": True, "restored": restored, "restore_red": bool(record),
|
|
131
|
+
"findings": [finding["title"] for finding in findings]})
|
|
130
132
|
job.tip, job.previous, job.corrected = driver_slots.head(job.slot), "", True
|
|
131
133
|
entry = run.state.read()["tickets"][number]
|
|
132
134
|
run.state.ticket(number, tip=job.tip, **driver_review.written(job.slot, entry, report["locked"], job.tip))
|
|
135
|
+
return _uncorrected(job, findings, report.get("not_addressed", []))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _uncorrected(job: Job, findings: list[dict], not_addressed: list[dict]) -> list[dict]:
|
|
139
|
+
"""Log what the correction did with each finding. Returns each one it did not correct as a finding of kind
|
|
140
|
+
code, with its reason as the evidence: the test needs a production change first, so a fixer closes it."""
|
|
141
|
+
number = job.ticket.number
|
|
142
|
+
reasons = {item["finding"]: item["reason"] for item in not_addressed}
|
|
133
143
|
for finding in findings:
|
|
144
|
+
result = (f"The test-writer did not correct it: {reasons[finding['title']]}. It goes to the fixer as a finding "
|
|
145
|
+
"of kind code." if finding["title"] in reasons else f"The test-writer corrected it at {job.tip[:7]}.")
|
|
134
146
|
ledger(f"Ticket {number}: {finding['kind']} correction in {finding['file']}, outside the fix rounds: "
|
|
135
|
-
f"{finding['title']}: {finding['evidence']}.
|
|
147
|
+
f"{finding['title']}: {finding['evidence']}. {result}")
|
|
148
|
+
code = {finding["title"]: {**finding, "kind": "code", "evidence": f"the test-writer did not correct it: {reason}"}
|
|
149
|
+
for finding in findings if (reason := reasons.get(finding["title"]))}
|
|
150
|
+
if code:
|
|
151
|
+
# The fixer reads the findings file of this review, so the finding is of kind code there too.
|
|
152
|
+
path = OUT / number / f"findings-{job.round}.json"
|
|
153
|
+
written = [code.get(finding["title"], finding) if finding in findings else finding
|
|
154
|
+
for finding in json.loads(path.read_text())]
|
|
155
|
+
path.write_text(json.dumps(written, indent=1) + "\n")
|
|
156
|
+
return list(code.values())
|
|
136
157
|
|
|
137
158
|
|
|
138
159
|
def _fix(job: Job) -> None:
|
|
@@ -170,7 +191,7 @@ def _act_on(job: Job, found: list[dict]) -> bool:
|
|
|
170
191
|
# A changed locked file is fixed first: a correction on top of it would make the changed file the reference.
|
|
171
192
|
locked_file_changed = any(finding["source"] == "driver" for finding in to_fix)
|
|
172
193
|
if to_correct and not locked_file_changed:
|
|
173
|
-
_correct(job, to_correct)
|
|
194
|
+
to_fix += [finding for finding in _correct(job, to_correct) if finding["severity"] in driver_review.OPEN]
|
|
174
195
|
if not to_fix:
|
|
175
196
|
return False
|
|
176
197
|
if not to_fix:
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
context: repo (slot path), branch, start (HEAD when the role started), log, full (a full review: no fix range), locked,
|
|
4
4
|
correction (a test-writer corrects locked tests), restored (locked files the driver put back before it),
|
|
5
|
-
restore_red (the gate is red on the commit of that restore),
|
|
5
|
+
restore_red (the gate is red on the commit of that restore), findings (the titles a correction gets),
|
|
6
6
|
owners (ticket numbers, for the final fix batch), merged (locked files of merged tickets, not its own), pr (tip)."""
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
@@ -81,7 +81,8 @@ def _correction_errors(report: dict, context: dict) -> list[str]:
|
|
|
81
81
|
errors.append("the gate is red on the restore commit, so a restored file fails with the code of this ticket: "
|
|
82
82
|
"update that file and list it in `locked`, or report BLOCKED with a concern that starts with "
|
|
83
83
|
"`locked test contradicts spec:`")
|
|
84
|
-
return errors
|
|
84
|
+
return errors + [f"not_addressed: {item['finding']} is not the title of a finding of this correction"
|
|
85
|
+
for item in report.get("not_addressed", []) if item["finding"] not in context.get("findings", [])]
|
|
85
86
|
|
|
86
87
|
|
|
87
88
|
def _merged_errors(report: dict, context: dict) -> list[str]:
|
|
@@ -3,4 +3,5 @@ This call is a correction of the locked tests, not a new test set. Read in {find
|
|
|
3
3
|
A `fixture` finding: a locked test encodes a stale path, format, fixture layout or setup detail. Correct only that detail and keep what each test asserts.
|
|
4
4
|
A `test` finding: a locked test is too weak, for example it stays green when the production line it protects is broken. Make the test assert the behavior of its criterion so that it fails on that break. Do not remove an assertion.
|
|
5
5
|
For this you may read the implementation.
|
|
6
|
+
When a finding needs a production change first, so that no correct test can be green, leave the test as it is and add the finding to `not_addressed`: `finding` is its `title` as the findings file gives it, `reason` is the production change it needs. The driver sends it to the fixer as a finding of kind `code`. A correction whose findings are all in `not_addressed` may report DONE with no commit.
|
|
6
7
|
The implementation exists, so a corrected test is green: for `red`, break the production line that the test protects, run the test, print its failing output, then revert the edit. In `locked`, list each test file you corrected: test file paths only, not test node ids. Unlike `mutations[].test` of the reviewer, `locked` takes no `::<test name>`.
|
|
@@ -25,9 +25,10 @@ Write one JSON report at {report} that matches this schema:
|
|
|
25
25
|
"files": {"type": "array", "items": {"type": "string"}},
|
|
26
26
|
"concerns": {"type": "array", "items": {"type": "string"}},
|
|
27
27
|
"locked": {"type": "array", "items": {"type": "string"}},
|
|
28
|
-
"red": {"type": "array", "items": {"type": "object", "required": ["test", "line"], "properties": {"test": {"type": "string", "minLength": 1}, "line": {"type": "string", "minLength": 1}}}}
|
|
28
|
+
"red": {"type": "array", "items": {"type": "object", "required": ["test", "line"], "properties": {"test": {"type": "string", "minLength": 1}, "line": {"type": "string", "minLength": 1}}}},
|
|
29
|
+
"not_addressed": {"type": "array", "items": {"type": "object", "required": ["finding", "reason"], "properties": {"finding": {"type": "string", "minLength": 1}, "reason": {"type": "string", "minLength": 1}}}}
|
|
29
30
|
},
|
|
30
|
-
"if": {"properties": {"status": {"enum": ["DONE", "DONE_WITH_CONCERNS"]}}},
|
|
31
|
+
"if": {"properties": {"status": {"enum": ["DONE", "DONE_WITH_CONCERNS"]}}, "not": {"required": ["not_addressed"], "properties": {"not_addressed": {"minItems": 1}}}},
|
|
31
32
|
"then": {"properties": {"commits": {"minItems": 1}, "locked": {"minItems": 1}, "red": {"minItems": 1}}}
|
|
32
33
|
}
|
|
33
34
|
```
|
|
@@ -177,6 +177,56 @@ def test_a_test_finding_on_a_locked_file_gets_the_test_writer_correction_not_a_f
|
|
|
177
177
|
assert driver_review.route([vacuous, minor, elsewhere], ["tests/test_01.py"]) == ([vacuous], [elsewhere])
|
|
178
178
|
|
|
179
179
|
|
|
180
|
+
NEEDS_CODE = "needs production code: the code has no multi-PR view"
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def not_addressing(*titles, **report):
|
|
184
|
+
"""A correction of tests/test_01.py that reports `titles` under `not_addressed`."""
|
|
185
|
+
step = {**tw("01"), "files": {"tests/test_01.py": "# corrected\n"}, "commit": "test: 01 fixture fix"}
|
|
186
|
+
reasons = [{"finding": title, "reason": NEEDS_CODE} for title in titles]
|
|
187
|
+
return {**step, "report": {**step["report"], **report, "not_addressed": reasons}}
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def test_a_test_finding_the_correction_did_not_address_goes_to_the_fixer_as_a_code_finding(tmp_path):
|
|
191
|
+
# Issue #111: the ledger said "corrected it at <sha>" for every finding, and the finding came back each review.
|
|
192
|
+
stale = finding("STALE", kind="fixture", file="tests/test_01.py")
|
|
193
|
+
vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
|
|
194
|
+
rig = Inside(tmp_path).run(plan(**{"01/test-writer": [tw("01"), not_addressing("VACUOUS")],
|
|
195
|
+
"01/fixer": [fix("01", 1)],
|
|
196
|
+
"01/reviewer": [review("01", [stale, vacuous]), review("01")]}))
|
|
197
|
+
assert rig.code == 0, rig.ledger()
|
|
198
|
+
order = [call["role"] for call in rig.calls("01")]
|
|
199
|
+
assert order == ["test-writer", "implementer", "reviewer", "test-writer", "fixer", "reviewer"]
|
|
200
|
+
assert rig.led("STALE: STALE. The test-writer corrected it at")
|
|
201
|
+
assert rig.led("VACUOUS", "corrected it at") == []
|
|
202
|
+
assert rig.led(f"test correction in tests/test_01.py, outside the fix rounds: VACUOUS: VACUOUS. The test-writer "
|
|
203
|
+
f"did not correct it: {NEEDS_CODE}. It goes to the fixer as a finding of kind code.")
|
|
204
|
+
# The fixer reads the finding as one of kind code, with the reason of the test-writer as its evidence.
|
|
205
|
+
(routed,) = [f for f in json.loads((rig.folder() / "findings-0.json").read_text()) if f["title"] == "VACUOUS"]
|
|
206
|
+
assert (routed["kind"], routed["evidence"]) == ("code", f"the test-writer did not correct it: {NEEDS_CODE}")
|
|
207
|
+
assert rig.state()["tickets"]["01"]["round"] == 1 and "findings-0.json" in rig.prompt("fixer-1")
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def test_a_correction_that_addresses_no_finding_needs_no_commit(tmp_path):
|
|
211
|
+
vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
|
|
212
|
+
none = not_addressing("VACUOUS", commits=[], files=[], locked=[], red=[])
|
|
213
|
+
del none["commit"], none["files"]
|
|
214
|
+
rig = Inside(tmp_path).run(plan(**{"01/test-writer": [tw("01"), none], "01/fixer": [fix("01", 1)],
|
|
215
|
+
"01/reviewer": [review("01", [vacuous]), review("01")]}))
|
|
216
|
+
assert rig.code == 0, rig.ledger()
|
|
217
|
+
assert [call["role"] for call in rig.calls("01")].count("test-writer") == 2 # no rerun
|
|
218
|
+
assert rig.led("VACUOUS. The test-writer did not correct it") and rig.state()["tickets"]["01"]["round"] == 1
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def test_a_correction_that_names_a_finding_it_was_not_given_is_rerun(tmp_path):
|
|
222
|
+
vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
|
|
223
|
+
rig = Inside(tmp_path).run(plan(**{"01/test-writer": [tw("01"), not_addressing("OTHER"), not_addressing()],
|
|
224
|
+
"01/reviewer": [review("01", [vacuous]), review("01")]}))
|
|
225
|
+
assert rig.code == 0, rig.ledger()
|
|
226
|
+
assert "not_addressed: OTHER is not the title of a finding of this correction" in rig.prompt("test-writer-1")
|
|
227
|
+
assert rig.led("VACUOUS: VACUOUS. The test-writer corrected it at")
|
|
228
|
+
|
|
229
|
+
|
|
180
230
|
def test_a_test_writer_that_lists_test_node_ids_in_locked_locks_their_files(tmp_path):
|
|
181
231
|
# Issue #71: a node id `<file>::<test>` in `locked` names its file, for the first run and for a correction.
|
|
182
232
|
vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: test-prune
|
|
3
|
+
description: >-
|
|
4
|
+
Mutation-guided test pruning: a fixer proposes deleting redundant tests and strengthening
|
|
5
|
+
weak ones, an independent reviewer accepts or rejects each proposal, and prune.py applies
|
|
6
|
+
the accepted batch behind a cold mutation run and the ratchet. Use when the user says
|
|
7
|
+
"prune tests", "test-prune", "mutation test pruning" or "strengthen weak tests", or when a
|
|
8
|
+
scheduled job runs it with `--unattended`.
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
# test-prune: delete what the mutation ratchet proves redundant, strengthen what it proves weak
|
|
12
|
+
|
|
13
|
+
You are the orchestrator. `prune.py` owns every git, make and file operation and every safety
|
|
14
|
+
guard; the subagents only write JSON files or test commits. Each role is a fresh subagent with no
|
|
15
|
+
context from the others: the reviewer reads the fixer's proposals, reasons included, and judges
|
|
16
|
+
them against the code itself.
|
|
17
|
+
|
|
18
|
+
**Publishing belongs to prune.py.** In both modes you never run `git push`, `git merge`,
|
|
19
|
+
`gh pr create` or `gh pr merge` yourself, and neither does any subagent; `prune` and
|
|
20
|
+
`strengthen-finish` push their own `chore/test-*` branch and open its PR, and nothing merges.
|
|
21
|
+
|
|
22
|
+
The target repo provides the make targets `test`, `mutation`, `mutation-check`,
|
|
23
|
+
`mutation-baseline`, `test-candidates`, `prune-gate` and `crap`, and the layout `backend/tests/`,
|
|
24
|
+
`backend/mutation_baseline.json`. Every command:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
python3 "<skill-dir>/prune.py" <command> [--state DIR] [--date YYYY-MM-DD]
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Run it from inside the target repo. State lives in `<git dir>/test-prune/` (outside the working
|
|
31
|
+
tree); each command reads what the previous one wrote there. Exit 0 is done (including nothing to
|
|
32
|
+
do), 1 is a failed make target, gate or guard, 2 is bad input or an unmet precondition; the
|
|
33
|
+
message names the offending ids or files. On exit 2 fix the input and rerun the same command; on
|
|
34
|
+
exit 1 stop and report the output.
|
|
35
|
+
|
|
36
|
+
Every command records itself in `<state>/status.json`. A CI job reads it after the headless
|
|
37
|
+
session (which cannot set the job's exit code) and fails the job when its `exit` is not 0; an
|
|
38
|
+
unattended run stays red there until the strengthen loop ends (`nothing to strengthen`, or
|
|
39
|
+
`strengthen-finish` exit 0). End a headless session by reporting its `reason` as the final line.
|
|
40
|
+
`--state` must be empty, new, or an earlier run's state dir.
|
|
41
|
+
|
|
42
|
+
When a push succeeded but `gh pr create` failed (exit 1, the branch is on origin with no PR),
|
|
43
|
+
open the PR by hand from `<state>/prune-pr.md` or `<state>/strengthen-pr.md` after fixing `gh`;
|
|
44
|
+
rerunning the command refuses because the branch exists.
|
|
45
|
+
|
|
46
|
+
**A run needs a checkout dedicated to it** for its whole duration: the CI checkout, or a worktree.
|
|
47
|
+
If anything else edits a file there mid-run, the run stops with exit 1 rather than overwrite it,
|
|
48
|
+
leaves the checkout as it is (HEAD may stay detached), names the files, and nothing is lost; keep
|
|
49
|
+
or discard the change, then check out the base branch. Races beyond that check are out of scope.
|
|
50
|
+
|
|
51
|
+
**Runtime.** Each cold `make mutation` takes about 30 minutes on a real repo, and a run makes
|
|
52
|
+
several (collect, each safety-check run, each strengthen proof). Run long commands with a timeout of
|
|
53
|
+
an hour or more, or in the background, and wait for them.
|
|
54
|
+
|
|
55
|
+
## Modes
|
|
56
|
+
|
|
57
|
+
- **Interactive** (default): nothing in the working tree changes before the human approves the
|
|
58
|
+
batch. Stop after `batch`, show its output, and run `apply` only on an explicit yes.
|
|
59
|
+
- **Unattended** (`collect --unattended`, run headless by the weekly CI job): only protected
|
|
60
|
+
candidates, at most 10 deleted test ids per run (a parametrized test goes whole, every case
|
|
61
|
+
counted; one with more than 10 cases is skipped, the rest left for next week), one PR from
|
|
62
|
+
`chore/test-prune-<date>` and one from `chore/test-strengthen-<date>`, both from the base
|
|
63
|
+
commit. The base branch never moves. A merge is the human's approval, so the run ends with open
|
|
64
|
+
PRs.
|
|
65
|
+
|
|
66
|
+
Two kinds of accepted proposal reach a human only: equivalent mutants (the skill writes no
|
|
67
|
+
equivalent id into the baseline) and "strengthen this candidate with this assertion" verdicts
|
|
68
|
+
(the skill applies no assertion to an existing candidate). `batch` prints both; unattended, every
|
|
69
|
+
PR body and the job summary carry them.
|
|
70
|
+
|
|
71
|
+
## Prune loop
|
|
72
|
+
|
|
73
|
+
1. `collect [--unattended | --protected-only]`. Needs a clean tracked tree on a branch. Runs the
|
|
74
|
+
base checks, a cold mutation run and `make test-candidates`, then writes
|
|
75
|
+
`fixer-input.json`. When it prints `no candidates` / `no protected candidates`, the weak spots
|
|
76
|
+
may still need the fixer; continue.
|
|
77
|
+
2. **Fixer.** Dispatch a subagent with `<state>/fixer-input.json`, read access to the repo, and
|
|
78
|
+
this brief: for every candidate decide `keep`, `delete` into one named test that contains it,
|
|
79
|
+
or `strengthen` with one named assertion; for every weak-spot group decide `strengthen`
|
|
80
|
+
(which test to add to or strengthen) or `equivalent` (with the argument why no test can tell
|
|
81
|
+
the mutant apart). A group with both kinds of mutant takes two entries, one `strengthen` and
|
|
82
|
+
one `equivalent`, each listing its own disjoint `mutant_ids`. Compare assertions, not coverage: a candidate whose assertions pin a value,
|
|
83
|
+
boundary or error the container never checks is `keep` or `strengthen`. It writes:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
{"candidates": [{"test_id": "...", "verdict": "keep|delete|strengthen",
|
|
87
|
+
"into": "<test id, delete only>", "assertion": "<strengthen only>", "reason": "..."}],
|
|
88
|
+
"weak_spots": [{"group": "...", "verdict": "strengthen|equivalent",
|
|
89
|
+
"mutant_ids": ["<subset of the group's ids, omit for all>"],
|
|
90
|
+
"test": "<test id or test file>", "reason": "..."}]}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
One entry per candidate, one per group (or the strengthen + equivalent pair), no others.
|
|
94
|
+
3. `review-input --proposals <fixer file>` validates the file and writes
|
|
95
|
+
`<state>/review-input.json`: each proposal with the candidate's source and assertions and,
|
|
96
|
+
for a deletion, the containing test's source and assertions. A proposal's `id` is the
|
|
97
|
+
candidate's test id or the group; a split group's two proposals are `<group>:strengthen` and
|
|
98
|
+
`<group>:equivalent`.
|
|
99
|
+
4. **Reviewer.** Dispatch a *different* fresh subagent with `<state>/review-input.json` and read
|
|
100
|
+
access to the repo. Brief: accept a deletion only when every assertion of the candidate is
|
|
101
|
+
implied by an assertion of the `into` test (same inputs or a superset, an equal or stricter
|
|
102
|
+
check); covering the same lines is not enough. Accept a strengthen proposal only when the
|
|
103
|
+
named assertion checks behaviour a user would notice. Accept an equivalent only with a
|
|
104
|
+
concrete argument that no input distinguishes the mutant. Every reason names the assertion or
|
|
105
|
+
input it rests on. It writes, one entry per proposal:
|
|
106
|
+
|
|
107
|
+
```json
|
|
108
|
+
{"reviews": [{"id": "<proposal id>", "verdict": "accept|reject", "reason": "..."}]}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
5. `batch --reviews <reviewer file>` writes `batch.json` and prints the accepted deletions (each
|
|
112
|
+
with its containing test and the reviewer's reason), strengthen items, equivalent proposals,
|
|
113
|
+
skipped deletions with the reason, and rejections. It edits nothing.
|
|
114
|
+
- Interactive: show that output to the human and stop. After approval, `apply` applies the
|
|
115
|
+
deletions, runs the safety check, restores every culprit and commits on the current branch.
|
|
116
|
+
- Unattended: `prune` applies the batch, runs the safety check, `make test`, commits, runs
|
|
117
|
+
`make prune-gate` and, only when it passes, pushes the branch and opens the PR.
|
|
118
|
+
|
|
119
|
+
**Safety check** (inside `apply` and `prune`): a cold mutation run plus `make mutation-check`
|
|
120
|
+
on the batch; every deletion that lets a module exceed its baseline is restored and reported.
|
|
121
|
+
|
|
122
|
+
## Strengthen loop
|
|
123
|
+
|
|
124
|
+
Runs after the prune loop, also in unattended mode, independently of the prune PR: run it even
|
|
125
|
+
when `prune` exited 1 (the status keeps that failure, so the job still goes red).
|
|
126
|
+
|
|
127
|
+
1. `strengthen-start` writes `<state>/strengthen-input.json` (each accepted weak-spot strengthen
|
|
128
|
+
item with the mutant ids it must kill) and prints the commit to build on; unattended it
|
|
129
|
+
detaches HEAD at the base commit. `nothing to strengthen` ends the loop.
|
|
130
|
+
2. **Strengthen agent.** Dispatch a fresh subagent with `strengthen-input.json`. Brief: per item,
|
|
131
|
+
test-first, write the assertion or test that fails against each named mutant and passes on
|
|
132
|
+
the real code; touch only `backend/tests/`; make one or more commits per item, each ending
|
|
133
|
+
with the trailer line `Test-Prune-Item: <item id>`; leave the tree clean.
|
|
134
|
+
3. `strengthen-verify` rejects commits without a known trailer or outside `backend/tests/`, then
|
|
135
|
+
proves each item on a cold run: every named id survived on the base and is killed on the
|
|
136
|
+
head. Unproven items are reverted and closed: a later commit for a closed item is refused. It
|
|
137
|
+
writes `<state>/strengthen-proof.json`, including the proven head. On failure HEAD stays
|
|
138
|
+
where the commits are (detached, unattended): fix and rerun. To abandon the loop instead,
|
|
139
|
+
`git checkout <base branch>`; the item commits stay reachable through `git reflog`.
|
|
140
|
+
4. **Strengthen reviewer.** Dispatch a fresh subagent with `strengthen-proof.json` and the diff
|
|
141
|
+
of the proven items (`git log -p <start>..HEAD`). Brief: accept an item only when its test
|
|
142
|
+
asserts observable behaviour through the public interface, rather than restating the
|
|
143
|
+
implementation; one review per proven item, same schema as the prune reviewer.
|
|
144
|
+
5. `strengthen-finish --reviews <file>` refuses a HEAD other than the proven one (rerun
|
|
145
|
+
`strengthen-verify` after any new commit); on any exit 2 HEAD stays at the proven head, so fix
|
|
146
|
+
the input and rerun it. It reverts rejected items, reruns the proof cold, runs
|
|
147
|
+
`make mutation-check`, `make test`, `make crap` and `make mutation-baseline`, refuses a
|
|
148
|
+
baseline that gained an equivalent id, and commits the lower baseline. Unattended it opens the
|
|
149
|
+
strengthen PR listing each changed test with the ids it now kills; interactive it leaves the
|
|
150
|
+
commits on the current branch for the human.
|
|
151
|
+
|
|
152
|
+
## Report
|
|
153
|
+
|
|
154
|
+
End with what the last command printed: deleted, restored, skipped and left-for-next-week tests,
|
|
155
|
+
strengthened tests with their killed ids, the PR links, and the equivalent proposals. In CI the
|
|
156
|
+
same summaries land in `$GITHUB_STEP_SUMMARY`.
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""test-prune: drive the fixer/reviewer prune loop and the strengthen loop over a target repo.
|
|
3
|
+
|
|
4
|
+
collect → (fixer) → review-input → (reviewer) → batch → apply (interactive) | prune (unattended);
|
|
5
|
+
strengthen-start → (strengthen agent) → strengthen-verify → (reviewer) → strengthen-finish.
|
|
6
|
+
The script owns every git, make and file operation; the agents only write JSON or test commits.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from prune_repo import BadInput, Fail, Repo, State, read_agent_json
|
|
18
|
+
from prune_review import batch, collect, review_input
|
|
19
|
+
from prune_runs import apply, prune
|
|
20
|
+
from prune_strengthen import strengthen_finish, strengthen_start, strengthen_verify
|
|
21
|
+
|
|
22
|
+
COMMANDS = {
|
|
23
|
+
"collect": collect, "review-input": review_input, "batch": batch, "apply": apply, "prune": prune,
|
|
24
|
+
"strengthen-start": strengthen_start, "strengthen-verify": strengthen_verify,
|
|
25
|
+
"strengthen-finish": strengthen_finish,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Parser(argparse.ArgumentParser):
|
|
30
|
+
"""An argument error raises BadInput (exit 2, recorded in the run status) instead of exiting."""
|
|
31
|
+
|
|
32
|
+
def error(self, message):
|
|
33
|
+
raise BadInput(f"{self.prog}: {message}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _common():
|
|
37
|
+
common = Parser(add_help=False)
|
|
38
|
+
common.add_argument("--state", help="run state directory (default: <git dir>/test-prune)")
|
|
39
|
+
common.add_argument("--date", help="branch date YYYY-MM-DD (default: today, UTC)")
|
|
40
|
+
return common
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def build_parser():
|
|
44
|
+
common = _common()
|
|
45
|
+
parser = Parser(prog="prune.py", description=__doc__)
|
|
46
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
47
|
+
|
|
48
|
+
collect_cmd = sub.add_parser("collect", parents=[common])
|
|
49
|
+
collect_cmd.add_argument("--unattended", action="store_true")
|
|
50
|
+
collect_cmd.add_argument("--protected-only", action="store_true")
|
|
51
|
+
|
|
52
|
+
sub.add_parser("review-input", parents=[common]).add_argument("--proposals", required=True)
|
|
53
|
+
sub.add_parser("batch", parents=[common]).add_argument("--reviews", required=True)
|
|
54
|
+
sub.add_parser("apply", parents=[common])
|
|
55
|
+
sub.add_parser("prune", parents=[common])
|
|
56
|
+
sub.add_parser("strengthen-start", parents=[common])
|
|
57
|
+
sub.add_parser("strengthen-verify", parents=[common])
|
|
58
|
+
sub.add_parser("strengthen-finish", parents=[common]).add_argument("--reviews", required=True)
|
|
59
|
+
return parser
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Context:
|
|
63
|
+
"""What every command gets: the repo, the run state, the branch date and its arguments."""
|
|
64
|
+
|
|
65
|
+
def __init__(self, args):
|
|
66
|
+
self.args = args
|
|
67
|
+
self.repo = Repo(Path.cwd())
|
|
68
|
+
self.state = State(Path(args.state).resolve() if args.state else self.repo.git_dir() / "test-prune")
|
|
69
|
+
self.date = args.date or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
70
|
+
self.finished = False # the run is complete: the strengthen loop ended (set by its commands)
|
|
71
|
+
|
|
72
|
+
def validate(self) -> None:
|
|
73
|
+
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", self.date):
|
|
74
|
+
raise BadInput(f"--date must be YYYY-MM-DD, got {self.date!r}")
|
|
75
|
+
|
|
76
|
+
def base(self, mode: str | None = None) -> dict:
|
|
77
|
+
base = self.state.load("base.json")
|
|
78
|
+
if mode and base["mode"] != mode:
|
|
79
|
+
raise BadInput(f"`{self.args.command}` runs only in {mode} mode; this run was collected in {base['mode']} mode")
|
|
80
|
+
return base
|
|
81
|
+
|
|
82
|
+
def reviews(self) -> list:
|
|
83
|
+
return read_agent_json(self.args.reviews, "reviews")
|
|
84
|
+
|
|
85
|
+
def proposals(self) -> dict:
|
|
86
|
+
return {"candidates": read_agent_json(self.args.proposals, "candidates"),
|
|
87
|
+
"weak_spots": read_agent_json(self.args.proposals, "weak_spots")}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _run(ctx: Context) -> tuple[int, str]:
|
|
91
|
+
"""The command's exit code and the summary its status reason comes from. Staying off the base branch
|
|
92
|
+
(leave_on found changes the run did not make) is reported and fails a command that would pass."""
|
|
93
|
+
try:
|
|
94
|
+
ctx.validate()
|
|
95
|
+
code, summary = COMMANDS[ctx.args.command](ctx), "\n\n".join(ctx.state.summaries)
|
|
96
|
+
except BadInput as exc:
|
|
97
|
+
print(f"test-prune: {exc}", file=sys.stderr)
|
|
98
|
+
code, summary = 2, f"test-prune: {exc}"
|
|
99
|
+
except Fail as exc:
|
|
100
|
+
ctx.state.emit(f"test-prune: {exc}")
|
|
101
|
+
code, summary = 1, f"test-prune: {exc}"
|
|
102
|
+
except Exception as exc: # a bug or an operational surprise still fails the run in status.json
|
|
103
|
+
summary = f"test-prune: unexpected {type(exc).__name__}: {exc}"
|
|
104
|
+
ctx.state.emit(summary)
|
|
105
|
+
code = 1
|
|
106
|
+
if ctx.repo.stranded:
|
|
107
|
+
ctx.state.emit(ctx.repo.stranded)
|
|
108
|
+
if code == 0:
|
|
109
|
+
code, summary = 1, ctx.repo.stranded
|
|
110
|
+
return code, summary
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
RUN = "run" # the marker entry an unattended run stays red on until it finishes
|
|
114
|
+
INCOMPLETE = "test-prune: run incomplete: the strengthen loop has not finished"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _open(commands: list) -> list:
|
|
118
|
+
"""The entries that still fail the run: every exit 1 but the run marker, and every exit 2 (or the run
|
|
119
|
+
marker) its command has not since passed. On exit 2 the session fixes the input and reruns the
|
|
120
|
+
command; the run marker passes when the strengthen loop ends."""
|
|
121
|
+
def clears(c: dict) -> bool:
|
|
122
|
+
return c["exit"] == 2 or c["command"] == RUN
|
|
123
|
+
return [c for i, c in enumerate(commands) if c["exit"] and not (clears(c) and any(
|
|
124
|
+
later["command"] == c["command"] and later["exit"] == 0 for later in commands[i + 1:]))]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _earlier(previous, command: str) -> list:
|
|
128
|
+
"""The run's commands so far: none for `collect` (a new run) or a missing or unreadable status."""
|
|
129
|
+
commands = previous.get("commands") if isinstance(previous, dict) and command != "collect" else None
|
|
130
|
+
return commands if isinstance(commands, list) else []
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def run_status(previous, command: str, code: int, summary: str, extra=()) -> dict:
|
|
134
|
+
"""The run's status, sticky across commands: the highest exit code still open and its reason (the
|
|
135
|
+
latest on a tie), plus every command in order, then the extra entries. `collect` starts a new run; a
|
|
136
|
+
missing or unreadable status starts fresh rather than failing the command."""
|
|
137
|
+
reason = next((line for line in summary.splitlines() if line.strip()), "")
|
|
138
|
+
commands = _earlier(previous, command) + [{"command": command, "exit": code, "reason": reason}]
|
|
139
|
+
ran_prune = any(c["command"] == "prune" for c in commands)
|
|
140
|
+
commands += [e for e in extra if e["exit"] or ran_prune] # the run finishes only after prune ran
|
|
141
|
+
worst = max(reversed(_open(commands)), key=lambda c: c["exit"], default={"exit": 0, "reason": reason})
|
|
142
|
+
return {"exit": worst["exit"], "reason": worst["reason"], "commands": commands}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _record(state: State, command: str, code: int, summary: str, extra=()) -> None:
|
|
146
|
+
"""Add the command to <state>/status.json, when the state dir is the run's (see State.owned)."""
|
|
147
|
+
if not state.owned():
|
|
148
|
+
return
|
|
149
|
+
path = state.path / "status.json"
|
|
150
|
+
try:
|
|
151
|
+
previous = json.loads(path.read_text()) if path.exists() else None
|
|
152
|
+
except (OSError, ValueError):
|
|
153
|
+
previous = None
|
|
154
|
+
state.save("status.json", run_status(previous, command, code, summary, extra))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _markers(ctx: Context) -> list:
|
|
158
|
+
"""The run marker: opened by an unattended collect, passed when the strengthen loop ends after
|
|
159
|
+
prune ran (see run_status)."""
|
|
160
|
+
if ctx.args.command == "collect" and ctx.args.unattended:
|
|
161
|
+
return [{"command": RUN, "exit": 1, "reason": INCOMPLETE}]
|
|
162
|
+
return [{"command": RUN, "exit": 0, "reason": "run finished"}] if ctx.finished else []
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _state_arg(argv) -> str | None:
|
|
166
|
+
"""The --state value, read off argv without a parser (the arguments just failed to parse)."""
|
|
167
|
+
for i, arg in enumerate(argv):
|
|
168
|
+
if arg == "--state" and i + 1 < len(argv):
|
|
169
|
+
return argv[i + 1]
|
|
170
|
+
if arg.startswith("--state="):
|
|
171
|
+
return arg[len("--state="):]
|
|
172
|
+
return None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _record_bad_arguments(argv, exc: BadInput) -> None:
|
|
176
|
+
"""An argument error is recorded as exit 2 when the command and the state dir can still be told."""
|
|
177
|
+
command = next((a for a in argv if a in COMMANDS), None)
|
|
178
|
+
state = _state_arg(argv)
|
|
179
|
+
try:
|
|
180
|
+
path = Path(state).resolve() if state else Repo(Path.cwd()).git_dir() / "test-prune"
|
|
181
|
+
except BadInput:
|
|
182
|
+
return
|
|
183
|
+
if command:
|
|
184
|
+
_record(State(path), command, 2, f"test-prune: {exc}")
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def main(argv=None):
|
|
188
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
189
|
+
try:
|
|
190
|
+
args = build_parser().parse_args(argv)
|
|
191
|
+
ctx = Context(args)
|
|
192
|
+
except BadInput as exc:
|
|
193
|
+
print(f"test-prune: {exc}", file=sys.stderr)
|
|
194
|
+
_record_bad_arguments(argv, exc)
|
|
195
|
+
return 2
|
|
196
|
+
code, summary = _run(ctx)
|
|
197
|
+
_record(ctx.state, args.command, code, summary, _markers(ctx))
|
|
198
|
+
return code
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
if __name__ == "__main__":
|
|
202
|
+
sys.exit(main())
|