@jesdi/skills 0.0.50 → 0.0.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/package.json +2 -2
  2. package/skills/implement-spec/ARCHITECTURE.md +3 -0
  3. package/skills/implement-spec/README.md +2 -2
  4. package/skills/implement-spec/driver_fixloop.py +27 -6
  5. package/skills/implement-spec/driver_report.py +3 -2
  6. package/skills/implement-spec/prompts/correction.md +1 -0
  7. package/skills/implement-spec/prompts/test-writer.md +3 -2
  8. package/skills/implement-spec/tests/test_fixloop.py +50 -0
  9. package/skills/test-prune/SKILL.md +156 -0
  10. package/skills/test-prune/prune.py +202 -0
  11. package/skills/test-prune/prune_repo.py +319 -0
  12. package/skills/test-prune/prune_review.py +363 -0
  13. package/skills/test-prune/prune_runs.py +240 -0
  14. package/skills/test-prune/prune_source.py +237 -0
  15. package/skills/test-prune/prune_strengthen.py +301 -0
  16. package/skills/test-prune/tests/conftest.py +15 -0
  17. package/skills/test-prune/tests/prune_harness.py +205 -0
  18. package/skills/test-prune/tests/test_prune_guards.py +871 -0
  19. package/skills/test-prune/tests/test_prune_inputs.py +97 -0
  20. package/skills/test-prune/tests/test_prune_review.py +113 -0
  21. package/skills/test-prune/tests/test_prune_source.py +84 -0
  22. package/skills/test-prune/tests/test_prune_strengthen.py +183 -0
  23. package/skills/test-prune/tests/test_prune_unattended.py +225 -0
  24. package/skills/test-prune/tests/testdata/gh +16 -0
  25. package/skills/test-prune/tests/testdata/repo/Makefile +25 -0
  26. package/skills/test-prune/tests/testdata/repo/backend/mutation_baseline.json +16 -0
  27. package/skills/test-prune/tests/testdata/repo/backend/services/concentration.py +2 -0
  28. package/skills/test-prune/tests/testdata/repo/backend/services/rebalancer.py +11 -0
  29. package/skills/test-prune/tests/testdata/repo/backend/tests/conftest.py +6 -0
  30. package/skills/test-prune/tests/testdata/repo/backend/tests/test_concentration.py +73 -0
  31. package/skills/test-prune/tests/testdata/repo/backend/tests/test_rebalancer.py +77 -0
  32. package/skills/test-prune/tests/testdata/repo/fake_engine.py +289 -0
  33. package/skills/test-prune/tests/testdata/repo/kill_map.json +131 -0
  34. package/skills-manifest.json +9 -3
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jesdi/skills",
3
- "version": "0.0.50",
3
+ "version": "0.0.52",
4
4
  "description": "jesdi's agent skills (content package consumed by @jesdi/skills-cli)",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -29,7 +29,7 @@
29
29
  "scripts": {
30
30
  "test": "vitest run",
31
31
  "test:coverage": "vitest run --coverage --coverage.reportOnFailure",
32
- "test:python": "mkdir -p .crap && python3 -m coverage run -m pytest skills/backlog skills/crap-gate -q && python3 -m coverage run -a -m pytest skills/implement-spec -q && python3 -m coverage json -o .crap/python.json",
32
+ "test:python": "mkdir -p .crap && python3 -m coverage erase && python3 -m coverage run -m pytest skills/backlog skills/crap-gate -q && python3 -m coverage run -a -m pytest skills/implement-spec -q && COVERAGE_SUBPROCESS_PATCH=subprocess python3 -m coverage run -m pytest skills/test-prune -q && python3 -m coverage combine --append -q && python3 -m coverage json -o .crap/python.json",
33
33
  "crap-gate": "sh skills/crap-gate/run.sh",
34
34
  "check": "sh scripts/check.sh",
35
35
  "generate-manifest": "tsx scripts/generate-manifest.ts"
@@ -132,6 +132,9 @@ and do not know about the run, except `driver_run.py`, which the three deciding
132
132
  it blocks only after round 4. A red gate on a test-writer correction blocks.
133
133
  - The implementer and each fixer get the `--gate` command and run it before they report. The gate of the driver
134
134
  stays the source of truth: an agent's run of the gate counts for nothing.
135
+ - A finding that a correction reports in `not_addressed` goes to the fixer of the same round as kind `code`
136
+ (`driver_fixloop._correct` rewrites it in the findings file): the test needs a production change first, so a
137
+ second correction could not close it.
135
138
  - No test-writer correction runs after round 4 (`driver_fixloop._act_on`): its findings become `finding` questions
136
139
  with the others. A correction there would add red locked tests that no fix round makes green, and `accept` could
137
140
  not merge the ticket.
@@ -117,7 +117,7 @@ Rules for each agent:
117
117
 
118
118
  | Finding | What the driver does |
119
119
  |---|---|
120
- | kind `fixture`, any severity, or kind `test`, `critical` or `important`, on a locked file of the ticket | One test-writer correction, outside the fix rounds. The fixer may not edit a locked file, so a fix round cannot close a `test` finding there. The corrected files are the new locked versions, and the next review is a full review. A finding of this kind that is open after the correction blocks the ticket. No correction runs after round 4. |
120
+ | kind `fixture`, any severity, or kind `test`, `critical` or `important`, on a locked file of the ticket | One test-writer correction, outside the fix rounds. The fixer may not edit a locked file, so a fix round cannot close a `test` finding there. The corrected files are the new locked versions, and the next review is a full review. A finding that the correction reports in `not_addressed` (it needs a production change first) becomes a finding of kind `code`, with the reason as its evidence, and goes to a fix round at once; the ledger says it was not corrected. A finding of this kind that is open after the correction blocks the ticket. No correction runs after round 4. |
121
121
  | `critical` or `important` | One fix round. |
122
122
  | `minor` | It stays in the ledger and in the findings file. The final reviewer gets the list. |
123
123
 
@@ -176,7 +176,7 @@ Every report has `status` (`DONE`, `DONE_WITH_CONCERNS`, `NEEDS_CONTEXT`, `BLOCK
176
176
 
177
177
  | Role | Template | More fields | Checks after the schema |
178
178
  |---|---|---|---|
179
- | test-writer | `prompts/test-writer.md` | `locked` (test file paths), `red` (`test`, `line`) | The driver reads a test node id `<file>::<test>` in `locked` as its file. With DONE: one commit, one locked file and one red line or more. Each locked path is in a commit that the role made. Each `line` is in the log. A locked file of a merged ticket that the role changed is in `locked`. A correction (after `overrule-test`, or for a review finding) changes only the files that it lists in `locked` and the files that the driver restored before it. When the gate is red on the restore commit, a DONE correction lists a restored file in `locked`. |
179
+ | test-writer | `prompts/test-writer.md` | `locked` (test file paths), `red` (`test`, `line`) | The driver reads a test node id `<file>::<test>` in `locked` as its file. With DONE: one commit, one locked file and one red line or more. Each locked path is in a commit that the role made. Each `line` is in the log. A locked file of a merged ticket that the role changed is in `locked`. A correction (after `overrule-test`, or for a review finding) changes only the files that it lists in `locked` and the files that the driver restored before it. When the gate is red on the restore commit, a DONE correction lists a restored file in `locked`. Each `not_addressed[].finding` is the title of a finding of the correction; a DONE correction with each of its findings there needs no commit. |
180
180
  | implementer | `prompts/implementer.md` | `green` | With DONE: one commit or more. |
181
181
  | reviewer | `prompts/reviewer.md` | `findings` (`severity`, `file`, `line`, `title`, `evidence`, `kind`), `mutations` (`test`, `file`, `line`, `red`); `commits` is empty | In a full review (one with no fix range: round 1, and the review after a test-writer correction): `mutations` is required, with a proof for each locked test file (`test` is `<locked file>::<test name>`). Each `red` is in the log. |
182
182
  | correctness | `prompts/correctness.md` | the same as the reviewer; `commits` is empty | `mutations` is optional. |
@@ -104,8 +104,9 @@ def _review(job: Job) -> list[dict]:
104
104
  return found
105
105
 
106
106
 
107
- def _correct(job: Job, findings: list[dict]) -> None:
108
- """One test-writer correction for the findings on the locked tests. It is outside the fix rounds."""
107
+ def _correct(job: Job, findings: list[dict]) -> list[dict]:
108
+ """One test-writer correction for the findings on the locked tests. It is outside the fix rounds. Returns the
109
+ findings that it did not correct, as findings of kind code for the fixer (issue #111)."""
109
110
  run, number = job.run, job.ticket.number
110
111
  if job.corrected:
111
112
  titles = "; ".join(finding["title"] for finding in findings)
@@ -120,19 +121,39 @@ def _correct(job: Job, findings: list[dict]) -> None:
120
121
  job.previous = "" # no review has seen the restore commit
121
122
  ledger(f"Ticket {number}: the code does not need the change to {', '.join(restored)}: the restore closes it, "
122
123
  "with no correction.")
123
- return
124
+ return []
124
125
  values = {**job.pointers(job.round), "restored": " ".join(restored),
125
126
  "record": record and os.path.relpath(record, job.slot)}
126
127
  notes = ["correction"] + ["restored"] * bool(restored) + ["restored-red"] * bool(record)
127
128
  with during("the test-writer correction"):
128
129
  report = run.agent("test-writer", number, job.slot, values, notes=notes, fix_round=job.round + 1,
129
- check={"correction": True, "restored": restored, "restore_red": bool(record)})
130
+ check={"correction": True, "restored": restored, "restore_red": bool(record),
131
+ "findings": [finding["title"] for finding in findings]})
130
132
  job.tip, job.previous, job.corrected = driver_slots.head(job.slot), "", True
131
133
  entry = run.state.read()["tickets"][number]
132
134
  run.state.ticket(number, tip=job.tip, **driver_review.written(job.slot, entry, report["locked"], job.tip))
135
+ return _uncorrected(job, findings, report.get("not_addressed", []))
136
+
137
+
138
+ def _uncorrected(job: Job, findings: list[dict], not_addressed: list[dict]) -> list[dict]:
139
+ """Log what the correction did with each finding. Returns each one it did not correct as a finding of kind
140
+ code, with its reason as the evidence: the test needs a production change first, so a fixer closes it."""
141
+ number = job.ticket.number
142
+ reasons = {item["finding"]: item["reason"] for item in not_addressed}
133
143
  for finding in findings:
144
+ result = (f"The test-writer did not correct it: {reasons[finding['title']]}. It goes to the fixer as a finding "
145
+ "of kind code." if finding["title"] in reasons else f"The test-writer corrected it at {job.tip[:7]}.")
134
146
  ledger(f"Ticket {number}: {finding['kind']} correction in {finding['file']}, outside the fix rounds: "
135
- f"{finding['title']}: {finding['evidence']}. The test-writer corrected it at {job.tip[:7]}.")
147
+ f"{finding['title']}: {finding['evidence']}. {result}")
148
+ code = {finding["title"]: {**finding, "kind": "code", "evidence": f"the test-writer did not correct it: {reason}"}
149
+ for finding in findings if (reason := reasons.get(finding["title"]))}
150
+ if code:
151
+ # The fixer reads the findings file of this review, so the finding is of kind code there too.
152
+ path = OUT / number / f"findings-{job.round}.json"
153
+ written = [code.get(finding["title"], finding) if finding in findings else finding
154
+ for finding in json.loads(path.read_text())]
155
+ path.write_text(json.dumps(written, indent=1) + "\n")
156
+ return list(code.values())
136
157
 
137
158
 
138
159
  def _fix(job: Job) -> None:
@@ -170,7 +191,7 @@ def _act_on(job: Job, found: list[dict]) -> bool:
170
191
  # A changed locked file is fixed first: a correction on top of it would make the changed file the reference.
171
192
  locked_file_changed = any(finding["source"] == "driver" for finding in to_fix)
172
193
  if to_correct and not locked_file_changed:
173
- _correct(job, to_correct)
194
+ to_fix += [finding for finding in _correct(job, to_correct) if finding["severity"] in driver_review.OPEN]
174
195
  if not to_fix:
175
196
  return False
176
197
  if not to_fix:
@@ -2,7 +2,7 @@
2
2
 
3
3
  context: repo (slot path), branch, start (HEAD when the role started), log, full (a full review: no fix range), locked,
4
4
  correction (a test-writer corrects locked tests), restored (locked files the driver put back before it),
5
- restore_red (the gate is red on the commit of that restore),
5
+ restore_red (the gate is red on the commit of that restore), findings (the titles a correction gets),
6
6
  owners (ticket numbers, for the final fix batch), merged (locked files of merged tickets, not its own), pr (tip)."""
7
7
  from __future__ import annotations
8
8
 
@@ -81,7 +81,8 @@ def _correction_errors(report: dict, context: dict) -> list[str]:
81
81
  errors.append("the gate is red on the restore commit, so a restored file fails with the code of this ticket: "
82
82
  "update that file and list it in `locked`, or report BLOCKED with a concern that starts with "
83
83
  "`locked test contradicts spec:`")
84
- return errors
84
+ return errors + [f"not_addressed: {item['finding']} is not the title of a finding of this correction"
85
+ for item in report.get("not_addressed", []) if item["finding"] not in context.get("findings", [])]
85
86
 
86
87
 
87
88
  def _merged_errors(report: dict, context: dict) -> list[str]:
@@ -3,4 +3,5 @@ This call is a correction of the locked tests, not a new test set. Read in {find
3
3
  A `fixture` finding: a locked test encodes a stale path, format, fixture layout or setup detail. Correct only that detail and keep what each test asserts.
4
4
  A `test` finding: a locked test is too weak, for example it stays green when the production line it protects is broken. Make the test assert the behavior of its criterion so that it fails on that break. Do not remove an assertion.
5
5
  For this you may read the implementation.
6
+ When a finding needs a production change first, so that no correct test can be green, leave the test as it is and add the finding to `not_addressed`: `finding` is its `title` as the findings file gives it, `reason` is the production change it needs. The driver sends it to the fixer as a finding of kind `code`. A correction whose findings are all in `not_addressed` may report DONE with no commit.
6
7
  The implementation exists, so a corrected test is green: for `red`, break the production line that the test protects, run the test, print its failing output, then revert the edit. In `locked`, list each test file you corrected: test file paths only, not test node ids. Unlike `mutations[].test` of the reviewer, `locked` takes no `::<test name>`.
@@ -25,9 +25,10 @@ Write one JSON report at {report} that matches this schema:
25
25
  "files": {"type": "array", "items": {"type": "string"}},
26
26
  "concerns": {"type": "array", "items": {"type": "string"}},
27
27
  "locked": {"type": "array", "items": {"type": "string"}},
28
- "red": {"type": "array", "items": {"type": "object", "required": ["test", "line"], "properties": {"test": {"type": "string", "minLength": 1}, "line": {"type": "string", "minLength": 1}}}}
28
+ "red": {"type": "array", "items": {"type": "object", "required": ["test", "line"], "properties": {"test": {"type": "string", "minLength": 1}, "line": {"type": "string", "minLength": 1}}}},
29
+ "not_addressed": {"type": "array", "items": {"type": "object", "required": ["finding", "reason"], "properties": {"finding": {"type": "string", "minLength": 1}, "reason": {"type": "string", "minLength": 1}}}}
29
30
  },
30
- "if": {"properties": {"status": {"enum": ["DONE", "DONE_WITH_CONCERNS"]}}},
31
+ "if": {"properties": {"status": {"enum": ["DONE", "DONE_WITH_CONCERNS"]}}, "not": {"required": ["not_addressed"], "properties": {"not_addressed": {"minItems": 1}}}},
31
32
  "then": {"properties": {"commits": {"minItems": 1}, "locked": {"minItems": 1}, "red": {"minItems": 1}}}
32
33
  }
33
34
  ```
@@ -177,6 +177,56 @@ def test_a_test_finding_on_a_locked_file_gets_the_test_writer_correction_not_a_f
177
177
  assert driver_review.route([vacuous, minor, elsewhere], ["tests/test_01.py"]) == ([vacuous], [elsewhere])
178
178
 
179
179
 
180
+ NEEDS_CODE = "needs production code: the code has no multi-PR view"
181
+
182
+
183
+ def not_addressing(*titles, **report):
184
+ """A correction of tests/test_01.py that reports `titles` under `not_addressed`."""
185
+ step = {**tw("01"), "files": {"tests/test_01.py": "# corrected\n"}, "commit": "test: 01 fixture fix"}
186
+ reasons = [{"finding": title, "reason": NEEDS_CODE} for title in titles]
187
+ return {**step, "report": {**step["report"], **report, "not_addressed": reasons}}
188
+
189
+
190
+ def test_a_test_finding_the_correction_did_not_address_goes_to_the_fixer_as_a_code_finding(tmp_path):
191
+ # Issue #111: the ledger said "corrected it at <sha>" for every finding, and the finding came back each review.
192
+ stale = finding("STALE", kind="fixture", file="tests/test_01.py")
193
+ vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
194
+ rig = Inside(tmp_path).run(plan(**{"01/test-writer": [tw("01"), not_addressing("VACUOUS")],
195
+ "01/fixer": [fix("01", 1)],
196
+ "01/reviewer": [review("01", [stale, vacuous]), review("01")]}))
197
+ assert rig.code == 0, rig.ledger()
198
+ order = [call["role"] for call in rig.calls("01")]
199
+ assert order == ["test-writer", "implementer", "reviewer", "test-writer", "fixer", "reviewer"]
200
+ assert rig.led("STALE: STALE. The test-writer corrected it at")
201
+ assert rig.led("VACUOUS", "corrected it at") == []
202
+ assert rig.led(f"test correction in tests/test_01.py, outside the fix rounds: VACUOUS: VACUOUS. The test-writer "
203
+ f"did not correct it: {NEEDS_CODE}. It goes to the fixer as a finding of kind code.")
204
+ # The fixer reads the finding as one of kind code, with the reason of the test-writer as its evidence.
205
+ (routed,) = [f for f in json.loads((rig.folder() / "findings-0.json").read_text()) if f["title"] == "VACUOUS"]
206
+ assert (routed["kind"], routed["evidence"]) == ("code", f"the test-writer did not correct it: {NEEDS_CODE}")
207
+ assert rig.state()["tickets"]["01"]["round"] == 1 and "findings-0.json" in rig.prompt("fixer-1")
208
+
209
+
210
+ def test_a_correction_that_addresses_no_finding_needs_no_commit(tmp_path):
211
+ vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
212
+ none = not_addressing("VACUOUS", commits=[], files=[], locked=[], red=[])
213
+ del none["commit"], none["files"]
214
+ rig = Inside(tmp_path).run(plan(**{"01/test-writer": [tw("01"), none], "01/fixer": [fix("01", 1)],
215
+ "01/reviewer": [review("01", [vacuous]), review("01")]}))
216
+ assert rig.code == 0, rig.ledger()
217
+ assert [call["role"] for call in rig.calls("01")].count("test-writer") == 2 # no rerun
218
+ assert rig.led("VACUOUS. The test-writer did not correct it") and rig.state()["tickets"]["01"]["round"] == 1
219
+
220
+
221
+ def test_a_correction_that_names_a_finding_it_was_not_given_is_rerun(tmp_path):
222
+ vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
223
+ rig = Inside(tmp_path).run(plan(**{"01/test-writer": [tw("01"), not_addressing("OTHER"), not_addressing()],
224
+ "01/reviewer": [review("01", [vacuous]), review("01")]}))
225
+ assert rig.code == 0, rig.ledger()
226
+ assert "not_addressed: OTHER is not the title of a finding of this correction" in rig.prompt("test-writer-1")
227
+ assert rig.led("VACUOUS: VACUOUS. The test-writer corrected it at")
228
+
229
+
180
230
  def test_a_test_writer_that_lists_test_node_ids_in_locked_locks_their_files(tmp_path):
181
231
  # Issue #71: a node id `<file>::<test>` in `locked` names its file, for the first run and for a correction.
182
232
  vacuous = finding("VACUOUS", "critical", kind="test", file="tests/test_01.py")
@@ -0,0 +1,156 @@
1
+ ---
2
+ name: test-prune
3
+ description: >-
4
+ Mutation-guided test pruning: a fixer proposes deleting redundant tests and strengthening
5
+ weak ones, an independent reviewer accepts or rejects each proposal, and prune.py applies
6
+ the accepted batch behind a cold mutation run and the ratchet. Use when the user says
7
+ "prune tests", "test-prune", "mutation test pruning" or "strengthen weak tests", or when a
8
+ scheduled job runs it with `--unattended`.
9
+ ---
10
+
11
+ # test-prune: delete what the mutation ratchet proves redundant, strengthen what it proves weak
12
+
13
+ You are the orchestrator. `prune.py` owns every git, make and file operation and every safety
14
+ guard; the subagents only write JSON files or test commits. Each role is a fresh subagent with no
15
+ context from the others: the reviewer reads the fixer's proposals, reasons included, and judges
16
+ them against the code itself.
17
+
18
+ **Publishing belongs to prune.py.** In both modes you never run `git push`, `git merge`,
19
+ `gh pr create` or `gh pr merge` yourself, and neither does any subagent; `prune` and
20
+ `strengthen-finish` push their own `chore/test-*` branch and open its PR, and nothing merges.
21
+
22
+ The target repo provides the make targets `test`, `mutation`, `mutation-check`,
23
+ `mutation-baseline`, `test-candidates`, `prune-gate` and `crap`, and the layout `backend/tests/`,
24
+ `backend/mutation_baseline.json`. Every command:
25
+
26
+ ```bash
27
+ python3 "<skill-dir>/prune.py" <command> [--state DIR] [--date YYYY-MM-DD]
28
+ ```
29
+
30
+ Run it from inside the target repo. State lives in `<git dir>/test-prune/` (outside the working
31
+ tree); each command reads what the previous one wrote there. Exit 0 is done (including nothing to
32
+ do), 1 is a failed make target, gate or guard, 2 is bad input or an unmet precondition; the
33
+ message names the offending ids or files. On exit 2 fix the input and rerun the same command; on
34
+ exit 1 stop and report the output.
35
+
36
+ Every command records itself in `<state>/status.json`. A CI job reads it after the headless
37
+ session (which cannot set the job's exit code) and fails the job when its `exit` is not 0; an
38
+ unattended run stays red there until the strengthen loop ends (`nothing to strengthen`, or
39
+ `strengthen-finish` exit 0). End a headless session by reporting its `reason` as the final line.
40
+ `--state` must be empty, new, or an earlier run's state dir.
41
+
42
+ When a push succeeded but `gh pr create` failed (exit 1, the branch is on origin with no PR),
43
+ open the PR by hand from `<state>/prune-pr.md` or `<state>/strengthen-pr.md` after fixing `gh`;
44
+ rerunning the command refuses because the branch exists.
45
+
46
+ **A run needs a checkout dedicated to it** for its whole duration: the CI checkout, or a worktree.
47
+ If anything else edits a file there mid-run, the run stops with exit 1 rather than overwrite it,
48
+ leaves the checkout as it is (HEAD may stay detached), names the files, and nothing is lost; keep
49
+ or discard the change, then check out the base branch. Races beyond that check are out of scope.
50
+
51
+ **Runtime.** Each cold `make mutation` takes about 30 minutes on a real repo, and a run makes
52
+ several (collect, each safety-check run, each strengthen proof). Run long commands with a timeout of
53
+ an hour or more, or in the background, and wait for them.
54
+
55
+ ## Modes
56
+
57
+ - **Interactive** (default): nothing in the working tree changes before the human approves the
58
+ batch. Stop after `batch`, show its output, and run `apply` only on an explicit yes.
59
+ - **Unattended** (`collect --unattended`, run headless by the weekly CI job): only protected
60
+ candidates, at most 10 deleted test ids per run (a parametrized test goes whole, every case
61
+ counted; one with more than 10 cases is skipped, the rest left for next week), one PR from
62
+ `chore/test-prune-<date>` and one from `chore/test-strengthen-<date>`, both from the base
63
+ commit. The base branch never moves. A merge is the human's approval, so the run ends with open
64
+ PRs.
65
+
66
+ Two kinds of accepted proposal reach a human only: equivalent mutants (the skill writes no
67
+ equivalent id into the baseline) and "strengthen this candidate with this assertion" verdicts
68
+ (the skill applies no assertion to an existing candidate). `batch` prints both; unattended, every
69
+ PR body and the job summary carry them.
70
+
71
+ ## Prune loop
72
+
73
+ 1. `collect [--unattended | --protected-only]`. Needs a clean tracked tree on a branch. Runs the
74
+ base checks, a cold mutation run and `make test-candidates`, then writes
75
+ `fixer-input.json`. When it prints `no candidates` / `no protected candidates`, the weak spots
76
+ may still need the fixer; continue.
77
+ 2. **Fixer.** Dispatch a subagent with `<state>/fixer-input.json`, read access to the repo, and
78
+ this brief: for every candidate decide `keep`, `delete` into one named test that contains it,
79
+ or `strengthen` with one named assertion; for every weak-spot group decide `strengthen`
80
+ (which test to add to or strengthen) or `equivalent` (with the argument why no test can tell
81
+ the mutant apart). A group with both kinds of mutant takes two entries, one `strengthen` and
82
+ one `equivalent`, each listing its own disjoint `mutant_ids`. Compare assertions, not coverage: a candidate whose assertions pin a value,
83
+ boundary or error the container never checks is `keep` or `strengthen`. It writes:
84
+
85
+ ```json
86
+ {"candidates": [{"test_id": "...", "verdict": "keep|delete|strengthen",
87
+ "into": "<test id, delete only>", "assertion": "<strengthen only>", "reason": "..."}],
88
+ "weak_spots": [{"group": "...", "verdict": "strengthen|equivalent",
89
+ "mutant_ids": ["<subset of the group's ids, omit for all>"],
90
+ "test": "<test id or test file>", "reason": "..."}]}
91
+ ```
92
+
93
+ One entry per candidate, one per group (or the strengthen + equivalent pair), no others.
94
+ 3. `review-input --proposals <fixer file>` validates the file and writes
95
+ `<state>/review-input.json`: each proposal with the candidate's source and assertions and,
96
+ for a deletion, the containing test's source and assertions. A proposal's `id` is the
97
+ candidate's test id or the group; a split group's two proposals are `<group>:strengthen` and
98
+ `<group>:equivalent`.
99
+ 4. **Reviewer.** Dispatch a *different* fresh subagent with `<state>/review-input.json` and read
100
+ access to the repo. Brief: accept a deletion only when every assertion of the candidate is
101
+ implied by an assertion of the `into` test (same inputs or a superset, an equal or stricter
102
+ check); covering the same lines is not enough. Accept a strengthen proposal only when the
103
+ named assertion checks behaviour a user would notice. Accept an equivalent only with a
104
+ concrete argument that no input distinguishes the mutant. Every reason names the assertion or
105
+ input it rests on. It writes, one entry per proposal:
106
+
107
+ ```json
108
+ {"reviews": [{"id": "<proposal id>", "verdict": "accept|reject", "reason": "..."}]}
109
+ ```
110
+
111
+ 5. `batch --reviews <reviewer file>` writes `batch.json` and prints the accepted deletions (each
112
+ with its containing test and the reviewer's reason), strengthen items, equivalent proposals,
113
+ skipped deletions with the reason, and rejections. It edits nothing.
114
+ - Interactive: show that output to the human and stop. After approval, `apply` applies the
115
+ deletions, runs the safety check, restores every culprit and commits on the current branch.
116
+ - Unattended: `prune` applies the batch, runs the safety check, `make test`, commits, runs
117
+ `make prune-gate` and, only when it passes, pushes the branch and opens the PR.
118
+
119
+ **Safety check** (inside `apply` and `prune`): a cold mutation run plus `make mutation-check`
120
+ on the batch; every deletion that lets a module exceed its baseline is restored and reported.
121
+
122
+ ## Strengthen loop
123
+
124
+ Runs after the prune loop, also in unattended mode, independently of the prune PR: run it even
125
+ when `prune` exited 1 (the status keeps that failure, so the job still goes red).
126
+
127
+ 1. `strengthen-start` writes `<state>/strengthen-input.json` (each accepted weak-spot strengthen
128
+ item with the mutant ids it must kill) and prints the commit to build on; unattended it
129
+ detaches HEAD at the base commit. `nothing to strengthen` ends the loop.
130
+ 2. **Strengthen agent.** Dispatch a fresh subagent with `strengthen-input.json`. Brief: per item,
131
+ test-first, write the assertion or test that fails against each named mutant and passes on
132
+ the real code; touch only `backend/tests/`; make one or more commits per item, each ending
133
+ with the trailer line `Test-Prune-Item: <item id>`; leave the tree clean.
134
+ 3. `strengthen-verify` rejects commits without a known trailer or outside `backend/tests/`, then
135
+ proves each item on a cold run: every named id survived on the base and is killed on the
136
+ head. Unproven items are reverted and closed: a later commit for a closed item is refused. It
137
+ writes `<state>/strengthen-proof.json`, including the proven head. On failure HEAD stays
138
+ where the commits are (detached, unattended): fix and rerun. To abandon the loop instead,
139
+ `git checkout <base branch>`; the item commits stay reachable through `git reflog`.
140
+ 4. **Strengthen reviewer.** Dispatch a fresh subagent with `strengthen-proof.json` and the diff
141
+ of the proven items (`git log -p <start>..HEAD`). Brief: accept an item only when its test
142
+ asserts observable behaviour through the public interface, rather than restating the
143
+ implementation; one review per proven item, same schema as the prune reviewer.
144
+ 5. `strengthen-finish --reviews <file>` refuses a HEAD other than the proven one (rerun
145
+ `strengthen-verify` after any new commit); on any exit 2 HEAD stays at the proven head, so fix
146
+ the input and rerun it. It reverts rejected items, reruns the proof cold, runs
147
+ `make mutation-check`, `make test`, `make crap` and `make mutation-baseline`, refuses a
148
+ baseline that gained an equivalent id, and commits the lower baseline. Unattended it opens the
149
+ strengthen PR listing each changed test with the ids it now kills; interactive it leaves the
150
+ commits on the current branch for the human.
151
+
152
+ ## Report
153
+
154
+ End with what the last command printed: deleted, restored, skipped and left-for-next-week tests,
155
+ strengthened tests with their killed ids, the PR links, and the equivalent proposals. In CI the
156
+ same summaries land in `$GITHUB_STEP_SUMMARY`.
@@ -0,0 +1,202 @@
1
+ #!/usr/bin/env python3
2
+ """test-prune: drive the fixer/reviewer prune loop and the strengthen loop over a target repo.
3
+
4
+ collect → (fixer) → review-input → (reviewer) → batch → apply (interactive) | prune (unattended);
5
+ strengthen-start → (strengthen agent) → strengthen-verify → (reviewer) → strengthen-finish.
6
+ The script owns every git, make and file operation; the agents only write JSON or test commits.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import json
12
+ import re
13
+ import sys
14
+ from datetime import datetime, timezone
15
+ from pathlib import Path
16
+
17
+ from prune_repo import BadInput, Fail, Repo, State, read_agent_json
18
+ from prune_review import batch, collect, review_input
19
+ from prune_runs import apply, prune
20
+ from prune_strengthen import strengthen_finish, strengthen_start, strengthen_verify
21
+
22
+ COMMANDS = {
23
+ "collect": collect, "review-input": review_input, "batch": batch, "apply": apply, "prune": prune,
24
+ "strengthen-start": strengthen_start, "strengthen-verify": strengthen_verify,
25
+ "strengthen-finish": strengthen_finish,
26
+ }
27
+
28
+
29
+ class Parser(argparse.ArgumentParser):
30
+ """An argument error raises BadInput (exit 2, recorded in the run status) instead of exiting."""
31
+
32
+ def error(self, message):
33
+ raise BadInput(f"{self.prog}: {message}")
34
+
35
+
36
+ def _common():
37
+ common = Parser(add_help=False)
38
+ common.add_argument("--state", help="run state directory (default: <git dir>/test-prune)")
39
+ common.add_argument("--date", help="branch date YYYY-MM-DD (default: today, UTC)")
40
+ return common
41
+
42
+
43
+ def build_parser():
44
+ common = _common()
45
+ parser = Parser(prog="prune.py", description=__doc__)
46
+ sub = parser.add_subparsers(dest="command", required=True)
47
+
48
+ collect_cmd = sub.add_parser("collect", parents=[common])
49
+ collect_cmd.add_argument("--unattended", action="store_true")
50
+ collect_cmd.add_argument("--protected-only", action="store_true")
51
+
52
+ sub.add_parser("review-input", parents=[common]).add_argument("--proposals", required=True)
53
+ sub.add_parser("batch", parents=[common]).add_argument("--reviews", required=True)
54
+ sub.add_parser("apply", parents=[common])
55
+ sub.add_parser("prune", parents=[common])
56
+ sub.add_parser("strengthen-start", parents=[common])
57
+ sub.add_parser("strengthen-verify", parents=[common])
58
+ sub.add_parser("strengthen-finish", parents=[common]).add_argument("--reviews", required=True)
59
+ return parser
60
+
61
+
62
+ class Context:
63
+ """What every command gets: the repo, the run state, the branch date and its arguments."""
64
+
65
+ def __init__(self, args):
66
+ self.args = args
67
+ self.repo = Repo(Path.cwd())
68
+ self.state = State(Path(args.state).resolve() if args.state else self.repo.git_dir() / "test-prune")
69
+ self.date = args.date or datetime.now(timezone.utc).strftime("%Y-%m-%d")
70
+ self.finished = False # the run is complete: the strengthen loop ended (set by its commands)
71
+
72
+ def validate(self) -> None:
73
+ if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", self.date):
74
+ raise BadInput(f"--date must be YYYY-MM-DD, got {self.date!r}")
75
+
76
+ def base(self, mode: str | None = None) -> dict:
77
+ base = self.state.load("base.json")
78
+ if mode and base["mode"] != mode:
79
+ raise BadInput(f"`{self.args.command}` runs only in {mode} mode; this run was collected in {base['mode']} mode")
80
+ return base
81
+
82
+ def reviews(self) -> list:
83
+ return read_agent_json(self.args.reviews, "reviews")
84
+
85
+ def proposals(self) -> dict:
86
+ return {"candidates": read_agent_json(self.args.proposals, "candidates"),
87
+ "weak_spots": read_agent_json(self.args.proposals, "weak_spots")}
88
+
89
+
90
+ def _run(ctx: Context) -> tuple[int, str]:
91
+ """The command's exit code and the summary its status reason comes from. Staying off the base branch
92
+ (leave_on found changes the run did not make) is reported and fails a command that would pass."""
93
+ try:
94
+ ctx.validate()
95
+ code, summary = COMMANDS[ctx.args.command](ctx), "\n\n".join(ctx.state.summaries)
96
+ except BadInput as exc:
97
+ print(f"test-prune: {exc}", file=sys.stderr)
98
+ code, summary = 2, f"test-prune: {exc}"
99
+ except Fail as exc:
100
+ ctx.state.emit(f"test-prune: {exc}")
101
+ code, summary = 1, f"test-prune: {exc}"
102
+ except Exception as exc: # a bug or an operational surprise still fails the run in status.json
103
+ summary = f"test-prune: unexpected {type(exc).__name__}: {exc}"
104
+ ctx.state.emit(summary)
105
+ code = 1
106
+ if ctx.repo.stranded:
107
+ ctx.state.emit(ctx.repo.stranded)
108
+ if code == 0:
109
+ code, summary = 1, ctx.repo.stranded
110
+ return code, summary
111
+
112
+
113
+ RUN = "run" # the marker entry an unattended run stays red on until it finishes
114
+ INCOMPLETE = "test-prune: run incomplete: the strengthen loop has not finished"
115
+
116
+
117
+ def _open(commands: list) -> list:
118
+ """The entries that still fail the run: every exit 1 but the run marker, and every exit 2 (or the run
119
+ marker) its command has not since passed. On exit 2 the session fixes the input and reruns the
120
+ command; the run marker passes when the strengthen loop ends."""
121
+ def clears(c: dict) -> bool:
122
+ return c["exit"] == 2 or c["command"] == RUN
123
+ return [c for i, c in enumerate(commands) if c["exit"] and not (clears(c) and any(
124
+ later["command"] == c["command"] and later["exit"] == 0 for later in commands[i + 1:]))]
125
+
126
+
127
+ def _earlier(previous, command: str) -> list:
128
+ """The run's commands so far: none for `collect` (a new run) or a missing or unreadable status."""
129
+ commands = previous.get("commands") if isinstance(previous, dict) and command != "collect" else None
130
+ return commands if isinstance(commands, list) else []
131
+
132
+
133
+ def run_status(previous, command: str, code: int, summary: str, extra=()) -> dict:
134
+ """The run's status, sticky across commands: the highest exit code still open and its reason (the
135
+ latest on a tie), plus every command in order, then the extra entries. `collect` starts a new run; a
136
+ missing or unreadable status starts fresh rather than failing the command."""
137
+ reason = next((line for line in summary.splitlines() if line.strip()), "")
138
+ commands = _earlier(previous, command) + [{"command": command, "exit": code, "reason": reason}]
139
+ ran_prune = any(c["command"] == "prune" for c in commands)
140
+ commands += [e for e in extra if e["exit"] or ran_prune] # the run finishes only after prune ran
141
+ worst = max(reversed(_open(commands)), key=lambda c: c["exit"], default={"exit": 0, "reason": reason})
142
+ return {"exit": worst["exit"], "reason": worst["reason"], "commands": commands}
143
+
144
+
145
+ def _record(state: State, command: str, code: int, summary: str, extra=()) -> None:
146
+ """Add the command to <state>/status.json, when the state dir is the run's (see State.owned)."""
147
+ if not state.owned():
148
+ return
149
+ path = state.path / "status.json"
150
+ try:
151
+ previous = json.loads(path.read_text()) if path.exists() else None
152
+ except (OSError, ValueError):
153
+ previous = None
154
+ state.save("status.json", run_status(previous, command, code, summary, extra))
155
+
156
+
157
+ def _markers(ctx: Context) -> list:
158
+ """The run marker: opened by an unattended collect, passed when the strengthen loop ends after
159
+ prune ran (see run_status)."""
160
+ if ctx.args.command == "collect" and ctx.args.unattended:
161
+ return [{"command": RUN, "exit": 1, "reason": INCOMPLETE}]
162
+ return [{"command": RUN, "exit": 0, "reason": "run finished"}] if ctx.finished else []
163
+
164
+
165
+ def _state_arg(argv) -> str | None:
166
+ """The --state value, read off argv without a parser (the arguments just failed to parse)."""
167
+ for i, arg in enumerate(argv):
168
+ if arg == "--state" and i + 1 < len(argv):
169
+ return argv[i + 1]
170
+ if arg.startswith("--state="):
171
+ return arg[len("--state="):]
172
+ return None
173
+
174
+
175
+ def _record_bad_arguments(argv, exc: BadInput) -> None:
176
+ """An argument error is recorded as exit 2 when the command and the state dir can still be told."""
177
+ command = next((a for a in argv if a in COMMANDS), None)
178
+ state = _state_arg(argv)
179
+ try:
180
+ path = Path(state).resolve() if state else Repo(Path.cwd()).git_dir() / "test-prune"
181
+ except BadInput:
182
+ return
183
+ if command:
184
+ _record(State(path), command, 2, f"test-prune: {exc}")
185
+
186
+
187
+ def main(argv=None):
188
+ argv = sys.argv[1:] if argv is None else argv
189
+ try:
190
+ args = build_parser().parse_args(argv)
191
+ ctx = Context(args)
192
+ except BadInput as exc:
193
+ print(f"test-prune: {exc}", file=sys.stderr)
194
+ _record_bad_arguments(argv, exc)
195
+ return 2
196
+ code, summary = _run(ctx)
197
+ _record(ctx.state, args.command, code, summary, _markers(ctx))
198
+ return code
199
+
200
+
201
+ if __name__ == "__main__":
202
+ sys.exit(main())