@mintwork/honest-tests 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +22 -0
- package/.claude-plugin/plugin.json +20 -0
- package/CHANGELOG.md +56 -0
- package/LICENSE +21 -0
- package/README.md +280 -2
- package/commands/audit.md +17 -0
- package/commands/mutate.md +10 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/js/ast.d.ts +24 -0
- package/dist/adapters/js/ast.js +199 -0
- package/dist/adapters/js/index.d.ts +25 -0
- package/dist/adapters/js/index.js +122 -0
- package/dist/adapters/js/mutants.d.ts +8 -0
- package/dist/adapters/js/mutants.js +190 -0
- package/dist/adapters/js/parse-tests.d.ts +5 -0
- package/dist/adapters/js/parse-tests.js +1101 -0
- package/dist/adapters/python/index.d.ts +17 -0
- package/dist/adapters/python/index.js +26 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +21 -0
- package/dist/core/engine.d.ts +44 -0
- package/dist/core/engine.js +205 -0
- package/dist/core/types.d.ts +195 -0
- package/dist/core/types.js +7 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +5 -0
- package/dist/mutate/runner.d.ts +59 -0
- package/dist/mutate/runner.js +379 -0
- package/dist/report/colors.d.ts +11 -0
- package/dist/report/colors.js +13 -0
- package/dist/report/human.d.ts +6 -0
- package/dist/report/human.js +79 -0
- package/dist/report/json.d.ts +4 -0
- package/dist/report/json.js +8 -0
- package/dist/rules/asserts-on-mock.d.ts +2 -0
- package/dist/rules/asserts-on-mock.js +53 -0
- package/dist/rules/asserts-on-own-literal.d.ts +2 -0
- package/dist/rules/asserts-on-own-literal.js +36 -0
- package/dist/rules/catch-only-assertion.d.ts +2 -0
- package/dist/rules/catch-only-assertion.js +22 -0
- package/dist/rules/conditional-assertion.d.ts +2 -0
- package/dist/rules/conditional-assertion.js +30 -0
- package/dist/rules/excessive-mocking.d.ts +3 -0
- package/dist/rules/excessive-mocking.js +22 -0
- package/dist/rules/focused-test.d.ts +2 -0
- package/dist/rules/focused-test.js +15 -0
- package/dist/rules/helpers.d.ts +5 -0
- package/dist/rules/helpers.js +18 -0
- package/dist/rules/index.d.ts +3 -0
- package/dist/rules/index.js +31 -0
- package/dist/rules/mocked-unit-under-test.d.ts +2 -0
- package/dist/rules/mocked-unit-under-test.js +32 -0
- package/dist/rules/never-failing-assertion.d.ts +2 -0
- package/dist/rules/never-failing-assertion.js +43 -0
- package/dist/rules/no-assertions.d.ts +2 -0
- package/dist/rules/no-assertions.js +16 -0
- package/dist/rules/skipped-test.d.ts +2 -0
- package/dist/rules/skipped-test.js +15 -0
- package/dist/rules/snapshot-only.d.ts +2 -0
- package/dist/rules/snapshot-only.js +18 -0
- package/dist/rules/swallowed-assertion.d.ts +2 -0
- package/dist/rules/swallowed-assertion.js +18 -0
- package/dist/rules/tautological-assertion.d.ts +2 -0
- package/dist/rules/tautological-assertion.js +42 -0
- package/dist/run.d.ts +12 -0
- package/dist/run.js +204 -0
- package/integrations/codex/AGENTS.md +20 -0
- package/integrations/cursor/honest-tests.mdc +24 -0
- package/package.json +73 -4
- package/skills/honest-tests/SKILL.md +141 -0
- package/skills/honest-tests/references/audit-procedure.md +128 -0
- package/skills/honest-tests/references/mutation-testing.md +68 -0
- package/skills/honest-tests/references/test-smells.md +268 -0
- package/skills/honest-tests/references/writing-honest-tests.md +96 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "honest-tests",
|
|
3
|
+
"owner": {
|
|
4
|
+
"name": "mintwork",
|
|
5
|
+
"url": "https://github.com/mintwork"
|
|
6
|
+
},
|
|
7
|
+
"metadata": {
|
|
8
|
+
"description": "Agent skill that stops AI coding agents from writing fake tests",
|
|
9
|
+
"version": "0.1.1"
|
|
10
|
+
},
|
|
11
|
+
"plugins": [
|
|
12
|
+
{
|
|
13
|
+
"name": "honest-tests",
|
|
14
|
+
"description": "Write and audit tests that actually verify behavior: detect fake tests and prove them with mutation testing",
|
|
15
|
+
"source": "./",
|
|
16
|
+
"strict": false,
|
|
17
|
+
"skills": [
|
|
18
|
+
"./skills/honest-tests"
|
|
19
|
+
]
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "honest-tests",
|
|
3
|
+
"displayName": "Honest Tests",
|
|
4
|
+
"version": "0.1.1",
|
|
5
|
+
"description": "Stop AI coding agents from writing fake tests. Skill + CLI that detects tests that can't fail and proves it with mutation testing.",
|
|
6
|
+
"author": {
|
|
7
|
+
"name": "mintwork",
|
|
8
|
+
"url": "https://github.com/mintwork"
|
|
9
|
+
},
|
|
10
|
+
"homepage": "https://github.com/mintwork/honest-tests#readme",
|
|
11
|
+
"repository": "https://github.com/mintwork/honest-tests",
|
|
12
|
+
"license": "MIT",
|
|
13
|
+
"keywords": [
|
|
14
|
+
"testing",
|
|
15
|
+
"test-smells",
|
|
16
|
+
"mutation-testing",
|
|
17
|
+
"vitest",
|
|
18
|
+
"jest"
|
|
19
|
+
]
|
|
20
|
+
}
|
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [0.1.1] - 2026-11-14
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- The npm package is now **`@mintwork/honest-tests`**; run it with `npx @mintwork/honest-tests`. The unscoped
|
|
12
|
+
`honest-tests` package is deprecated and points here.
|
|
13
|
+
- The repository moved to [github.com/mintwork/honest-tests](https://github.com/mintwork/honest-tests); old URLs
|
|
14
|
+
redirect.
|
|
15
|
+
- Skill, plugin and editor-integration instructions use the new package name.
|
|
16
|
+
|
|
17
|
+
## [0.1.0] - 2026-11-14
|
|
18
|
+
|
|
19
|
+
First public release.
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- **`honest-tests audit`**: static detection of JS/TS tests (vitest, jest, node:assert, chai) that
|
|
24
|
+
can't fail. It has 13 rules:
|
|
25
|
+
- errors: `no-assertions`, `tautological-assertion`, `asserts-on-mock`, `asserts-on-own-literal`,
|
|
26
|
+
`never-failing-assertion`, `swallowed-assertion`, `catch-only-assertion`,
|
|
27
|
+
`mocked-unit-under-test`, `focused-test`
|
|
28
|
+
- warnings: `excessive-mocking`, `snapshot-only`, `conditional-assertion`, `skipped-test`
|
|
29
|
+
- **`asserts-on-own-literal`**: flags assertions on a literal the test just built, e.g.
|
|
30
|
+
`const p = { service: 's3' }; expect(p.service).toBe('s3')`. These are common in runtime "type
|
|
31
|
+
tests". The rule only fires when the value traces back purely to literals declared in the same
|
|
32
|
+
test, and that binding is never passed to code, mutated or reassigned.
|
|
33
|
+
- **`honest-tests mutate <file>`**: built-in mutation testing. It works on a temporary copy of the
|
|
34
|
+
project and only runs the test files that reach the target, so the working tree is never touched,
|
|
35
|
+
even on Ctrl-C. Supports `--test-cmd`, `--max-mutants`, `--timeout` and `--min-score`.
|
|
36
|
+
`audit --mutate <file>` combines both checks in one run.
|
|
37
|
+
- Output for humans and machines (`--json`, schema version 1), plus suppression comments
|
|
38
|
+
(`honest-tests-disable-next-line`, `-disable-line` and file-level `disable`) and `--fail-on`.
|
|
39
|
+
- **Agent integrations**:
|
|
40
|
+
- Agent Skill (`skills/honest-tests`) with a smell catalog, a guide to writing honest tests, an
|
|
41
|
+
audit procedure and a mutation-testing reference.
|
|
42
|
+
- Claude Code plugin and marketplace, with `/honest-tests:audit` and `/honest-tests:mutate`.
|
|
43
|
+
- `honest-tests init claude|codex|cursor` installs the skill, an `AGENTS.md` section or a Cursor
|
|
44
|
+
rule.
|
|
45
|
+
- `npm run demo:gif` renders the README demo from the real CLI output.
|
|
46
|
+
|
|
47
|
+
### Known limitations
|
|
48
|
+
|
|
49
|
+
- JS/TS only. The pytest adapter is planned; Python test files are reported as unsupported rather
|
|
50
|
+
than skipped silently.
|
|
51
|
+
- `mutate` doesn't follow path-alias imports (`@/lib/x`) when picking related tests yet. Pass
|
|
52
|
+
`--test-cmd` in that case.
|
|
53
|
+
- Mutants run one at a time.
|
|
54
|
+
|
|
55
|
+
[0.1.1]: https://github.com/mintwork/honest-tests/releases/tag/v0.1.1
|
|
56
|
+
[0.1.0]: https://github.com/mintwork/honest-tests/releases/tag/v0.1.0
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 honest-tests contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,281 @@
|
|
|
1
|
-
#
|
|
1
|
+
# honest-tests
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://github.com/mintwork/honest-tests/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/@mintwork/honest-tests)
|
|
5
|
+
[](https://www.npmjs.com/package/@mintwork/honest-tests)
|
|
6
|
+
[](package.json)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
[](https://github.com/mintwork/honest-tests/stargazers)
|
|
9
|
+
|
|
10
|
+
### Your AI wrote 200 tests. Many of them can't fail.
|
|
11
|
+
|
|
12
|
+
**honest-tests** is an agent skill + CLI that stops Claude Code, Codex, Cursor and friends from writing
|
|
13
|
+
fake tests — and proves which of your existing tests are fake by mutating your code and watching them
|
|
14
|
+
stay green.
|
|
15
|
+
|
|
16
|
+
<p align="center">
|
|
17
|
+
<img src="docs/demo.gif" width="1200" alt="honest-tests audit finds 11 of 15 tests that can't fail in a sample project; honest-tests mutate then shows 22 of 26 code changes survive the test suite, a 15.4% mutation score">
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
npx @mintwork/honest-tests audit . # find tests that can't fail
|
|
22
|
+
npx @mintwork/honest-tests mutate src/pricing.ts # prove it: break the code, see who notices
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## The problem
|
|
28
|
+
|
|
29
|
+
Ask an agent to "add tests" and you get a wall of green checkmarks. Look closer:
|
|
30
|
+
|
|
31
|
+
```ts
|
|
32
|
+
it('handles multiple items', () => {
|
|
33
|
+
const result = subtotal(items);
|
|
34
|
+
expect(result).toBe(result); // compares a value to itself
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('applies a percentage discount', () => {
|
|
38
|
+
applyDiscount(100, 10); // no assertion at all
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it('computes the discounted price', () => {
|
|
42
|
+
try {
|
|
43
|
+
expect(applyDiscount(100, 20)).toBe(70); // WRONG (it's 80) — and still passes
|
|
44
|
+
} catch {}
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it('gives members free shipping', () => {
|
|
48
|
+
const qualifies = vi.fn().mockReturnValue(true);
|
|
49
|
+
expect(qualifies(10, true)).toBe(true); // tests the mock it just configured
|
|
50
|
+
});
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Every one of these passes. Coverage goes up. Nothing is verified.
|
|
54
|
+
|
|
55
|
+
## What honest-tests says about it
|
|
56
|
+
|
|
57
|
+
```text
|
|
58
|
+
$ npx @mintwork/honest-tests audit test/pricing.test.ts
|
|
59
|
+
|
|
60
|
+
test/pricing.test.ts
|
|
61
|
+
16:5 error Compares `result` to itself — this can never fail. tautological-assertion
|
|
62
|
+
│ expect(result).toBe(result);
|
|
63
|
+
25:1 error Test "applies a percentage discount" has no assertions — it can only fail by crashing. no-assertions
|
|
64
|
+
│ it('applies a percentage discount', () => {
|
|
65
|
+
39:7 error Assertion is inside a try/catch that swallows errors — when it fails, the catch eats the AssertionError and the test passes. swallowed-assertion
|
|
66
|
+
│ expect(applyDiscount(100, 20)).toBe(70); // wrong! but the catch hides it
|
|
67
|
+
53:5 error Asserts on what mock `qualifies` returns (configured on line 52) — this verifies the mock, not your code. asserts-on-mock
|
|
68
|
+
│ expect(qualifies(10, true)).toBe(true);
|
|
69
|
+
... 5 more
|
|
70
|
+
|
|
71
|
+
7 of 10 tests can't fail (or don't test what they claim).
|
|
72
|
+
1 file, 10 tests: 7 errors, 2 warnings. Run with --verbose for fixes.
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Then the killer demo — **mutation testing, built in**. honest-tests flips operators, booleans,
|
|
76
|
+
constants and return values in your source (on a temp copy), runs your tests, and shows every change
|
|
77
|
+
nobody noticed:
|
|
78
|
+
|
|
79
|
+
```text
|
|
80
|
+
$ npx @mintwork/honest-tests mutate src/pricing.ts
|
|
81
|
+
|
|
82
|
+
22 mutants survived — your tests don't actually check these lines:
|
|
83
|
+
|
|
84
|
+
src/pricing.ts:15 if (percent < 0 || percent > 100) {
|
|
85
|
+
✗ `<` → `<=`
|
|
86
|
+
✗ `||` → `&&`
|
|
87
|
+
✗ `>` → `>=`
|
|
88
|
+
✗ `100` → `101`
|
|
89
|
+
|
|
90
|
+
src/pricing.ts:22 return isMember || total >= 50;
|
|
91
|
+
✗ `>=` → `>`
|
|
92
|
+
✗ `50` → `51`
|
|
93
|
+
...
|
|
94
|
+
|
|
95
|
+
Mutation score: 15.4% (4 killed, 0 timed out, 22 survived of 26; baseline 0.5s)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## After: the same code, honest tests
|
|
99
|
+
|
|
100
|
+
```ts
|
|
101
|
+
it('takes the percentage off the amount', () => {
|
|
102
|
+
expect(applyDiscount(200, 25)).toBe(150);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('accepts the 0% and 100% boundaries', () => {
|
|
106
|
+
expect(applyDiscount(200, 0)).toBe(200);
|
|
107
|
+
expect(applyDiscount(200, 100)).toBe(0);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it('rejects percentages outside 0..100', () => {
|
|
111
|
+
expect(() => applyDiscount(200, -1)).toThrow(RangeError);
|
|
112
|
+
expect(() => applyDiscount(200, 101)).toThrow(RangeError);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('starts free shipping exactly at 50 for non-members', () => {
|
|
116
|
+
expect(qualifiesForFreeShipping(49.99, false)).toBe(false);
|
|
117
|
+
expect(qualifiesForFreeShipping(50, false)).toBe(true);
|
|
118
|
+
});
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
```text
|
|
122
|
+
$ npx @mintwork/honest-tests audit test && npx @mintwork/honest-tests mutate src/pricing.ts
|
|
123
|
+
✓ 12 tests in 2 files: no fake tests found.
|
|
124
|
+
Mutation score: 100% (26 killed, 0 timed out, 0 survived of 26; baseline 0.5s)
|
|
125
|
+
✓ Every mutation was caught. These tests actually test something.
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Both projects live in [`fixtures/`](fixtures) — run `npm run demo:bad` / `npm run demo:good` yourself
|
|
129
|
+
(`npm run demo:gif` re-renders the GIF above from the real CLI output).
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## Install
|
|
134
|
+
|
|
135
|
+
### Claude Code (plugin + skill)
|
|
136
|
+
|
|
137
|
+
```text
|
|
138
|
+
/plugin marketplace add mintwork/honest-tests
|
|
139
|
+
/plugin install honest-tests@honest-tests
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Or drop just the skill into a repo: `npx @mintwork/honest-tests init claude` (writes `.claude/skills/honest-tests/`).
|
|
143
|
+
|
|
144
|
+
Then ask: *"write tests for src/cart.ts"* or *"audit my tests"* — or use the slash commands
|
|
145
|
+
`/honest-tests:audit [path]` and `/honest-tests:mutate <file>`.
|
|
146
|
+
|
|
147
|
+
### Codex
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
npx @mintwork/honest-tests init codex # appends an honest-tests section to AGENTS.md (idempotent)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### Cursor
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
npx @mintwork/honest-tests init cursor # writes .cursor/rules/honest-tests.mdc (auto-attaches to test files)
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### Any agent / CI
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
npx @mintwork/honest-tests audit . --json # machine-readable findings
|
|
163
|
+
npx @mintwork/honest-tests mutate src/x.ts --min-score 80 # fail CI below 80% mutation score
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## How the skill works
|
|
169
|
+
|
|
170
|
+
The [skill](skills/honest-tests/SKILL.md) (Agent Skills format) teaches agents one rule — *a test is
|
|
171
|
+
honest only if it fails when the behavior it names is broken* — plus two workflows:
|
|
172
|
+
|
|
173
|
+
- **Writing tests:** name the bug each test catches, derive expected values independently, mock only
|
|
174
|
+
boundaries, cover boundaries and error paths, then self-audit with `audit` + `mutate` before saying
|
|
175
|
+
"done". Hard rules forbid the classic agent moves: weakening assertions to get green, changing the
|
|
176
|
+
expected value to match output, mocking the unit under test, `try/catch` around asserts.
|
|
177
|
+
- **Auditing tests**, in phases, like a security audit:
|
|
178
|
+
1. **Detect** — `honest-tests audit --json` plus a manual pass for smells static analysis can't see.
|
|
179
|
+
2. **Verify** — every finding is opened and marked *confirmed / false-positive / accepted*; key
|
|
180
|
+
findings are proven with surviving mutants.
|
|
181
|
+
3. **Report** — "N of M tests can't fail", confirmed findings, surviving mutants, next steps.
|
|
182
|
+
4. **Fix** — only when asked; re-run and show the before/after mutation score.
|
|
183
|
+
|
|
184
|
+
References the agent loads on demand: [test smell catalog](skills/honest-tests/references/test-smells.md),
|
|
185
|
+
[writing honest tests](skills/honest-tests/references/writing-honest-tests.md),
|
|
186
|
+
[audit procedure](skills/honest-tests/references/audit-procedure.md),
|
|
187
|
+
[mutation testing](skills/honest-tests/references/mutation-testing.md).
|
|
188
|
+
|
|
189
|
+
## Rules
|
|
190
|
+
|
|
191
|
+
| Rule | Severity | Catches |
|
|
192
|
+
| --- | --- | --- |
|
|
193
|
+
| `no-assertions` | error | Test body never asserts — passes unless it crashes |
|
|
194
|
+
| `tautological-assertion` | error | `expect(x).toBe(x)`, `expect(f(1)).toBe(f(1))`, `expect(true).toBe(true)`, `toHaveLength(list.length)` |
|
|
195
|
+
| `asserts-on-mock` | error | Asserting on the return value of a mock the test configured (`vi.fn().mockReturnValue`, `spyOn`, `vi.mocked`, `.mock.results`) |
|
|
196
|
+
| `asserts-on-own-literal` | error | Asserting on a literal the test just built: `const p = { service: 's3' }; expect(p.service).toBe('s3')` — common in runtime "type tests" |
|
|
197
|
+
| `never-failing-assertion` | error | `expect(x).toBeTruthy` (never called), bare `expect(x)`, un-awaited `.resolves`/`.rejects`, `typeof x` "defined", `length >= 0`, `{}` truthy |
|
|
198
|
+
| `swallowed-assertion` | error | Assertion inside `try` whose `catch` doesn't rethrow |
|
|
199
|
+
| `catch-only-assertion` | error | Assertions only in `catch` — passes when nothing throws |
|
|
200
|
+
| `mocked-unit-under-test` | error | `vi.mock('./cart')` inside `cart.test.ts`; spying on and replacing the unit's own exports |
|
|
201
|
+
| `focused-test` | error | `.only`, `fit`, `fdescribe` left in |
|
|
202
|
+
| `excessive-mocking` | warning | 5+ module mocks in one file |
|
|
203
|
+
| `snapshot-only` | warning | The only assertion is a snapshot |
|
|
204
|
+
| `conditional-assertion` | warning | Assertion behind `if` / `&&` / ternary |
|
|
205
|
+
| `skipped-test` | warning | `.skip`, `xit`, `xdescribe` |
|
|
206
|
+
|
|
207
|
+
`npx @mintwork/honest-tests rules` prints the list. Suppress a deliberate case with a reason:
|
|
208
|
+
|
|
209
|
+
```ts
|
|
210
|
+
// honest-tests-disable-next-line no-assertions -- smoke test: module must import cleanly
|
|
211
|
+
it('loads', () => { require('../src/plugin'); });
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
(`honest-tests-disable-line <rule>` and file-level `/* honest-tests-disable <rule> */` also work.)
|
|
215
|
+
|
|
216
|
+
## Mutation check
|
|
217
|
+
|
|
218
|
+
| Operator | Example |
|
|
219
|
+
| --- | --- |
|
|
220
|
+
| binary | `>=` → `>`, `<` → `<=`, `===` → `!==`, `+` → `-`, `*` → `/` |
|
|
221
|
+
| logical | `&&` ↔ `\|\|` |
|
|
222
|
+
| assignment | `+=` ↔ `-=` |
|
|
223
|
+
| boolean | `true` ↔ `false` |
|
|
224
|
+
| negation | `!x` → `x` |
|
|
225
|
+
| number | `50` → `51`, `0` → `1` |
|
|
226
|
+
| condition | `if (c)` → `if (true)` / `if (false)` |
|
|
227
|
+
| return | `return x` → `return undefined` |
|
|
228
|
+
|
|
229
|
+
**Safe by design:** the project is copied to a temp dir (`node_modules` symlinked), mutants are written
|
|
230
|
+
only to the copy and restored after every run, and the copy is removed at the end — even on Ctrl-C. A red
|
|
231
|
+
baseline aborts the run. Only test files that import the target — directly or through other local
|
|
232
|
+
modules — are run (vitest/jest), once and with coverage off, so it's fast: 26 mutants in ~12s on the
|
|
233
|
+
fixture. Imports through path aliases (`@/lib/x`) aren't followed yet; pass `--test-cmd` if your tests
|
|
234
|
+
reach the target that way.
|
|
235
|
+
|
|
236
|
+
```text
|
|
237
|
+
--test-cmd "<cmd>" custom command; {files} expands to related test files
|
|
238
|
+
--max-mutants <n> cap (default 100) --timeout <ms> per-run timeout
|
|
239
|
+
--min-score <0-100> exit 1 below this score --root <dir> project root
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
## CLI reference
|
|
243
|
+
|
|
244
|
+
```text
|
|
245
|
+
honest-tests audit [paths...] [--json] [-v] [--ignore <glob>] [--rule <id>] [--disable <id>]
|
|
246
|
+
[--fail-on error|warning|never] [--mutate <file> ...]
|
|
247
|
+
honest-tests mutate <file> [--json] [--test-cmd <cmd>] [--max-mutants <n>] [--min-score <n>]
|
|
248
|
+
honest-tests rules [--json]
|
|
249
|
+
honest-tests init <claude|codex|cursor> [--dir <repo>]
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
Exit codes: `0` clean, `1` findings at/above `--fail-on` (default `error`) or score below `--min-score`,
|
|
253
|
+
`2` usage/runtime error. JSON output schema is documented in
|
|
254
|
+
[audit-procedure.md](skills/honest-tests/references/audit-procedure.md#phase-1--detect).
|
|
255
|
+
|
|
256
|
+
## Dogfooding
|
|
257
|
+
|
|
258
|
+
honest-tests audits its own suite in CI (`npm run dogfood` → zero findings) and mutation-checks its own
|
|
259
|
+
rules. While building v1, `mutate src/rules/asserts-on-mock.ts` scored **50%** against our first rule
|
|
260
|
+
tests — the tool found the gaps in the tests for the tool. After adding the missing negative cases it's
|
|
261
|
+
at 84%, and `never-failing-assertion.ts` is at 100%; the remaining survivors are near-equivalent mutants
|
|
262
|
+
such as early-exit optimizations.
|
|
263
|
+
|
|
264
|
+
## Roadmap
|
|
265
|
+
|
|
266
|
+
- [ ] **pytest adapter** — the rule engine is language-neutral; an adapter only has to emit the
|
|
267
|
+
`TestFileModel` ([src/adapters/python](src/adapters/python/index.ts) documents the mapping)
|
|
268
|
+
- [ ] Parallel mutant runs and per-test coverage mapping (run only tests that execute the mutated line)
|
|
269
|
+
- [ ] SARIF output for GitHub code scanning; `--changed` mode for PR diffs
|
|
270
|
+
- [ ] More rules: oracle copied from output, re-implemented algorithm in test, weak `toHaveBeenCalled()`
|
|
271
|
+
- [ ] Mocha, node:test, Bun, Deno; Go and Rust adapters
|
|
272
|
+
- [ ] GitHub Action with a PR comment: "this PR adds 12 tests, 4 can't fail"
|
|
273
|
+
|
|
274
|
+
## Contributing
|
|
275
|
+
|
|
276
|
+
PRs welcome — especially new rules with a bad/good fixture and pytest adapter work. See
|
|
277
|
+
[CONTRIBUTING.md](CONTRIBUTING.md).
|
|
278
|
+
|
|
279
|
+
## License
|
|
280
|
+
|
|
281
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Audit tests for fake/weak tests (detect → verify → report) using the honest-tests skill
|
|
3
|
+
argument-hint: "[path] [--mutate <source-file>]"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Use the `honest-tests` skill to audit the tests in: $ARGUMENTS (default: the whole repo).
|
|
7
|
+
|
|
8
|
+
Follow the skill's phased procedure exactly:
|
|
9
|
+
|
|
10
|
+
1. **Detect** — run `npx @mintwork/honest-tests audit <path> --json` (add `--mutate <file>` if given), plus the manual
|
|
11
|
+
"beyond the detector" pass.
|
|
12
|
+
2. **Verify** — open every finding, classify it as confirmed / false-positive / accepted with a one-line note.
|
|
13
|
+
Prove the important ones with surviving mutants (`npx @mintwork/honest-tests mutate <source-file>`).
|
|
14
|
+
3. **Report** — use the report template from `references/audit-procedure.md`, leading with
|
|
15
|
+
"N of M tests can't fail" and the mutation score.
|
|
16
|
+
|
|
17
|
+
Do not modify any files unless I ask you to fix the findings.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Mutation-check a source file and explain which behaviors its tests don't actually verify
|
|
3
|
+
argument-hint: "<source-file>"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Run `npx @mintwork/honest-tests mutate $ARGUMENTS --json` (it works on a temp copy; the working tree is not modified).
|
|
7
|
+
|
|
8
|
+
Then, for every surviving mutant, explain in one line which behavior is untested and propose the exact
|
|
9
|
+
assertion that would kill it (following `references/writing-honest-tests.md` from the `honest-tests` skill).
|
|
10
|
+
Flag mutants you believe are equivalent, with a reason. Finish with the mutation score.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { extname } from 'node:path';
|
|
2
|
+
import { jsAdapter } from './js/index.js';
|
|
3
|
+
import { pythonAdapter } from './python/index.js';
|
|
4
|
+
export const adapters = [jsAdapter, pythonAdapter];
|
|
5
|
+
export function adapterFor(path) {
|
|
6
|
+
const ext = extname(path).toLowerCase();
|
|
7
|
+
return adapters.find((a) => a.extensions.includes(ext));
|
|
8
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type * as t from '@babel/types';
|
|
2
|
+
import type { ExprInfo } from '../../core/types.js';
|
|
3
|
+
export type Node = t.Node;
|
|
4
|
+
export declare function parseSource(path: string, source: string): t.File;
|
|
5
|
+
/**
|
|
6
|
+
* Depth-first walk. `visit` receives the node and its ancestors (outermost first).
|
|
7
|
+
* Return `false` from `visit` to skip the node's children.
|
|
8
|
+
*/
|
|
9
|
+
export declare function walk(root: Node, visit: (node: Node, ancestors: Node[]) => boolean | void, ancestors?: Node[]): void;
|
|
10
|
+
export declare function textOf(source: string, node: Node): string;
|
|
11
|
+
export declare function normalize(text: string): string;
|
|
12
|
+
export declare function normText(source: string, node: Node): string;
|
|
13
|
+
/** Strip wrappers that don't change runtime meaning for our purposes. */
|
|
14
|
+
export declare function unwrap(node: Node): Node;
|
|
15
|
+
export declare function isCall(node: Node | null | undefined): node is t.CallExpression | t.OptionalCallExpression;
|
|
16
|
+
export declare function isMember(node: Node | null | undefined): node is t.MemberExpression | t.OptionalMemberExpression;
|
|
17
|
+
export declare function isFunction(node: Node | null | undefined): node is t.ArrowFunctionExpression | t.FunctionExpression;
|
|
18
|
+
export declare function propName(member: t.MemberExpression | t.OptionalMemberExpression): string | null;
|
|
19
|
+
export declare function stringValue(node: Node | undefined): string | null;
|
|
20
|
+
/** Root identifier name of a member/call chain: `vi.fn().mockReturnValue(1)` -> `vi`. */
|
|
21
|
+
export declare function chainRoot(node: Node): Node;
|
|
22
|
+
/** All member property names along a call chain, outermost last. */
|
|
23
|
+
export declare function chainProps(node: Node): string[];
|
|
24
|
+
export declare function describeExpr(source: string, raw: Node): ExprInfo;
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import { parse } from '@babel/parser';
|
|
2
|
+
import { extname } from 'node:path';
|
|
3
|
+
export function parseSource(path, source) {
|
|
4
|
+
const ext = extname(path).toLowerCase();
|
|
5
|
+
const plugins = ['decorators-legacy', 'importAttributes'];
|
|
6
|
+
if (['.ts', '.mts', '.cts'].includes(ext))
|
|
7
|
+
plugins.push('typescript');
|
|
8
|
+
else if (ext === '.tsx')
|
|
9
|
+
plugins.push('typescript', 'jsx');
|
|
10
|
+
else
|
|
11
|
+
plugins.push('jsx');
|
|
12
|
+
return parse(source, {
|
|
13
|
+
sourceType: 'unambiguous',
|
|
14
|
+
plugins,
|
|
15
|
+
errorRecovery: true,
|
|
16
|
+
allowReturnOutsideFunction: true,
|
|
17
|
+
allowAwaitOutsideFunction: true,
|
|
18
|
+
allowImportExportEverywhere: true,
|
|
19
|
+
allowUndeclaredExports: true,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
const SKIP_KEYS = new Set([
|
|
23
|
+
'type',
|
|
24
|
+
'loc',
|
|
25
|
+
'start',
|
|
26
|
+
'end',
|
|
27
|
+
'range',
|
|
28
|
+
'extra',
|
|
29
|
+
'leadingComments',
|
|
30
|
+
'trailingComments',
|
|
31
|
+
'innerComments',
|
|
32
|
+
'comments',
|
|
33
|
+
'tokens',
|
|
34
|
+
]);
|
|
35
|
+
function isNode(value) {
|
|
36
|
+
return typeof value === 'object' && value !== null && typeof value.type === 'string';
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Depth-first walk. `visit` receives the node and its ancestors (outermost first).
|
|
40
|
+
* Return `false` from `visit` to skip the node's children.
|
|
41
|
+
*/
|
|
42
|
+
export function walk(root, visit, ancestors = []) {
|
|
43
|
+
if (visit(root, ancestors) === false)
|
|
44
|
+
return;
|
|
45
|
+
ancestors.push(root);
|
|
46
|
+
for (const key of Object.keys(root)) {
|
|
47
|
+
if (SKIP_KEYS.has(key))
|
|
48
|
+
continue;
|
|
49
|
+
const value = root[key];
|
|
50
|
+
if (Array.isArray(value)) {
|
|
51
|
+
for (const child of value)
|
|
52
|
+
if (isNode(child))
|
|
53
|
+
walk(child, visit, ancestors);
|
|
54
|
+
}
|
|
55
|
+
else if (isNode(value)) {
|
|
56
|
+
walk(value, visit, ancestors);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
ancestors.pop();
|
|
60
|
+
}
|
|
61
|
+
export function textOf(source, node) {
|
|
62
|
+
return source.slice(node.start ?? 0, node.end ?? 0);
|
|
63
|
+
}
|
|
64
|
+
export function normalize(text) {
|
|
65
|
+
return text
|
|
66
|
+
.replace(/\s+/g, ' ')
|
|
67
|
+
.replace(/\s*([(),.[\]{}])\s*/g, '$1')
|
|
68
|
+
.replace(/;+$/, '')
|
|
69
|
+
.trim();
|
|
70
|
+
}
|
|
71
|
+
export function normText(source, node) {
|
|
72
|
+
return normalize(textOf(source, node));
|
|
73
|
+
}
|
|
74
|
+
/** Strip wrappers that don't change runtime meaning for our purposes. */
|
|
75
|
+
export function unwrap(node) {
|
|
76
|
+
let cur = node;
|
|
77
|
+
for (;;) {
|
|
78
|
+
switch (cur.type) {
|
|
79
|
+
case 'TSAsExpression':
|
|
80
|
+
case 'TSSatisfiesExpression':
|
|
81
|
+
case 'TSNonNullExpression':
|
|
82
|
+
case 'TSTypeAssertion':
|
|
83
|
+
case 'ParenthesizedExpression':
|
|
84
|
+
case 'AwaitExpression':
|
|
85
|
+
cur = cur.expression ?? cur.argument;
|
|
86
|
+
continue;
|
|
87
|
+
default:
|
|
88
|
+
return cur;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
export function isCall(node) {
|
|
93
|
+
return !!node && (node.type === 'CallExpression' || node.type === 'OptionalCallExpression');
|
|
94
|
+
}
|
|
95
|
+
export function isMember(node) {
|
|
96
|
+
return !!node && (node.type === 'MemberExpression' || node.type === 'OptionalMemberExpression');
|
|
97
|
+
}
|
|
98
|
+
export function isFunction(node) {
|
|
99
|
+
return !!node && (node.type === 'ArrowFunctionExpression' || node.type === 'FunctionExpression');
|
|
100
|
+
}
|
|
101
|
+
export function propName(member) {
|
|
102
|
+
const p = member.property;
|
|
103
|
+
if (!member.computed && p.type === 'Identifier')
|
|
104
|
+
return p.name;
|
|
105
|
+
if (p.type === 'StringLiteral')
|
|
106
|
+
return p.value;
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
export function stringValue(node) {
|
|
110
|
+
if (!node)
|
|
111
|
+
return null;
|
|
112
|
+
if (node.type === 'StringLiteral')
|
|
113
|
+
return node.value;
|
|
114
|
+
if (node.type === 'TemplateLiteral' && node.expressions.length === 0) {
|
|
115
|
+
return node.quasis.map((q) => q.value.cooked ?? q.value.raw).join('');
|
|
116
|
+
}
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
/** Root identifier name of a member/call chain: `vi.fn().mockReturnValue(1)` -> `vi`. */
|
|
120
|
+
export function chainRoot(node) {
|
|
121
|
+
let cur = node;
|
|
122
|
+
for (;;) {
|
|
123
|
+
if (isCall(cur))
|
|
124
|
+
cur = cur.callee;
|
|
125
|
+
else if (isMember(cur))
|
|
126
|
+
cur = cur.object;
|
|
127
|
+
else
|
|
128
|
+
return cur;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/** All member property names along a call chain, outermost last. */
|
|
132
|
+
export function chainProps(node) {
|
|
133
|
+
const props = [];
|
|
134
|
+
let cur = node;
|
|
135
|
+
for (;;) {
|
|
136
|
+
if (isCall(cur))
|
|
137
|
+
cur = cur.callee;
|
|
138
|
+
else if (isMember(cur)) {
|
|
139
|
+
const name = propName(cur);
|
|
140
|
+
if (name)
|
|
141
|
+
props.unshift(name);
|
|
142
|
+
cur = cur.object;
|
|
143
|
+
}
|
|
144
|
+
else
|
|
145
|
+
return props;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
export function describeExpr(source, raw) {
|
|
149
|
+
const node = unwrap(raw);
|
|
150
|
+
const text = normText(source, node);
|
|
151
|
+
switch (node.type) {
|
|
152
|
+
case 'NumericLiteral':
|
|
153
|
+
case 'BigIntLiteral':
|
|
154
|
+
return { text, kind: 'literal', literalValue: Number(node.value) };
|
|
155
|
+
case 'StringLiteral':
|
|
156
|
+
return { text, kind: 'literal', literalValue: node.value };
|
|
157
|
+
case 'BooleanLiteral':
|
|
158
|
+
return { text, kind: 'literal', literalValue: node.value };
|
|
159
|
+
case 'NullLiteral':
|
|
160
|
+
return { text, kind: 'literal', literalValue: null };
|
|
161
|
+
case 'RegExpLiteral':
|
|
162
|
+
return { text, kind: 'object' };
|
|
163
|
+
case 'TemplateLiteral':
|
|
164
|
+
return node.expressions.length === 0
|
|
165
|
+
? { text, kind: 'literal', literalValue: stringValue(node) ?? '' }
|
|
166
|
+
: { text, kind: 'other' };
|
|
167
|
+
case 'Identifier':
|
|
168
|
+
if (node.name === 'undefined')
|
|
169
|
+
return { text, kind: 'literal', literalValue: undefined, isUndefinedLiteral: true };
|
|
170
|
+
return { text, kind: 'identifier' };
|
|
171
|
+
case 'UnaryExpression':
|
|
172
|
+
if (node.operator === 'typeof')
|
|
173
|
+
return { text, kind: 'typeof' };
|
|
174
|
+
if (node.operator === 'void')
|
|
175
|
+
return { text, kind: 'literal', literalValue: undefined, isUndefinedLiteral: true };
|
|
176
|
+
if ((node.operator === '-' || node.operator === '+') && node.argument.type === 'NumericLiteral') {
|
|
177
|
+
return { text, kind: 'literal', literalValue: Number(text) };
|
|
178
|
+
}
|
|
179
|
+
return { text, kind: 'other' };
|
|
180
|
+
case 'MemberExpression':
|
|
181
|
+
case 'OptionalMemberExpression':
|
|
182
|
+
return { text, kind: 'member' };
|
|
183
|
+
case 'CallExpression':
|
|
184
|
+
case 'OptionalCallExpression':
|
|
185
|
+
return { text, kind: 'call', calleeText: normText(source, node.callee) };
|
|
186
|
+
case 'ObjectExpression':
|
|
187
|
+
return { text, kind: 'object' };
|
|
188
|
+
case 'ArrayExpression':
|
|
189
|
+
return { text, kind: 'array' };
|
|
190
|
+
case 'ArrowFunctionExpression':
|
|
191
|
+
case 'FunctionExpression':
|
|
192
|
+
case 'ClassExpression':
|
|
193
|
+
return { text, kind: 'function' };
|
|
194
|
+
case 'NewExpression':
|
|
195
|
+
return { text, kind: 'new' };
|
|
196
|
+
default:
|
|
197
|
+
return { text, kind: 'other' };
|
|
198
|
+
}
|
|
199
|
+
}
|