@mintwork/honest-tests 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/.claude-plugin/marketplace.json +22 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/CHANGELOG.md +56 -0
  4. package/LICENSE +21 -0
  5. package/README.md +280 -2
  6. package/commands/audit.md +17 -0
  7. package/commands/mutate.md +10 -0
  8. package/dist/adapters/index.d.ts +3 -0
  9. package/dist/adapters/index.js +8 -0
  10. package/dist/adapters/js/ast.d.ts +24 -0
  11. package/dist/adapters/js/ast.js +199 -0
  12. package/dist/adapters/js/index.d.ts +25 -0
  13. package/dist/adapters/js/index.js +122 -0
  14. package/dist/adapters/js/mutants.d.ts +8 -0
  15. package/dist/adapters/js/mutants.js +190 -0
  16. package/dist/adapters/js/parse-tests.d.ts +5 -0
  17. package/dist/adapters/js/parse-tests.js +1101 -0
  18. package/dist/adapters/python/index.d.ts +17 -0
  19. package/dist/adapters/python/index.js +26 -0
  20. package/dist/cli.d.ts +2 -0
  21. package/dist/cli.js +21 -0
  22. package/dist/core/engine.d.ts +44 -0
  23. package/dist/core/engine.js +205 -0
  24. package/dist/core/types.d.ts +195 -0
  25. package/dist/core/types.js +7 -0
  26. package/dist/index.d.ts +6 -0
  27. package/dist/index.js +5 -0
  28. package/dist/mutate/runner.d.ts +59 -0
  29. package/dist/mutate/runner.js +379 -0
  30. package/dist/report/colors.d.ts +11 -0
  31. package/dist/report/colors.js +13 -0
  32. package/dist/report/human.d.ts +6 -0
  33. package/dist/report/human.js +79 -0
  34. package/dist/report/json.d.ts +4 -0
  35. package/dist/report/json.js +8 -0
  36. package/dist/rules/asserts-on-mock.d.ts +2 -0
  37. package/dist/rules/asserts-on-mock.js +53 -0
  38. package/dist/rules/asserts-on-own-literal.d.ts +2 -0
  39. package/dist/rules/asserts-on-own-literal.js +36 -0
  40. package/dist/rules/catch-only-assertion.d.ts +2 -0
  41. package/dist/rules/catch-only-assertion.js +22 -0
  42. package/dist/rules/conditional-assertion.d.ts +2 -0
  43. package/dist/rules/conditional-assertion.js +30 -0
  44. package/dist/rules/excessive-mocking.d.ts +3 -0
  45. package/dist/rules/excessive-mocking.js +22 -0
  46. package/dist/rules/focused-test.d.ts +2 -0
  47. package/dist/rules/focused-test.js +15 -0
  48. package/dist/rules/helpers.d.ts +5 -0
  49. package/dist/rules/helpers.js +18 -0
  50. package/dist/rules/index.d.ts +3 -0
  51. package/dist/rules/index.js +31 -0
  52. package/dist/rules/mocked-unit-under-test.d.ts +2 -0
  53. package/dist/rules/mocked-unit-under-test.js +32 -0
  54. package/dist/rules/never-failing-assertion.d.ts +2 -0
  55. package/dist/rules/never-failing-assertion.js +43 -0
  56. package/dist/rules/no-assertions.d.ts +2 -0
  57. package/dist/rules/no-assertions.js +16 -0
  58. package/dist/rules/skipped-test.d.ts +2 -0
  59. package/dist/rules/skipped-test.js +15 -0
  60. package/dist/rules/snapshot-only.d.ts +2 -0
  61. package/dist/rules/snapshot-only.js +18 -0
  62. package/dist/rules/swallowed-assertion.d.ts +2 -0
  63. package/dist/rules/swallowed-assertion.js +18 -0
  64. package/dist/rules/tautological-assertion.d.ts +2 -0
  65. package/dist/rules/tautological-assertion.js +42 -0
  66. package/dist/run.d.ts +12 -0
  67. package/dist/run.js +204 -0
  68. package/integrations/codex/AGENTS.md +20 -0
  69. package/integrations/cursor/honest-tests.mdc +24 -0
  70. package/package.json +73 -4
  71. package/skills/honest-tests/SKILL.md +141 -0
  72. package/skills/honest-tests/references/audit-procedure.md +128 -0
  73. package/skills/honest-tests/references/mutation-testing.md +68 -0
  74. package/skills/honest-tests/references/test-smells.md +268 -0
  75. package/skills/honest-tests/references/writing-honest-tests.md +96 -0
@@ -0,0 +1,22 @@
1
+ {
2
+ "name": "honest-tests",
3
+ "owner": {
4
+ "name": "mintwork",
5
+ "url": "https://github.com/mintwork"
6
+ },
7
+ "metadata": {
8
+ "description": "Agent skill that stops AI coding agents from writing fake tests",
9
+ "version": "0.1.1"
10
+ },
11
+ "plugins": [
12
+ {
13
+ "name": "honest-tests",
14
+ "description": "Write and audit tests that actually verify behavior: detect fake tests and prove them with mutation testing",
15
+ "source": "./",
16
+ "strict": false,
17
+ "skills": [
18
+ "./skills/honest-tests"
19
+ ]
20
+ }
21
+ ]
22
+ }
@@ -0,0 +1,20 @@
1
+ {
2
+ "name": "honest-tests",
3
+ "displayName": "Honest Tests",
4
+ "version": "0.1.1",
5
+ "description": "Stop AI coding agents from writing fake tests. Skill + CLI that detects tests that can't fail and proves it with mutation testing.",
6
+ "author": {
7
+ "name": "mintwork",
8
+ "url": "https://github.com/mintwork"
9
+ },
10
+ "homepage": "https://github.com/mintwork/honest-tests#readme",
11
+ "repository": "https://github.com/mintwork/honest-tests",
12
+ "license": "MIT",
13
+ "keywords": [
14
+ "testing",
15
+ "test-smells",
16
+ "mutation-testing",
17
+ "vitest",
18
+ "jest"
19
+ ]
20
+ }
package/CHANGELOG.md ADDED
@@ -0,0 +1,56 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
5
+ [Semantic Versioning](https://semver.org/).
6
+
7
+ ## [0.1.1] - 2026-11-14
8
+
9
+ ### Changed
10
+
11
+ - The npm package is now **`@mintwork/honest-tests`**; run it with `npx @mintwork/honest-tests`. The unscoped
12
+ `honest-tests` package is deprecated and points here.
13
+ - The repository moved to [github.com/mintwork/honest-tests](https://github.com/mintwork/honest-tests); old URLs
14
+ redirect.
15
+ - Skill, plugin and editor-integration instructions use the new package name.
16
+
17
+ ## [0.1.0] - 2026-11-14
18
+
19
+ First public release.
20
+
21
+ ### Added
22
+
23
+ - **`honest-tests audit`**: static detection of JS/TS tests (vitest, jest, node:assert, chai) that
24
+ can't fail. It has 13 rules:
25
+ - errors: `no-assertions`, `tautological-assertion`, `asserts-on-mock`, `asserts-on-own-literal`,
26
+ `never-failing-assertion`, `swallowed-assertion`, `catch-only-assertion`,
27
+ `mocked-unit-under-test`, `focused-test`
28
+ - warnings: `excessive-mocking`, `snapshot-only`, `conditional-assertion`, `skipped-test`
29
+ - **`asserts-on-own-literal`**: flags assertions on a literal the test just built, e.g.
30
+ `const p = { service: 's3' }; expect(p.service).toBe('s3')`. These are common in runtime "type
31
+ tests". The rule only fires when the value traces back purely to literals declared in the same
32
+ test, and that binding is never passed to code, mutated or reassigned.
33
+ - **`honest-tests mutate <file>`**: built-in mutation testing. It works on a temporary copy of the
34
+ project and only runs the test files that reach the target, so the working tree is never touched,
35
+ even on Ctrl-C. Supports `--test-cmd`, `--max-mutants`, `--timeout` and `--min-score`.
36
+ `audit --mutate <file>` combines both checks in one run.
37
+ - Output for humans and machines (`--json`, schema version 1), plus suppression comments
38
+ (`honest-tests-disable-next-line`, `-disable-line` and file-level `disable`) and `--fail-on`.
39
+ - **Agent integrations**:
40
+ - Agent Skill (`skills/honest-tests`) with a smell catalog, a guide to writing honest tests, an
41
+ audit procedure and a mutation-testing reference.
42
+ - Claude Code plugin and marketplace, with `/honest-tests:audit` and `/honest-tests:mutate`.
43
+ - `honest-tests init claude|codex|cursor` installs the skill, an `AGENTS.md` section or a Cursor
44
+ rule.
45
+ - `npm run demo:gif` renders the README demo from the real CLI output.
46
+
47
+ ### Known limitations
48
+
49
+ - JS/TS only. The pytest adapter is planned; Python test files are reported as unsupported rather
50
+ than skipped silently.
51
+ - `mutate` doesn't follow path-alias imports (`@/lib/x`) when picking related tests yet. Pass
52
+ `--test-cmd` in that case.
53
+ - Mutants run one at a time.
54
+
55
+ [0.1.1]: https://github.com/mintwork/honest-tests/releases/tag/v0.1.1
56
+ [0.1.0]: https://github.com/mintwork/honest-tests/releases/tag/v0.1.0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 honest-tests contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,281 @@
1
- # Temporary Holding Version
1
+ # honest-tests
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ [![CI](https://github.com/mintwork/honest-tests/actions/workflows/ci.yml/badge.svg)](https://github.com/mintwork/honest-tests/actions/workflows/ci.yml)
4
+ [![npm](https://img.shields.io/npm/v/@mintwork/honest-tests?color=cb3837&logo=npm)](https://www.npmjs.com/package/@mintwork/honest-tests)
5
+ [![npm downloads](https://img.shields.io/npm/dm/@mintwork/honest-tests?color=cb3837)](https://www.npmjs.com/package/@mintwork/honest-tests)
6
+ [![node](https://img.shields.io/badge/node-%3E%3D22-339933?logo=node.js&logoColor=white)](package.json)
7
+ [![license: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
8
+ [![GitHub stars](https://img.shields.io/github/stars/mintwork/honest-tests?style=social)](https://github.com/mintwork/honest-tests/stargazers)
9
+
10
+ ### Your AI wrote 200 tests. Many of them can't fail.
11
+
12
+ **honest-tests** is an agent skill + CLI that stops Claude Code, Codex, Cursor and friends from writing
13
+ fake tests — and proves which of your existing tests are fake by mutating your code and watching them
14
+ stay green.
15
+
16
+ <p align="center">
17
+ <img src="docs/demo.gif" width="1200" alt="honest-tests audit finds 11 of 15 tests that can't fail in a sample project; honest-tests mutate then shows 22 of 26 code changes survive the test suite, a 15.4% mutation score">
18
+ </p>
19
+
20
+ ```bash
21
+ npx @mintwork/honest-tests audit . # find tests that can't fail
22
+ npx @mintwork/honest-tests mutate src/pricing.ts # prove it: break the code, see who notices
23
+ ```
24
+
25
+ ---
26
+
27
+ ## The problem
28
+
29
+ Ask an agent to "add tests" and you get a wall of green checkmarks. Look closer:
30
+
31
+ ```ts
32
+ it('handles multiple items', () => {
33
+ const result = subtotal(items);
34
+ expect(result).toBe(result); // compares a value to itself
35
+ });
36
+
37
+ it('applies a percentage discount', () => {
38
+ applyDiscount(100, 10); // no assertion at all
39
+ });
40
+
41
+ it('computes the discounted price', () => {
42
+ try {
43
+ expect(applyDiscount(100, 20)).toBe(70); // WRONG (it's 80) — and still passes
44
+ } catch {}
45
+ });
46
+
47
+ it('gives members free shipping', () => {
48
+ const qualifies = vi.fn().mockReturnValue(true);
49
+ expect(qualifies(10, true)).toBe(true); // tests the mock it just configured
50
+ });
51
+ ```
52
+
53
+ Every one of these passes. Coverage goes up. Nothing is verified.
54
+
55
+ ## What honest-tests says about it
56
+
57
+ ```text
58
+ $ npx @mintwork/honest-tests audit test/pricing.test.ts
59
+
60
+ test/pricing.test.ts
61
+ 16:5 error Compares `result` to itself — this can never fail. tautological-assertion
62
+ │ expect(result).toBe(result);
63
+ 25:1 error Test "applies a percentage discount" has no assertions — it can only fail by crashing. no-assertions
64
+ │ it('applies a percentage discount', () => {
65
+ 39:7 error Assertion is inside a try/catch that swallows errors — when it fails, the catch eats the AssertionError and the test passes. swallowed-assertion
66
+ │ expect(applyDiscount(100, 20)).toBe(70); // wrong! but the catch hides it
67
+ 53:5 error Asserts on what mock `qualifies` returns (configured on line 52) — this verifies the mock, not your code. asserts-on-mock
68
+ │ expect(qualifies(10, true)).toBe(true);
69
+ ... 5 more
70
+
71
+ 7 of 10 tests can't fail (or don't test what they claim).
72
+ 1 file, 10 tests: 7 errors, 2 warnings. Run with --verbose for fixes.
73
+ ```
74
+
75
+ Then the killer demo — **mutation testing, built in**. honest-tests flips operators, booleans,
76
+ constants and return values in your source (on a temp copy), runs your tests, and shows every change
77
+ nobody noticed:
78
+
79
+ ```text
80
+ $ npx @mintwork/honest-tests mutate src/pricing.ts
81
+
82
+ 22 mutants survived — your tests don't actually check these lines:
83
+
84
+ src/pricing.ts:15 if (percent < 0 || percent > 100) {
85
+ ✗ `<` → `<=`
86
+ ✗ `||` → `&&`
87
+ ✗ `>` → `>=`
88
+ ✗ `100` → `101`
89
+
90
+ src/pricing.ts:22 return isMember || total >= 50;
91
+ ✗ `>=` → `>`
92
+ ✗ `50` → `51`
93
+ ...
94
+
95
+ Mutation score: 15.4% (4 killed, 0 timed out, 22 survived of 26; baseline 0.5s)
96
+ ```
97
+
98
+ ## After: the same code, honest tests
99
+
100
+ ```ts
101
+ it('takes the percentage off the amount', () => {
102
+ expect(applyDiscount(200, 25)).toBe(150);
103
+ });
104
+
105
+ it('accepts the 0% and 100% boundaries', () => {
106
+ expect(applyDiscount(200, 0)).toBe(200);
107
+ expect(applyDiscount(200, 100)).toBe(0);
108
+ });
109
+
110
+ it('rejects percentages outside 0..100', () => {
111
+ expect(() => applyDiscount(200, -1)).toThrow(RangeError);
112
+ expect(() => applyDiscount(200, 101)).toThrow(RangeError);
113
+ });
114
+
115
+ it('starts free shipping exactly at 50 for non-members', () => {
116
+ expect(qualifiesForFreeShipping(49.99, false)).toBe(false);
117
+ expect(qualifiesForFreeShipping(50, false)).toBe(true);
118
+ });
119
+ ```
120
+
121
+ ```text
122
+ $ npx @mintwork/honest-tests audit test && npx @mintwork/honest-tests mutate src/pricing.ts
123
+ ✓ 12 tests in 2 files: no fake tests found.
124
+ Mutation score: 100% (26 killed, 0 timed out, 0 survived of 26; baseline 0.5s)
125
+ ✓ Every mutation was caught. These tests actually test something.
126
+ ```
127
+
128
+ Both projects live in [`fixtures/`](fixtures) — run `npm run demo:bad` / `npm run demo:good` yourself
129
+ (`npm run demo:gif` re-renders the GIF above from the real CLI output).
130
+
131
+ ---
132
+
133
+ ## Install
134
+
135
+ ### Claude Code (plugin + skill)
136
+
137
+ ```text
138
+ /plugin marketplace add mintwork/honest-tests
139
+ /plugin install honest-tests@honest-tests
140
+ ```
141
+
142
+ Or drop just the skill into a repo: `npx @mintwork/honest-tests init claude` (writes `.claude/skills/honest-tests/`).
143
+
144
+ Then ask: *"write tests for src/cart.ts"* or *"audit my tests"* — or use the slash commands
145
+ `/honest-tests:audit [path]` and `/honest-tests:mutate <file>`.
146
+
147
+ ### Codex
148
+
149
+ ```bash
150
+ npx @mintwork/honest-tests init codex # appends an honest-tests section to AGENTS.md (idempotent)
151
+ ```
152
+
153
+ ### Cursor
154
+
155
+ ```bash
156
+ npx @mintwork/honest-tests init cursor # writes .cursor/rules/honest-tests.mdc (auto-attaches to test files)
157
+ ```
158
+
159
+ ### Any agent / CI
160
+
161
+ ```bash
162
+ npx @mintwork/honest-tests audit . --json # machine-readable findings
163
+ npx @mintwork/honest-tests mutate src/x.ts --min-score 80 # fail CI below 80% mutation score
164
+ ```
165
+
166
+ ---
167
+
168
+ ## How the skill works
169
+
170
+ The [skill](skills/honest-tests/SKILL.md) (Agent Skills format) teaches agents one rule — *a test is
171
+ honest only if it fails when the behavior it names is broken* — plus two workflows:
172
+
173
+ - **Writing tests:** name the bug each test catches, derive expected values independently, mock only
174
+ boundaries, cover boundaries and error paths, then self-audit with `audit` + `mutate` before saying
175
+ "done". Hard rules forbid the classic agent moves: weakening assertions to get green, changing the
176
+ expected value to match output, mocking the unit under test, `try/catch` around asserts.
177
+ - **Auditing tests**, in phases, like a security audit:
178
+ 1. **Detect** — `honest-tests audit --json` plus a manual pass for smells static analysis can't see.
179
+ 2. **Verify** — every finding is opened and marked *confirmed / false-positive / accepted*; key
180
+ findings are proven with surviving mutants.
181
+ 3. **Report** — "N of M tests can't fail", confirmed findings, surviving mutants, next steps.
182
+ 4. **Fix** — only when asked; re-run and show the before/after mutation score.
183
+
184
+ References the agent loads on demand: [test smell catalog](skills/honest-tests/references/test-smells.md),
185
+ [writing honest tests](skills/honest-tests/references/writing-honest-tests.md),
186
+ [audit procedure](skills/honest-tests/references/audit-procedure.md),
187
+ [mutation testing](skills/honest-tests/references/mutation-testing.md).
188
+
189
+ ## Rules
190
+
191
+ | Rule | Severity | Catches |
192
+ | --- | --- | --- |
193
+ | `no-assertions` | error | Test body never asserts — passes unless it crashes |
194
+ | `tautological-assertion` | error | `expect(x).toBe(x)`, `expect(f(1)).toBe(f(1))`, `expect(true).toBe(true)`, `toHaveLength(list.length)` |
195
+ | `asserts-on-mock` | error | Asserting on the return value of a mock the test configured (`vi.fn().mockReturnValue`, `spyOn`, `vi.mocked`, `.mock.results`) |
196
+ | `asserts-on-own-literal` | error | Asserting on a literal the test just built: `const p = { service: 's3' }; expect(p.service).toBe('s3')` — common in runtime "type tests" |
197
+ | `never-failing-assertion` | error | `expect(x).toBeTruthy` (never called), bare `expect(x)`, un-awaited `.resolves`/`.rejects`, `typeof x` "defined", `length >= 0`, `{}` truthy |
198
+ | `swallowed-assertion` | error | Assertion inside `try` whose `catch` doesn't rethrow |
199
+ | `catch-only-assertion` | error | Assertions only in `catch` — passes when nothing throws |
200
+ | `mocked-unit-under-test` | error | `vi.mock('./cart')` inside `cart.test.ts`; spying on and replacing the unit's own exports |
201
+ | `focused-test` | error | `.only`, `fit`, `fdescribe` left in |
202
+ | `excessive-mocking` | warning | 5+ module mocks in one file |
203
+ | `snapshot-only` | warning | The only assertion is a snapshot |
204
+ | `conditional-assertion` | warning | Assertion behind `if` / `&&` / ternary |
205
+ | `skipped-test` | warning | `.skip`, `xit`, `xdescribe` |
206
+
207
+ `npx @mintwork/honest-tests rules` prints the list. Suppress a deliberate case with a reason:
208
+
209
+ ```ts
210
+ // honest-tests-disable-next-line no-assertions -- smoke test: module must import cleanly
211
+ it('loads', () => { require('../src/plugin'); });
212
+ ```
213
+
214
+ (`honest-tests-disable-line <rule>` and file-level `/* honest-tests-disable <rule> */` also work.)
215
+
216
+ ## Mutation check
217
+
218
+ | Operator | Example |
219
+ | --- | --- |
220
+ | binary | `>=` → `>`, `<` → `<=`, `===` → `!==`, `+` → `-`, `*` → `/` |
221
+ | logical | `&&` ↔ `\|\|` |
222
+ | assignment | `+=` ↔ `-=` |
223
+ | boolean | `true` ↔ `false` |
224
+ | negation | `!x` → `x` |
225
+ | number | `50` → `51`, `0` → `1` |
226
+ | condition | `if (c)` → `if (true)` / `if (false)` |
227
+ | return | `return x` → `return undefined` |
228
+
229
+ **Safe by design:** the project is copied to a temp dir (`node_modules` symlinked), mutants are written
230
+ only to the copy and restored after every run, and the copy is removed at the end — even on Ctrl-C. A red
231
+ baseline aborts the run. Only test files that import the target — directly or through other local
232
+ modules — are run (vitest/jest), once and with coverage off, so it's fast: 26 mutants in ~12s on the
233
+ fixture. Imports through path aliases (`@/lib/x`) aren't followed yet; pass `--test-cmd` if your tests
234
+ reach the target that way.
235
+
236
+ ```text
237
+ --test-cmd "<cmd>" custom command; {files} expands to related test files
238
+ --max-mutants <n> cap (default 100) --timeout <ms> per-run timeout
239
+ --min-score <0-100> exit 1 below this score --root <dir> project root
240
+ ```
241
+
242
+ ## CLI reference
243
+
244
+ ```text
245
+ honest-tests audit [paths...] [--json] [-v] [--ignore <glob>] [--rule <id>] [--disable <id>]
246
+ [--fail-on error|warning|never] [--mutate <file> ...]
247
+ honest-tests mutate <file> [--json] [--test-cmd <cmd>] [--max-mutants <n>] [--min-score <n>]
248
+ honest-tests rules [--json]
249
+ honest-tests init <claude|codex|cursor> [--dir <repo>]
250
+ ```
251
+
252
+ Exit codes: `0` clean, `1` findings at/above `--fail-on` (default `error`) or score below `--min-score`,
253
+ `2` usage/runtime error. JSON output schema is documented in
254
+ [audit-procedure.md](skills/honest-tests/references/audit-procedure.md#phase-1--detect).
255
+
256
+ ## Dogfooding
257
+
258
+ honest-tests audits its own suite in CI (`npm run dogfood` → zero findings) and mutation-checks its own
259
+ rules. While building v1, `mutate src/rules/asserts-on-mock.ts` scored **50%** against our first rule
260
+ tests — the tool found the gaps in the tests for the tool. After adding the missing negative cases it's
261
+ at 84%, and `never-failing-assertion.ts` is at 100%; the remaining survivors are near-equivalent mutants
262
+ such as early-exit optimizations.
263
+
264
+ ## Roadmap
265
+
266
+ - [ ] **pytest adapter** — the rule engine is language-neutral; an adapter only has to emit the
267
+ `TestFileModel` ([src/adapters/python](src/adapters/python/index.ts) documents the mapping)
268
+ - [ ] Parallel mutant runs and per-test coverage mapping (run only tests that execute the mutated line)
269
+ - [ ] SARIF output for GitHub code scanning; `--changed` mode for PR diffs
270
+ - [ ] More rules: oracle copied from output, re-implemented algorithm in test, weak `toHaveBeenCalled()`
271
+ - [ ] Mocha, node:test, Bun, Deno; Go and Rust adapters
272
+ - [ ] GitHub Action with a PR comment: "this PR adds 12 tests, 4 can't fail"
273
+
274
+ ## Contributing
275
+
276
+ PRs welcome — especially new rules with a bad/good fixture and pytest adapter work. See
277
+ [CONTRIBUTING.md](CONTRIBUTING.md).
278
+
279
+ ## License
280
+
281
+ [MIT](LICENSE)
@@ -0,0 +1,17 @@
1
+ ---
2
+ description: Audit tests for fake/weak tests (detect → verify → report) using the honest-tests skill
3
+ argument-hint: "[path] [--mutate <source-file>]"
4
+ ---
5
+
6
+ Use the `honest-tests` skill to audit the tests in: $ARGUMENTS (default: the whole repo).
7
+
8
+ Follow the skill's phased procedure exactly:
9
+
10
+ 1. **Detect** — run `npx @mintwork/honest-tests audit <path> --json` (add `--mutate <file>` if given), plus the manual
11
+ "beyond the detector" pass.
12
+ 2. **Verify** — open every finding, classify it as confirmed / false-positive / accepted with a one-line note.
13
+ Prove the important ones with surviving mutants (`npx @mintwork/honest-tests mutate <source-file>`).
14
+ 3. **Report** — use the report template from `references/audit-procedure.md`, leading with
15
+ "N of M tests can't fail" and the mutation score.
16
+
17
+ Do not modify any files unless I ask you to fix the findings.
@@ -0,0 +1,10 @@
1
+ ---
2
+ description: Mutation-check a source file and explain which behaviors its tests don't actually verify
3
+ argument-hint: "<source-file>"
4
+ ---
5
+
6
+ Run `npx @mintwork/honest-tests mutate $ARGUMENTS --json` (it works on a temp copy; the working tree is not modified).
7
+
8
+ Then, for every surviving mutant, explain in one line which behavior is untested and propose the exact
9
+ assertion that would kill it (following `references/writing-honest-tests.md` from the `honest-tests` skill).
10
+ Flag mutants you believe are equivalent, with a reason. Finish with the mutation score.
@@ -0,0 +1,3 @@
1
+ import type { LanguageAdapter } from '../core/types.js';
2
+ export declare const adapters: LanguageAdapter[];
3
+ export declare function adapterFor(path: string): LanguageAdapter | undefined;
@@ -0,0 +1,8 @@
1
+ import { extname } from 'node:path';
2
+ import { jsAdapter } from './js/index.js';
3
+ import { pythonAdapter } from './python/index.js';
4
+ export const adapters = [jsAdapter, pythonAdapter];
5
+ export function adapterFor(path) {
6
+ const ext = extname(path).toLowerCase();
7
+ return adapters.find((a) => a.extensions.includes(ext));
8
+ }
@@ -0,0 +1,24 @@
1
+ import type * as t from '@babel/types';
2
+ import type { ExprInfo } from '../../core/types.js';
3
+ export type Node = t.Node;
4
+ export declare function parseSource(path: string, source: string): t.File;
5
+ /**
6
+ * Depth-first walk. `visit` receives the node and its ancestors (outermost first).
7
+ * Return `false` from `visit` to skip the node's children.
8
+ */
9
+ export declare function walk(root: Node, visit: (node: Node, ancestors: Node[]) => boolean | void, ancestors?: Node[]): void;
10
+ export declare function textOf(source: string, node: Node): string;
11
+ export declare function normalize(text: string): string;
12
+ export declare function normText(source: string, node: Node): string;
13
+ /** Strip wrappers that don't change runtime meaning for our purposes. */
14
+ export declare function unwrap(node: Node): Node;
15
+ export declare function isCall(node: Node | null | undefined): node is t.CallExpression | t.OptionalCallExpression;
16
+ export declare function isMember(node: Node | null | undefined): node is t.MemberExpression | t.OptionalMemberExpression;
17
+ export declare function isFunction(node: Node | null | undefined): node is t.ArrowFunctionExpression | t.FunctionExpression;
18
+ export declare function propName(member: t.MemberExpression | t.OptionalMemberExpression): string | null;
19
+ export declare function stringValue(node: Node | undefined): string | null;
20
+ /** Root identifier name of a member/call chain: `vi.fn().mockReturnValue(1)` -> `vi`. */
21
+ export declare function chainRoot(node: Node): Node;
22
+ /** All member property names along a call chain, outermost last. */
23
+ export declare function chainProps(node: Node): string[];
24
+ export declare function describeExpr(source: string, raw: Node): ExprInfo;
@@ -0,0 +1,199 @@
1
+ import { parse } from '@babel/parser';
2
+ import { extname } from 'node:path';
3
+ export function parseSource(path, source) {
4
+ const ext = extname(path).toLowerCase();
5
+ const plugins = ['decorators-legacy', 'importAttributes'];
6
+ if (['.ts', '.mts', '.cts'].includes(ext))
7
+ plugins.push('typescript');
8
+ else if (ext === '.tsx')
9
+ plugins.push('typescript', 'jsx');
10
+ else
11
+ plugins.push('jsx');
12
+ return parse(source, {
13
+ sourceType: 'unambiguous',
14
+ plugins,
15
+ errorRecovery: true,
16
+ allowReturnOutsideFunction: true,
17
+ allowAwaitOutsideFunction: true,
18
+ allowImportExportEverywhere: true,
19
+ allowUndeclaredExports: true,
20
+ });
21
+ }
22
+ const SKIP_KEYS = new Set([
23
+ 'type',
24
+ 'loc',
25
+ 'start',
26
+ 'end',
27
+ 'range',
28
+ 'extra',
29
+ 'leadingComments',
30
+ 'trailingComments',
31
+ 'innerComments',
32
+ 'comments',
33
+ 'tokens',
34
+ ]);
35
+ function isNode(value) {
36
+ return typeof value === 'object' && value !== null && typeof value.type === 'string';
37
+ }
38
+ /**
39
+ * Depth-first walk. `visit` receives the node and its ancestors (outermost first).
40
+ * Return `false` from `visit` to skip the node's children.
41
+ */
42
+ export function walk(root, visit, ancestors = []) {
43
+ if (visit(root, ancestors) === false)
44
+ return;
45
+ ancestors.push(root);
46
+ for (const key of Object.keys(root)) {
47
+ if (SKIP_KEYS.has(key))
48
+ continue;
49
+ const value = root[key];
50
+ if (Array.isArray(value)) {
51
+ for (const child of value)
52
+ if (isNode(child))
53
+ walk(child, visit, ancestors);
54
+ }
55
+ else if (isNode(value)) {
56
+ walk(value, visit, ancestors);
57
+ }
58
+ }
59
+ ancestors.pop();
60
+ }
61
+ export function textOf(source, node) {
62
+ return source.slice(node.start ?? 0, node.end ?? 0);
63
+ }
64
+ export function normalize(text) {
65
+ return text
66
+ .replace(/\s+/g, ' ')
67
+ .replace(/\s*([(),.[\]{}])\s*/g, '$1')
68
+ .replace(/;+$/, '')
69
+ .trim();
70
+ }
71
+ export function normText(source, node) {
72
+ return normalize(textOf(source, node));
73
+ }
74
+ /** Strip wrappers that don't change runtime meaning for our purposes. */
75
+ export function unwrap(node) {
76
+ let cur = node;
77
+ for (;;) {
78
+ switch (cur.type) {
79
+ case 'TSAsExpression':
80
+ case 'TSSatisfiesExpression':
81
+ case 'TSNonNullExpression':
82
+ case 'TSTypeAssertion':
83
+ case 'ParenthesizedExpression':
84
+ case 'AwaitExpression':
85
+ cur = cur.expression ?? cur.argument;
86
+ continue;
87
+ default:
88
+ return cur;
89
+ }
90
+ }
91
+ }
92
+ export function isCall(node) {
93
+ return !!node && (node.type === 'CallExpression' || node.type === 'OptionalCallExpression');
94
+ }
95
+ export function isMember(node) {
96
+ return !!node && (node.type === 'MemberExpression' || node.type === 'OptionalMemberExpression');
97
+ }
98
+ export function isFunction(node) {
99
+ return !!node && (node.type === 'ArrowFunctionExpression' || node.type === 'FunctionExpression');
100
+ }
101
+ export function propName(member) {
102
+ const p = member.property;
103
+ if (!member.computed && p.type === 'Identifier')
104
+ return p.name;
105
+ if (p.type === 'StringLiteral')
106
+ return p.value;
107
+ return null;
108
+ }
109
+ export function stringValue(node) {
110
+ if (!node)
111
+ return null;
112
+ if (node.type === 'StringLiteral')
113
+ return node.value;
114
+ if (node.type === 'TemplateLiteral' && node.expressions.length === 0) {
115
+ return node.quasis.map((q) => q.value.cooked ?? q.value.raw).join('');
116
+ }
117
+ return null;
118
+ }
119
+ /** Root identifier name of a member/call chain: `vi.fn().mockReturnValue(1)` -> `vi`. */
120
+ export function chainRoot(node) {
121
+ let cur = node;
122
+ for (;;) {
123
+ if (isCall(cur))
124
+ cur = cur.callee;
125
+ else if (isMember(cur))
126
+ cur = cur.object;
127
+ else
128
+ return cur;
129
+ }
130
+ }
131
+ /** All member property names along a call chain, outermost last. */
132
+ export function chainProps(node) {
133
+ const props = [];
134
+ let cur = node;
135
+ for (;;) {
136
+ if (isCall(cur))
137
+ cur = cur.callee;
138
+ else if (isMember(cur)) {
139
+ const name = propName(cur);
140
+ if (name)
141
+ props.unshift(name);
142
+ cur = cur.object;
143
+ }
144
+ else
145
+ return props;
146
+ }
147
+ }
148
+ export function describeExpr(source, raw) {
149
+ const node = unwrap(raw);
150
+ const text = normText(source, node);
151
+ switch (node.type) {
152
+ case 'NumericLiteral':
153
+ case 'BigIntLiteral':
154
+ return { text, kind: 'literal', literalValue: Number(node.value) };
155
+ case 'StringLiteral':
156
+ return { text, kind: 'literal', literalValue: node.value };
157
+ case 'BooleanLiteral':
158
+ return { text, kind: 'literal', literalValue: node.value };
159
+ case 'NullLiteral':
160
+ return { text, kind: 'literal', literalValue: null };
161
+ case 'RegExpLiteral':
162
+ return { text, kind: 'object' };
163
+ case 'TemplateLiteral':
164
+ return node.expressions.length === 0
165
+ ? { text, kind: 'literal', literalValue: stringValue(node) ?? '' }
166
+ : { text, kind: 'other' };
167
+ case 'Identifier':
168
+ if (node.name === 'undefined')
169
+ return { text, kind: 'literal', literalValue: undefined, isUndefinedLiteral: true };
170
+ return { text, kind: 'identifier' };
171
+ case 'UnaryExpression':
172
+ if (node.operator === 'typeof')
173
+ return { text, kind: 'typeof' };
174
+ if (node.operator === 'void')
175
+ return { text, kind: 'literal', literalValue: undefined, isUndefinedLiteral: true };
176
+ if ((node.operator === '-' || node.operator === '+') && node.argument.type === 'NumericLiteral') {
177
+ return { text, kind: 'literal', literalValue: Number(text) };
178
+ }
179
+ return { text, kind: 'other' };
180
+ case 'MemberExpression':
181
+ case 'OptionalMemberExpression':
182
+ return { text, kind: 'member' };
183
+ case 'CallExpression':
184
+ case 'OptionalCallExpression':
185
+ return { text, kind: 'call', calleeText: normText(source, node.callee) };
186
+ case 'ObjectExpression':
187
+ return { text, kind: 'object' };
188
+ case 'ArrayExpression':
189
+ return { text, kind: 'array' };
190
+ case 'ArrowFunctionExpression':
191
+ case 'FunctionExpression':
192
+ case 'ClassExpression':
193
+ return { text, kind: 'function' };
194
+ case 'NewExpression':
195
+ return { text, kind: 'new' };
196
+ default:
197
+ return { text, kind: 'other' };
198
+ }
199
+ }