@mintwork/honest-tests 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/.claude-plugin/marketplace.json +22 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/CHANGELOG.md +56 -0
  4. package/LICENSE +21 -0
  5. package/README.md +280 -2
  6. package/commands/audit.md +17 -0
  7. package/commands/mutate.md +10 -0
  8. package/dist/adapters/index.d.ts +3 -0
  9. package/dist/adapters/index.js +8 -0
  10. package/dist/adapters/js/ast.d.ts +24 -0
  11. package/dist/adapters/js/ast.js +199 -0
  12. package/dist/adapters/js/index.d.ts +25 -0
  13. package/dist/adapters/js/index.js +122 -0
  14. package/dist/adapters/js/mutants.d.ts +8 -0
  15. package/dist/adapters/js/mutants.js +190 -0
  16. package/dist/adapters/js/parse-tests.d.ts +5 -0
  17. package/dist/adapters/js/parse-tests.js +1101 -0
  18. package/dist/adapters/python/index.d.ts +17 -0
  19. package/dist/adapters/python/index.js +26 -0
  20. package/dist/cli.d.ts +2 -0
  21. package/dist/cli.js +21 -0
  22. package/dist/core/engine.d.ts +44 -0
  23. package/dist/core/engine.js +205 -0
  24. package/dist/core/types.d.ts +195 -0
  25. package/dist/core/types.js +7 -0
  26. package/dist/index.d.ts +6 -0
  27. package/dist/index.js +5 -0
  28. package/dist/mutate/runner.d.ts +59 -0
  29. package/dist/mutate/runner.js +379 -0
  30. package/dist/report/colors.d.ts +11 -0
  31. package/dist/report/colors.js +13 -0
  32. package/dist/report/human.d.ts +6 -0
  33. package/dist/report/human.js +79 -0
  34. package/dist/report/json.d.ts +4 -0
  35. package/dist/report/json.js +8 -0
  36. package/dist/rules/asserts-on-mock.d.ts +2 -0
  37. package/dist/rules/asserts-on-mock.js +53 -0
  38. package/dist/rules/asserts-on-own-literal.d.ts +2 -0
  39. package/dist/rules/asserts-on-own-literal.js +36 -0
  40. package/dist/rules/catch-only-assertion.d.ts +2 -0
  41. package/dist/rules/catch-only-assertion.js +22 -0
  42. package/dist/rules/conditional-assertion.d.ts +2 -0
  43. package/dist/rules/conditional-assertion.js +30 -0
  44. package/dist/rules/excessive-mocking.d.ts +3 -0
  45. package/dist/rules/excessive-mocking.js +22 -0
  46. package/dist/rules/focused-test.d.ts +2 -0
  47. package/dist/rules/focused-test.js +15 -0
  48. package/dist/rules/helpers.d.ts +5 -0
  49. package/dist/rules/helpers.js +18 -0
  50. package/dist/rules/index.d.ts +3 -0
  51. package/dist/rules/index.js +31 -0
  52. package/dist/rules/mocked-unit-under-test.d.ts +2 -0
  53. package/dist/rules/mocked-unit-under-test.js +32 -0
  54. package/dist/rules/never-failing-assertion.d.ts +2 -0
  55. package/dist/rules/never-failing-assertion.js +43 -0
  56. package/dist/rules/no-assertions.d.ts +2 -0
  57. package/dist/rules/no-assertions.js +16 -0
  58. package/dist/rules/skipped-test.d.ts +2 -0
  59. package/dist/rules/skipped-test.js +15 -0
  60. package/dist/rules/snapshot-only.d.ts +2 -0
  61. package/dist/rules/snapshot-only.js +18 -0
  62. package/dist/rules/swallowed-assertion.d.ts +2 -0
  63. package/dist/rules/swallowed-assertion.js +18 -0
  64. package/dist/rules/tautological-assertion.d.ts +2 -0
  65. package/dist/rules/tautological-assertion.js +42 -0
  66. package/dist/run.d.ts +12 -0
  67. package/dist/run.js +204 -0
  68. package/integrations/codex/AGENTS.md +20 -0
  69. package/integrations/cursor/honest-tests.mdc +24 -0
  70. package/package.json +73 -4
  71. package/skills/honest-tests/SKILL.md +141 -0
  72. package/skills/honest-tests/references/audit-procedure.md +128 -0
  73. package/skills/honest-tests/references/mutation-testing.md +68 -0
  74. package/skills/honest-tests/references/test-smells.md +268 -0
  75. package/skills/honest-tests/references/writing-honest-tests.md +96 -0
@@ -0,0 +1,3 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const EXCESSIVE_MOCK_THRESHOLD = 5;
3
+ export declare const excessiveMocking: Rule;
@@ -0,0 +1,22 @@
1
+ export const EXCESSIVE_MOCK_THRESHOLD = 5;
2
+ export const excessiveMocking = {
3
+ id: 'excessive-mocking',
4
+ severity: 'warning',
5
+ description: `File mocks ${EXCESSIVE_MOCK_THRESHOLD}+ modules — the test mostly exercises mocks wired to each other.`,
6
+ fix: 'Mock at the system boundary (HTTP, DB, clock) and let internal modules run for real, or split the unit so it needs fewer collaborators.',
7
+ check(file) {
8
+ // Re-mocking the same module in several tests (vi.doMock per case) is still one collaborator.
9
+ const specs = [...new Set(file.moduleMocks.map((m) => m.specifier))];
10
+ if (specs.length < EXCESSIVE_MOCK_THRESHOLD)
11
+ return [];
12
+ const first = file.moduleMocks[0];
13
+ return [
14
+ {
15
+ line: first.line,
16
+ column: 1,
17
+ evidence: specs.join(', '),
18
+ message: `Mocks ${specs.length} modules — little real code is left under test.`,
19
+ },
20
+ ];
21
+ },
22
+ };
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const focusedTest: Rule;
@@ -0,0 +1,15 @@
1
+ export const focusedTest = {
2
+ id: 'focused-test',
3
+ severity: 'error',
4
+ description: '.only / fit / fdescribe left in — every other test in the file silently stops running.',
5
+ fix: 'Remove .only before committing.',
6
+ check(file) {
7
+ const out = [];
8
+ for (const t of file.tests) {
9
+ if (t.modifier === 'only') {
10
+ out.push({ line: t.line, column: 1, evidence: t.headText, test: t.name, message: `${t.kind === 'suite' ? 'Suite' : 'Test'} "${t.name}" is focused — the rest of the file is skipped.` });
11
+ }
12
+ }
13
+ return out;
14
+ },
15
+ };
@@ -0,0 +1,5 @@
1
+ import type { Assertion, RawFinding, TestCase, TestFileModel } from '../core/types.js';
2
+ export declare function eachTest(file: TestFileModel): Generator<TestCase>;
3
+ export declare function atAssertion(t: TestCase, a: Assertion, message: string, extra?: Partial<RawFinding>): RawFinding;
4
+ export declare function atTest(t: TestCase, message: string, extra?: Partial<RawFinding>): RawFinding;
5
+ export declare function code(s: string, max?: number): string;
@@ -0,0 +1,18 @@
1
+ export function* eachTest(file) {
2
+ for (const t of file.tests) {
3
+ if (t.kind !== 'test' || !t.analyzable)
4
+ continue;
5
+ if (t.modifier === 'todo' || t.modifier === 'skip')
6
+ continue;
7
+ yield t;
8
+ }
9
+ }
10
+ export function atAssertion(t, a, message, extra = {}) {
11
+ return { line: a.line, column: a.column, message, evidence: a.text, test: t.name, ...extra };
12
+ }
13
+ export function atTest(t, message, extra = {}) {
14
+ return { line: t.line, column: 1, message, evidence: t.headText, test: t.name, ...extra };
15
+ }
16
+ export function code(s, max = 60) {
17
+ return '`' + (s.length > max ? s.slice(0, max - 1) + '…' : s) + '`';
18
+ }
@@ -0,0 +1,3 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const rules: Rule[];
3
+ export declare function ruleById(id: string): Rule | undefined;
@@ -0,0 +1,31 @@
1
+ import { assertsOnMock } from './asserts-on-mock.js';
2
+ import { assertsOnOwnLiteral } from './asserts-on-own-literal.js';
3
+ import { catchOnlyAssertion } from './catch-only-assertion.js';
4
+ import { conditionalAssertion } from './conditional-assertion.js';
5
+ import { excessiveMocking } from './excessive-mocking.js';
6
+ import { focusedTest } from './focused-test.js';
7
+ import { mockedUnitUnderTest } from './mocked-unit-under-test.js';
8
+ import { neverFailingAssertion } from './never-failing-assertion.js';
9
+ import { noAssertions } from './no-assertions.js';
10
+ import { skippedTest } from './skipped-test.js';
11
+ import { snapshotOnly } from './snapshot-only.js';
12
+ import { swallowedAssertion } from './swallowed-assertion.js';
13
+ import { tautologicalAssertion } from './tautological-assertion.js';
14
+ export const rules = [
15
+ noAssertions,
16
+ tautologicalAssertion,
17
+ assertsOnMock,
18
+ assertsOnOwnLiteral,
19
+ neverFailingAssertion,
20
+ swallowedAssertion,
21
+ catchOnlyAssertion,
22
+ mockedUnitUnderTest,
23
+ excessiveMocking,
24
+ snapshotOnly,
25
+ conditionalAssertion,
26
+ focusedTest,
27
+ skippedTest,
28
+ ];
29
+ export function ruleById(id) {
30
+ return rules.find((r) => r.id === id);
31
+ }
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const mockedUnitUnderTest: Rule;
@@ -0,0 +1,32 @@
1
+ import { code } from './helpers.js';
2
+ export const mockedUnitUnderTest = {
3
+ id: 'mocked-unit-under-test',
4
+ severity: 'error',
5
+ description: 'The module the test file is named after is itself mocked, so the real implementation never runs.',
6
+ fix: 'Mock only the unit\'s dependencies (network, clock, filesystem, other modules) — never the unit itself. Import the real module and assert on its behavior.',
7
+ check(file) {
8
+ const out = [];
9
+ for (const m of file.moduleMocks) {
10
+ if (!m.isUnitUnderTest)
11
+ continue;
12
+ out.push({
13
+ line: m.line,
14
+ column: 1,
15
+ evidence: m.text,
16
+ severity: m.partial ? 'warning' : 'error',
17
+ message: m.partial
18
+ ? `Partially mocks ${code(m.specifier)}, the module under test — the overridden exports are never really tested here.`
19
+ : `Mocks ${code(m.specifier)}, the module this file is testing — every assertion runs against the mock, not the real code.`,
20
+ });
21
+ }
22
+ for (const s of file.unitSpies) {
23
+ out.push({
24
+ line: s.line,
25
+ column: 1,
26
+ evidence: s.text,
27
+ message: `Replaces ${code(s.target)} (an export of the module under test) with a mock implementation.`,
28
+ });
29
+ }
30
+ return out;
31
+ },
32
+ };
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const neverFailingAssertion: Rule;
@@ -0,0 +1,43 @@
1
+ import { atAssertion, code, eachTest } from './helpers.js';
2
+ const ALWAYS_DEFINED = new Set(['object', 'array', 'function', 'new', 'typeof']);
3
+ const ALWAYS_TRUTHY = new Set(['object', 'array', 'function', 'new']);
4
+ export const neverFailingAssertion = {
5
+ id: 'never-failing-assertion',
6
+ severity: 'error',
7
+ description: 'Assertion is structurally unable to fail: missing matcher call, un-awaited async matcher, or a check that is always true.',
8
+ fix: 'Make the assertion specific: call the matcher, await async matchers, and assert the exact expected value instead of "is defined"/"is truthy".',
9
+ check(file) {
10
+ const out = [];
11
+ for (const t of eachTest(file)) {
12
+ for (const a of t.assertions) {
13
+ const s = a.subject;
14
+ if (a.style === 'expect' && a.matcherKind === 'none') {
15
+ out.push(atAssertion(t, a, a.matcher
16
+ ? `Matcher ${code('.' + a.matcher)} is referenced but never called — add \`()\`. As written it asserts nothing.`
17
+ : '`expect(...)` without a matcher asserts nothing.'));
18
+ continue;
19
+ }
20
+ if (a.async && !a.awaitedOrReturned) {
21
+ out.push(atAssertion(t, a, `Async matcher (.resolves/.rejects) is not awaited or returned — the test can finish before it runs.`));
22
+ continue;
23
+ }
24
+ if (!s || a.negated)
25
+ continue;
26
+ if (a.matcherKind === 'defined' && ALWAYS_DEFINED.has(s.kind)) {
27
+ out.push(atAssertion(t, a, `${code(s.text)} is always defined — this assertion can never fail.`));
28
+ }
29
+ else if (a.matcherKind === 'truthy' && ALWAYS_TRUTHY.has(s.kind)) {
30
+ out.push(atAssertion(t, a, `${code(s.text)} is always truthy — this assertion can never fail.`));
31
+ }
32
+ else if ((a.matcherKind === 'compare-gte' || a.matcherKind === 'compare-other') &&
33
+ /\.(length|size)$/.test(s.text) &&
34
+ a.args[0] &&
35
+ ((a.matcherKind === 'compare-gte' && a.args[0].literalValue === 0) ||
36
+ (a.matcher === 'toBeGreaterThan' && typeof a.args[0].literalValue === 'number' && a.args[0].literalValue < 0))) {
37
+ out.push(atAssertion(t, a, `${code(s.text)} is never negative — this assertion can never fail.`));
38
+ }
39
+ }
40
+ }
41
+ return out;
42
+ },
43
+ };
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const noAssertions: Rule;
@@ -0,0 +1,16 @@
1
+ import { atTest, eachTest } from './helpers.js';
2
+ export const noAssertions = {
3
+ id: 'no-assertions',
4
+ severity: 'error',
5
+ description: 'Test contains no assertions, so it passes as long as nothing throws.',
6
+ fix: 'Assert on the observable result (return value, state change, emitted call). If "does not throw" is the real contract, say so: expect(() => fn()).not.toThrow().',
7
+ check(file) {
8
+ const out = [];
9
+ for (const t of eachTest(file)) {
10
+ if (t.assertions.length === 0 && t.helperAssertionCalls === 0 && !t.hasAssertionCountGuard) {
11
+ out.push(atTest(t, `Test "${t.name}" has no assertions — it can only fail by crashing.`));
12
+ }
13
+ }
14
+ return out;
15
+ },
16
+ };
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const skippedTest: Rule;
@@ -0,0 +1,15 @@
1
+ export const skippedTest = {
2
+ id: 'skipped-test',
3
+ severity: 'warning',
4
+ description: '.skip / xit / xdescribe left in — the test counts toward the total but verifies nothing.',
5
+ fix: 'Fix the test, delete it, or turn it into it.todo("...") with a tracking issue.',
6
+ check(file) {
7
+ const out = [];
8
+ for (const t of file.tests) {
9
+ if (t.modifier === 'skip') {
10
+ out.push({ line: t.line, column: 1, evidence: t.headText, test: t.name, message: `${t.kind === 'suite' ? 'Suite' : 'Test'} "${t.name}" is skipped.` });
11
+ }
12
+ }
13
+ return out;
14
+ },
15
+ };
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const snapshotOnly: Rule;
@@ -0,0 +1,18 @@
1
+ import { atTest, eachTest } from './helpers.js';
2
+ export const snapshotOnly = {
3
+ id: 'snapshot-only',
4
+ severity: 'warning',
5
+ description: 'The only assertions are snapshots, which record whatever the code does today — bugs included.',
6
+ fix: 'Add at least one explicit assertion on the property that matters (a value, a field, a call). Keep the snapshot only as a change detector.',
7
+ check(file) {
8
+ const out = [];
9
+ for (const t of eachTest(file)) {
10
+ if (t.assertions.length === 0 || t.helperAssertionCalls > 0)
11
+ continue;
12
+ if (t.assertions.every((a) => a.matcherKind === 'snapshot')) {
13
+ out.push(atTest(t, `Test "${t.name}" only asserts a snapshot — it will happily approve a wrong result the first time it runs.`));
14
+ }
15
+ }
16
+ return out;
17
+ },
18
+ };
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const swallowedAssertion: Rule;
@@ -0,0 +1,18 @@
1
+ import { atAssertion, eachTest } from './helpers.js';
2
+ export const swallowedAssertion = {
3
+ id: 'swallowed-assertion',
4
+ severity: 'error',
5
+ description: 'Assertion sits inside a try block whose catch does not rethrow, so a failing assertion is caught and the test passes.',
6
+ fix: 'Remove the try/catch. For expected errors use expect(() => fn()).toThrow(...) or await expect(p).rejects.toThrow(...).',
7
+ check(file) {
8
+ const out = [];
9
+ for (const t of eachTest(file)) {
10
+ for (const a of t.assertions) {
11
+ if (a.context.inSwallowingTry) {
12
+ out.push(atAssertion(t, a, 'Assertion is inside a try/catch that swallows errors — when it fails, the catch eats the AssertionError and the test passes.'));
13
+ }
14
+ }
15
+ }
16
+ return out;
17
+ },
18
+ };
@@ -0,0 +1,2 @@
1
+ import type { Rule } from '../core/types.js';
2
+ export declare const tautologicalAssertion: Rule;
@@ -0,0 +1,42 @@
1
+ import { atAssertion, code, eachTest } from './helpers.js';
2
+ /**
3
+ * Expected values that are the actual value restated: `toHaveLength(list.length)`, `toEqual(list.slice())`.
4
+ * Transformations (`toEqual(keys.toSorted())`, `toEqual(xs.map(lower))`) are real property checks.
5
+ */
6
+ function isCopyOf(actual, expected, kind) {
7
+ if (kind === 'length')
8
+ return expected === `${actual}.length` || expected === `${actual}.size`;
9
+ return [`${actual}.slice()`, `${actual}.concat()`, `${actual}.valueOf()`, `[...${actual}]`, `{...${actual}}`, `structuredClone(${actual})`].includes(expected);
10
+ }
11
+ export const tautologicalAssertion = {
12
+ id: 'tautological-assertion',
13
+ severity: 'error',
14
+ description: 'Assertion compares a value to itself or asserts on a constant, so its outcome is fixed before any code runs.',
15
+ fix: 'Compare the output of the code under test against an independently known expected value (a literal you worked out by hand, or a fixture).',
16
+ check(file) {
17
+ const out = [];
18
+ for (const t of eachTest(file)) {
19
+ for (const a of t.assertions) {
20
+ const s = a.subject;
21
+ if (!s || a.matcherKind === 'none')
22
+ continue;
23
+ if (s.kind === 'literal') {
24
+ out.push(atAssertion(t, a, `Asserts on the constant ${code(s.text)} — no code under test is involved, so this can never fail.`));
25
+ continue;
26
+ }
27
+ const expected = a.args[0];
28
+ if (!expected)
29
+ continue;
30
+ if (a.matcherKind !== 'equal' && a.matcherKind !== 'length')
31
+ continue;
32
+ if (expected.text === s.text) {
33
+ out.push(atAssertion(t, a, `Compares ${code(s.text)} to itself — this can never fail.`));
34
+ }
35
+ else if ((s.kind === 'identifier' || s.kind === 'member') && isCopyOf(s.text, expected.text, a.matcherKind)) {
36
+ out.push(atAssertion(t, a, `Expected value ${code(expected.text)} is derived from the actual value ${code(s.text)} — the assertion checks nothing.`));
37
+ }
38
+ }
39
+ }
40
+ return out;
41
+ },
42
+ };
package/dist/run.d.ts ADDED
@@ -0,0 +1,12 @@
1
+ export declare const PACKAGE_ROOT: string;
2
+ export declare function version(): string;
3
+ export interface IO {
4
+ stdout: (s: string) => void;
5
+ stderr: (s: string) => void;
6
+ cwd: string;
7
+ /** Show mutation progress on stderr. */
8
+ progress?: boolean;
9
+ }
10
+ export declare function run(argv: string[], io: IO): Promise<number>;
11
+ /** First and last lines of a test run: the error is usually at the top, the summary at the bottom. */
12
+ export declare function excerpt(output: string, headLines?: number, tailLines?: number): string;
package/dist/run.js ADDED
@@ -0,0 +1,204 @@
1
+ import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { dirname, join, resolve } from 'node:path';
3
+ import { fileURLToPath } from 'node:url';
4
+ import { parseArgs } from 'node:util';
5
+ import { audit } from './core/engine.js';
6
+ import { mutate } from './mutate/runner.js';
7
+ import { c } from './report/colors.js';
8
+ import { formatAudit, formatMutation } from './report/human.js';
9
+ import { toJson } from './report/json.js';
10
+ import { rules } from './rules/index.js';
11
+ export const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
12
+ export function version() {
13
+ try {
14
+ return JSON.parse(readFileSync(join(PACKAGE_ROOT, 'package.json'), 'utf8')).version;
15
+ }
16
+ catch {
17
+ return '0.0.0';
18
+ }
19
+ }
20
+ const HELP = `honest-tests — find tests that can't fail.
21
+
22
+ Usage
23
+ honest-tests audit [paths...] Detect fake/weak tests (default: .)
24
+ honest-tests audit [paths] --mutate <file>
25
+ ...then mutation-check a source file
26
+ honest-tests mutate <file> Mutation check only
27
+ honest-tests rules List rules
28
+ honest-tests init <claude|codex|cursor> Install the agent skill/rules into this repo
29
+
30
+ Audit options
31
+ --json Machine-readable output (findings with file/line/rule/severity/evidence)
32
+ -v, --verbose Show fix advice for each finding
33
+ --ignore <glob> Ignore paths (repeatable), e.g. --ignore "fixtures/**"
34
+ --rule <id> Only run these rules (repeatable)
35
+ --disable <id> Skip these rules (repeatable)
36
+ --fail-on <level> error (default) | warning | never
37
+
38
+ Mutation options
39
+ --mutate <file> Source file to mutate (with audit)
40
+ --test-cmd <cmd> Test command (default: package.json "test" script, scoped to related tests).
41
+ "{files}" is replaced with test files that import the target.
42
+ --root <dir> Project root (default: nearest package.json above the target)
43
+ --max-mutants <n> Cap on mutants to run (default 100)
44
+ --timeout <ms> Per-run timeout (default: 4x baseline + 5s)
45
+ --min-score <0-100> Exit 1 if mutation score is below this (default: 0, report only)
46
+
47
+ Suppress a finding: // honest-tests-disable-next-line <rule-id>`;
48
+ export async function run(argv, io) {
49
+ let parsed;
50
+ try {
51
+ parsed = parseArgs({
52
+ args: argv,
53
+ allowPositionals: true,
54
+ options: {
55
+ json: { type: 'boolean' },
56
+ verbose: { type: 'boolean', short: 'v' },
57
+ ignore: { type: 'string', multiple: true },
58
+ rule: { type: 'string', multiple: true },
59
+ disable: { type: 'string', multiple: true },
60
+ 'fail-on': { type: 'string' },
61
+ mutate: { type: 'string' },
62
+ 'test-cmd': { type: 'string' },
63
+ root: { type: 'string' },
64
+ 'max-mutants': { type: 'string' },
65
+ timeout: { type: 'string' },
66
+ 'min-score': { type: 'string' },
67
+ dir: { type: 'string' },
68
+ help: { type: 'boolean', short: 'h' },
69
+ version: { type: 'boolean' },
70
+ },
71
+ });
72
+ }
73
+ catch (e) {
74
+ io.stderr(`${e.message}\n\n${HELP}\n`);
75
+ return 2;
76
+ }
77
+ const { values: v, positionals } = parsed;
78
+ if (v.version) {
79
+ io.stdout(version() + '\n');
80
+ return 0;
81
+ }
82
+ const [command, ...rest] = positionals;
83
+ if (v.help || !command) {
84
+ io.stdout(HELP + '\n');
85
+ return command || v.help ? 0 : 2;
86
+ }
87
+ if (command === 'rules') {
88
+ if (v.json)
89
+ io.stdout(JSON.stringify(rules.map(({ id, severity, description, fix }) => ({ id, severity, description, fix })), null, 2) + '\n');
90
+ else
91
+ for (const r of rules)
92
+ io.stdout(`${c.bold(r.id.padEnd(26))} ${r.severity.padEnd(8)} ${r.description}\n`);
93
+ return 0;
94
+ }
95
+ if (command === 'init')
96
+ return init(rest[0], v.dir ?? io.cwd, io);
97
+ if (command !== 'audit' && command !== 'mutate') {
98
+ io.stderr(`Unknown command "${command}".\n\n${HELP}\n`);
99
+ return 2;
100
+ }
101
+ const unknownRules = [...(v.rule ?? []), ...(v.disable ?? [])].filter((id) => !rules.some((r) => r.id === id));
102
+ if (unknownRules.length) {
103
+ io.stderr(`Unknown rule(s): ${unknownRules.join(', ')}. Run "honest-tests rules".\n`);
104
+ return 2;
105
+ }
106
+ const failOn = v['fail-on'] ?? 'error';
107
+ if (!['error', 'warning', 'never'].includes(failOn)) {
108
+ io.stderr(`--fail-on must be error, warning or never.\n`);
109
+ return 2;
110
+ }
111
+ let auditResult = null;
112
+ let exit = 0;
113
+ if (command === 'audit') {
114
+ auditResult = audit({ paths: rest, cwd: io.cwd, ignore: v.ignore, only: v.rule, disable: v.disable }, version());
115
+ const { error, warning } = auditResult.summary;
116
+ if ((failOn === 'error' && error > 0) || (failOn === 'warning' && error + warning > 0))
117
+ exit = 1;
118
+ if (!v.json)
119
+ io.stdout(formatAudit(auditResult, { verbose: v.verbose }) + '\n');
120
+ }
121
+ const target = command === 'mutate' ? rest[0] : v.mutate;
122
+ if (command === 'mutate' && !target) {
123
+ io.stderr('Usage: honest-tests mutate <source-file>\n');
124
+ return 2;
125
+ }
126
+ let mutation = null;
127
+ if (target) {
128
+ const num = (s) => (s === undefined ? undefined : Number(s));
129
+ const showProgress = !v.json && io.progress;
130
+ try {
131
+ mutation = await mutate({
132
+ target,
133
+ cwd: io.cwd,
134
+ root: v.root,
135
+ testCmd: v['test-cmd'],
136
+ maxMutants: num(v['max-mutants']),
137
+ timeoutMs: num(v.timeout),
138
+ onProgress: showProgress
139
+ ? (done, total, r) => io.stderr(`\r mutating ${target}: ${done}/${total} ${r.status === 'survived' ? c.red('survived') : c.green(r.status)} `)
140
+ : undefined,
141
+ });
142
+ if (showProgress)
143
+ io.stderr('\r\x1b[2K');
144
+ }
145
+ catch (e) {
146
+ if (showProgress)
147
+ io.stderr('\n');
148
+ const err = e;
149
+ io.stderr(c.red(`Mutation check failed: ${err.message}\n`));
150
+ if (err.output)
151
+ io.stderr(c.dim(excerpt(err.output)) + '\n');
152
+ return 2;
153
+ }
154
+ if (!v.json)
155
+ io.stdout(formatMutation(mutation) + '\n');
156
+ const minScore = num(v['min-score']) ?? 0;
157
+ if (mutation.score < minScore)
158
+ exit = 1;
159
+ }
160
+ if (v.json)
161
+ io.stdout(toJson(auditResult, mutation) + '\n');
162
+ return exit;
163
+ }
164
+ /** First and last lines of a test run: the error is usually at the top, the summary at the bottom. */
165
+ export function excerpt(output, headLines = 20, tailLines = 20) {
166
+ const lines = output.replace(/\n{3,}/g, '\n\n').trimEnd().split('\n');
167
+ if (lines.length <= headLines + tailLines)
168
+ return lines.join('\n');
169
+ return [...lines.slice(0, headLines), ` … ${lines.length - headLines - tailLines} lines …`, ...lines.slice(-tailLines)].join('\n');
170
+ }
171
+ const MARK_START = '<!-- honest-tests:start -->';
172
+ const MARK_END = '<!-- honest-tests:end -->';
173
+ function init(target, dir, io) {
174
+ const base = resolve(io.cwd, dir);
175
+ switch (target) {
176
+ case 'claude': {
177
+ const dest = join(base, '.claude', 'skills', 'honest-tests');
178
+ cpSync(join(PACKAGE_ROOT, 'skills', 'honest-tests'), dest, { recursive: true });
179
+ io.stdout(`Installed skill → ${dest}\nClaude Code will pick it up automatically. Try: "audit my tests with honest-tests".\n`);
180
+ return 0;
181
+ }
182
+ case 'codex': {
183
+ const snippet = readFileSync(join(PACKAGE_ROOT, 'integrations', 'codex', 'AGENTS.md'), 'utf8').trim();
184
+ const dest = join(base, 'AGENTS.md');
185
+ const block = `${MARK_START}\n${snippet}\n${MARK_END}`;
186
+ let content = existsSync(dest) ? readFileSync(dest, 'utf8') : '';
187
+ const re = new RegExp(`${MARK_START}[\\s\\S]*?${MARK_END}`);
188
+ content = re.test(content) ? content.replace(re, block) : (content ? content.trimEnd() + '\n\n' : '') + block + '\n';
189
+ writeFileSync(dest, content);
190
+ io.stdout(`Added honest-tests section → ${dest}\n`);
191
+ return 0;
192
+ }
193
+ case 'cursor': {
194
+ const destDir = join(base, '.cursor', 'rules');
195
+ mkdirSync(destDir, { recursive: true });
196
+ cpSync(join(PACKAGE_ROOT, 'integrations', 'cursor', 'honest-tests.mdc'), join(destDir, 'honest-tests.mdc'));
197
+ io.stdout(`Installed rule → ${join(destDir, 'honest-tests.mdc')}\n`);
198
+ return 0;
199
+ }
200
+ default:
201
+ io.stderr('Usage: honest-tests init <claude|codex|cursor> [--dir <repo>]\n');
202
+ return 2;
203
+ }
204
+ }
@@ -0,0 +1,20 @@
1
+ ## Testing: honest tests only
2
+
3
+ A test is honest only if it **fails when the behavior it names is broken**. When writing or changing tests:
4
+
5
+ - Never weaken an assertion, change an expected value, add `.skip`, or wrap an assertion in `try/catch` to make a test pass. When a test fails, first assume the code is wrong.
6
+ - Expected values must come from a spec or a hand calculation, never from re-running the code under test (`expect(f(x)).toBe(f(x))`) or copying its output unverified.
7
+ - Never mock the module under test. Mock only boundaries (network, DB, clock, randomness, filesystem).
8
+ - Assert on what the unit returns or changes, not on what a mock was configured to return. Use mocks for interaction checks (`toHaveBeenCalledWith`). Never assert on a literal the test just built (`const p = { a: 1 }; expect(p.a).toBe(1)`) — pass it through the unit.
9
+ - Every test asserts something specific: exact values over `toBeDefined()`/`toBeTruthy()`; no snapshot-only tests; `await` every `.resolves`/`.rejects`; errors via `expect(() => f()).toThrow(Type)`.
10
+ - Cover boundaries (`>=` vs `>`), empty inputs, and error paths.
11
+ - Never commit `.only`.
12
+
13
+ Before reporting that tests are done, run and fix:
14
+
15
+ ```bash
16
+ npx @mintwork/honest-tests audit <test files> # must report 0 errors
17
+ npx @mintwork/honest-tests mutate <source file> # report the mutation score; add tests for surviving mutants
18
+ ```
19
+
20
+ To audit an existing suite: run `npx @mintwork/honest-tests audit . --json`, verify each finding by reading the test (confirmed / false-positive / accepted), prove key findings with `npx @mintwork/honest-tests mutate <file>`, then report "N of M tests can't fail" with the confirmed findings and surviving mutants. Do not edit files during an audit unless asked.
@@ -0,0 +1,24 @@
1
+ ---
2
+ description: Write and audit tests that actually verify behavior (no fake tests). Apply when writing, editing, or reviewing test files.
3
+ globs: ["**/*.test.*", "**/*.spec.*", "**/__tests__/**", "**/test_*.py", "**/*_test.py"]
4
+ alwaysApply: false
5
+ ---
6
+
7
+ # Honest tests
8
+
9
+ A test is honest only if it **fails when the behavior it names is broken**.
10
+
11
+ Rules:
12
+ - Never weaken an assertion, change an expected value, `.skip` a test, or wrap an assertion in `try/catch` to get green. When a test fails, first suspect the code.
13
+ - Expected values come from a spec or hand calculation — never `expect(f(x)).toBe(f(x))`, never output copied unverified.
14
+ - Never mock the module under test; mock only boundaries (network, DB, clock, randomness, fs).
15
+ - Assert on what the unit produces, not on what a mock was told to return. Use mocks for `toHaveBeenCalledWith` checks. Never assert on a literal the test just built — pass it through the unit.
16
+ - Prefer exact matchers over `toBeDefined()` / `toBeTruthy()`; no snapshot-only tests; `await` `.resolves`/`.rejects`; use `expect(() => f()).toThrow(Type)` for errors.
17
+ - Test boundaries (`>=` vs `>`), empty inputs, and error paths. Never commit `.only`.
18
+
19
+ Before saying tests are done, run in the terminal and fix all errors:
20
+
21
+ ```bash
22
+ npx @mintwork/honest-tests audit <test files>
23
+ npx @mintwork/honest-tests mutate <source file> # add assertions for each surviving mutant
24
+ ```