@mintwork/honest-tests 0.0.0-stage → 0.1.1

package/dist/adapters/python/index.d.ts

@@ -0,0 +1,17 @@
+import type { LanguageAdapter } from '../../core/types.js';
+/**
+ * pytest adapter — PLANNED (see ROADMAP in README).
+ *
+ * The rule engine is language-neutral: an adapter only has to turn a test file
+ * into a `TestFileModel`. For pytest that means mapping:
+ *   - `def test_*` / `class Test*` methods        -> TestCase
+ *   - `assert a == b`, `assert x`, `pytest.raises` -> Assertion (matcherKind 'equal' | 'truthy' | 'throws')
+ *   - `@pytest.mark.skip` / `xfail`               -> modifier 'skip'
+ *   - `mocker.patch('pkg.unit.fn')` / `@patch`    -> ModuleMock / MockSetup
+ *   - `mock.return_value = x`                     -> MockSetup { configuresReturn: true }
+ * Mutation: operator flips via Python's own `ast` module in a subprocess.
+ *
+ * Until then, Python test files are discovered and reported as "not yet supported"
+ * instead of being silently ignored.
+ */
+export declare const pythonAdapter: LanguageAdapter;

package/dist/adapters/python/index.js

@@ -0,0 +1,26 @@
+/**
+ * pytest adapter — PLANNED (see ROADMAP in README).
+ *
+ * The rule engine is language-neutral: an adapter only has to turn a test file
+ * into a `TestFileModel`. For pytest that means mapping:
+ *   - `def test_*` / `class Test*` methods        -> TestCase
+ *   - `assert a == b`, `assert x`, `pytest.raises` -> Assertion (matcherKind 'equal' | 'truthy' | 'throws')
+ *   - `@pytest.mark.skip` / `xfail`               -> modifier 'skip'
+ *   - `mocker.patch('pkg.unit.fn')` / `@patch`    -> ModuleMock / MockSetup
+ *   - `mock.return_value = x`                     -> MockSetup { configuresReturn: true }
+ * Mutation: operator flips via Python's own `ast` module in a subprocess.
+ *
+ * Until then, Python test files are discovered and reported as "not yet supported"
+ * instead of being silently ignored.
+ */
+export const pythonAdapter = {
+    id: 'python',
+    extensions: ['.py'],
+    planned: true,
+    isTestFile(path) {
+        return /(^|\/)(test_[^/]*|[^/]*_test)\.py$/.test(path.replace(/\\/g, '/'));
+    },
+    parseTestFile(path, source) {
+        return { file: path, language: 'python', tests: [], moduleMocks: [], unitSpies: [], lines: source.split(/\r?\n/) };
+    },
+};

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+import { run } from './run.js';
+// `honest-tests audit | head` closes the pipe early; that's not an error worth a stack trace.
+for (const stream of [process.stdout, process.stderr]) {
+    stream.on('error', (err) => {
+        if (err.code === 'EPIPE')
+            process.exit(process.exitCode ?? 0);
+        throw err;
+    });
+}
+run(process.argv.slice(2), {
+    stdout: (s) => process.stdout.write(s),
+    stderr: (s) => process.stderr.write(s),
+    cwd: process.cwd(),
+    progress: process.stderr.isTTY,
+}).then((code) => {
+    process.exitCode = code;
+}, (err) => {
+    process.stderr.write(`honest-tests: ${err?.stack ?? String(err)}\n`);
+    process.exitCode = 2;
+});

package/dist/core/engine.d.ts

@@ -0,0 +1,44 @@
+import type { Finding, Rule, TestFileModel } from './types.js';
+export declare const DEFAULT_IGNORED_DIRS: Set<string>;
+export interface AuditOptions {
+    paths: string[];
+    cwd?: string;
+    /** Glob-ish patterns relative to cwd (`fixtures/**`, `**\/legacy/*`, or a bare dir name). */
+    ignore?: string[];
+    /** Restrict to these rule ids. */
+    only?: string[];
+    /** Exclude these rule ids. */
+    disable?: string[];
+}
+export interface AuditResult {
+    tool: 'honest-tests';
+    version: string;
+    cwd: string;
+    filesScanned: number;
+    testsScanned: number;
+    findings: Finding[];
+    /** Test files found for languages whose adapter is still planned (e.g. pytest). */
+    unsupported: Array<{
+        file: string;
+        language: string;
+    }>;
+    parseErrors: Array<{
+        file: string;
+        message: string;
+    }>;
+    summary: {
+        error: number;
+        warning: number;
+        info: number;
+        /** Distinct tests with at least one error-level finding. */
+        suspiciousTests: number;
+    };
+}
+export declare function globToRegExp(pattern: string): RegExp;
+/** Path relative to cwd, or absolute when the file is outside it (`../../../tmp/x` helps nobody). */
+export declare function displayPath(cwd: string, abs: string): string;
+export declare function discoverTestFiles(paths: string[], cwd: string, ignore?: string[]): string[];
+export declare function runRules(model: TestFileModel, rules: Rule[], displayPath: string): Finding[];
+export declare function selectRules(opts: Pick<AuditOptions, 'only' | 'disable'>): Rule[];
+export declare function auditSource(path: string, source: string, opts?: Pick<AuditOptions, 'only' | 'disable'>): Finding[];
+export declare function audit(opts: AuditOptions, version?: string): AuditResult;

package/dist/core/engine.js

@@ -0,0 +1,205 @@
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, relative, resolve, sep } from 'node:path';
+import { adapterFor } from '../adapters/index.js';
+import { rules as allRules } from '../rules/index.js';
+export const DEFAULT_IGNORED_DIRS = new Set([
+    'node_modules',
+    '.git',
+    'dist',
+    'build',
+    'coverage',
+    '.next',
+    '.nuxt',
+    'out',
+    '.turbo',
+    '.cache',
+    'vendor',
+    '.venv',
+    'venv',
+    '__pycache__',
+]);
+export function globToRegExp(pattern) {
+    let p = pattern.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/$/, '');
+    if (!/[*?]/.test(p)) {
+        // bare path/dir name: match it as a path segment sequence anywhere
+        const esc = p.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+        return new RegExp(`(^|/)${esc}(/|$)`);
+    }
+    let re = '';
+    for (let i = 0; i < p.length; i++) {
+        const c = p[i];
+        if (c === '*') {
+            if (p[i + 1] === '*') {
+                re += '.*';
+                i++;
+                if (p[i + 1] === '/')
+                    i++;
+            }
+            else
+                re += '[^/]*';
+        }
+        else if (c === '?')
+            re += '[^/]';
+        else
+            re += c.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+    }
+    return new RegExp(`^${re}(/|$)`);
+}
+function toPosix(p) {
+    return p.split(sep).join('/');
+}
+/** Path relative to cwd, or absolute when the file is outside it (`../../../tmp/x` helps nobody). */
+export function displayPath(cwd, abs) {
+    const rel = toPosix(relative(cwd, abs));
+    return !rel || rel.startsWith('../') || rel === '..' ? toPosix(abs) : rel;
+}
+export function discoverTestFiles(paths, cwd, ignore = []) {
+    const ignoreRes = ignore.map(globToRegExp);
+    const isIgnored = (abs) => {
+        const rel = toPosix(relative(cwd, abs));
+        return ignoreRes.some((re) => re.test(rel));
+    };
+    const out = new Set();
+    const visit = (abs, explicit) => {
+        let st;
+        try {
+            st = statSync(abs);
+        }
+        catch {
+            return;
+        }
+        if (st.isDirectory()) {
+            if (!explicit && (DEFAULT_IGNORED_DIRS.has(abs.split(sep).pop()) || isIgnored(abs)))
+                return;
+            for (const entry of readdirSync(abs).sort())
+                visit(join(abs, entry), false);
+            return;
+        }
+        if (!st.isFile())
+            return;
+        if (!explicit && isIgnored(abs))
+            return;
+        const adapter = adapterFor(abs);
+        if (adapter && (explicit || adapter.isTestFile(toPosix(abs))))
+            out.add(abs);
+    };
+    for (const p of paths)
+        visit(resolve(cwd, p), true);
+    return [...out];
+}
+/** Parse `honest-tests-disable*` comments. Works for `//`, `/* *\/` and `#` comments. */
+function suppressions(lines) {
+    const fileLevel = new Set();
+    let fileAll = false;
+    const perLine = new Map();
+    const parseIds = (rest) => rest
+        .replace(/\*\/.*$/, '')
+        .split(/[\s,]+/)
+        .map((s) => s.trim())
+        .filter((s) => /^[a-z][a-z0-9-]*$/.test(s));
+    lines.forEach((line, i) => {
+        const m = /honest-tests-(disable-next-line|disable-line|disable)\b(.*)$/.exec(line);
+        if (!m)
+            return;
+        const ids = parseIds(m[2] ?? '');
+        if (m[1] === 'disable') {
+            if (ids.length === 0)
+                fileAll = true;
+            ids.forEach((id) => fileLevel.add(id));
+            return;
+        }
+        const target = m[1] === 'disable-next-line' ? i + 2 : i + 1;
+        perLine.set(target, ids.length === 0 ? 'all' : new Set(ids));
+    });
+    return (f) => {
+        if (fileAll || fileLevel.has(f.ruleId))
+            return true;
+        const s = perLine.get(f.line);
+        return s === 'all' || (s instanceof Set && s.has(f.ruleId));
+    };
+}
+export function runRules(model, rules, displayPath) {
+    const suppressed = suppressions(model.lines);
+    const findings = [];
+    for (const rule of rules) {
+        for (const raw of rule.check(model)) {
+            const f = {
+                ruleId: rule.id,
+                severity: raw.severity ?? rule.severity,
+                file: displayPath,
+                line: raw.line,
+                column: raw.column,
+                message: raw.message,
+                evidence: raw.evidence,
+                ...(raw.test !== undefined ? { test: raw.test } : {}),
+                fix: raw.fix ?? rule.fix,
+            };
+            if (!suppressed(f))
+                findings.push(f);
+        }
+    }
+    return findings.sort((a, b) => a.line - b.line || a.column - b.column || a.ruleId.localeCompare(b.ruleId));
+}
+export function selectRules(opts) {
+    let selected = allRules;
+    if (opts.only?.length)
+        selected = selected.filter((r) => opts.only.includes(r.id));
+    if (opts.disable?.length)
+        selected = selected.filter((r) => !opts.disable.includes(r.id));
+    return selected;
+}
+export function auditSource(path, source, opts = {}) {
+    const adapter = adapterFor(path);
+    if (!adapter)
+        throw new Error(`No adapter for ${path}`);
+    return runRules(adapter.parseTestFile(path, source), selectRules(opts), path);
+}
+const SEVERITY_ORDER = { error: 0, warning: 1, info: 2 };
+export function audit(opts, version = '0.0.0') {
+    const cwd = resolve(opts.cwd ?? process.cwd());
+    const files = discoverTestFiles(opts.paths.length ? opts.paths : ['.'], cwd, opts.ignore);
+    const rules = selectRules(opts);
+    const findings = [];
+    const unsupported = [];
+    const parseErrors = [];
+    let testsScanned = 0;
+    let filesScanned = 0;
+    for (const abs of files) {
+        const adapter = adapterFor(abs);
+        const display = displayPath(cwd, abs);
+        if (adapter.planned) {
+            unsupported.push({ file: display, language: adapter.id });
+            continue;
+        }
+        let model;
+        try {
+            model = adapter.parseTestFile(abs, readFileSync(abs, 'utf8'));
+        }
+        catch (e) {
+            parseErrors.push({ file: display, message: e.message });
+            continue;
+        }
+        filesScanned++;
+        testsScanned += model.tests.filter((t) => t.kind === 'test').length;
+        findings.push(...runRules(model, rules, display));
+    }
+    findings.sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line || SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity] || a.ruleId.localeCompare(b.ruleId));
+    // "Can't fail" = an error-level finding about the test's verification (focus markers don't count).
+    const suspicious = new Set(findings.filter((f) => f.severity === 'error' && f.test && f.ruleId !== 'focused-test').map((f) => `${f.file}::${f.test}`));
+    return {
+        tool: 'honest-tests',
+        version,
+        cwd,
+        filesScanned,
+        testsScanned,
+        findings,
+        unsupported,
+        parseErrors,
+        summary: {
+            error: findings.filter((f) => f.severity === 'error').length,
+            warning: findings.filter((f) => f.severity === 'warning').length,
+            info: findings.filter((f) => f.severity === 'info').length,
+            suspiciousTests: suspicious.size,
+        },
+    };
+}

package/dist/core/types.d.ts

@@ -0,0 +1,195 @@
+/**
+ * Language-neutral model of a test file.
+ *
+ * Language adapters (JS/TS today, pytest next) parse source into this model.
+ * Rules only ever see the model, so a new adapter gets every rule for free.
+ */
+export type Severity = 'error' | 'warning' | 'info';
+export type SubjectKind = 'literal' | 'identifier' | 'member' | 'call' | 'object' | 'array' | 'function' | 'new' | 'typeof' | 'other';
+export interface ExprInfo {
+    /** Normalized source text (whitespace collapsed). */
+    text: string;
+    kind: SubjectKind;
+    /** For calls: normalized callee text, e.g. `mockFetch` or `api.get`. */
+    calleeText?: string;
+    /**
+     * If the subject is an identifier bound in the test to the result of a call
+     * (`const r = fn()`), the callee text of that call.
+     */
+    aliasCalleeText?: string;
+    /** For literals: the literal's runtime truthiness / undefined-ness. */
+    literalValue?: string | number | boolean | null | undefined;
+    isUndefinedLiteral?: boolean;
+    /**
+     * Set when the value is built purely from literals constructed in this test
+     * (`const p = { service: 's3' }; expect(p.service)`): no call, import or outer value is involved,
+     * so the value is fixed before any code under test runs. `name`/`line` locate the literal binding.
+     */
+    ownLiteral?: {
+        name: string;
+        line: number;
+    };
+    /** Built only from literals and test-local literal bindings (`{ a: 1 }`, `[x, 'y']`); implied by `ownLiteral`. */
+    pureLiteral?: boolean;
+}
+/** Normalized intent of an assertion matcher, independent of framework spelling. */
+export type MatcherKind = 'equal' | 'truthy' | 'falsy' | 'defined' | 'undefined' | 'null' | 'length' | 'compare-gte' | 'compare-other' | 'snapshot' | 'throws' | 'called' | 'type' | 'other'
+/** `expect(x)` with no matcher, or matcher referenced but never called. */
+ | 'none';
+export interface AssertionContext {
+    /** Inside a `try` block whose `catch` does not rethrow. */
+    inSwallowingTry: boolean;
+    /** Inside a `catch` clause. */
+    inCatch: boolean;
+    /** Inside a `catch` whose `try` block calls fail()/throws after the risky call, so skipping the catch fails the test. */
+    catchGuarded: boolean;
+    /** Inside an `if`/ternary/`&&` guard (branches where every path asserts don't count). */
+    inConditional: boolean;
+    /**
+     * Normalized text of the value each enclosing guard tests (`!r.success` -> `r.success`).
+     * Lets rules excuse type-narrowing guards whose condition was already asserted.
+     */
+    guards?: string[];
+}
+export interface Assertion {
+    line: number;
+    column: number;
+    /** Source line(s) of the assertion, trimmed — used as evidence. */
+    text: string;
+    style: 'expect' | 'assert';
+    subject: ExprInfo | null;
+    matcher: string | null;
+    matcherKind: MatcherKind;
+    negated: boolean;
+    args: ExprInfo[];
+    /** `.resolves` / `.rejects` used. */
+    async: boolean;
+    /** Awaited or returned (only meaningful when `async`). */
+    awaitedOrReturned: boolean;
+    context: AssertionContext;
+}
+export interface MockSetup {
+    /** Normalized text of the mocked callable, e.g. `getUser` or `api.get`. */
+    target: string;
+    line: number;
+    /** Whether a return value / implementation was configured. */
+    configuresReturn: boolean;
+}
+/** `spy.mockRestore()` / `fn.mockReset()` (target set) or `vi.restoreAllMocks()` (target null). */
+export interface MockRestore {
+    target: string | null;
+    line: number;
+}
+export type TestModifier = 'only' | 'skip' | 'todo' | null;
+export interface TestCase {
+    name: string;
+    kind: 'test' | 'suite';
+    line: number;
+    endLine: number;
+    modifier: TestModifier;
+    /** Source text of the call head, e.g. `it.only('adds', ...` — evidence. */
+    headText: string;
+    /** False when the test body is not an inline function we could analyze. */
+    analyzable: boolean;
+    assertions: Assertion[];
+    /** Calls to helpers that look like assertions (`expectValid(x)`, `assertOk()`, `request.expect(200)`). */
+    helperAssertionCalls: number;
+    /** `expect.assertions(n)` / `expect.hasAssertions()` present. */
+    hasAssertionCountGuard: boolean;
+    /** Mocks configured inside this test (or in file-level setup visible to it). */
+    mockSetups: MockSetup[];
+    /** Mock restores/resets inside this test body; a configured mock is inactive after one. */
+    mockRestores?: MockRestore[];
+}
+export interface ModuleMock {
+    specifier: string;
+    line: number;
+    text: string;
+    /** The adapter's judgement that this specifier is the module under test. */
+    isUnitUnderTest: boolean;
+    /** The mock keeps the real module and overrides only some exports. */
+    partial: boolean;
+}
+export interface SpyOnUnit {
+    /** e.g. `pricing.calculateTotal` */
+    target: string;
+    line: number;
+    text: string;
+}
+export interface TestFileModel {
+    file: string;
+    language: string;
+    tests: TestCase[];
+    moduleMocks: ModuleMock[];
+    /** spies on exports of the module under test that replace its behavior. */
+    unitSpies: SpyOnUnit[];
+    /** Raw source lines (for evidence + disable comments). */
+    lines: string[];
+}
+export interface Finding {
+    ruleId: string;
+    severity: Severity;
+    file: string;
+    line: number;
+    column: number;
+    message: string;
+    /** The offending source line(s). */
+    evidence: string;
+    /** Name of the enclosing test, if any. */
+    test?: string;
+    /** Short actionable advice. */
+    fix: string;
+}
+/** What a rule returns; the engine fills in ruleId/file/defaults. */
+export interface RawFinding {
+    line: number;
+    column: number;
+    message: string;
+    evidence: string;
+    test?: string;
+    severity?: Severity;
+    fix?: string;
+}
+export interface Rule {
+    id: string;
+    severity: Severity;
+    /** One-line description for `honest-tests rules`. */
+    description: string;
+    /** Default fix advice. */
+    fix: string;
+    check(file: TestFileModel): RawFinding[];
+}
+/** A single source mutation. Offsets index into the original source string. */
+export interface Mutant {
+    id: number;
+    file: string;
+    line: number;
+    column: number;
+    start: number;
+    end: number;
+    original: string;
+    replacement: string;
+    operator: string;
+    description: string;
+}
+export interface LanguageAdapter {
+    id: string;
+    /** File extensions this adapter understands. */
+    extensions: string[];
+    /** True if `path` looks like a test file for this language. */
+    isTestFile(path: string): boolean;
+    /** Parse a test file into the language-neutral model. */
+    parseTestFile(path: string, source: string): TestFileModel;
+    /** Generate mutants for a source (non-test) file. Optional. */
+    generateMutants?(path: string, source: string): Mutant[];
+    /** Return true if `testSource` (a test file) appears to import `targetPath`. */
+    testImportsTarget?(testPath: string, testSource: string, targetPath: string): boolean;
+    /** Normalized ids of local modules `source` imports, comparable with `moduleId`. */
+    localImports?(path: string, source: string): string[];
+    /** Normalized id of the module at `path` (e.g. extension and `/index` stripped). */
+    moduleId?(path: string): string;
+    /** Default command to run tests in `root`, or null if unknown. */
+    defaultTestCommand?(root: string): string | null;
+    /** Set when the adapter is a placeholder (parses nothing yet). */
+    planned?: boolean;
+}

package/dist/core/types.js

@@ -0,0 +1,7 @@
+/**
+ * Language-neutral model of a test file.
+ *
+ * Language adapters (JS/TS today, pytest next) parse source into this model.
+ * Rules only ever see the model, so a new adapter gets every rule for free.
+ */
+export {};

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { audit, auditSource, discoverTestFiles, type AuditOptions, type AuditResult } from './core/engine.js';
+export { mutate, MutationError, type MutateOptions, type MutationReport, type MutantResult } from './mutate/runner.js';
+export { rules } from './rules/index.js';
+export { adapters, adapterFor } from './adapters/index.js';
+export { generateJsMutants, applyMutant } from './adapters/js/mutants.js';
+export type * from './core/types.js';

package/dist/index.js

@@ -0,0 +1,5 @@
+export { audit, auditSource, discoverTestFiles } from './core/engine.js';
+export { mutate, MutationError } from './mutate/runner.js';
+export { rules } from './rules/index.js';
+export { adapters, adapterFor } from './adapters/index.js';
+export { generateJsMutants, applyMutant } from './adapters/js/mutants.js';

package/dist/mutate/runner.d.ts

@@ -0,0 +1,59 @@
+import type { LanguageAdapter, Mutant } from '../core/types.js';
+export type MutantStatus = 'killed' | 'survived' | 'timeout';
+export interface MutantResult extends Mutant {
+    status: MutantStatus;
+    /** Original source line, trimmed. */
+    sourceLine: string;
+    durationMs: number;
+}
+export interface MutateOptions {
+    /** Source file to mutate. */
+    target: string;
+    cwd?: string;
+    /** Project root (dir with package.json). Defaults to nearest ancestor of target. */
+    root?: string;
+    /** Test command. `{files}` is replaced with the related test files. Defaults to the package's `test` script. */
+    testCmd?: string;
+    /** Stop after this many mutants (default 100). */
+    maxMutants?: number;
+    /** Per-run timeout; default derived from the baseline run. */
+    timeoutMs?: number;
+    onProgress?: (done: number, total: number, result: MutantResult) => void;
+}
+export interface MutationReport {
+    target: string;
+    root: string;
+    command: string;
+    relatedTests: string[];
+    baselineMs: number;
+    generated: number;
+    tested: number;
+    killed: number;
+    survived: number;
+    timedOut: number;
+    /** 0-100, (killed + timeout) / tested. */
+    score: number;
+    mutants: MutantResult[];
+}
+export declare class MutationError extends Error {
+    output: string;
+    constructor(message: string, output?: string);
+}
+/**
+ * Test files that import `target` directly or through other local modules
+ * (`test -> Page.jsx -> lib/format.js`). Direct-import-only selection misses
+ * tests that exercise the target through its callers, which shows up as
+ * false survivors.
+ */
+export declare function findRelatedTests(root: string, target: string, adapter: LanguageAdapter): string[];
+/** Kill every in-flight test run (and its children). Used on Ctrl-C/SIGTERM before removing the copy. */
+export declare function killRunningTests(): void;
+/**
+ * Mutation-test one source file.
+ *
+ * Safety: the project is copied to a fresh temp directory (node_modules is
+ * symlinked, never copied or written by us). Mutants are written only inside
+ * the copy, restored after every run, and the copy is deleted at the end —
+ * including on Ctrl-C. Your working tree is never modified.
+ */
+export declare function mutate(opts: MutateOptions): Promise<MutationReport>;