@safetnsr/vet 1.1.0 → 1.3.0

package/dist/checks/tests.d.ts

@@ -0,0 +1,2 @@
+import type { CheckResult } from '../types.js';
+export declare function checkTests(cwd: string, ignore: string[]): CheckResult;

package/dist/checks/tests.js

@@ -0,0 +1,228 @@
+import { join } from 'node:path';
+import { readFileSync } from 'node:fs';
+import { walkFiles } from '../util.js';
+const TEST_FILE_RE = /\.(test|spec)\.(ts|js|tsx|jsx)$/;
+const TEST_DIR_RE = /(?:^|[/\\])(__tests__|tests?)[/\\]/;
+function isTestFile(relPath) {
+    return TEST_FILE_RE.test(relPath) || TEST_DIR_RE.test(relPath);
+}
+// Pattern 1: Tautological assertions
+function findTautological(lines, file) {
+    const issues = [];
+    // expect(literal).toBe(literal) or .toEqual(literal)
+    const expectLiteral = /expect\(([^)]+)\)\s*\.\s*(?:toBe|toEqual)\(\s*([^)]+)\s*\)/;
+    // assert.strictEqual(x, x)
+    const assertStrictEqual = /assert\.strictEqual\(\s*([^,]+?)\s*,\s*([^)]+?)\s*\)/;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const m1 = line.match(expectLiteral);
+        if (m1) {
+            const left = m1[1].trim();
+            const right = m1[2].trim();
+            if (left === right) {
+                issues.push({
+                    severity: 'error',
+                    message: `tautological assertion: expect(${left}).toBe/toEqual(${right})`,
+                    file, line: i + 1, fixable: false,
+                    fixHint: 'assert on actual behavior, not constant values',
+                });
+            }
+        }
+        const m2 = line.match(assertStrictEqual);
+        if (m2) {
+            const left = m2[1].trim();
+            const right = m2[2].trim();
+            if (left === right) {
+                issues.push({
+                    severity: 'error',
+                    message: `tautological assertion: assert.strictEqual(${left}, ${left})`,
+                    file, line: i + 1, fixable: false,
+                    fixHint: 'compare different values — input vs expected output',
+                });
+            }
+        }
+    }
+    return issues;
+}
+// Pattern 2: Empty test bodies
+function findEmptyBodies(content, file) {
+    const issues = [];
+    // Match it/test with arrow or function, empty body
+    const re = /(?:^|\n)([ \t]*(?:it|test)\s*\([^,]+,\s*(?:(?:async\s+)?(?:\(\)\s*=>|\([^)]*\)\s*=>|function\s*\([^)]*\)))\s*\{([\s]*)\}\s*\))/g;
+    let m;
+    while ((m = re.exec(content)) !== null) {
+        const body = m[2];
+        // body should be empty or whitespace/comments only
+        const stripped = body.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '').trim();
+        if (stripped === '') {
+            const line = content.substring(0, m.index).split('\n').length;
+            issues.push({
+                severity: 'error',
+                message: 'empty test body — test does nothing',
+                file, line, fixable: false,
+                fixHint: 'add actual test logic or remove the test',
+            });
+        }
+    }
+    return issues;
+}
+// Pattern 3: Todo / skipped tests
+function findTodoSkipped(lines, file) {
+    const issues = [];
+    const todoRe = /(?:it|test)\.todo\s*\(/;
+    const skippedRe = /(?:^|\s)(?:xit|xtest|xdescribe)\s*\(/;
+    for (let i = 0; i < lines.length; i++) {
+        if (todoRe.test(lines[i])) {
+            issues.push({
+                severity: 'warning',
+                message: 'todo test — placeholder with no implementation',
+                file, line: i + 1, fixable: false,
+                fixHint: 'implement the test or remove the placeholder',
+            });
+        }
+        if (skippedRe.test(lines[i])) {
+            issues.push({
+                severity: 'warning',
+                message: 'skipped test — disabled with x prefix',
+                file, line: i + 1, fixable: false,
+                fixHint: 'fix and re-enable or remove the skipped test',
+            });
+        }
+    }
+    return issues;
+}
+// Pattern 4: Zero-assertion tests
+// We need to find test blocks with code but no assertions
+function findZeroAssertionTests(content, file) {
+    const issues = [];
+    // Find it(...) or test(...) blocks - simplified regex for the opening
+    const testBlockRe = /(?:^|\n)([ \t]*)(?:it|test)\s*\(\s*(?:'[^']*'|"[^"]*"|`[^`]*`)\s*,\s*(?:async\s+)?(?:\(\)\s*=>|\([^)]*\)\s*=>|function\s*\([^)]*\))\s*\{/g;
+    let m;
+    while ((m = testBlockRe.exec(content)) !== null) {
+        const startIdx = m.index + m[0].length;
+        // Find matching closing brace
+        let depth = 1;
+        let i = startIdx;
+        while (i < content.length && depth > 0) {
+            if (content[i] === '{')
+                depth++;
+            else if (content[i] === '}')
+                depth--;
+            i++;
+        }
+        if (depth !== 0)
+            continue;
+        const body = content.substring(startIdx, i - 1);
+        const stripped = body.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '').trim();
+        if (stripped === '')
+            continue; // empty body handled elsewhere
+        // Check for assertion calls
+        const assertionRe = /(?:expect\s*\(|assert\.|\.should\.|toBe\s*\(|toEqual\s*\(|toMatch\s*\(|toThrow\s*\()/;
+        if (!assertionRe.test(body)) {
+            const line = content.substring(0, m.index).split('\n').length;
+            issues.push({
+                severity: 'warning',
+                message: 'test has code but no assertions',
+                file, line, fixable: false,
+                fixHint: 'add expect() or assert calls to verify behavior',
+            });
+        }
+    }
+    return issues;
+}
+// Pattern 5: Mock-only tests
+function findMockOnlyTests(content, file) {
+    const issues = [];
+    const testBlockRe = /(?:^|\n)([ \t]*)(?:it|test)\s*\(\s*(?:'[^']*'|"[^"]*"|`[^`]*`)\s*,\s*(?:async\s+)?(?:\(\)\s*=>|\([^)]*\)\s*=>|function\s*\([^)]*\))\s*\{/g;
+    let m;
+    while ((m = testBlockRe.exec(content)) !== null) {
+        const startIdx = m.index + m[0].length;
+        let depth = 1;
+        let i = startIdx;
+        while (i < content.length && depth > 0) {
+            if (content[i] === '{')
+                depth++;
+            else if (content[i] === '}')
+                depth--;
+            i++;
+        }
+        if (depth !== 0)
+            continue;
+        const body = content.substring(startIdx, i - 1);
+        // Find all expect lines
+        const expectLines = body.split('\n').filter(l => /expect\s*\(/.test(l));
+        if (expectLines.length === 0)
+            continue;
+        const mockRe = /\.mock|mockFn|jest\.fn|vi\.fn/;
+        const allMock = expectLines.every(l => mockRe.test(l));
+        if (allMock) {
+            const line = content.substring(0, m.index).split('\n').length;
+            issues.push({
+                severity: 'info',
+                message: 'test only asserts on mocks — no real behavior verified',
+                file, line, fixable: false,
+                fixHint: 'add assertions on actual return values or side effects',
+            });
+        }
+    }
+    return issues;
+}
+// Pattern 6: Duplicate describe blocks
+function findDuplicateDescribes(lines, file) {
+    const issues = [];
+    const describeRe = /describe\s*\(\s*(['"`])([^'"`]+)\1/;
+    const seen = new Map();
+    for (let i = 0; i < lines.length; i++) {
+        const m = lines[i].match(describeRe);
+        if (m) {
+            const name = m[2];
+            if (seen.has(name)) {
+                issues.push({
+                    severity: 'info',
+                    message: `duplicate describe block: "${name}"`,
+                    file, line: i + 1, fixable: false,
+                    fixHint: 'merge duplicate describe blocks into one',
+                });
+            }
+            else {
+                seen.set(name, i + 1);
+            }
+        }
+    }
+    return issues;
+}
+export function checkTests(cwd, ignore) {
+    const allFiles = walkFiles(cwd, ignore);
+    const testFiles = allFiles.filter(f => isTestFile(f));
+    const issues = [];
+    for (const rel of testFiles) {
+        let content;
+        try {
+            content = readFileSync(join(cwd, rel), 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        const lines = content.split('\n');
+        issues.push(...findTautological(lines, rel));
+        issues.push(...findEmptyBodies(content, rel));
+        issues.push(...findTodoSkipped(lines, rel));
+        issues.push(...findZeroAssertionTests(content, rel));
+        issues.push(...findMockOnlyTests(content, rel));
+        issues.push(...findDuplicateDescribes(lines, rel));
+    }
+    let score = 100;
+    for (const issue of issues) {
+        if (issue.severity === 'error')
+            score -= 8;
+        else if (issue.severity === 'warning')
+            score -= 4;
+        else
+            score -= 2;
+    }
+    score = Math.max(0, score);
+    const summary = issues.length > 0
+        ? `${issues.length} test anti-pattern${issues.length !== 1 ? 's' : ''} found across ${testFiles.length} test file${testFiles.length !== 1 ? 's' : ''}`
+        : 'no test anti-patterns found';
+    return { name: 'tests', score, maxScore: 100, issues, summary };
+}

package/dist/checks/verify.d.ts

@@ -0,0 +1,2 @@
+import type { CheckResult } from '../types.js';
+export declare function checkVerify(cwd: string, since?: string): CheckResult;

package/dist/checks/verify.js

@@ -0,0 +1,219 @@
+import { join, basename } from 'node:path';
+import { readFileSync, existsSync, statSync } from 'node:fs';
+import { execSync } from 'node:child_process';
+// ── Helpers ──────────────────────────────────────────────────────────────────
+function safeExec(cmd, cwd) {
+    try {
+        return execSync(cmd, { cwd, encoding: 'utf-8', timeout: 10_000, stdio: ['pipe', 'pipe', 'pipe'] });
+    }
+    catch {
+        return '';
+    }
+}
+function isTestFile(filePath) {
+    const base = basename(filePath);
+    if (/\.(test|spec)\.[a-z]+$/i.test(base))
+        return true;
+    const normalized = filePath.replace(/\\/g, '/');
+    // Match __tests__/ anywhere in path (including at root)
+    if (normalized.includes('__tests__/') || normalized.includes('/__tests__'))
+        return true;
+    if (normalized.includes('/test/') || normalized.startsWith('test/'))
+        return true;
+    if (normalized.includes('/tests/') || normalized.startsWith('tests/'))
+        return true;
+    return false;
+}
+function hasAssertions(content) {
+    return /\b(assert|expect\s*\(|it\s*\(|test\s*\(|describe\s*\(|should\.|toBe\(|toEqual\(|assertEqual|assertStrictEqual)\b/i.test(content);
+}
+function countLines(content) {
+    return content.split('\n').filter(l => l.trim().length > 0).length;
+}
+/** Extract file names mentioned in commit messages as claims */
+function extractClaimsFromMessages(messages) {
+    const claims = [];
+    // All patterns require a file extension (dot in name) to avoid false positives
+    const patterns = [
+        /\b(?:creat\w*|add\w*|implement\w*|wrot\w*|built|generat\w*|scaffold\w*)\s+([\w./\\-]+\.[a-z]{1,5})/gi,
+        /\b(?:fix\w*|resolv\w*|updat\w*|modify|modified)\s+([\w./\\-]+\.[a-z]{1,5})/gi,
+        /\badd\w*\s+tests?\s+(?:for\s+)?([\w./\\-]+\.[a-z]{1,5})/gi,
+    ];
+    for (const msg of messages) {
+        for (const pattern of patterns) {
+            pattern.lastIndex = 0;
+            let m;
+            while ((m = pattern.exec(msg)) !== null) {
+                const candidate = m[1].replace(/[,.:;)]+$/, '');
+                if (candidate && candidate.length > 2 && !candidate.startsWith('-')) {
+                    claims.push(candidate);
+                }
+            }
+        }
+    }
+    return [...new Set(claims)];
+}
+/** Get files changed in recent agent session (git diff against since or HEAD~1) */
+function getChangedFiles(cwd, since) {
+    let raw = '';
+    if (since) {
+        raw = safeExec(`git diff ${since} --name-only`, cwd);
+    }
+    else {
+        // Try HEAD~1 first
+        raw = safeExec(`git diff HEAD~1 --name-only`, cwd);
+        if (!raw.trim()) {
+            // Fall back to last commit's added/modified files
+            raw = safeExec(`git show --name-only --format="" HEAD`, cwd);
+        }
+    }
+    return raw
+        .split('\n')
+        .map(l => l.trim())
+        .filter(l => l.length > 0 && !l.startsWith('diff') && !l.startsWith('index'));
+}
+/** Get recent git log messages */
+function getRecentMessages(cwd, since) {
+    let raw = '';
+    if (since) {
+        raw = safeExec(`git log ${since}..HEAD --oneline`, cwd);
+    }
+    else {
+        raw = safeExec(`git log -10 --oneline`, cwd);
+    }
+    return raw.split('\n').map(l => l.replace(/^[a-f0-9]+\s+/, '').trim()).filter(l => l.length > 0);
+}
+// ── Main check ───────────────────────────────────────────────────────────────
+export function checkVerify(cwd, since) {
+    const issues = [];
+    let deductions = 0;
+    // Check if git repo
+    const isGit = safeExec('git rev-parse --is-inside-work-tree', cwd).trim();
+    if (isGit !== 'true') {
+        return {
+            name: 'verify',
+            score: 100,
+            maxScore: 100,
+            issues: [],
+            summary: 'not a git repository — skipped',
+        };
+    }
+    // Check if any commits exist
+    const hasCommits = safeExec('git rev-parse HEAD', cwd).trim();
+    if (!hasCommits) {
+        return {
+            name: 'verify',
+            score: 100,
+            maxScore: 100,
+            issues: [],
+            summary: 'no commits found — skipped',
+        };
+    }
+    // Get changed files from git diff
+    const changedFiles = getChangedFiles(cwd, since);
+    // Get commit messages for claim extraction
+    const messages = getRecentMessages(cwd, since);
+    // Extract explicit claims from commit messages
+    const explicitClaims = extractClaimsFromMessages(messages);
+    // Build unified file list to verify: changed files + explicitly claimed files
+    const toVerify = new Set();
+    for (const f of changedFiles)
+        toVerify.add(f);
+    for (const f of explicitClaims)
+        toVerify.add(f);
+    if (toVerify.size === 0) {
+        return {
+            name: 'verify',
+            score: 100,
+            maxScore: 100,
+            issues: [],
+            summary: 'no agent claims found in recent git history',
+        };
+    }
+    let verified = 0;
+    let failed = 0;
+    for (const relPath of toVerify) {
+        const absPath = join(cwd, relPath);
+        // 1. File must exist
+        if (!existsSync(absPath)) {
+            // Only flag files that were explicitly in claims from messages (not just diff-referenced)
+            // Changed files that don't exist could be deletions — only flag if explicitly claimed
+            if (explicitClaims.includes(relPath)) {
+                issues.push({
+                    severity: 'error',
+                    message: `Claimed file missing: ${relPath}`,
+                    file: relPath,
+                    fixable: false,
+                    fixHint: 'Agent claimed to create this file but it does not exist',
+                });
+                deductions += 15;
+                failed++;
+            }
+            continue;
+        }
+        let content = '';
+        try {
+            const stat = statSync(absPath);
+            if (!stat.isFile()) {
+                verified++;
+                continue;
+            }
+            content = readFileSync(absPath, 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        const lineCount = countLines(content);
+        // 2. File must have meaningful content (>10 non-empty lines)
+        if (lineCount < 10 && lineCount > 0) {
+            issues.push({
+                severity: 'warning',
+                message: `Thin file: ${relPath} (${lineCount} non-empty lines)`,
+                file: relPath,
+                fixable: false,
+                fixHint: 'Agent claimed to create/modify this file but it has minimal content',
+            });
+            deductions += 8;
+            failed++;
+            continue;
+        }
+        if (lineCount === 0) {
+            issues.push({
+                severity: 'error',
+                message: `Empty file: ${relPath}`,
+                file: relPath,
+                fixable: false,
+                fixHint: 'Agent claimed to create this file but it is empty',
+            });
+            deductions += 15;
+            failed++;
+            continue;
+        }
+        // 3. Test files must have actual assertions
+        if (isTestFile(relPath)) {
+            if (!hasAssertions(content)) {
+                issues.push({
+                    severity: 'error',
+                    message: `Test file has no assertions: ${relPath}`,
+                    file: relPath,
+                    fixable: false,
+                    fixHint: 'Test file exists but contains no expect(), assert(), or test() calls',
+                });
+                deductions += 12;
+                failed++;
+                continue;
+            }
+        }
+        verified++;
+    }
+    const finalScore = Math.max(0, 100 - deductions);
+    return {
+        name: 'verify',
+        score: finalScore,
+        maxScore: 100,
+        issues,
+        summary: failed === 0
+            ? `${verified} agent claim${verified !== 1 ? 's' : ''} verified clean`
+            : `${failed} claim${failed !== 1 ? 's' : ''} failed verification (${verified} passed)`,
+    };
+}

package/dist/cli.js

@@ -15,6 +15,8 @@ import { checkDebt } from './checks/debt.js';
 import { checkIntegrity } from './checks/integrity.js';
 import { checkReceipt, runReceiptCommand } from './checks/receipt.js';
 import { checkMemory } from './checks/memory.js';
+import { checkVerify } from './checks/verify.js';
+import { checkTests } from './checks/tests.js';
 import { checkMap, renderMapReport } from './checks/map.js';
 import { score } from './scorer.js';
 import { reportPretty, reportJSON, reportBadge } from './reporter.js';
@@ -166,9 +168,13 @@ async function runChecks() {
     const receiptResult = await checkReceipt(cwd);
     // Memory: stale facts in agent memory files
     const memoryResult = checkMemory(cwd);
+    // Verify: agent claim validation
+    const verifyResult = checkVerify(cwd, since);
+    // Tests: test theater detection
+    const testsResult = checkTests(cwd, ignore);
     return score(cwd, {
         security: [scanResult, secretsResult, configResult, modelsResult, owaspResult],
-        integrity: [diffResult, integrityResult, receiptResult, memoryResult],
+        integrity: [diffResult, integrityResult, receiptResult, memoryResult, verifyResult, testsResult],
         debt: [readyResult, historyResult, debtResult],
         deps: [depsResult],
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@safetnsr/vet",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "vet your AI-generated code — one command, one score card, one letter grade",
   "type": "module",
   "bin": {