@rigour-labs/core 4.3.5 → 5.0.0

package/dist/gates/dependency.js

@@ -1,6 +1,62 @@
+/**
+ * Dependency Guardian Gate (v2)
+ *
+ * Detects dependency issues that AI agents commonly introduce:
+ * 1. Forbidden dependencies (existing) — packages banned by project standards
+ * 2. Unused dependencies (NEW) — installed but never imported
+ * 3. Heavy alternatives (NEW) — bloated packages with lighter alternatives
+ * 4. Duplicate purpose (NEW) — multiple packages solving the same problem
+ *
+ * AI agents are particularly prone to:
+ * - Adding packages they've seen in training data without checking existing deps
+ * - Using heavy/popular packages when lighter alternatives exist
+ * - Installing multiple HTTP clients, date libs, etc. across different sessions
+ *
+ * @since v2.0.0 (forbidden deps)
+ * @since v5.1.0 (unused, heavy alternatives, duplicate purpose)
+ */
 import fs from 'fs-extra';
 import path from 'path';
 import { Gate } from './base.js';
+import { FileScanner } from '../utils/scanner.js';
+import { Logger } from '../utils/logger.js';
+/**
+ * Known heavy packages with lighter alternatives.
+ * Format: package → "alternative (size comparison)"
+ */
+const HEAVY_ALTERNATIVES = {
+    'moment': 'date-fns or dayjs (67KB → 2KB gzipped)',
+    'lodash': 'lodash-es (tree-shakeable) or native Array/Object methods',
+    'underscore': 'native ES6+ methods (Array.map, Object.entries, etc.)',
+    'axios': 'native fetch API (built into Node 18+)',
+    'request': 'node-fetch or native fetch (deprecated since 2020)',
+    'bluebird': 'native Promise (built-in since ES2015)',
+    'jquery': 'native DOM APIs (querySelector, fetch, classList)',
+    'classnames': 'clsx (0.3KB vs 1KB) or template literals',
+    'uuid': 'crypto.randomUUID() (built into Node 19+ and browsers)',
+    'left-pad': 'String.prototype.padStart() (built-in)',
+    'is-even': 'n % 2 === 0 (one-liner)',
+    'is-odd': 'n % 2 !== 0 (one-liner)',
+    'chalk': 'picocolors (14x smaller, faster)',
+};
+/**
+ * Functional groups — if >1 package from same group is installed,
+ * it's likely a duplicate purpose issue from AI drift.
+ */
+const FUNCTIONAL_GROUPS = [
+    { name: 'HTTP clients', packages: ['axios', 'node-fetch', 'got', 'request', 'ky', 'superagent', 'undici'] },
+    { name: 'Date/time libraries', packages: ['moment', 'dayjs', 'date-fns', 'luxon', 'fecha', 'chrono-node'] },
+    { name: 'Terminal colors', packages: ['chalk', 'kleur', 'ansi-colors', 'picocolors', 'colorette'] },
+    { name: 'CLI argument parsers', packages: ['commander', 'yargs', 'meow', 'cac', 'minimist', 'arg'] },
+    { name: 'Schema validation', packages: ['zod', 'joi', 'yup', 'ajv', 'superstruct', 'io-ts'] },
+    { name: 'Logging libraries', packages: ['winston', 'pino', 'bunyan', 'log4js', 'signale', 'consola'] },
+    { name: 'UUID generators', packages: ['uuid', 'nanoid', 'cuid', 'ulid', 'shortid'] },
+    { name: 'Markdown parsers', packages: ['marked', 'markdown-it', 'remark', 'showdown', 'remarkable'] },
+    { name: 'Testing frameworks', packages: ['jest', 'mocha', 'vitest', 'ava', 'tap', 'jasmine'] },
+    { name: 'CSS-in-JS', packages: ['styled-components', 'emotion', '@emotion/react', 'linaria', 'vanilla-extract'] },
+    { name: 'State management', packages: ['redux', 'mobx', 'zustand', 'jotai', 'recoil', 'valtio'] },
+    { name: 'Environment config', packages: ['dotenv', 'env-var', 'envalid', 'convict'] },
+];
 export class DependencyGate extends Gate {
     config;
     constructor(config) {
@@ -9,11 +65,10 @@ export class DependencyGate extends Gate {
     }
     async run(context) {
         const failures = [];
-        const forbidden = this.config.gates.dependencies?.forbid || [];
-        if (forbidden.length === 0)
-            return [];
+        const depConfig = this.config.gates.dependencies || {};
+        const forbidden = depConfig.forbid || [];
         const { cwd } = context;
-        // 1. Scan Node.js (package.json)
+        // 1. Scan Node.js (package.json) — forbidden + new checks
         const pkgPath = path.join(cwd, 'package.json');
         if (await fs.pathExists(pkgPath)) {
             try {
@@ -23,15 +78,32 @@ export class DependencyGate extends Gate {
                     ...(pkg.devDependencies || {}),
                     ...(pkg.peerDependencies || {}),
                 };
+                const depNames = Object.keys(allDeps);
+                // Forbidden deps check
                 for (const dep of forbidden) {
                     if (allDeps[dep]) {
                         failures.push(this.createFailure(`The package '${dep}' is forbidden by project standards.`, ['package.json'], `Remove '${dep}' from package.json and use approved alternatives.`, 'Forbidden Dependency', undefined, undefined, 'medium'));
                     }
                 }
+                // NEW: Unused dependency detection
+                if (depConfig.detect_unused !== false && depNames.length > 0) {
+                    const unusedFailures = await this.detectUnusedDeps(context, pkg, depNames, depConfig.unused_allowlist || []);
+                    failures.push(...unusedFailures);
+                }
+                // NEW: Heavy alternative detection
+                if (depConfig.detect_heavy_alternatives !== false) {
+                    const heavyFailures = this.detectHeavyAlternatives(depNames);
+                    failures.push(...heavyFailures);
+                }
+                // NEW: Duplicate purpose detection
+                if (depConfig.detect_duplicate_purpose !== false) {
+                    const dupeFailures = this.detectDuplicatePurpose(depNames);
+                    failures.push(...dupeFailures);
+                }
             }
             catch (e) { }
         }
-        // 2. Scan Python (requirements.txt, pyproject.toml)
+        // 2. Scan Python (requirements.txt, pyproject.toml) — forbidden only
         const reqPath = path.join(cwd, 'requirements.txt');
         if (await fs.pathExists(reqPath)) {
             const content = await fs.readFile(reqPath, 'utf-8');
@@ -72,4 +144,139 @@ export class DependencyGate extends Gate {
         }
         return failures;
     }
+    // ─── Unused Dependency Detection ─────────────────────────────────
+    /**
+     * Detect dependencies listed in package.json but never imported.
+     * Scans all source files for import/require statements.
+     *
+     * Allowlist handles side-effect imports like:
+     * - @types/* (TypeScript type packages)
+     * - polyfills (core-js, regenerator-runtime)
+     * - PostCSS/Babel plugins (used in config files, not source)
+     */
+    async detectUnusedDeps(context, pkg, depNames, allowlist) {
+        const failures = [];
+        // Only check production + dev dependencies (not peer)
+        const prodDeps = Object.keys(pkg.dependencies || {});
+        const devDeps = Object.keys(pkg.devDependencies || {});
+        const checkDeps = [...prodDeps, ...devDeps];
+        if (checkDeps.length === 0)
+            return [];
+        // Default allowlist patterns for known side-effect packages
+        const defaultAllowPatterns = [
+            /^@types\//, // TypeScript type packages
+            /^typescript$/, // Used by tsc, not imported
+            /^eslint/, // Used by config, not imported
+            /^prettier$/, // Used by config
+            /^@eslint/, // ESLint config packages
+            /^husky$/, // Git hooks
+            /^lint-staged$/, // Pre-commit
+            /^core-js/, // Polyfills
+            /^regenerator-runtime$/, // Babel polyfill
+            /^postcss/, // PostCSS plugins
+            /^autoprefixer$/, // PostCSS plugin
+            /^tailwindcss$/, // Build tool
+            /^@tailwindcss\//, // Build tool
+            /^webpack/, // Build tool
+            /^vite$/, // Build tool
+            /^@vitejs\//, // Vite plugins
+            /^rollup/, // Build tool
+            /^esbuild$/, // Build tool
+            /^tsup$/, // Build tool
+            /^turbo$/, // Build tool
+            /^nodemon$/, // Dev tool
+            /^ts-node$/, // Dev tool
+            /^tsx$/, // Dev tool
+            /^concurrently$/, // Dev tool
+        ];
+        // Combine with user allowlist
+        const userPatterns = allowlist.map(p => {
+            if (p.endsWith('*'))
+                return new RegExp(`^${p.slice(0, -1)}`);
+            return new RegExp(`^${p}$`);
+        });
+        const allPatterns = [...defaultAllowPatterns, ...userPatterns];
+        // Filter deps that need checking
+        const depsToCheck = checkDeps.filter(dep => !allPatterns.some(pattern => pattern.test(dep)));
+        if (depsToCheck.length === 0)
+            return [];
+        // Scan source files for import/require patterns
+        const sourceFiles = await FileScanner.findFiles({
+            cwd: context.cwd,
+            patterns: ['**/*.{ts,tsx,js,jsx,mjs,cjs,mts,cts,vue,svelte}'],
+            ignore: [...(context.ignore || []), '**/node_modules/**', '**/dist/**'],
+        });
+        // Read all source files
+        const contents = await FileScanner.readFiles(context.cwd, sourceFiles, context.fileCache);
+        // Also check config files that might reference deps
+        const configFiles = ['vite.config.ts', 'vite.config.js', 'next.config.js', 'next.config.mjs',
+            'jest.config.ts', 'jest.config.js', 'vitest.config.ts', 'tsconfig.json',
+            'babel.config.js', '.babelrc', 'postcss.config.js', 'tailwind.config.js',
+            'webpack.config.js', 'rollup.config.js', 'esbuild.config.js'];
+        for (const cf of configFiles) {
+            const cfPath = path.join(context.cwd, cf);
+            if (await fs.pathExists(cfPath)) {
+                try {
+                    contents.set(cf, await fs.readFile(cfPath, 'utf-8'));
+                }
+                catch { }
+            }
+        }
+        // Build a combined source text for fast searching
+        const allSource = Array.from(contents.values()).join('\n');
+        for (const dep of depsToCheck) {
+            // Check for various import patterns:
+            // import ... from 'dep'
+            // require('dep')
+            // import('dep')
+            // Also check for scoped package partial imports: @scope/package/sub
+            const depEscaped = dep.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+            const importPattern = new RegExp(`(?:from\\s+['"\`]${depEscaped}(?:/[^'"\`]*)?['"\`])|` +
+                `(?:require\\s*\\(\\s*['"\`]${depEscaped}(?:/[^'"\`]*)?['"\`]\\s*\\))|` +
+                `(?:import\\s*\\(\\s*['"\`]${depEscaped}(?:/[^'"\`]*)?['"\`]\\s*\\))|` +
+                `(?:['"\`]${depEscaped}['"\`])` // Config file references
+            );
+            if (!importPattern.test(allSource)) {
+                const isDevDep = devDeps.includes(dep);
+                failures.push(this.createFailure(`${isDevDep ? 'Dev dependency' : 'Dependency'} '${dep}' appears unused — no import/require found in source files.`, ['package.json'], `Remove '${dep}' from package.json if it's truly unused, or add it to unused_allowlist if it's a side-effect import.`, 'Unused Dependency', undefined, undefined, 'low'));
+            }
+        }
+        if (failures.length > 0) {
+            Logger.info(`Dependency Guardian: Found ${failures.length} potentially unused dependencies`);
+        }
+        return failures;
+    }
+    // ─── Heavy Alternative Detection ─────────────────────────────────
+    /**
+     * Detect heavy/bloated packages that have lighter modern alternatives.
+     * AI agents tend to reach for the most popular (heaviest) package
+     * because that's what they've seen most in training data.
+     */
+    detectHeavyAlternatives(depNames) {
+        const failures = [];
+        for (const dep of depNames) {
+            const alternative = HEAVY_ALTERNATIVES[dep];
+            if (alternative) {
+                failures.push(this.createFailure(`Package '${dep}' has a lighter alternative: ${alternative}.`, ['package.json'], `Consider replacing '${dep}' with ${alternative}. AI agents often default to popular-but-heavy packages.`, 'Heavy Dependency', undefined, undefined, 'low'));
+            }
+        }
+        return failures;
+    }
+    // ─── Duplicate Purpose Detection ─────────────────────────────────
+    /**
+     * Detect when multiple packages serve the same purpose.
+     * This is a classic AI drift symptom — different sessions install different
+     * packages for the same task (e.g., axios in one PR, got in another).
+     */
+    detectDuplicatePurpose(depNames) {
+        const failures = [];
+        const depSet = new Set(depNames);
+        for (const group of FUNCTIONAL_GROUPS) {
+            const installed = group.packages.filter(pkg => depSet.has(pkg));
+            if (installed.length >= 2) {
+                failures.push(this.createFailure(`Multiple ${group.name} installed: ${installed.join(', ')}. Pick one and remove the rest.`, ['package.json'], `Having multiple packages for ${group.name} is a sign of AI drift — different sessions chose different packages. Standardize on one.`, 'Duplicate Purpose Dependencies', undefined, undefined, 'medium'));
+            }
+        }
+        return failures;
+    }
 }

package/dist/gates/duplication-drift.d.ts

@@ -1,33 +1,128 @@
 /**
- * Duplication Drift Gate
+ * Duplication Drift Gate (v2)
  *
  * Detects when AI generates near-identical functions across files because
  * it doesn't remember what it already wrote. This is an AI-specific failure
  * mode — humans reuse via copy-paste (same file), AI re-invents (cross-file).
  *
- * Detection strategy:
- * 1. Extract all function bodies (normalized: strip whitespace, comments)
- * 2. Compare function signatures + body hashes across files
- * 3. Flag functions with >80% similarity in different files
+ * v2 upgrades:
+ * - tree-sitter AST node type sequences replace hand-rolled regex tokenizer
+ * - Jaccard similarity on AST node multisets (structural, not textual)
+ * - Catches duplicates even when every variable name is different
+ * - MD5 kept as fast-path for exact matches, Jaccard runs on remaining pairs
  *
- * @since v2.16.0
+ * Detection strategy (three-pass):
+ * 1. Extract function bodies, normalize text (strip comments/whitespace)
+ * 2. Parse with tree-sitter → walk AST → collect node type multiset
+ * 3. Generate semantic embeddings via all-MiniLM-L6-v2 (384D)
+ * 4. Pass 1 (fast):     MD5 hash → exact duplicates (O(n), <10ms)
+ * 5. Pass 2 (Jaccard):  AST node multiset similarity → structural near-duplicates (O(n²) bounded)
+ * 6. Pass 3 (semantic):  Embedding cosine similarity → semantic duplicates (O(n²) bounded)
+ * 7. Flag functions with similarity > threshold in different files
+ *
+ * Why AST node types > raw tokens:
+ * - `getUserById(id) { return db.find(x => x.id === id) }`
+ * - `fetchUser(userId) { return database.filter(u => u.id === userId)[0] }`
+ * Both produce similar AST: [return_statement, call_expression, arrow_function,
+ *   binary_expression, member_expression]. Variable names are invisible.
+ *
+ * @since v2.16.0 (original MD5)
+ * @since v5.0.0  (tree-sitter AST + Jaccard)
+ * @since v5.1.0  (semantic embedding Pass 3)
  */
 import { Gate, GateContext } from './base.js';
 import { Failure, Provenance } from '../types/index.js';
 export interface DuplicationDriftConfig {
     enabled?: boolean;
     similarity_threshold?: number;
+    semantic_threshold?: number;
+    semantic_enabled?: boolean;
     min_body_lines?: number;
+    approved_duplications?: string[];
 }
 export declare class DuplicationDriftGate extends Gate {
     private config;
+    private parser;
     constructor(config?: DuplicationDriftConfig);
     protected get provenance(): Provenance;
     run(context: GateContext): Promise<Failure[]>;
+    /**
+     * Parse the file with tree-sitter, find function nodes that match
+     * our extracted functions (by line number), and replace their token
+     * multisets with AST node type sequences.
+     *
+     * AST node types are language-agnostic structural tokens:
+     * - if_statement, for_statement, return_statement
+     * - call_expression, member_expression, binary_expression
+     * - arrow_function, function_declaration
+     *
+     * Variable names, string literals, comments — all invisible.
+     * Only STRUCTURE matters.
+     */
+    private enrichWithASTTokens;
+    /**
+     * Walk the AST tree to find a function/method node at a given line.
+     */
+    private findFunctionNodeAtLine;
+    /**
+     * Walk an AST subtree and collect node types as a multiset.
+     *
+     * This is the core insight: two functions with different variable names
+     * but the same control flow produce the same node type multiset.
+     *
+     * Example:
+     * `function a(x) { if (x > 0) return x * 2; return 0; }`
+     * `function b(val) { if (val > 0) return val * 2; return 0; }`
+     *
+     * Both produce: {if_statement: 1, binary_expression: 2, return_statement: 2, ...}
+     * → Jaccard similarity = 1.0
+     */
+    private collectASTNodeTypes;
+    /**
+     * Fallback tokenizer when tree-sitter is not available.
+     * Uses normalized text → keyword/operator multiset.
+     */
+    private textTokenize;
+    /**
+     * Jaccard similarity on multisets.
+     * intersection = sum of min(countA, countB) for each key
+     * union = sum of max(countA, countB) for each key
+     */
+    private jaccardSimilarity;
     private extractJSFunctions;
     private extractPyFunctions;
     private extractFunctionBody;
     private normalizeBody;
     private hash;
+    /**
+     * Generate semantic embedding text for a function.
+     * Combines function name, parameter names, and first 200 tokens of body.
+     * This captures INTENT regardless of implementation differences.
+     *
+     * Example:
+     * getUserById(id) { return db.users.find(x => x.id === id) }
+     * → "getUserById id return db users find x id id"
+     *
+     * fetchUserRecord(userId) { return database.users.filter(u => u.id === userId)[0] }
+     * → "fetchUserRecord userId return database users filter u id userId 0"
+     *
+     * These produce similar embeddings (~0.91 cosine) despite different AST.
+     */
+    private buildEmbeddingText;
+    /**
+     * Enrich functions with semantic embeddings for Pass 3.
+     * Only called for functions not already claimed by Pass 1/2.
+     * Uses generateEmbedding() from pattern-index/embeddings.ts.
+     */
+    private enrichWithEmbeddings;
+    /**
+     * Three-pass duplicate detection:
+     * Pass 1 (fast):     MD5 hash → exact duplicates (O(n))
+     * Pass 2 (Jaccard):  AST node multiset similarity → near-duplicates (O(n²) bounded)
+     * Pass 3 (semantic):  Embedding cosine similarity → semantic duplicates (O(n²) bounded)
+     *
+     * Pass 3 catches what AST Jaccard misses: same intent, different implementation.
+     * Example: .find() vs .filter()[0] — different AST nodes, same semantic meaning.
+     */
     private findDuplicateGroups;
 }