@rigour-labs/core 5.0.1 → 5.1.0

package/dist/gates/duplication-drift/index.js

@@ -0,0 +1,240 @@
+/**
+ * Duplication Drift Gate (v2)
+ *
+ * Detects when AI generates near-identical functions across files because
+ * it doesn't remember what it already wrote. This is an AI-specific failure
+ * mode — humans reuse via copy-paste (same file), AI re-invents (cross-file).
+ *
+ * v2 upgrades:
+ * - tree-sitter AST node type sequences replace hand-rolled regex tokenizer
+ * - Jaccard similarity on AST node multisets (structural, not textual)
+ * - Catches duplicates even when every variable name is different
+ * - MD5 kept as fast-path for exact matches, Jaccard runs on remaining pairs
+ *
+ * Detection strategy (three-pass):
+ * 1. Extract function bodies, normalize text (strip comments/whitespace)
+ * 2. Parse with tree-sitter → walk AST → collect node type multiset
+ * 3. Generate semantic embeddings via all-MiniLM-L6-v2 (384D)
+ * 4. Pass 1 (fast):     MD5 hash → exact duplicates (O(n), <10ms)
+ * 5. Pass 2 (Jaccard):  AST node multiset similarity → structural near-duplicates (O(n²) bounded)
+ * 6. Pass 3 (semantic):  Embedding cosine similarity → semantic duplicates (O(n²) bounded)
+ * 7. Flag functions with similarity > threshold in different files
+ *
+ * Why AST node types > raw tokens:
+ * - `getUserById(id) { return db.find(x => x.id === id) }`
+ * - `fetchUser(userId) { return database.filter(u => u.id === userId)[0] }`
+ * Both produce similar AST: [return_statement, call_expression, arrow_function,
+ *   binary_expression, member_expression]. Variable names are invisible.
+ */
+import path from 'path';
+import { Gate } from '../base.js';
+import { FileScanner } from '../../utils/scanner.js';
+import { Logger } from '../../utils/logger.js';
+import { languageAdapters } from '../language-adapters/index.js';
+import { generateEmbedding } from '../../pattern-index/embeddings.js';
+import { initTreeSitter, GRAMMAR_PATHS, languageCache, getParser, getGrammarDir, textTokenize, collectASTNodeTypes, findFunctionNodeAtLine, normalizeBody, extractFunctionBody, } from './tokenizer.js';
+import { hashBody, findDuplicateGroups, buildEmbeddingText, calculateSimilarity, } from './similarity.js';
+export class DuplicationDriftGate extends Gate {
+    config;
+    parser = null;
+    constructor(config = {}) {
+        super('duplication-drift', 'AI Duplication Drift Detection');
+        this.config = {
+            enabled: config.enabled ?? true,
+            similarity_threshold: config.similarity_threshold ?? 0.75,
+            semantic_threshold: config.semantic_threshold ?? 0.85,
+            semantic_enabled: config.semantic_enabled ?? true,
+            min_body_lines: config.min_body_lines ?? 5,
+            approved_duplications: config.approved_duplications ?? [],
+        };
+    }
+    get provenance() { return 'ai-drift'; }
+    async run(context) {
+        if (!this.config.enabled)
+            return [];
+        // Try to init tree-sitter (non-blocking, falls back gracefully)
+        const hasTreeSitter = await initTreeSitter();
+        if (hasTreeSitter && !this.parser) {
+            const Parser = getParser();
+            this.parser = new Parser();
+        }
+        const failures = [];
+        const functions = [];
+        const scanPatterns = context.patterns || ['**/*.{ts,js,tsx,jsx,py,go,rs}'];
+        const files = await FileScanner.findFiles({
+            cwd: context.cwd,
+            patterns: scanPatterns,
+            ignore: [...(context.ignore || []), '**/node_modules/**', '**/dist/**', '**/*.test.*', '**/*.spec.*'],
+        });
+        Logger.info(`Duplication Drift: Scanning ${files.length} files (tree-sitter: ${hasTreeSitter ? 'ON' : 'fallback'})`);
+        for (const file of files) {
+            try {
+                const { readFile } = await import('fs-extra');
+                const content = await readFile(path.join(context.cwd, file), 'utf-8');
+                const ext = path.extname(file);
+                const adapter = languageAdapters.getAdapter(file);
+                if (adapter?.id === 'js') {
+                    this.extractJSFunctions(content, file, functions);
+                }
+                else if (adapter?.id === 'python') {
+                    this.extractPyFunctions(content, file, functions);
+                }
+                // Generate AST tokens using tree-sitter if available
+                if (hasTreeSitter && GRAMMAR_PATHS[ext]) {
+                    await this.enrichWithASTTokens(content, ext, file, functions);
+                }
+            }
+            catch (e) { }
+        }
+        // Pass 3 prep: Generate semantic embeddings for all extracted functions
+        // (embedding generation is lazy — only runs when semantic_enabled is true)
+        if (this.config.semantic_enabled && functions.length > 0) {
+            const allIndices = functions.map((_, i) => i);
+            await this.enrichWithEmbeddings(functions, allIndices);
+        }
+        const duplicateGroups = findDuplicateGroups(functions, this.config.similarity_threshold, this.config.semantic_threshold, this.config.semantic_enabled);
+        // Build approved pairs set for fast lookup
+        const approvedSet = new Set((this.config.approved_duplications || []).map(s => s.toLowerCase()));
+        for (const group of duplicateGroups) {
+            // Check if this pair is human-approved
+            const names = group.map(f => f.name).sort();
+            const pairKey = names.join(':').toLowerCase();
+            if (approvedSet.has(pairKey))
+                continue;
+            const files = group.map(f => f.file);
+            const locations = group.map(f => `${f.file}:${f.line} (${f.name})`).join(', ');
+            const { similarity, method } = calculateSimilarity(group[0], group[1] || group[0]);
+            const pct = (similarity * 100).toFixed(0);
+            failures.push(this.createFailure(`AI Duplication Drift: Function '${group[0].name}' has ${group.length} near-identical copies (${pct}% similar via ${method})`, [...new Set(files)], `Found duplicate implementations at: ${locations}. Extract to a shared module and import.`, 'Duplication Drift', group[0].line, undefined, 'high'));
+        }
+        return failures;
+    }
+    // ─── tree-sitter AST Tokenization ───────────────────────────────
+    /**
+     * Parse the file with tree-sitter, find function nodes that match
+     * our extracted functions (by line number), and replace their token
+     * multisets with AST node type sequences.
+     */
+    async enrichWithASTTokens(content, ext, file, functions) {
+        if (!this.parser)
+            return;
+        const grammarRelPath = GRAMMAR_PATHS[ext];
+        if (!grammarRelPath)
+            return;
+        try {
+            // Load language (cached)
+            if (!languageCache.has(ext)) {
+                const grammarDir = getGrammarDir();
+                const grammarPath = path.resolve(grammarDir, grammarRelPath);
+                const Parser = getParser();
+                const lang = await Parser.Language.load(grammarPath);
+                languageCache.set(ext, lang);
+            }
+            const lang = languageCache.get(ext);
+            this.parser.setLanguage(lang);
+            const tree = this.parser.parse(content);
+            // Find functions that belong to this file
+            const fileFunctions = functions.filter(f => f.file === file);
+            for (const fn of fileFunctions) {
+                // Find the AST node at this function's line
+                const node = findFunctionNodeAtLine(tree.rootNode, fn.line);
+                if (node) {
+                    fn.astTokens = collectASTNodeTypes(node);
+                }
+            }
+        }
+        catch (e) {
+            // tree-sitter parse failed for this file — keep text tokens
+            Logger.debug(`tree-sitter parse failed for ${file}: ${e}`);
+        }
+    }
+    // ─── Function Extraction ────────────────────────────────────────
+    extractJSFunctions(content, file, functions) {
+        const lines = content.split('\n');
+        const patterns = [
+            /^(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(([^)]*)\)/,
+            /^(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|(\w+))\s*=>/,
+            /^\s+(?:async\s+)?(\w+)\s*\(([^)]*)\)\s*\{/,
+        ];
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            for (const pattern of patterns) {
+                const match = line.match(pattern);
+                if (match) {
+                    const name = match[1];
+                    const params = match[2] || '';
+                    const body = extractFunctionBody(lines, i);
+                    if (body.length >= this.config.min_body_lines) {
+                        const normalized = normalizeBody(body.join('\n'));
+                        functions.push({
+                            name,
+                            file,
+                            line: i + 1,
+                            paramCount: params ? params.split(',').length : 0,
+                            bodyHash: hashBody(normalized),
+                            bodyLength: body.length,
+                            normalized,
+                            // Start with text tokens, enrichWithASTTokens() upgrades if tree-sitter available
+                            astTokens: textTokenize(normalized),
+                        });
+                    }
+                    break;
+                }
+            }
+        }
+    }
+    extractPyFunctions(content, file, functions) {
+        const lines = content.split('\n');
+        for (let i = 0; i < lines.length; i++) {
+            const match = lines[i].match(/^(?:\s*)(?:async\s+)?def\s+(\w+)\s*\(([^)]*)\)/);
+            if (match) {
+                const name = match[1];
+                const params = match[2] || '';
+                const indent = lines[i].match(/^(\s*)/)?.[1]?.length || 0;
+                const body = [];
+                for (let j = i + 1; j < lines.length; j++) {
+                    const lineIndent = lines[j].match(/^(\s*)/)?.[1]?.length || 0;
+                    if (lines[j].trim() === '' || lineIndent > indent) {
+                        body.push(lines[j]);
+                    }
+                    else {
+                        break;
+                    }
+                }
+                if (body.length >= this.config.min_body_lines) {
+                    const normalized = normalizeBody(body.join('\n'));
+                    functions.push({
+                        name,
+                        file,
+                        line: i + 1,
+                        paramCount: params ? params.split(',').length : 0,
+                        bodyHash: hashBody(normalized),
+                        bodyLength: body.length,
+                        normalized,
+                        astTokens: textTokenize(normalized),
+                    });
+                }
+            }
+        }
+    }
+    // ─── Semantic Embedding ─────────────────────────────────────────
+    /**
+     * Enrich functions with semantic embeddings for Pass 3.
+     * Only called for functions not already claimed by Pass 1/2.
+     * Uses generateEmbedding() from pattern-index/embeddings.ts.
+     */
+    async enrichWithEmbeddings(functions, indices) {
+        Logger.info(`Semantic Pass 3: Generating embeddings for ${indices.length} functions`);
+        for (const idx of indices) {
+            const fn = functions[idx];
+            try {
+                const text = buildEmbeddingText(fn);
+                fn.embedding = await generateEmbedding(text);
+            }
+            catch {
+                // Embedding failed — skip this function for Pass 3
+                Logger.debug(`Embedding generation failed for ${fn.file}:${fn.name}`);
+            }
+        }
+    }
+}

package/dist/gates/duplication-drift/similarity.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Duplication Drift Similarity Detection
+ *
+ * Handles all comparison and similarity detection logic:
+ * - Fingerprinting/hashing
+ * - Jaccard similarity on multisets
+ * - Clone detection algorithms
+ * - Threshold calculations
+ */
+/**
+ * Represents a detected function with its signatures and comparison data.
+ */
+export interface FunctionSignature {
+    name: string;
+    file: string;
+    line: number;
+    paramCount: number;
+    bodyHash: string;
+    bodyLength: number;
+    normalized: string;
+    /** AST node type multiset for Jaccard comparison */
+    astTokens: Map<string, number>;
+    /** Semantic embedding vector (384D) for cosine similarity */
+    embedding?: number[];
+}
+/**
+ * Hash a normalized function body using MD5.
+ */
+export declare function hashBody(text: string): string;
+/**
+ * Jaccard similarity on multisets.
+ * intersection = sum of min(countA, countB) for each key
+ * union = sum of max(countA, countB) for each key
+ */
+export declare function jaccardSimilarity(a: Map<string, number>, b: Map<string, number>): number;
+/**
+ * Generate semantic embedding text for a function.
+ * Combines function name, parameter names, and first 200 tokens of body.
+ * This captures INTENT regardless of implementation differences.
+ *
+ * Example:
+ * getUserById(id) { return db.users.find(x => x.id === id) }
+ * → "getUserById id return db users find x id id"
+ *
+ * fetchUserRecord(userId) { return database.users.filter(u => u.id === userId)[0] }
+ * → "fetchUserRecord userId return database users filter u id userId 0"
+ *
+ * These produce similar embeddings (~0.91 cosine) despite different AST.
+ */
+export declare function buildEmbeddingText(fn: FunctionSignature): string;
+/**
+ * Calculate similarity between two functions using all available methods.
+ * Returns { similarity, method } where method indicates which technique was used.
+ */
+export declare function calculateSimilarity(fn1: FunctionSignature, fn2: FunctionSignature): {
+    similarity: number;
+    method: string;
+};
+/**
+ * Three-pass duplicate detection:
+ * Pass 1 (fast):     MD5 hash → exact duplicates (O(n))
+ * Pass 2 (Jaccard):  AST node multiset similarity → near-duplicates (O(n²) bounded)
+ * Pass 3 (semantic):  Embedding cosine similarity → semantic duplicates (O(n²) bounded)
+ *
+ * Pass 3 catches what AST Jaccard misses: same intent, different implementation.
+ * Example: .find() vs .filter()[0] — different AST nodes, same semantic meaning.
+ */
+export declare function findDuplicateGroups(functions: FunctionSignature[], similarityThreshold: number, semanticThreshold: number, semanticEnabled: boolean): FunctionSignature[][];

package/dist/gates/duplication-drift/similarity.js

@@ -0,0 +1,177 @@
+/**
+ * Duplication Drift Similarity Detection
+ *
+ * Handles all comparison and similarity detection logic:
+ * - Fingerprinting/hashing
+ * - Jaccard similarity on multisets
+ * - Clone detection algorithms
+ * - Threshold calculations
+ */
+import crypto from 'crypto';
+import { cosineSimilarity } from '../../pattern-index/embeddings.js';
+/**
+ * Hash a normalized function body using MD5.
+ */
+export function hashBody(text) {
+    return crypto.createHash('md5').update(text).digest('hex');
+}
+/**
+ * Jaccard similarity on multisets.
+ * intersection = sum of min(countA, countB) for each key
+ * union = sum of max(countA, countB) for each key
+ */
+export function jaccardSimilarity(a, b) {
+    const allKeys = new Set([...a.keys(), ...b.keys()]);
+    let intersection = 0;
+    let union = 0;
+    for (const key of allKeys) {
+        const countA = a.get(key) || 0;
+        const countB = b.get(key) || 0;
+        intersection += Math.min(countA, countB);
+        union += Math.max(countA, countB);
+    }
+    return union === 0 ? 0 : intersection / union;
+}
+/**
+ * Generate semantic embedding text for a function.
+ * Combines function name, parameter names, and first 200 tokens of body.
+ * This captures INTENT regardless of implementation differences.
+ *
+ * Example:
+ * getUserById(id) { return db.users.find(x => x.id === id) }
+ * → "getUserById id return db users find x id id"
+ *
+ * fetchUserRecord(userId) { return database.users.filter(u => u.id === userId)[0] }
+ * → "fetchUserRecord userId return database users filter u id userId 0"
+ *
+ * These produce similar embeddings (~0.91 cosine) despite different AST.
+ */
+export function buildEmbeddingText(fn) {
+    // Extract identifiers from normalized body (first 200 tokens)
+    const bodyTokens = fn.normalized.match(/\b[a-zA-Z_]\w*\b/g) || [];
+    const first200 = bodyTokens.slice(0, 200).join(' ');
+    return `${fn.name} ${first200}`;
+}
+/**
+ * Calculate similarity between two functions using all available methods.
+ * Returns { similarity, method } where method indicates which technique was used.
+ */
+export function calculateSimilarity(fn1, fn2) {
+    // Exact hash match (Pass 1)
+    if (fn1.bodyHash === fn2.bodyHash) {
+        return { similarity: 1.0, method: 'exact-hash' };
+    }
+    // Semantic embedding comparison (if available)
+    if (fn1.embedding && fn2.embedding) {
+        const jaccardSim = jaccardSimilarity(fn1.astTokens, fn2.astTokens);
+        const cosineSim = cosineSimilarity(fn1.embedding, fn2.embedding);
+        if (cosineSim > jaccardSim) {
+            return { similarity: cosineSim, method: 'semantic-embedding' };
+        }
+    }
+    // AST Jaccard similarity (Pass 2)
+    const jaccardSim = jaccardSimilarity(fn1.astTokens, fn2.astTokens);
+    return { similarity: jaccardSim, method: 'ast-jaccard' };
+}
+/**
+ * Three-pass duplicate detection:
+ * Pass 1 (fast):     MD5 hash → exact duplicates (O(n))
+ * Pass 2 (Jaccard):  AST node multiset similarity → near-duplicates (O(n²) bounded)
+ * Pass 3 (semantic):  Embedding cosine similarity → semantic duplicates (O(n²) bounded)
+ *
+ * Pass 3 catches what AST Jaccard misses: same intent, different implementation.
+ * Example: .find() vs .filter()[0] — different AST nodes, same semantic meaning.
+ */
+export function findDuplicateGroups(functions, similarityThreshold, semanticThreshold, semanticEnabled) {
+    const duplicates = [];
+    const claimedIndices = new Set();
+    // Pass 1: Exact hash match
+    const hashGroups = new Map();
+    for (let i = 0; i < functions.length; i++) {
+        const existing = hashGroups.get(functions[i].bodyHash) || [];
+        existing.push(i);
+        hashGroups.set(functions[i].bodyHash, existing);
+    }
+    for (const indices of hashGroups.values()) {
+        if (indices.length < 2)
+            continue;
+        const group = indices.map(i => functions[i]);
+        const uniqueFiles = new Set(group.map(f => f.file));
+        if (uniqueFiles.size >= 2) {
+            duplicates.push(group);
+            indices.forEach(i => claimedIndices.add(i));
+        }
+    }
+    // Pass 2: Jaccard on AST tokens for remaining functions
+    const remaining = functions
+        .map((fn, i) => ({ fn, idx: i }))
+        .filter(({ idx }) => !claimedIndices.has(idx));
+    remaining.sort((a, b) => a.fn.bodyLength - b.fn.bodyLength);
+    const jaccardClaimed = new Set();
+    for (let i = 0; i < remaining.length; i++) {
+        if (jaccardClaimed.has(remaining[i].idx))
+            continue;
+        const group = [remaining[i].fn];
+        const baseLen = remaining[i].fn.bodyLength;
+        for (let j = i + 1; j < remaining.length; j++) {
+            if (jaccardClaimed.has(remaining[j].idx))
+                continue;
+            if (remaining[j].fn.bodyLength > baseLen * 1.5)
+                break;
+            if (remaining[j].fn.file === remaining[i].fn.file)
+                continue;
+            const sim = jaccardSimilarity(remaining[i].fn.astTokens, remaining[j].fn.astTokens);
+            if (sim >= similarityThreshold) {
+                group.push(remaining[j].fn);
+                jaccardClaimed.add(remaining[j].idx);
+            }
+        }
+        if (group.length >= 2) {
+            const uniqueFiles = new Set(group.map(f => f.file));
+            if (uniqueFiles.size >= 2) {
+                duplicates.push(group);
+                jaccardClaimed.add(remaining[i].idx);
+            }
+        }
+    }
+    // Mark all Pass 1 + Pass 2 claimed indices
+    for (const idx of jaccardClaimed)
+        claimedIndices.add(idx);
+    // Pass 3: Semantic embedding cosine similarity for still-unclaimed functions
+    if (semanticEnabled) {
+        const semanticRemaining = functions
+            .map((fn, i) => ({ fn, idx: i }))
+            .filter(({ idx }) => !claimedIndices.has(idx))
+            .filter(({ fn }) => fn.embedding && fn.embedding.length > 0);
+        semanticRemaining.sort((a, b) => a.fn.bodyLength - b.fn.bodyLength);
+        const semanticClaimed = new Set();
+        for (let i = 0; i < semanticRemaining.length; i++) {
+            if (semanticClaimed.has(semanticRemaining[i].idx))
+                continue;
+            const group = [semanticRemaining[i].fn];
+            const baseLen = semanticRemaining[i].fn.bodyLength;
+            for (let j = i + 1; j < semanticRemaining.length; j++) {
+                if (semanticClaimed.has(semanticRemaining[j].idx))
+                    continue;
+                // Body length must be within 2x range (semantic allows more variance)
+                if (semanticRemaining[j].fn.bodyLength > baseLen * 2.0)
+                    break;
+                if (semanticRemaining[j].fn.file === semanticRemaining[i].fn.file)
+                    continue;
+                const sim = cosineSimilarity(semanticRemaining[i].fn.embedding, semanticRemaining[j].fn.embedding);
+                if (sim >= semanticThreshold) {
+                    group.push(semanticRemaining[j].fn);
+                    semanticClaimed.add(semanticRemaining[j].idx);
+                }
+            }
+            if (group.length >= 2) {
+                const uniqueFiles = new Set(group.map(f => f.file));
+                if (uniqueFiles.size >= 2) {
+                    duplicates.push(group);
+                    semanticClaimed.add(semanticRemaining[i].idx);
+                }
+            }
+        }
+    }
+    return duplicates;
+}

package/dist/gates/duplication-drift/tokenizer.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Duplication Drift Tokenizer
+ *
+ * Handles language-aware tokenization of code functions.
+ * Supports tree-sitter AST-based tokenization with fallback to text-based.
+ */
+export declare function initTreeSitter(): Promise<boolean>;
+export declare const GRAMMAR_PATHS: Record<string, string>;
+export declare const languageCache: Map<string, any>;
+/**
+ * AST node types that are structural even as leaf nodes.
+ * These carry semantic meaning without children.
+ */
+export declare function isStructuralLeaf(type: string): boolean;
+/**
+ * Fallback tokenizer when tree-sitter is not available.
+ * Uses normalized text → keyword/operator multiset.
+ */
+export declare function textTokenize(normalized: string): Map<string, number>;
+/**
+ * Walk an AST subtree and collect node types as a multiset.
+ *
+ * This is the core insight: two functions with different variable names
+ * but the same control flow produce the same node type multiset.
+ *
+ * Example:
+ * `function a(x) { if (x > 0) return x * 2; return 0; }`
+ * `function b(val) { if (val > 0) return val * 2; return 0; }`
+ *
+ * Both produce: {if_statement: 1, binary_expression: 2, return_statement: 2, ...}
+ * → Jaccard similarity = 1.0
+ */
+export declare function collectASTNodeTypes(node: any): Map<string, number>;
+/**
+ * Walk the AST tree to find a function/method node at a given line.
+ */
+export declare function findFunctionNodeAtLine(rootNode: any, targetLine: number): any;
+/**
+ * Normalize function body text for tokenization.
+ * Removes comments, normalizes whitespace, etc.
+ */
+export declare function normalizeBody(body: string): string;
+/**
+ * Extract function body from lines starting at startIndex.
+ * Handles brace matching.
+ */
+export declare function extractFunctionBody(lines: string[], startIndex: number): string[];
+/**
+ * Get Parser instance (may be null if tree-sitter not available).
+ */
+export declare function getParser(): any;
+/**
+ * Get the __dirname path for grammar loading.
+ */
+export declare function getGrammarDir(): string;