@rigour-labs/core 5.0.1 → 5.1.0

package/dist/gates/duplication-drift/tokenizer.js

@@ -0,0 +1,195 @@
+/**
+ * Duplication Drift Tokenizer
+ *
+ * Handles language-aware tokenization of code functions.
+ * Supports tree-sitter AST-based tokenization with fallback to text-based.
+ */
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { Logger } from '../../utils/logger.js';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+// tree-sitter is optional — graceful fallback to text tokenization
+let Parser = null;
+let treeSitterReady = false;
+let treeSitterFailed = false;
+export async function initTreeSitter() {
+    if (treeSitterReady)
+        return true;
+    if (treeSitterFailed)
+        return false;
+    try {
+        const mod = await import('web-tree-sitter');
+        Parser = mod.default || mod;
+        await Parser.init();
+        treeSitterReady = true;
+        return true;
+    }
+    catch {
+        treeSitterFailed = true;
+        Logger.debug('tree-sitter not available, falling back to text tokenization');
+        return false;
+    }
+}
+export const GRAMMAR_PATHS = {
+    '.ts': '../../vendor/grammars/tree-sitter-typescript.wasm',
+    '.tsx': '../../vendor/grammars/tree-sitter-tsx.wasm',
+    '.js': '../../vendor/grammars/tree-sitter-javascript.wasm',
+    '.jsx': '../../vendor/grammars/tree-sitter-javascript.wasm',
+    '.py': '../../vendor/grammars/tree-sitter-python.wasm',
+    '.go': '../../vendor/grammars/tree-sitter-go.wasm',
+    '.rs': '../../vendor/grammars/tree-sitter-rust.wasm',
+};
+// Cache loaded languages
+export const languageCache = new Map();
+/**
+ * AST node types that are structural even as leaf nodes.
+ * These carry semantic meaning without children.
+ */
+export function isStructuralLeaf(type) {
+    const structural = new Set([
+        'return', 'break', 'continue', 'yield', 'throw',
+        'true', 'false', 'null', 'undefined', 'none',
+        'self', 'this', 'super',
+        'string', 'number', 'template_string',
+        // Operators
+        '=', '==', '===', '!=', '!==', '<', '>', '<=', '>=',
+        '+', '-', '*', '/', '%', '**',
+        '&&', '||', '!', '??',
+        '=>', '...', '?', ':',
+    ]);
+    return structural.has(type);
+}
+/**
+ * Fallback tokenizer when tree-sitter is not available.
+ * Uses normalized text → keyword/operator multiset.
+ */
+export function textTokenize(normalized) {
+    const tokens = new Map();
+    const structural = normalized.match(/\b(if|else|for|while|return|const|let|var|function|class|import|export|async|await|try|catch|throw|new|switch|case|break|continue|yield|def|self)\b|[{}()\[\];,.:=<>!&|+\-*/%?]+/g) || [];
+    for (const token of structural) {
+        tokens.set(token, (tokens.get(token) || 0) + 1);
+    }
+    // Normalize all identifiers to a count (variable names don't matter)
+    const keywords = new Set([
+        'if', 'else', 'for', 'while', 'return', 'const', 'let', 'var',
+        'function', 'class', 'import', 'export', 'async', 'await',
+        'try', 'catch', 'throw', 'new', 'switch', 'case', 'break',
+        'continue', 'yield', 'def', 'self', 'true', 'false', 'null', 'undefined',
+    ]);
+    const identifiers = normalized.match(/\b[a-zA-Z_]\w*\b/g) || [];
+    let idCount = 0;
+    for (const id of identifiers) {
+        if (!keywords.has(id))
+            idCount++;
+    }
+    if (idCount > 0)
+        tokens.set('_ID_', idCount);
+    return tokens;
+}
+/**
+ * Walk an AST subtree and collect node types as a multiset.
+ *
+ * This is the core insight: two functions with different variable names
+ * but the same control flow produce the same node type multiset.
+ *
+ * Example:
+ * `function a(x) { if (x > 0) return x * 2; return 0; }`
+ * `function b(val) { if (val > 0) return val * 2; return 0; }`
+ *
+ * Both produce: {if_statement: 1, binary_expression: 2, return_statement: 2, ...}
+ * → Jaccard similarity = 1.0
+ */
+export function collectASTNodeTypes(node) {
+    const types = new Map();
+    const walk = (n) => {
+        // Skip leaf nodes that are just identifiers/literals (noise)
+        // Keep structural node types only
+        if (n.childCount > 0 || isStructuralLeaf(n.type)) {
+            types.set(n.type, (types.get(n.type) || 0) + 1);
+        }
+        for (let i = 0; i < n.childCount; i++) {
+            walk(n.child(i));
+        }
+    };
+    walk(node);
+    return types;
+}
+/**
+ * Walk the AST tree to find a function/method node at a given line.
+ */
+export function findFunctionNodeAtLine(rootNode, targetLine) {
+    const functionTypes = new Set([
+        'function_declaration', 'method_definition', 'arrow_function',
+        'function_definition', // Python
+        'function_item', // Rust
+        'method_declaration', // Java/C#
+        'lexical_declaration', // const x = () => {}
+    ]);
+    let bestMatch = null;
+    const walk = (node) => {
+        // tree-sitter lines are 0-indexed, our lines are 1-indexed
+        if (functionTypes.has(node.type) && node.startPosition.row + 1 === targetLine) {
+            bestMatch = node;
+            return;
+        }
+        for (let i = 0; i < node.childCount; i++) {
+            walk(node.child(i));
+            if (bestMatch)
+                return;
+        }
+    };
+    walk(rootNode);
+    return bestMatch;
+}
+/**
+ * Normalize function body text for tokenization.
+ * Removes comments, normalizes whitespace, etc.
+ */
+export function normalizeBody(body) {
+    return body
+        .replace(/\/\/.*/g, '')
+        .replace(/\/\*[\s\S]*?\*\//g, '')
+        .replace(/#.*/g, '')
+        .replace(/`[^`]*`/g, '"STR"')
+        .replace(/\basync\s+/g, '')
+        .replace(/\s+/g, ' ')
+        .replace(/['"]/g, '"')
+        .trim();
+}
+/**
+ * Extract function body from lines starting at startIndex.
+ * Handles brace matching.
+ */
+export function extractFunctionBody(lines, startIndex) {
+    let braceDepth = 0;
+    let started = false;
+    const body = [];
+    for (let i = startIndex; i < lines.length; i++) {
+        const line = lines[i];
+        for (const ch of line) {
+            if (ch === '{') {
+                braceDepth++;
+                started = true;
+            }
+            if (ch === '}')
+                braceDepth--;
+        }
+        if (started)
+            body.push(line);
+        if (started && braceDepth === 0)
+            break;
+    }
+    return body;
+}
+/**
+ * Get Parser instance (may be null if tree-sitter not available).
+ */
+export function getParser() {
+    return Parser;
+}
+/**
+ * Get the __dirname path for grammar loading.
+ */
+export function getGrammarDir() {
+    return __dirname;
+}

package/dist/gates/frontend-secret-exposure.d.ts

@@ -13,9 +13,6 @@
  *      where VARNAME matches secret_env_name_patterns and lacks a safe public prefix.
  *   2. Literal key detection: actual live API key values embedded in source
  *      (sk_live_..., sk-proj-..., AKIA..., ghp_..., etc.)
- *
- * @since v4.4.0
- * @see CWE-312 Cleartext Storage of Sensitive Information
  */
 import { Gate, GateContext } from './base.js';
 import { Failure, Provenance } from '../types/index.js';

package/dist/gates/frontend-secret-exposure.js

@@ -13,126 +13,13 @@
  *      where VARNAME matches secret_env_name_patterns and lacks a safe public prefix.
  *   2. Literal key detection: actual live API key values embedded in source
  *      (sk_live_..., sk-proj-..., AKIA..., ghp_..., etc.)
- *
- * @since v4.4.0
- * @see CWE-312 Cleartext Storage of Sensitive Information
  */
 import { Gate } from './base.js';
 import { FileScanner } from '../utils/scanner.js';
 import { Logger } from '../utils/logger.js';
+import { LITERAL_KEY_PATTERNS, SEVERITY_ORDER, extractEnvVarName, extractDestructuredEnvVars, } from './frontend-secret-patterns.js';
 import fs from 'fs-extra';
 import path from 'path';
-/**
- * Literal API key patterns — actual values that are always dangerous in frontend code.
- * These bypass the env-name heuristic and fire regardless of variable naming.
- */
-const LITERAL_KEY_PATTERNS = [
-    {
-        label: 'Stripe Live Secret Key',
-        regex: /\bsk_live_[a-zA-Z0-9]{24,}\b/g,
-        severity: 'critical',
-        hint: 'Live Stripe secret key (sk_live_...) exposed in client bundle. ' +
-            'Attackers can charge cards without a frontend. Move to server-side only.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'Stripe Live Restricted Key',
-        regex: /\brk_live_[a-zA-Z0-9]{24,}\b/g,
-        severity: 'critical',
-        hint: 'Live Stripe restricted key (rk_live_...) must never appear in frontend code.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'OpenAI API Key',
-        regex: /\bsk-proj-[a-zA-Z0-9_-]{40,}\b/g,
-        severity: 'critical',
-        hint: 'OpenAI project API key (sk-proj-...) exposed in client bundle. ' +
-            'Attackers can make unlimited API calls at your expense.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'OpenAI Legacy API Key',
-        regex: /\bsk-[a-zA-Z0-9]{48,}\b/g,
-        severity: 'critical',
-        hint: 'OpenAI API key (sk-...) exposed in client bundle. Move to server-side proxy.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'AWS Access Key ID',
-        regex: /\bAKIA[A-Z2-7]{16}\b/g,
-        severity: 'critical',
-        hint: 'AWS Access Key ID exposed in client bundle. This grants AWS API access.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'GitHub Personal Access Token',
-        regex: /\bghp_[a-zA-Z0-9]{36,}\b/g,
-        severity: 'critical',
-        hint: 'GitHub PAT (ghp_...) in frontend code. Grants repo access to anyone who views bundle.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'GitHub Fine-Grained PAT',
-        regex: /\bgithub_pat_[a-zA-Z0-9_]{55,}\b/g,
-        severity: 'critical',
-        hint: 'GitHub fine-grained PAT (github_pat_...) must never appear in frontend code.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'Anthropic API Key',
-        regex: /\bsk-ant-api\d{2}-[a-zA-Z0-9_-]{90,}\b/g,
-        severity: 'critical',
-        hint: 'Anthropic API key exposed in client bundle. Move to a server-side proxy.',
-        cwe: 'CWE-312',
-    },
-    {
-        label: 'SendGrid API Key',
-        regex: /\bSG\.[a-zA-Z0-9_-]{22,}\.[a-zA-Z0-9_-]{43,}\b/g,
-        severity: 'critical',
-        hint: 'SendGrid API key exposed in client bundle. Attackers can send spam as you.',
-        cwe: 'CWE-312',
-    },
-];
-/** Severity ordering for threshold comparison */
-const SEVERITY_ORDER = { critical: 0, high: 1, medium: 2, low: 3 };
-/**
- * Extracts the env var name from a single-reference line.
- *   process.env.VARNAME          → "VARNAME"
- *   process.env["VARNAME"]       → "VARNAME"
- *   import.meta.env.VARNAME      → "VARNAME"
- * Returns null if no match.
- */
-function extractEnvVarName(line, checkProcessEnv, checkImportMetaEnv) {
-    if (checkProcessEnv) {
-        const dot = line.match(/process\.env\.([A-Z][A-Z0-9_]*)/);
-        if (dot)
-            return dot[1];
-        const bracket = line.match(/process\.env\[['"]([A-Z][A-Z0-9_]*)['"]\]/);
-        if (bracket)
-            return bracket[1];
-    }
-    if (checkImportMetaEnv) {
-        const vite = line.match(/import\.meta\.env\.([A-Z_][A-Z0-9_]*)/);
-        if (vite)
-            return vite[1];
-    }
-    return null;
-}
-/**
- * Finds all var names from destructuring:
- *   const { STRIPE_SECRET_KEY, OPENAI_KEY } = process.env
- */
-function extractDestructuredEnvVars(line) {
-    if (!/=\s*process\.env\b/.test(line))
-        return [];
-    const brace = line.match(/\{\s*([^}]+)\s*\}/);
-    if (!brace)
-        return [];
-    return brace[1]
-        .split(',')
-        .map(s => s.split(':')[0].trim()) // handle "SECRET_KEY: renamed"
-        .filter(s => /^[A-Z_][A-Z0-9_]*$/.test(s));
-}
 export class FrontendSecretExposureGate extends Gate {
     cfg;
     secretNamePatterns;

package/dist/gates/frontend-secret-patterns.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Frontend Secret Exposure Gate — Environment Variable and Literal Key Patterns
+ *
+ * Contains environment variable pattern definitions, framework-specific env patterns
+ * (Next.js, Vite, SvelteKit, etc.), and literal API key detection patterns.
+ */
+/**
+ * Literal API key patterns — actual values that are always dangerous in frontend code.
+ * These bypass the env-name heuristic and fire regardless of variable naming.
+ */
+export declare const LITERAL_KEY_PATTERNS: {
+    label: string;
+    regex: RegExp;
+    severity: 'critical' | 'high';
+    hint: string;
+    cwe: string;
+}[];
+/** Severity ordering for threshold comparison */
+export declare const SEVERITY_ORDER: {
+    readonly critical: 0;
+    readonly high: 1;
+    readonly medium: 2;
+    readonly low: 3;
+};
+/**
+ * Extracts the env var name from a single-reference line.
+ */
+export declare function extractEnvVarName(line: string, checkProcessEnv: boolean, checkImportMetaEnv: boolean): string | null;
+/**
+ * Finds all var names from destructuring:
+ * const { STRIPE_SECRET_KEY, OPENAI_KEY } = process.env
+ */
+export declare function extractDestructuredEnvVars(line: string): string[];

package/dist/gates/frontend-secret-patterns.js

@@ -0,0 +1,119 @@
+/**
+ * Frontend Secret Exposure Gate — Environment Variable and Literal Key Patterns
+ *
+ * Contains environment variable pattern definitions, framework-specific env patterns
+ * (Next.js, Vite, SvelteKit, etc.), and literal API key detection patterns.
+ */
+/**
+ * Literal API key patterns — actual values that are always dangerous in frontend code.
+ * These bypass the env-name heuristic and fire regardless of variable naming.
+ */
+export const LITERAL_KEY_PATTERNS = [
+    {
+        label: 'Stripe Live Secret Key',
+        regex: /\bsk_live_[a-zA-Z0-9]{24,}\b/g,
+        severity: 'critical',
+        hint: 'Live Stripe secret key (sk_live_...) exposed in client bundle. ' +
+            'Attackers can charge cards without a frontend. Move to server-side only.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'Stripe Live Restricted Key',
+        regex: /\brk_live_[a-zA-Z0-9]{24,}\b/g,
+        severity: 'critical',
+        hint: 'Live Stripe restricted key (rk_live_...) must never appear in frontend code.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'OpenAI API Key',
+        regex: /\bsk-proj-[a-zA-Z0-9_-]{40,}\b/g,
+        severity: 'critical',
+        hint: 'OpenAI project API key (sk-proj-...) exposed in client bundle. ' +
+            'Attackers can make unlimited API calls at your expense.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'OpenAI Legacy API Key',
+        regex: /\bsk-[a-zA-Z0-9]{48,}\b/g,
+        severity: 'critical',
+        hint: 'OpenAI API key (sk-...) exposed in client bundle. Move to server-side proxy.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'AWS Access Key ID',
+        regex: /\bAKIA[A-Z2-7]{16}\b/g,
+        severity: 'critical',
+        hint: 'AWS Access Key ID exposed in client bundle. This grants AWS API access.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'GitHub Personal Access Token',
+        regex: /\bghp_[a-zA-Z0-9]{36,}\b/g,
+        severity: 'critical',
+        hint: 'GitHub PAT (ghp_...) in frontend code. Grants repo access to anyone who views bundle.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'GitHub Fine-Grained PAT',
+        regex: /\bgithub_pat_[a-zA-Z0-9_]{55,}\b/g,
+        severity: 'critical',
+        hint: 'GitHub fine-grained PAT (github_pat_...) must never appear in frontend code.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'Anthropic API Key',
+        regex: /\bsk-ant-api\d{2}-[a-zA-Z0-9_-]{90,}\b/g,
+        severity: 'critical',
+        hint: 'Anthropic API key exposed in client bundle. Move to a server-side proxy.',
+        cwe: 'CWE-312',
+    },
+    {
+        label: 'SendGrid API Key',
+        regex: /\bSG\.[a-zA-Z0-9_-]{22,}\.[a-zA-Z0-9_-]{43,}\b/g,
+        severity: 'critical',
+        hint: 'SendGrid API key exposed in client bundle. Attackers can send spam as you.',
+        cwe: 'CWE-312',
+    },
+];
+/** Severity ordering for threshold comparison */
+export const SEVERITY_ORDER = { critical: 0, high: 1, medium: 2, low: 3 };
+/**
+ * Extracts the env var name from a single-reference line.
+ */
+export function extractEnvVarName(line, checkProcessEnv, checkImportMetaEnv) {
+    if (checkProcessEnv) {
+        const dot = line.match(/process\.env\.([A-Z][A-Z0-9_]*)/);
+        if (dot)
+            return dot[1];
+        const bracket = line.match(/process\.env\[['"]([A-Z][A-Z0-9_]*)['"]\]/);
+        if (bracket)
+            return bracket[1];
+    }
+    if (checkImportMetaEnv) {
+        const vite = line.match(/import\.meta\.env\.([A-Z_][A-Z0-9_]*)/);
+        if (vite)
+            return vite[1];
+    }
+    const sveltePrivate = line.match(/import\s+\{([^}]+)\}\s+from\s+['"]?\$env\/(?:static|dynamic)\/private['"]?/);
+    if (sveltePrivate) {
+        const firstVar = sveltePrivate[1].split(',')[0].trim();
+        if (/^[A-Z_][A-Z0-9_]*$/.test(firstVar))
+            return firstVar;
+    }
+    return null;
+}
+/**
+ * Finds all var names from destructuring:
+ * const { STRIPE_SECRET_KEY, OPENAI_KEY } = process.env
+ */
+export function extractDestructuredEnvVars(line) {
+    if (!/=\s*process\.env\b/.test(line))
+        return [];
+    const brace = line.match(/\{\s*([^}]+)\s*\}/);
+    if (!brace)
+        return [];
+    return brace[1]
+        .split(',')
+        .map(s => s.split(':')[0].trim())
+        .filter(s => /^[A-Z_][A-Z0-9_]*$/.test(s));
+}

package/dist/gates/{hallucinated-imports.d.ts → hallucinated-imports/index.d.ts}

@@ -15,12 +15,9 @@
  *   Rust   — std/core/alloc crates, Cargo.toml deps, use/extern crate statements
  *   Java   — java/javax/jakarta stdlib, build.gradle + pom.xml deps, import statements
  *   Kotlin — kotlin/kotlinx stdlib, Gradle deps, import statements
- *
- * @since v2.16.0
- * @since v3.0.1 — Go stdlib fix, Ruby/C# strengthened, Rust/Java/Kotlin added
  */
-import { Gate, GateContext } from './base.js';
-import { Failure, Provenance } from '../types/index.js';
+import { Gate, GateContext } from '../base.js';
+import { Failure, Provenance } from '../../types/index.js';
 export interface HallucinatedImport {
     file: string;
     line: number;
@@ -39,33 +36,9 @@ export declare class HallucinatedImportsGate extends Gate {
     constructor(config?: HallucinatedImportsConfig);
     protected get provenance(): Provenance;
     run(context: GateContext): Promise<Failure[]>;
-    private checkJSImports;
-    private collectJSImportSpecs;
-    private resolveJSDepsForFile;
-    private resolveTsPathAlias;
-    private matchTsPathRule;
-    private resolveTsPathTarget;
-    private resolveTsPathConfigForFile;
-    private loadTsPathConfig;
-    private readLooseJson;
-    private checkPyImports;
-    /**
-     * Find Python source roots (directories that are on sys.path) by looking at
-     * pyproject.toml, setup.cfg, or common patterns like src/ layouts.
-     */
-    private findPythonSourceRoots;
-    /**
-     * Load installed Python package names from pyproject.toml dependencies,
-     * requirements.txt, setup.cfg, or Pipfile.
-     */
-    private loadPythonInstalledPackages;
     private resolveRelativeImport;
     private extractPackageName;
     private shouldIgnore;
-    /**
-     * Build candidate source paths for an import.
-     * Handles ESM-style TS source imports like "./foo.js" that map to "./foo.ts" pre-build.
-     */
     private buildImportCandidates;
     private shouldSkipFile;
 }