driftdetect-core 0.4.1 → 0.4.3

package/dist/lake/examples-store.js

@@ -0,0 +1,389 @@
+/**
+ * Examples Store
+ *
+ * Manages pre-extracted code examples for patterns.
+ * This allows drift_code_examples to return instantly without reading source files.
+ *
+ * Storage structure:
+ * .drift/lake/examples/
+ *   ├── index.json           # Example index (pattern -> file mapping)
+ *   └── patterns/
+ *       ├── {pattern-id}.json  # Pre-extracted examples for each pattern
+ *       └── ...
+ *
+ * Key features:
+ * - Pre-extracted code snippets with context
+ * - Quality scoring for example selection
+ * - Incremental updates per pattern
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
+import { EventEmitter } from 'node:events';
+import { LAKE_DIRS, DEFAULT_DATA_LAKE_CONFIG, } from './types.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const EXAMPLES_DIR = 'examples';
+const PATTERNS_SUBDIR = 'patterns';
+const INDEX_FILE = 'index.json';
+const STORE_VERSION = '1.0.0';
+// Default context lines around examples
+const DEFAULT_CONTEXT_LINES = 5;
+// ============================================================================
+// Helper Functions
+// ============================================================================
+async function fileExists(filePath) {
+    try {
+        await fs.access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function ensureDir(dirPath) {
+    await fs.mkdir(dirPath, { recursive: true });
+}
+function generateChecksum(data) {
+    const content = JSON.stringify(data);
+    return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+function sanitizePatternId(id) {
+    // Convert pattern ID to safe filename
+    return id.replace(/[^a-zA-Z0-9_-]/g, '_').toLowerCase();
+}
+// ============================================================================
+// Examples Store Class
+// ============================================================================
+export class ExamplesStore extends EventEmitter {
+    config;
+    examplesDir;
+    patternsDir;
+    // In-memory cache
+    indexCache = null;
+    examplesCache = new Map();
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_DATA_LAKE_CONFIG, ...config };
+        this.examplesDir = path.join(this.config.rootDir, LAKE_DIRS.root, LAKE_DIRS.lake, EXAMPLES_DIR);
+        this.patternsDir = path.join(this.examplesDir, PATTERNS_SUBDIR);
+    }
+    // ==========================================================================
+    // Initialization
+    // ==========================================================================
+    async initialize() {
+        await ensureDir(this.examplesDir);
+        await ensureDir(this.patternsDir);
+    }
+    // ==========================================================================
+    // Index Operations
+    // ==========================================================================
+    /**
+     * Get the examples index
+     */
+    async getIndex() {
+        if (this.indexCache) {
+            return this.indexCache;
+        }
+        const indexPath = path.join(this.examplesDir, INDEX_FILE);
+        if (!(await fileExists(indexPath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(indexPath, 'utf-8');
+            this.indexCache = JSON.parse(content);
+            return this.indexCache;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Save the examples index
+     */
+    async saveIndex(index) {
+        await ensureDir(this.examplesDir);
+        const indexPath = path.join(this.examplesDir, INDEX_FILE);
+        await fs.writeFile(indexPath, JSON.stringify(index, null, 2));
+        this.indexCache = index;
+        this.emit('index:saved');
+    }
+    /**
+     * Build examples index from pattern examples
+     */
+    async buildIndex() {
+        const patternIds = await this.listPatterns();
+        const entries = [];
+        let totalExamples = 0;
+        for (const patternId of patternIds) {
+            const examples = await this.getPatternExamples(patternId);
+            if (!examples)
+                continue;
+            const bestQuality = examples.examples.length > 0
+                ? Math.max(...examples.examples.map(e => e.quality))
+                : 0;
+            totalExamples += examples.examples.length;
+            entries.push({
+                patternId: examples.patternId,
+                patternName: examples.patternName,
+                category: examples.category,
+                exampleCount: examples.examples.length,
+                bestQuality,
+                lastUpdated: examples.generatedAt,
+            });
+        }
+        const index = {
+            version: STORE_VERSION,
+            generatedAt: new Date().toISOString(),
+            totalPatterns: entries.length,
+            totalExamples,
+            patterns: entries,
+        };
+        await this.saveIndex(index);
+        return index;
+    }
+    // ==========================================================================
+    // Pattern Examples Operations
+    // ==========================================================================
+    /**
+     * Get examples for a specific pattern
+     */
+    async getPatternExamples(patternId) {
+        // Check cache first
+        const cached = this.examplesCache.get(patternId);
+        if (cached) {
+            return cached;
+        }
+        const filePath = path.join(this.patternsDir, `${sanitizePatternId(patternId)}.json`);
+        if (!(await fileExists(filePath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            const data = JSON.parse(content);
+            const examples = {
+                patternId: data.patternId,
+                patternName: data.patternName,
+                category: data.category,
+                generatedAt: data.generatedAt,
+                examples: data.examples,
+            };
+            this.examplesCache.set(patternId, examples);
+            return examples;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Save examples for a pattern
+     */
+    async savePatternExamples(examples) {
+        await ensureDir(this.patternsDir);
+        const filePath = path.join(this.patternsDir, `${sanitizePatternId(examples.patternId)}.json`);
+        const examplesFile = {
+            version: STORE_VERSION,
+            checksum: generateChecksum(examples.examples),
+            ...examples,
+        };
+        await fs.writeFile(filePath, JSON.stringify(examplesFile, null, 2));
+        this.examplesCache.set(examples.patternId, examples);
+        this.emit('examples:saved', examples.patternId);
+    }
+    /**
+     * Get examples for multiple patterns
+     */
+    async getMultiplePatternExamples(patternIds) {
+        const results = [];
+        const examples = await Promise.all(patternIds.map(id => this.getPatternExamples(id)));
+        for (const ex of examples) {
+            if (ex) {
+                results.push(ex);
+            }
+        }
+        return results;
+    }
+    /**
+     * Get examples by category
+     */
+    async getExamplesByCategory(category) {
+        const index = await this.getIndex();
+        if (!index)
+            return [];
+        const patternIds = index.patterns
+            .filter(p => p.category === category)
+            .map(p => p.patternId);
+        return this.getMultiplePatternExamples(patternIds);
+    }
+    /**
+     * List all patterns with examples
+     */
+    async listPatterns() {
+        if (!(await fileExists(this.patternsDir))) {
+            return [];
+        }
+        const files = await fs.readdir(this.patternsDir);
+        return files
+            .filter(f => f.endsWith('.json'))
+            .map(f => f.replace('.json', ''));
+    }
+    /**
+     * Delete examples for a pattern
+     */
+    async deletePatternExamples(patternId) {
+        const filePath = path.join(this.patternsDir, `${sanitizePatternId(patternId)}.json`);
+        try {
+            await fs.unlink(filePath);
+            this.examplesCache.delete(patternId);
+            this.emit('examples:deleted', patternId);
+        }
+        catch {
+            // Ignore if doesn't exist
+        }
+    }
+    // ==========================================================================
+    // Example Extraction
+    // ==========================================================================
+    /**
+     * Extract examples from source code for a pattern
+     *
+     * This method reads source files and extracts code snippets with context.
+     * It's called during scan/materialize, not at query time.
+     */
+    async extractExamples(patternId, patternName, category, locations, outliers, options = {}) {
+        const { maxExamples = 5, contextLines = DEFAULT_CONTEXT_LINES, minQuality = 0.3, includeOutliers = true, } = options;
+        const examples = [];
+        // Extract from regular locations
+        for (const loc of locations) {
+            if (examples.length >= maxExamples)
+                break;
+            const example = await this.extractSingleExample(loc.file, loc.line, loc.endLine, contextLines, false);
+            if (example && example.quality >= minQuality) {
+                examples.push(example);
+            }
+        }
+        // Extract from outliers if requested
+        if (includeOutliers && examples.length < maxExamples) {
+            for (const out of outliers) {
+                if (examples.length >= maxExamples)
+                    break;
+                const example = await this.extractSingleExample(out.file, out.line, out.endLine, contextLines, true);
+                if (example && example.quality >= minQuality) {
+                    examples.push(example);
+                }
+            }
+        }
+        // Sort by quality (best first)
+        examples.sort((a, b) => b.quality - a.quality);
+        const patternExamples = {
+            patternId,
+            patternName,
+            category,
+            generatedAt: new Date().toISOString(),
+            examples,
+        };
+        await this.savePatternExamples(patternExamples);
+        return patternExamples;
+    }
+    /**
+     * Extract a single example from a source file
+     */
+    async extractSingleExample(file, line, endLine, contextLines, isOutlier) {
+        const filePath = path.join(this.config.rootDir, file);
+        if (!(await fileExists(filePath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            const lines = content.split('\n');
+            const actualEndLine = endLine ?? line;
+            const startContext = Math.max(0, line - contextLines - 1);
+            const endContext = Math.min(lines.length, actualEndLine + contextLines);
+            // Extract the main code
+            const codeLines = lines.slice(line - 1, actualEndLine);
+            const code = codeLines.join('\n');
+            // Extract context (before and after)
+            const contextBefore = lines.slice(startContext, line - 1).join('\n');
+            const contextAfter = lines.slice(actualEndLine, endContext).join('\n');
+            const context = [contextBefore, code, contextAfter].filter(Boolean).join('\n');
+            // Calculate quality score
+            const quality = this.calculateExampleQuality(code, lines.length, isOutlier);
+            return {
+                file,
+                line,
+                endLine: actualEndLine,
+                code,
+                context,
+                quality,
+                isOutlier,
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Calculate quality score for an example
+     */
+    calculateExampleQuality(code, totalLines, isOutlier) {
+        let score = 0.5; // Base score
+        // Prefer shorter, focused examples
+        const codeLines = code.split('\n').length;
+        if (codeLines >= 3 && codeLines <= 20) {
+            score += 0.2;
+        }
+        else if (codeLines > 50) {
+            score -= 0.2;
+        }
+        // Prefer examples from larger files (more context)
+        if (totalLines > 100) {
+            score += 0.1;
+        }
+        // Penalize outliers slightly
+        if (isOutlier) {
+            score -= 0.1;
+        }
+        // Prefer examples with meaningful content
+        if (code.includes('function') || code.includes('class') || code.includes('const')) {
+            score += 0.1;
+        }
+        // Penalize examples that are mostly comments
+        const commentLines = code.split('\n').filter(l => l.trim().startsWith('//') || l.trim().startsWith('*') || l.trim().startsWith('/*')).length;
+        if (commentLines > codeLines / 2) {
+            score -= 0.2;
+        }
+        return Math.max(0, Math.min(1, score));
+    }
+    // ==========================================================================
+    // Cache Management
+    // ==========================================================================
+    /**
+     * Invalidate caches
+     */
+    invalidateCache(patternId) {
+        if (patternId) {
+            this.examplesCache.delete(patternId);
+        }
+        else {
+            this.indexCache = null;
+            this.examplesCache.clear();
+        }
+    }
+    /**
+     * Get cache stats
+     */
+    getCacheStats() {
+        return {
+            cachedPatterns: this.examplesCache.size,
+            hasIndex: this.indexCache !== null,
+        };
+    }
+}
+// ============================================================================
+// Factory Function
+// ============================================================================
+export function createExamplesStore(config = {}) {
+    return new ExamplesStore(config);
+}
+//# sourceMappingURL=examples-store.js.map

package/dist/lake/examples-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"examples-store.js","sourceRoot":"","sources":["../../src/lake/examples-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAQ3C,OAAO,EACL,SAAS,EACT,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAIpB,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,YAAY,GAAG,UAAU,CAAC;AAChC,MAAM,eAAe,GAAG,UAAU,CAAC;AACnC,MAAM,UAAU,GAAG,YAAY,CAAC;AAChC,MAAM,aAAa,GAAG,OAAO,CAAC;AAE9B,wCAAwC;AACxC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAkChC,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAe;IACtC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAC,EAAU;IACnC,sCAAsC;IACtC,OAAO,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED,+EAA+E;AAC/E,uBAAuB;AACvB,+EAA+E;AAE/E,MAAM,OAAO,aAAc,SAAQ,YAAY;IAC5B,MAAM,CAAiB;IACvB,WAAW,CAAS;IACpB,WAAW,CAAS;IAErC,kBAAkB;IACV,UAAU,GAAyB,IAAI,CAAC;IACxC,aAAa,GAAiC,IAAI,GAAG,EAAE,CAAC;IAEhE,YAAY,SAAkC,EAAE;QAC9C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAC1B,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,SAAS,CAAC,IAAI,EACd,SAAS,CAAC,IAAI,EACd,YAAY,CACb,CAAC;QACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAClE,CAAC;IAED,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E,KAAK,CAAC,UAAU;QACd,MAAM,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClC,MAAM,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpC,CAAC;IAED,6EAA6E;IAC7E,mBAAmB;IACnB,6EAA6E;IAE7E;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,UAAU,CAAC;QACzB,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAC1D,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACtD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,OAAO,IAAI,CAAC,UAAU,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,KAAoB;QAClC,MAAM,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAElC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAC1D,MAAM,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAE9D,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAwB,EAAE,CAAC;QACxC,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC1D,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;gBAC9C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBACpD,CAAC,CAAC,CAAC,CAAC;YAEN,aAAa,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM;gBACtC,WAAW;gBACX,WAAW,EAAE,QAAQ,CAAC,WAAW;aAClC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAkB;YAC3B,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,aAAa;YACb,QAAQ,EAAE,OAAO;SAClB,CAAC;QAEF,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,6EAA6E;IAC7E,8BAA8B;IAC9B,6EAA6E;IAE7E;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,SAAiB;QACxC,oBAAoB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,iBAAiB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrF,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAoB;gBAChC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC;YAEF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC5C,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,QAAyB;QACjD,MAAM,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAElC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9F,MAAM,YAAY,GAAG;YACnB,OAAO,EAAE,aAAa;YACtB,QAAQ,EAAE,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAC7C,GAAG,QAAQ;SACZ,CAAC;QAEF,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,0BAA0B,CAAC,UAAoB;QACnD,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,CAClD,CAAC;QAEF,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,EAAE,EAAE,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB,CAAC,QAAyB;QACnD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ;aAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;aACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAEzB,OAAO,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjD,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aAChC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB,CAAC,SAAiB;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,iBAAiB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAErF,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC1B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,qBAAqB;IACrB,6EAA6E;IAE7E;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CACnB,SAAiB,EACjB,WAAmB,EACnB,QAAyB,EACzB,SAAkE,EAClE,QAAiE,EACjE,UAAoC,EAAE;QAEtC,MAAM,EACJ,WAAW,GAAG,CAAC,EACf,YAAY,GAAG,qBAAqB,EACpC,UAAU,GAAG,GAAG,EAChB,eAAe,GAAG,IAAI,GACvB,GAAG,OAAO,CAAC;QAEZ,MAAM,QAAQ,GAAkB,EAAE,CAAC;QAEnC,iCAAiC;QACjC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,WAAW;gBAAE,MAAM;YAE1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAC7C,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,OAAO,EACX,YAAY,EACZ,KAAK,CACN,CAAC;YAEF,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,UAAU,EAAE,CAAC;gBAC7C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,IAAI,eAAe,IAAI,QAAQ,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YACrD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,IAAI,QAAQ,CAAC,MAAM,IAAI,WAAW;oBAAE,MAAM;gBAE1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAC7C,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,OAAO,EACX,YAAY,EACZ,IAAI,CACL,CAAC;gBAEF,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,UAAU,EAAE,CAAC;oBAC7C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAE/C,MAAM,eAAe,GAAoB;YACvC,SAAS;YACT,WAAW;YACX,QAAQ;YACR,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,QAAQ;SACT,CAAC;QAEF,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAChD,OAAO,eAAe,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,IAAY,EACZ,IAAY,EACZ,OAA2B,EAC3B,YAAoB,EACpB,SAAkB;QAElB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAEtD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAElC,MAAM,aAAa,GAAG,OAAO,IAAI,IAAI,CAAC;YACtC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,GAAG,YAAY,CAAC,CAAC;YAExE,wBAAwB;YACxB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,aAAa,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAElC,qCAAqC;YACrC,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrE,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvE,MAAM,OAAO,GAAG,CAAC,aAAa,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE/E,0BAA0B;YAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,uBAAuB,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAE5E,OAAO;gBACL,IAAI;gBACJ,IAAI;gBACJ,OAAO,EAAE,aAAa;gBACtB,IAAI;gBACJ,OAAO;gBACP,OAAO;gBACP,SAAS;aACV,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,IAAY,EACZ,UAAkB,EAClB,SAAkB;QAElB,IAAI,KAAK,GAAG,GAAG,CAAC,CAAC,aAAa;QAE9B,mCAAmC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1C,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACtC,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;aAAM,IAAI,SAAS,GAAG,EAAE,EAAE,CAAC;YAC1B,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,mDAAmD;QACnD,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;YACrB,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,6BAA6B;QAC7B,IAAI,SAAS,EAAE,CAAC;YACd,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,0CAA0C;QAC1C,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAClF,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC/C,CAAC,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CACnF,CAAC,MAAM,CAAC;QACT,IAAI,YAAY,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC;YACjC,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,6EAA6E;IAC7E,mBAAmB;IACnB,6EAA6E;IAE7E;;OAEG;IACH,eAAe,CAAC,SAAkB;QAChC,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;YACvC,QAAQ,EAAE,IAAI,CAAC,UAAU,KAAK,IAAI;SACnC,CAAC;IACJ,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,MAAM,UAAU,mBAAmB,CAAC,SAAkC,EAAE;IACtE,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC"}

package/dist/lake/index-store.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Index Store
+ *
+ * Manages pre-built indexes for fast lookups.
+ * Indexes map common query dimensions to IDs for O(1) access.
+ *
+ * Key indexes:
+ * - by-file: file path → pattern/function/access IDs
+ * - by-category: category → pattern IDs
+ * - by-table: table name → access point/function IDs
+ * - by-entry-point: entry point → reachable function IDs
+ */
+import { EventEmitter } from 'node:events';
+import type { FileIndex, CategoryIndex, TableIndex, EntryPointIndex, DataLakeConfig, IndexType } from './types.js';
+import type { Pattern, PatternCategory } from '../store/types.js';
+export declare class IndexStore extends EventEmitter {
+    private readonly config;
+    private readonly indexesDir;
+    private fileIndexCache;
+    private categoryIndexCache;
+    private tableIndexCache;
+    private entryPointIndexCache;
+    constructor(config?: Partial<DataLakeConfig>);
+    initialize(): Promise<void>;
+    getFileIndex(): Promise<FileIndex | null>;
+    saveFileIndex(index: FileIndex): Promise<void>;
+    /**
+     * Build file index from patterns
+     */
+    buildFileIndex(patterns: Pattern[]): FileIndex;
+    /**
+     * Get pattern IDs for a specific file
+     */
+    getPatternIdsForFile(file: string): Promise<string[]>;
+    /**
+     * Update file index for specific files (incremental)
+     */
+    updateFileIndex(file: string, patternIds: string[], accessPointIds?: string[], functionIds?: string[]): Promise<void>;
+    getCategoryIndex(): Promise<CategoryIndex | null>;
+    saveCategoryIndex(index: CategoryIndex): Promise<void>;
+    /**
+     * Build category index from patterns
+     */
+    buildCategoryIndex(patterns: Pattern[]): CategoryIndex;
+    /**
+     * Get pattern IDs for a specific category
+     */
+    getPatternIdsForCategory(category: PatternCategory): Promise<string[]>;
+    getTableIndex(): Promise<TableIndex | null>;
+    saveTableIndex(index: TableIndex): Promise<void>;
+    /**
+     * Get access point IDs for a specific table
+     */
+    getAccessPointIdsForTable(table: string): Promise<string[]>;
+    /**
+     * Get function IDs that access a specific table
+     */
+    getAccessorIdsForTable(table: string): Promise<string[]>;
+    getEntryPointIndex(): Promise<EntryPointIndex | null>;
+    saveEntryPointIndex(index: EntryPointIndex): Promise<void>;
+    /**
+     * Get reachable function IDs from an entry point
+     */
+    getReachableFunctions(entryPointId: string): Promise<string[]>;
+    /**
+     * Get tables reachable from an entry point
+     */
+    getReachableTables(entryPointId: string): Promise<string[]>;
+    /**
+     * Get sensitive data reachable from an entry point
+     */
+    getReachableSensitiveData(entryPointId: string): Promise<string[]>;
+    invalidateCache(index?: IndexType): void;
+    hasIndex(index: IndexType): Promise<boolean>;
+    deleteIndex(index: IndexType): Promise<void>;
+    /**
+     * Rebuild all indexes from patterns
+     */
+    rebuildAllIndexes(patterns: Pattern[]): Promise<void>;
+}
+export declare function createIndexStore(config?: Partial<DataLakeConfig>): IndexStore;
+//# sourceMappingURL=index-store.d.ts.map

package/dist/lake/index-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/lake/index-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EACV,SAAS,EACT,aAAa,EACb,UAAU,EACV,eAAe,EACf,cAAc,EACd,SAAS,EACV,MAAM,YAAY,CAAC;AAQpB,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAuBlE,qBAAa,UAAW,SAAQ,YAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAGpC,OAAO,CAAC,cAAc,CAA0B;IAChD,OAAO,CAAC,kBAAkB,CAA8B;IACxD,OAAO,CAAC,eAAe,CAA2B;IAClD,OAAO,CAAC,oBAAoB,CAAgC;gBAEhD,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM;IAc1C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,YAAY,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAmBzC,aAAa,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAQpD;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,SAAS;IAgC9C;;OAEG;IACG,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAK3D;;OAEG;IACG,eAAe,CACnB,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAAE,EACpB,cAAc,CAAC,EAAE,MAAM,EAAE,EACzB,WAAW,CAAC,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,IAAI,CAAC;IA2BV,gBAAgB,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmBjD,iBAAiB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ5D;;OAEG;IACH,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,aAAa;IAoBtD;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAStE,aAAa,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAmB3C,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtD;;OAEG;IACG,yBAAyB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKjE;;OAEG;IACG,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IASxD,kBAAkB,IAAI,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAmBrD,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAQhE;;OAEG;IACG,qBAAqB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKpE;;OAEG;IACG,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKjE;;OAEG;IACG,yBAAyB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IASxE,eAAe,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,IAAI;IA6BlC,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAY5C,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBlD;;OAEG;IACG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAS5D;AAMD,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GAAG,UAAU,CAEjF"}