driftdetect-core 0.4.1 → 0.4.3

package/dist/lake/index-store.js

@@ -0,0 +1,359 @@
+/**
+ * Index Store
+ *
+ * Manages pre-built indexes for fast lookups.
+ * Indexes map common query dimensions to IDs for O(1) access.
+ *
+ * Key indexes:
+ * - by-file: file path → pattern/function/access IDs
+ * - by-category: category → pattern IDs
+ * - by-table: table name → access point/function IDs
+ * - by-entry-point: entry point → reachable function IDs
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { EventEmitter } from 'node:events';
+import { LAKE_DIRS, INDEX_FILES, DEFAULT_DATA_LAKE_CONFIG, } from './types.js';
+// ============================================================================
+// Helper Functions
+// ============================================================================
+async function fileExists(filePath) {
+    try {
+        await fs.access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function ensureDir(dirPath) {
+    await fs.mkdir(dirPath, { recursive: true });
+}
+// ============================================================================
+// Index Store Class
+// ============================================================================
+export class IndexStore extends EventEmitter {
+    config;
+    indexesDir;
+    // In-memory cache
+    fileIndexCache = null;
+    categoryIndexCache = null;
+    tableIndexCache = null;
+    entryPointIndexCache = null;
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_DATA_LAKE_CONFIG, ...config };
+        this.indexesDir = path.join(this.config.rootDir, LAKE_DIRS.root, LAKE_DIRS.indexes);
+    }
+    // ==========================================================================
+    // Initialization
+    // ==========================================================================
+    async initialize() {
+        await ensureDir(this.indexesDir);
+    }
+    // ==========================================================================
+    // File Index
+    // ==========================================================================
+    async getFileIndex() {
+        if (this.fileIndexCache) {
+            return this.fileIndexCache;
+        }
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byFile);
+        if (!(await fileExists(filePath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            this.fileIndexCache = JSON.parse(content);
+            return this.fileIndexCache;
+        }
+        catch {
+            return null;
+        }
+    }
+    async saveFileIndex(index) {
+        await ensureDir(this.indexesDir);
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byFile);
+        await fs.writeFile(filePath, JSON.stringify(index, null, 2));
+        this.fileIndexCache = index;
+        this.emit('index:saved', 'byFile', index);
+    }
+    /**
+     * Build file index from patterns
+     */
+    buildFileIndex(patterns) {
+        const patternsByFile = {};
+        for (const pattern of patterns) {
+            for (const location of pattern.locations) {
+                if (!patternsByFile[location.file]) {
+                    patternsByFile[location.file] = [];
+                }
+                const filePatterns = patternsByFile[location.file];
+                if (filePatterns && !filePatterns.includes(pattern.id)) {
+                    filePatterns.push(pattern.id);
+                }
+            }
+            for (const outlier of pattern.outliers) {
+                if (!patternsByFile[outlier.file]) {
+                    patternsByFile[outlier.file] = [];
+                }
+                const filePatterns = patternsByFile[outlier.file];
+                if (filePatterns && !filePatterns.includes(pattern.id)) {
+                    filePatterns.push(pattern.id);
+                }
+            }
+        }
+        return {
+            generatedAt: new Date().toISOString(),
+            patterns: patternsByFile,
+            accessPoints: {}, // Populated by boundary scanner
+            functions: {}, // Populated by call graph builder
+        };
+    }
+    /**
+     * Get pattern IDs for a specific file
+     */
+    async getPatternIdsForFile(file) {
+        const index = await this.getFileIndex();
+        return index?.patterns[file] ?? [];
+    }
+    /**
+     * Update file index for specific files (incremental)
+     */
+    async updateFileIndex(file, patternIds, accessPointIds, functionIds) {
+        let index = await this.getFileIndex();
+        if (!index) {
+            index = {
+                generatedAt: new Date().toISOString(),
+                patterns: {},
+                accessPoints: {},
+                functions: {},
+            };
+        }
+        index.patterns[file] = patternIds;
+        if (accessPointIds) {
+            index.accessPoints[file] = accessPointIds;
+        }
+        if (functionIds) {
+            index.functions[file] = functionIds;
+        }
+        index.generatedAt = new Date().toISOString();
+        await this.saveFileIndex(index);
+    }
+    // ==========================================================================
+    // Category Index
+    // ==========================================================================
+    async getCategoryIndex() {
+        if (this.categoryIndexCache) {
+            return this.categoryIndexCache;
+        }
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byCategory);
+        if (!(await fileExists(filePath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            this.categoryIndexCache = JSON.parse(content);
+            return this.categoryIndexCache;
+        }
+        catch {
+            return null;
+        }
+    }
+    async saveCategoryIndex(index) {
+        await ensureDir(this.indexesDir);
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byCategory);
+        await fs.writeFile(filePath, JSON.stringify(index, null, 2));
+        this.categoryIndexCache = index;
+        this.emit('index:saved', 'byCategory', index);
+    }
+    /**
+     * Build category index from patterns
+     */
+    buildCategoryIndex(patterns) {
+        const patternsByCategory = {};
+        const counts = {};
+        for (const pattern of patterns) {
+            if (!patternsByCategory[pattern.category]) {
+                patternsByCategory[pattern.category] = [];
+                counts[pattern.category] = 0;
+            }
+            patternsByCategory[pattern.category].push(pattern.id);
+            counts[pattern.category]++;
+        }
+        return {
+            generatedAt: new Date().toISOString(),
+            patterns: patternsByCategory,
+            counts,
+        };
+    }
+    /**
+     * Get pattern IDs for a specific category
+     */
+    async getPatternIdsForCategory(category) {
+        const index = await this.getCategoryIndex();
+        return index?.patterns[category] ?? [];
+    }
+    // ==========================================================================
+    // Table Index
+    // ==========================================================================
+    async getTableIndex() {
+        if (this.tableIndexCache) {
+            return this.tableIndexCache;
+        }
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byTable);
+        if (!(await fileExists(filePath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            this.tableIndexCache = JSON.parse(content);
+            return this.tableIndexCache;
+        }
+        catch {
+            return null;
+        }
+    }
+    async saveTableIndex(index) {
+        await ensureDir(this.indexesDir);
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byTable);
+        await fs.writeFile(filePath, JSON.stringify(index, null, 2));
+        this.tableIndexCache = index;
+        this.emit('index:saved', 'byTable', index);
+    }
+    /**
+     * Get access point IDs for a specific table
+     */
+    async getAccessPointIdsForTable(table) {
+        const index = await this.getTableIndex();
+        return index?.accessPoints[table] ?? [];
+    }
+    /**
+     * Get function IDs that access a specific table
+     */
+    async getAccessorIdsForTable(table) {
+        const index = await this.getTableIndex();
+        return index?.accessors[table] ?? [];
+    }
+    // ==========================================================================
+    // Entry Point Index
+    // ==========================================================================
+    async getEntryPointIndex() {
+        if (this.entryPointIndexCache) {
+            return this.entryPointIndexCache;
+        }
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byEntryPoint);
+        if (!(await fileExists(filePath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            this.entryPointIndexCache = JSON.parse(content);
+            return this.entryPointIndexCache;
+        }
+        catch {
+            return null;
+        }
+    }
+    async saveEntryPointIndex(index) {
+        await ensureDir(this.indexesDir);
+        const filePath = path.join(this.indexesDir, INDEX_FILES.byEntryPoint);
+        await fs.writeFile(filePath, JSON.stringify(index, null, 2));
+        this.entryPointIndexCache = index;
+        this.emit('index:saved', 'byEntryPoint', index);
+    }
+    /**
+     * Get reachable function IDs from an entry point
+     */
+    async getReachableFunctions(entryPointId) {
+        const index = await this.getEntryPointIndex();
+        return index?.reachable[entryPointId] ?? [];
+    }
+    /**
+     * Get tables reachable from an entry point
+     */
+    async getReachableTables(entryPointId) {
+        const index = await this.getEntryPointIndex();
+        return index?.tables[entryPointId] ?? [];
+    }
+    /**
+     * Get sensitive data reachable from an entry point
+     */
+    async getReachableSensitiveData(entryPointId) {
+        const index = await this.getEntryPointIndex();
+        return index?.sensitiveData[entryPointId] ?? [];
+    }
+    // ==========================================================================
+    // Cache Management
+    // ==========================================================================
+    invalidateCache(index) {
+        if (!index) {
+            this.fileIndexCache = null;
+            this.categoryIndexCache = null;
+            this.tableIndexCache = null;
+            this.entryPointIndexCache = null;
+            return;
+        }
+        switch (index) {
+            case 'byFile':
+                this.fileIndexCache = null;
+                break;
+            case 'byCategory':
+                this.categoryIndexCache = null;
+                break;
+            case 'byTable':
+                this.tableIndexCache = null;
+                break;
+            case 'byEntryPoint':
+                this.entryPointIndexCache = null;
+                break;
+        }
+    }
+    // ==========================================================================
+    // Bulk Operations
+    // ==========================================================================
+    async hasIndex(index) {
+        const files = {
+            byFile: INDEX_FILES.byFile,
+            byCategory: INDEX_FILES.byCategory,
+            byTable: INDEX_FILES.byTable,
+            byEntryPoint: INDEX_FILES.byEntryPoint,
+        };
+        const filePath = path.join(this.indexesDir, files[index]);
+        return fileExists(filePath);
+    }
+    async deleteIndex(index) {
+        const files = {
+            byFile: INDEX_FILES.byFile,
+            byCategory: INDEX_FILES.byCategory,
+            byTable: INDEX_FILES.byTable,
+            byEntryPoint: INDEX_FILES.byEntryPoint,
+        };
+        const filePath = path.join(this.indexesDir, files[index]);
+        try {
+            await fs.unlink(filePath);
+        }
+        catch {
+            // Ignore if doesn't exist
+        }
+        this.invalidateCache(index);
+    }
+    /**
+     * Rebuild all indexes from patterns
+     */
+    async rebuildAllIndexes(patterns) {
+        const fileIndex = this.buildFileIndex(patterns);
+        const categoryIndex = this.buildCategoryIndex(patterns);
+        await Promise.all([
+            this.saveFileIndex(fileIndex),
+            this.saveCategoryIndex(categoryIndex),
+        ]);
+    }
+}
+// ============================================================================
+// Factory Function
+// ============================================================================
+export function createIndexStore(config = {}) {
+    return new IndexStore(config);
+}
+//# sourceMappingURL=index-store.js.map

package/dist/lake/index-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-store.js","sourceRoot":"","sources":["../../src/lake/index-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAW3C,OAAO,EACL,SAAS,EACT,WAAW,EACX,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAIpB,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAe;IACtC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E,MAAM,OAAO,UAAW,SAAQ,YAAY;IACzB,MAAM,CAAiB;IACvB,UAAU,CAAS;IAEpC,kBAAkB;IACV,cAAc,GAAqB,IAAI,CAAC;IACxC,kBAAkB,GAAyB,IAAI,CAAC;IAChD,eAAe,GAAsB,IAAI,CAAC;IAC1C,oBAAoB,GAA2B,IAAI,CAAC;IAE5D,YAAY,SAAkC,EAAE;QAC9C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QACzD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CACzB,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,SAAS,CAAC,IAAI,EACd,SAAS,CAAC,OAAO,CAClB,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E,KAAK,CAAC,UAAU;QACd,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IAED,6EAA6E;IAC7E,aAAa;IACb,6EAA6E;IAE7E,KAAK,CAAC,YAAY;QAChB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,cAAc,CAAC;QAC7B,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;YACvD,OAAO,IAAI,CAAC,cAAc,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAgB;QAClC,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAChE,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,QAAmB;QAChC,MAAM,cAAc,GAA6B,EAAE,CAAC;QAEpD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACzC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACrC,CAAC;gBACD,MAAM,YAAY,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACnD,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;oBACvD,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBAClC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACpC,CAAC;gBACD,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;oBACvD,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,QAAQ,EAAE,cAAc;YACxB,YAAY,EAAE,EAAE,EAAE,gCAAgC;YAClD,SAAS,EAAE,EAAE,EAAE,kCAAkC;SAClD,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,IAAY;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACxC,OAAO,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CACnB,IAAY,EACZ,UAAoB,EACpB,cAAyB,EACzB,WAAsB;QAEtB,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG;gBACN,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,QAAQ,EAAE,EAAE;gBACZ,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aACd,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;QAClC,IAAI,cAAc,EAAE,CAAC;YACnB,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC;QAC5C,CAAC;QACD,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC;QACtC,CAAC;QACD,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE7C,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E,KAAK,CAAC,gBAAgB;QACpB,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACjC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;QACpE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;YAC/D,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,KAAoB;QAC1C,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;QACpE,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,QAAmB;QACpC,MAAM,kBAAkB,GAAsC,EAAuC,CAAC;QACtG,MAAM,MAAM,GAAoC,EAAqC,CAAC;QAEtF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC1C,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAC1C,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;YACD,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACtD,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,OAAO;YACL,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,QAAQ,EAAE,kBAAkB;YAC5B,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,wBAAwB,CAAC,QAAyB;QACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC5C,OAAO,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,6EAA6E;IAC7E,cAAc;IACd,6EAA6E;IAE7E,KAAK,CAAC,aAAa;QACjB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;QACjE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAe,CAAC;YACzD,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAiB;QACpC,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,yBAAyB,CAAC,KAAa;QAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QACzC,OAAO,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB,CAAC,KAAa;QACxC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QACzC,OAAO,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED,6EAA6E;IAC7E,oBAAoB;IACpB,6EAA6E;IAE7E,KAAK,CAAC,kBAAkB;QACtB,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,oBAAoB,CAAC;QACnC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;QACtE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoB,CAAC;YACnE,OAAO,IAAI,CAAC,oBAAoB,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAsB;QAC9C,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;QACtE,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB,CAAC,YAAoB;QAC9C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC9C,OAAO,KAAK,EAAE,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,YAAoB;QAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC9C,OAAO,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,yBAAyB,CAAC,YAAoB;QAClD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC9C,OAAO,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IAED,6EAA6E;IAC7E,mBAAmB;IACnB,6EAA6E;IAE7E,eAAe,CAAC,KAAiB;QAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC3B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAC/B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;YACjC,OAAO;QACT,CAAC;QAED,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,QAAQ;gBACX,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,MAAM;YACR,KAAK,YAAY;gBACf,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;gBAC/B,MAAM;YACR,KAAK,SAAS;gBACZ,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;gBAC5B,MAAM;YACR,KAAK,cAAc;gBACjB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;gBACjC,MAAM;QACV,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAE7E,KAAK,CAAC,QAAQ,CAAC,KAAgB;QAC7B,MAAM,KAAK,GAA8B;YACvC,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,OAAO,EAAE,WAAW,CAAC,OAAO;YAC5B,YAAY,EAAE,WAAW,CAAC,YAAY;SACvC,CAAC;QAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1D,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,KAAgB;QAChC,MAAM,KAAK,GAA8B;YACvC,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,OAAO,EAAE,WAAW,CAAC,OAAO;YAC5B,YAAY,EAAE,WAAW,CAAC,YAAY;SACvC,CAAC;QAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,QAAmB;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC;YAC7B,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC;SACtC,CAAC,CAAC;IACL,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,MAAM,UAAU,gBAAgB,CAAC,SAAkC,EAAE;IACnE,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC"}

package/dist/lake/index.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Data Lake Module
+ *
+ * Enterprise-grade data storage with materialized views, indexes, and query optimization.
+ *
+ * Architecture:
+ * - ManifestStore: Quick-load index of everything (instant drift_status)
+ * - ViewStore: Pre-computed views for common queries
+ * - IndexStore: Fast lookups by file, category, table, entry point
+ * - QueryEngine: Unified query API with automatic view/index selection
+ * - ViewMaterializer: Rebuilds views after scans
+ *
+ * Usage:
+ * ```typescript
+ * import { createDataLake } from './lake';
+ *
+ * const lake = createDataLake({ rootDir: '/path/to/project' });
+ * await lake.initialize();
+ *
+ * // Instant status (reads pre-computed view)
+ * const status = await lake.query.getStatus();
+ *
+ * // Fast pattern listing (uses index + view)
+ * const patterns = await lake.query.getPatterns({ categories: ['api'] });
+ *
+ * // After scan, materialize views
+ * await lake.materializer.materialize(patterns, {}, { lastScan: scanInfo });
+ * ```
+ */
+export type { DriftManifest, ManifestStats, PatternStats as LakePatternStats, SecurityStats as LakeSecurityStats, CallGraphStats as LakeCallGraphStats, ContractStats as LakeContractStats, DNAStats as LakeDNAStats, LastScanInfo, ViewFreshness, ViewMeta, StatusView, PatternIndexView, PatternSummary as LakePatternSummary, SecuritySummaryView, TrendsView, HealthFactor, TopIssue, SensitiveTableSummary, ViolationSummary, TrendItem, CategoryTrend, FileIndex, CategoryIndex, TableIndex, EntryPointIndex, Shard, PatternShard, PatternShardEntry, PatternLocation as LakePatternLocation, PatternMetadata as LakePatternMetadata, AccessMapShard, SensitiveFieldInfo, AccessPointEntry, CallGraphShard, FunctionEntry, DataAccessRef, PatternExamples, CodeExample, IncrementalUpdate, ViewType, IndexType, DataLakeConfig, } from './types.js';
+export { LAKE_VERSION, LAKE_DIRS, VIEW_FILES, INDEX_FILES, DEFAULT_DATA_LAKE_CONFIG, } from './types.js';
+export { ManifestStore as LakeManifestStore, createManifestStore } from './manifest-store.js';
+export { ViewStore, createViewStore } from './view-store.js';
+export { IndexStore, createIndexStore } from './index-store.js';
+export { PatternShardStore, createPatternShardStore } from './pattern-shard-store.js';
+export { SecurityShardStore, createSecurityShardStore, type SecurityIndex, type TableIndexEntry, type SensitiveTableEntry, type ViolationEntry, type SensitiveFieldRegistry, type SensitiveFieldRecord, } from './security-shard-store.js';
+export { ExamplesStore, createExamplesStore, type ExamplesIndex, type ExampleIndexEntry, type ExampleExtractionOptions, } from './examples-store.js';
+export { CallGraphShardStore, createCallGraphShardStore, type CallGraphIndex, type FileIndexEntry as CallGraphFileIndexEntry, type EntryPointSummary, type DataAccessorSummary, type EntryPointsData, type EntryPointDetail, } from './callgraph-shard-store.js';
+export { QueryEngine, createQueryEngine, type PaginationOptions, type PaginatedResult, type PatternQueryOptions, type QueryStats, } from './query-engine.js';
+export { ViewMaterializer, createViewMaterializer, type MaterializeOptions, type MaterializeResult, } from './view-materializer.js';
+import type { DataLakeConfig } from './types.js';
+import { ManifestStore } from './manifest-store.js';
+import { ViewStore } from './view-store.js';
+import { IndexStore } from './index-store.js';
+import { PatternShardStore } from './pattern-shard-store.js';
+import { SecurityShardStore } from './security-shard-store.js';
+import { ExamplesStore } from './examples-store.js';
+import { CallGraphShardStore } from './callgraph-shard-store.js';
+import { QueryEngine } from './query-engine.js';
+import { ViewMaterializer } from './view-materializer.js';
+import type { Pattern } from '../store/types.js';
+/**
+ * Data Lake - Unified interface for all data lake operations
+ */
+export declare class DataLake {
+    readonly config: DataLakeConfig;
+    readonly manifest: ManifestStore;
+    readonly views: ViewStore;
+    readonly indexes: IndexStore;
+    readonly patternShards: PatternShardStore;
+    readonly securityShards: SecurityShardStore;
+    readonly examples: ExamplesStore;
+    readonly callGraphShards: CallGraphShardStore;
+    readonly query: QueryEngine;
+    readonly materializer: ViewMaterializer;
+    private initialized;
+    constructor(config?: Partial<DataLakeConfig>);
+    /**
+     * Initialize the data lake
+     */
+    initialize(): Promise<void>;
+    /**
+     * Set the raw pattern loader for fallback queries
+     */
+    setPatternLoader(loader: () => Promise<Pattern[]>): void;
+    /**
+     * Check if the data lake has been initialized with data
+     */
+    hasData(): boolean;
+    /**
+     * Get quick stats (instant, from manifest)
+     */
+    getQuickStats(): import("./types.js").ManifestStats;
+    /**
+     * Invalidate all caches and views
+     */
+    invalidateAll(): void;
+}
+/**
+ * Create a new DataLake instance
+ */
+export declare function createDataLake(config?: Partial<DataLakeConfig>): DataLake;
+//# sourceMappingURL=index.d.ts.map

package/dist/lake/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lake/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,YAAY,EAEV,aAAa,EACb,aAAa,EACb,YAAY,IAAI,gBAAgB,EAChC,aAAa,IAAI,iBAAiB,EAClC,cAAc,IAAI,kBAAkB,EACpC,aAAa,IAAI,iBAAiB,EAClC,QAAQ,IAAI,YAAY,EACxB,YAAY,EACZ,aAAa,EACb,QAAQ,EAGR,UAAU,EACV,gBAAgB,EAChB,cAAc,IAAI,kBAAkB,EACpC,mBAAmB,EACnB,UAAU,EACV,YAAY,EACZ,QAAQ,EACR,qBAAqB,EACrB,gBAAgB,EAChB,SAAS,EACT,aAAa,EAGb,SAAS,EACT,aAAa,EACb,UAAU,EACV,eAAe,EAGf,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,eAAe,IAAI,mBAAmB,EACtC,eAAe,IAAI,mBAAmB,EACtC,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,aAAa,EAGb,eAAe,EACf,WAAW,EAGX,iBAAiB,EACjB,QAAQ,EACR,SAAS,EAGT,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,YAAY,EACZ,SAAS,EACT,UAAU,EACV,WAAW,EACX,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9F,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACtF,OAAO,EACL,kBAAkB,EAClB,wBAAwB,EACxB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,GAC1B,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EACL,aAAa,EACb,mBAAmB,EACnB,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,GAC9B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,mBAAmB,EACnB,yBAAyB,EACzB,KAAK,cAAc,EACnB,KAAK,cAAc,IAAI,uBAAuB,EAC9C,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,gBAAgB,GACtB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,UAAU,GAChB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,GACvB,MAAM,wBAAwB,CAAC;AAMhC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAEjD;;GAEG;AACH,qBAAa,QAAQ;IACnB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAC;IAC1C,QAAQ,CAAC,cAAc,EAAE,kBAAkB,CAAC;IAC5C,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,mBAAmB,CAAC;IAC9C,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IAExC,OAAO,CAAC,WAAW,CAAS;gBAEhB,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM;IA8BhD;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBjC;;OAEG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;IAIxD;;OAEG;IACH,OAAO,IAAI,OAAO;IAKlB;;OAEG;IACH,aAAa;IAIb;;OAEG;IACH,aAAa,IAAI,IAAI;CAStB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GAAG,QAAQ,CAE7E"}

package/dist/lake/index.js

@@ -0,0 +1,138 @@
+/**
+ * Data Lake Module
+ *
+ * Enterprise-grade data storage with materialized views, indexes, and query optimization.
+ *
+ * Architecture:
+ * - ManifestStore: Quick-load index of everything (instant drift_status)
+ * - ViewStore: Pre-computed views for common queries
+ * - IndexStore: Fast lookups by file, category, table, entry point
+ * - QueryEngine: Unified query API with automatic view/index selection
+ * - ViewMaterializer: Rebuilds views after scans
+ *
+ * Usage:
+ * ```typescript
+ * import { createDataLake } from './lake';
+ *
+ * const lake = createDataLake({ rootDir: '/path/to/project' });
+ * await lake.initialize();
+ *
+ * // Instant status (reads pre-computed view)
+ * const status = await lake.query.getStatus();
+ *
+ * // Fast pattern listing (uses index + view)
+ * const patterns = await lake.query.getPatterns({ categories: ['api'] });
+ *
+ * // After scan, materialize views
+ * await lake.materializer.materialize(patterns, {}, { lastScan: scanInfo });
+ * ```
+ */
+export { LAKE_VERSION, LAKE_DIRS, VIEW_FILES, INDEX_FILES, DEFAULT_DATA_LAKE_CONFIG, } from './types.js';
+// Stores - Use Lake prefix to avoid conflicts
+export { ManifestStore as LakeManifestStore, createManifestStore } from './manifest-store.js';
+export { ViewStore, createViewStore } from './view-store.js';
+export { IndexStore, createIndexStore } from './index-store.js';
+export { PatternShardStore, createPatternShardStore } from './pattern-shard-store.js';
+export { SecurityShardStore, createSecurityShardStore, } from './security-shard-store.js';
+export { ExamplesStore, createExamplesStore, } from './examples-store.js';
+export { CallGraphShardStore, createCallGraphShardStore, } from './callgraph-shard-store.js';
+// Query Engine
+export { QueryEngine, createQueryEngine, } from './query-engine.js';
+// View Materializer
+export { ViewMaterializer, createViewMaterializer, } from './view-materializer.js';
+import { DEFAULT_DATA_LAKE_CONFIG } from './types.js';
+import { ManifestStore } from './manifest-store.js';
+import { ViewStore } from './view-store.js';
+import { IndexStore } from './index-store.js';
+import { PatternShardStore } from './pattern-shard-store.js';
+import { SecurityShardStore } from './security-shard-store.js';
+import { ExamplesStore } from './examples-store.js';
+import { CallGraphShardStore } from './callgraph-shard-store.js';
+import { QueryEngine } from './query-engine.js';
+import { ViewMaterializer } from './view-materializer.js';
+/**
+ * Data Lake - Unified interface for all data lake operations
+ */
+export class DataLake {
+    config;
+    manifest;
+    views;
+    indexes;
+    patternShards;
+    securityShards;
+    examples;
+    callGraphShards;
+    query;
+    materializer;
+    initialized = false;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_DATA_LAKE_CONFIG, ...config };
+        // Create stores
+        this.manifest = new ManifestStore(this.config);
+        this.views = new ViewStore(this.config);
+        this.indexes = new IndexStore(this.config);
+        this.patternShards = new PatternShardStore(this.config);
+        this.securityShards = new SecurityShardStore(this.config);
+        this.examples = new ExamplesStore(this.config);
+        this.callGraphShards = new CallGraphShardStore(this.config);
+        // Create query engine with stores
+        this.query = new QueryEngine(this.config, this.manifest, this.views, this.indexes, this.patternShards);
+        // Create materializer with stores
+        this.materializer = new ViewMaterializer(this.config, this.manifest, this.views, this.indexes);
+    }
+    /**
+     * Initialize the data lake
+     */
+    async initialize() {
+        if (this.initialized)
+            return;
+        await Promise.all([
+            this.manifest.initialize(),
+            this.views.initialize(),
+            this.indexes.initialize(),
+            this.patternShards.initialize(),
+            this.securityShards.initialize(),
+            this.examples.initialize(),
+            this.callGraphShards.initialize(),
+        ]);
+        this.initialized = true;
+    }
+    /**
+     * Set the raw pattern loader for fallback queries
+     */
+    setPatternLoader(loader) {
+        this.query.setRawPatternLoader(loader);
+    }
+    /**
+     * Check if the data lake has been initialized with data
+     */
+    hasData() {
+        const stats = this.manifest.getStats();
+        return stats.patterns.total > 0;
+    }
+    /**
+     * Get quick stats (instant, from manifest)
+     */
+    getQuickStats() {
+        return this.manifest.getStats();
+    }
+    /**
+     * Invalidate all caches and views
+     */
+    invalidateAll() {
+        this.manifest.markAllViewsStale('full invalidation');
+        this.views.invalidateCache();
+        this.indexes.invalidateCache();
+        this.patternShards.invalidateCache();
+        this.securityShards.invalidateCache();
+        this.examples.invalidateCache();
+        this.callGraphShards.invalidateCache();
+    }
+}
+/**
+ * Create a new DataLake instance
+ */
+export function createDataLake(config = {}) {
+    return new DataLake(config);
+}
+//# sourceMappingURL=index.js.map

package/dist/lake/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/lake/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AA6DH,OAAO,EACL,YAAY,EACZ,SAAS,EACT,UAAU,EACV,WAAW,EACX,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAEpB,8CAA8C;AAC9C,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9F,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACtF,OAAO,EACL,kBAAkB,EAClB,wBAAwB,GAOzB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EACL,aAAa,EACb,mBAAmB,GAIpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,mBAAmB,EACnB,yBAAyB,GAO1B,MAAM,4BAA4B,CAAC;AAEpC,eAAe;AACf,OAAO,EACL,WAAW,EACX,iBAAiB,GAKlB,MAAM,mBAAmB,CAAC;AAE3B,oBAAoB;AACpB,OAAO,EACL,gBAAgB,EAChB,sBAAsB,GAGvB,MAAM,wBAAwB,CAAC;AAOhC,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAG1D;;GAEG;AACH,MAAM,OAAO,QAAQ;IACV,MAAM,CAAiB;IACvB,QAAQ,CAAgB;IACxB,KAAK,CAAY;IACjB,OAAO,CAAa;IACpB,aAAa,CAAoB;IACjC,cAAc,CAAqB;IACnC,QAAQ,CAAgB;IACxB,eAAe,CAAsB;IACrC,KAAK,CAAc;IACnB,YAAY,CAAmB;IAEhC,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,SAAkC,EAAE;QAC9C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QAEzD,gBAAgB;QAChB,IAAI,CAAC,QAAQ,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,KAAK,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,aAAa,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,eAAe,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5D,kCAAkC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,WAAW,CAC1B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,aAAa,CACnB,CAAC;QAEF,kCAAkC;QAClC,IAAI,CAAC,YAAY,GAAG,IAAI,gBAAgB,CACtC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE;YAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE;YACvB,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE;YACzB,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE;YAChC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE;YAC1B,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,MAAgC;QAC/C,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,OAAO;QACL,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC/B,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,CAAC;QACrC,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,CAAC;IACzC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,SAAkC,EAAE;IACjE,OAAO,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC9B,CAAC"}