driftdetect-core 0.4.1 → 0.4.2

package/dist/lake/pattern-shard-store.js

@@ -0,0 +1,347 @@
+/**
+ * Pattern Shard Store
+ *
+ * Manages sharded pattern storage - one file per category.
+ * This allows loading only the patterns you need instead of all patterns.
+ *
+ * Storage structure:
+ * .drift/lake/patterns/
+ *   ├── api.json
+ *   ├── auth.json
+ *   ├── security.json
+ *   └── ...
+ *
+ * Key features:
+ * - Load patterns by category (not all at once)
+ * - Incremental updates per category
+ * - Category-level checksums for cache invalidation
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
+import { EventEmitter } from 'node:events';
+import { LAKE_DIRS, DEFAULT_DATA_LAKE_CONFIG, } from './types.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const PATTERNS_SHARD_DIR = 'patterns';
+const SHARD_VERSION = '1.0.0';
+// ============================================================================
+// Helper Functions
+// ============================================================================
+async function fileExists(filePath) {
+    try {
+        await fs.access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function ensureDir(dirPath) {
+    await fs.mkdir(dirPath, { recursive: true });
+}
+function generateChecksum(data) {
+    const content = JSON.stringify(data);
+    return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+// ============================================================================
+// Pattern Shard Store Class
+// ============================================================================
+export class PatternShardStore extends EventEmitter {
+    config;
+    shardsDir;
+    // In-memory cache of loaded shards
+    shardCache = new Map();
+    shardChecksums = new Map();
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_DATA_LAKE_CONFIG, ...config };
+        this.shardsDir = path.join(this.config.rootDir, LAKE_DIRS.root, LAKE_DIRS.lake, PATTERNS_SHARD_DIR);
+    }
+    // ==========================================================================
+    // Initialization
+    // ==========================================================================
+    async initialize() {
+        await ensureDir(this.shardsDir);
+    }
+    // ==========================================================================
+    // Reading Shards
+    // ==========================================================================
+    /**
+     * Get patterns for a specific category (loads shard if needed)
+     */
+    async getByCategory(category) {
+        const shard = await this.loadShard(category);
+        return shard?.patterns ?? [];
+    }
+    /**
+     * Get patterns for multiple categories
+     */
+    async getByCategories(categories) {
+        const results = [];
+        // Load shards in parallel
+        const shards = await Promise.all(categories.map(cat => this.loadShard(cat)));
+        for (const shard of shards) {
+            if (shard) {
+                results.push(...shard.patterns);
+            }
+        }
+        return results;
+    }
+    /**
+     * Get a single pattern by ID (searches all shards or specific category)
+     */
+    async getById(id, category) {
+        if (category) {
+            const patterns = await this.getByCategory(category);
+            return patterns.find(p => p.id === id) ?? null;
+        }
+        // Search all cached shards first
+        for (const shard of this.shardCache.values()) {
+            const pattern = shard.patterns.find(p => p.id === id);
+            if (pattern)
+                return pattern;
+        }
+        // Load all shards and search
+        const categories = await this.listCategories();
+        for (const cat of categories) {
+            const patterns = await this.getByCategory(cat);
+            const pattern = patterns.find(p => p.id === id);
+            if (pattern)
+                return pattern;
+        }
+        return null;
+    }
+    /**
+     * Get all patterns (loads all shards)
+     */
+    async getAll() {
+        const categories = await this.listCategories();
+        return this.getByCategories(categories);
+    }
+    /**
+     * List available categories (based on shard files)
+     */
+    async listCategories() {
+        if (!(await fileExists(this.shardsDir))) {
+            return [];
+        }
+        const files = await fs.readdir(this.shardsDir);
+        return files
+            .filter(f => f.endsWith('.json'))
+            .map(f => f.replace('.json', ''));
+    }
+    /**
+     * Get pattern count by category (without loading full patterns)
+     */
+    async getCategoryCounts() {
+        const counts = {};
+        const categories = await this.listCategories();
+        for (const category of categories) {
+            const shard = await this.loadShard(category);
+            counts[category] = shard?.patterns.length ?? 0;
+        }
+        return counts;
+    }
+    // ==========================================================================
+    // Writing Shards
+    // ==========================================================================
+    /**
+     * Save patterns for a category (creates/updates shard)
+     */
+    async saveShard(category, patterns) {
+        await ensureDir(this.shardsDir);
+        const shard = {
+            category,
+            patterns,
+        };
+        const filePath = path.join(this.shardsDir, `${category}.json`);
+        const checksum = generateChecksum(patterns);
+        const shardFile = {
+            version: SHARD_VERSION,
+            generatedAt: new Date().toISOString(),
+            checksum,
+            ...shard,
+        };
+        await fs.writeFile(filePath, JSON.stringify(shardFile, null, 2));
+        // Update cache
+        this.shardCache.set(category, shard);
+        this.shardChecksums.set(category, checksum);
+        this.emit('shard:saved', category, patterns.length);
+    }
+    /**
+     * Save all patterns (shards by category)
+     */
+    async saveAll(patterns) {
+        // Group patterns by category
+        const byCategory = new Map();
+        for (const pattern of patterns) {
+            if (!byCategory.has(pattern.category)) {
+                byCategory.set(pattern.category, []);
+            }
+            byCategory.get(pattern.category).push(this.patternToShardEntry(pattern));
+        }
+        // Save each category shard
+        const savePromises = [];
+        for (const [category, categoryPatterns] of byCategory) {
+            savePromises.push(this.saveShard(category, categoryPatterns));
+        }
+        await Promise.all(savePromises);
+        this.emit('shards:saved', byCategory.size);
+    }
+    /**
+     * Delete a category shard
+     */
+    async deleteShard(category) {
+        const filePath = path.join(this.shardsDir, `${category}.json`);
+        try {
+            await fs.unlink(filePath);
+            this.shardCache.delete(category);
+            this.shardChecksums.delete(category);
+            this.emit('shard:deleted', category);
+        }
+        catch {
+            // Ignore if doesn't exist
+        }
+    }
+    // ==========================================================================
+    // Cache Management
+    // ==========================================================================
+    /**
+     * Check if a shard has changed (by checksum)
+     */
+    async hasShardChanged(category, newChecksum) {
+        const currentChecksum = this.shardChecksums.get(category);
+        if (!currentChecksum) {
+            // Load from disk to get checksum
+            await this.loadShard(category);
+            return this.shardChecksums.get(category) !== newChecksum;
+        }
+        return currentChecksum !== newChecksum;
+    }
+    /**
+     * Invalidate cache for a category
+     */
+    invalidateCache(category) {
+        if (category) {
+            this.shardCache.delete(category);
+            this.shardChecksums.delete(category);
+        }
+        else {
+            this.shardCache.clear();
+            this.shardChecksums.clear();
+        }
+    }
+    /**
+     * Get cache stats
+     */
+    getCacheStats() {
+        let totalPatterns = 0;
+        for (const shard of this.shardCache.values()) {
+            totalPatterns += shard.patterns.length;
+        }
+        return {
+            cachedCategories: this.shardCache.size,
+            totalPatterns,
+        };
+    }
+    // ==========================================================================
+    // Private Helpers
+    // ==========================================================================
+    async loadShard(category) {
+        // Check cache first
+        const cached = this.shardCache.get(category);
+        if (cached) {
+            return cached;
+        }
+        const filePath = path.join(this.shardsDir, `${category}.json`);
+        if (!(await fileExists(filePath))) {
+            return null;
+        }
+        try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            const data = JSON.parse(content);
+            const shard = {
+                category: data.category,
+                patterns: data.patterns,
+            };
+            // Cache it
+            this.shardCache.set(category, shard);
+            if (data.checksum) {
+                this.shardChecksums.set(category, data.checksum);
+            }
+            return shard;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Convert a full Pattern to a PatternShardEntry
+     */
+    patternToShardEntry(pattern) {
+        // Build metadata object, only including defined optional properties
+        const metadata = {
+            firstSeen: pattern.metadata.firstSeen,
+            lastSeen: pattern.metadata.lastSeen,
+        };
+        if (pattern.metadata.approvedAt !== undefined) {
+            metadata.approvedAt = pattern.metadata.approvedAt;
+        }
+        if (pattern.metadata.approvedBy !== undefined) {
+            metadata.approvedBy = pattern.metadata.approvedBy;
+        }
+        if (pattern.metadata.tags !== undefined) {
+            metadata.tags = pattern.metadata.tags;
+        }
+        return {
+            id: pattern.id,
+            name: pattern.name,
+            description: pattern.description,
+            subcategory: pattern.subcategory,
+            status: pattern.status,
+            confidence: {
+                score: pattern.confidence.score,
+                level: pattern.confidence.level,
+            },
+            severity: pattern.severity,
+            locations: pattern.locations.map(loc => {
+                const result = {
+                    file: loc.file,
+                    line: loc.line,
+                    column: loc.column,
+                };
+                if (loc.endLine !== undefined) {
+                    result.endLine = loc.endLine;
+                }
+                if (loc.endColumn !== undefined) {
+                    result.endColumn = loc.endColumn;
+                }
+                return result;
+            }),
+            outliers: pattern.outliers.map(out => {
+                const result = {
+                    file: out.file,
+                    line: out.line,
+                    column: out.column,
+                };
+                if (out.endLine !== undefined) {
+                    result.endLine = out.endLine;
+                }
+                if (out.endColumn !== undefined) {
+                    result.endColumn = out.endColumn;
+                }
+                return result;
+            }),
+            metadata,
+        };
+    }
+}
+// ============================================================================
+// Factory Function
+// ============================================================================
+export function createPatternShardStore(config = {}) {
+    return new PatternShardStore(config);
+}
+//# sourceMappingURL=pattern-shard-store.js.map

package/dist/lake/pattern-shard-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pattern-shard-store.js","sourceRoot":"","sources":["../../src/lake/pattern-shard-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAU3C,OAAO,EACL,SAAS,EACT,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAIpB,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,kBAAkB,GAAG,UAAU,CAAC;AACtC,MAAM,aAAa,GAAG,OAAO,CAAC;AAE9B,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAe;IACtC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,+EAA+E;AAC/E,4BAA4B;AAC5B,+EAA+E;AAE/E,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IAChC,MAAM,CAAiB;IACvB,SAAS,CAAS;IAEnC,mCAAmC;IAC3B,UAAU,GAAuC,IAAI,GAAG,EAAE,CAAC;IAC3D,cAAc,GAAiC,IAAI,GAAG,EAAE,CAAC;IAEjE,YAAY,SAAkC,EAAE;QAC9C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QACzD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,SAAS,CAAC,IAAI,EACd,SAAS,CAAC,IAAI,EACd,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E,KAAK,CAAC,UAAU;QACd,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAED,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,QAAyB;QAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC7C,OAAO,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,UAA6B;QACjD,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,0BAA0B;QAC1B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAC3C,CAAC;QAEF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,QAA0B;QAClD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YACpD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC;QACjD,CAAC;QAED,iCAAiC;QACjC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YACtD,IAAI,OAAO;gBAAE,OAAO,OAAO,CAAC;QAC9B,CAAC;QAED,6BAA6B;QAC7B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC/C,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAChD,IAAI,OAAO;gBAAE,OAAO,OAAO,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACxC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/C,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aAChC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAoB,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE/C,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,MAAyC,CAAC;IACnD,CAAC;IAED,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,QAA6B;QACtE,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhC,MAAM,KAAK,GAAiB;YAC1B,QAAQ;YACR,QAAQ;SACT,CAAC;QAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,QAAQ,OAAO,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAE5C,MAAM,SAAS,GAAG;YAChB,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,QAAQ;YACR,GAAG,KAAK;SACT,CAAC;QAEF,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjE,eAAe;QACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,QAAmB;QAC/B,6BAA6B;QAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;QAEnE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACtC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACvC,CAAC;YACD,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,2BAA2B;QAC3B,MAAM,YAAY,GAAoB,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC,IAAI,UAAU,EAAE,CAAC;YACtD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,QAAyB;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,QAAQ,OAAO,CAAC,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,mBAAmB;IACnB,6EAA6E;IAE7E;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,QAAyB,EAAE,WAAmB;QAClE,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1D,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,iCAAiC;YACjC,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,WAAW,CAAC;QAC3D,CAAC;QACD,OAAO,eAAe,KAAK,WAAW,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,QAA0B;QACxC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACxB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC7C,aAAa,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;QACzC,CAAC;QACD,OAAO;YACL,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YACtC,aAAa;SACd,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAErE,KAAK,CAAC,SAAS,CAAC,QAAyB;QAC/C,oBAAoB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,QAAQ,OAAO,CAAC,CAAC;QAE/D,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAEjC,MAAM,KAAK,GAAiB;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC;YAEF,WAAW;YACX,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACrC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnD,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,OAAgB;QAC1C,oEAAoE;QACpE,MAAM,QAAQ,GAAoB;YAChC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS;YACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ;SACpC,CAAC;QACF,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC9C,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;QACpD,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC9C,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;QACpD,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACxC,QAAQ,CAAC,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;QACxC,CAAC;QAED,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE;gBACV,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,KAAK;gBAC/B,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,KAAK;aAChC;YACD,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;gBACrC,MAAM,MAAM,GAAoB;oBAC9B,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,MAAM,EAAE,GAAG,CAAC,MAAM;iBACnB,CAAC;gBACF,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC9B,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC/B,CAAC;gBACD,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBAChC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;gBACnC,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC;YACF,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;gBACnC,MAAM,MAAM,GAAoB;oBAC9B,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,MAAM,EAAE,GAAG,CAAC,MAAM;iBACnB,CAAC;gBACF,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC9B,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC/B,CAAC;gBACD,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBAChC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;gBACnC,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC;YACF,QAAQ;SACT,CAAC;IACJ,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,MAAM,UAAU,uBAAuB,CAAC,SAAkC,EAAE;IAC1E,OAAO,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC"}

package/dist/lake/query-engine.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Query Engine
+ *
+ * Unified query API that knows the optimal path to answer any query.
+ * Routes queries to views, indexes, or raw data based on availability.
+ *
+ * Key features:
+ * - Automatic view/index selection
+ * - Fallback to raw data when views unavailable
+ * - Pagination with cursor support
+ * - Query result caching
+ */
+import { EventEmitter } from 'node:events';
+import type { StatusView, PatternSummary, SecuritySummaryView, TrendsView, DataLakeConfig, ViewType } from './types.js';
+import { ManifestStore } from './manifest-store.js';
+import { ViewStore } from './view-store.js';
+import { IndexStore } from './index-store.js';
+import { PatternShardStore } from './pattern-shard-store.js';
+import type { Pattern, PatternCategory, PatternStatus } from '../store/types.js';
+export interface PaginationOptions {
+    limit?: number;
+    cursor?: string;
+    offset?: number;
+}
+export interface PaginatedResult<T> {
+    items: T[];
+    total: number;
+    hasMore: boolean;
+    nextCursor?: string | undefined;
+    executionTime: number;
+    source: 'view' | 'index' | 'raw';
+}
+export interface PatternQueryOptions extends PaginationOptions {
+    categories?: PatternCategory[];
+    status?: PatternStatus;
+    minConfidence?: number;
+    search?: string;
+    file?: string;
+}
+export interface QueryStats {
+    viewHits: number;
+    indexHits: number;
+    rawHits: number;
+    shardHits: number;
+    avgResponseTime: number;
+}
+export declare class QueryEngine extends EventEmitter {
+    private readonly config;
+    private readonly manifestStore;
+    private readonly viewStore;
+    private readonly indexStore;
+    private readonly patternShardStore;
+    private stats;
+    private queryCount;
+    private rawPatternLoader?;
+    constructor(config?: Partial<DataLakeConfig>, manifestStore?: ManifestStore, viewStore?: ViewStore, indexStore?: IndexStore, patternShardStore?: PatternShardStore);
+    initialize(): Promise<void>;
+    /**
+     * Set the raw pattern loader for fallback queries
+     */
+    setRawPatternLoader(loader: () => Promise<Pattern[]>): void;
+    /**
+     * Get status - uses pre-computed view if available
+     */
+    getStatus(): Promise<StatusView | null>;
+    /**
+     * Get pattern list with pagination
+     *
+     * Query strategy:
+     * 1. If pattern index view is fresh, use it (fastest for full list)
+     * 2. If querying specific categories, use pattern shards (loads only needed data)
+     * 3. Fall back to raw data loader
+     */
+    getPatterns(options?: PatternQueryOptions): Promise<PaginatedResult<PatternSummary>>;
+    /**
+     * Get patterns for a specific file
+     */
+    getFilePatterns(file: string): Promise<PatternSummary[]>;
+    /**
+     * Get patterns by category (uses shards for efficiency)
+     */
+    getCategoryPatterns(category: PatternCategory): Promise<PatternSummary[]>;
+    /**
+     * Get security summary - uses pre-computed view if available
+     */
+    getSecuritySummary(): Promise<SecuritySummaryView | null>;
+    /**
+     * Get trends - uses pre-computed view if available
+     */
+    getTrends(): Promise<TrendsView | null>;
+    /**
+     * Get quick stats from manifest (instant)
+     */
+    getQuickStats(): import("./types.js").ManifestStats;
+    /**
+     * Get pattern stats from manifest (instant)
+     */
+    getPatternStats(): import("./types.js").PatternStats;
+    /**
+     * Get security stats from manifest (instant)
+     */
+    getSecurityStats(): import("./types.js").SecurityStats;
+    /**
+     * Check if a view is stale
+     */
+    isViewStale(view: ViewType): boolean;
+    /**
+     * Invalidate views (triggers rebuild on next query)
+     */
+    invalidateViews(views?: ViewType[]): void;
+    getQueryStats(): QueryStats;
+    resetStats(): void;
+    /**
+     * Convert a PatternShardEntry to PatternSummary
+     */
+    private shardEntryToSummary;
+    private filterPatternSummaries;
+    private encodeCursor;
+    /** Decode cursor - kept for future pagination implementation */
+    private decodeCursor;
+    private recordQuery;
+}
+export declare function createQueryEngine(config?: Partial<DataLakeConfig>, manifestStore?: ManifestStore, viewStore?: ViewStore, indexStore?: IndexStore, patternShardStore?: PatternShardStore): QueryEngine;
+//# sourceMappingURL=query-engine.d.ts.map

package/dist/lake/query-engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-engine.d.ts","sourceRoot":"","sources":["../../src/lake/query-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EACV,UAAU,EACV,cAAc,EACd,mBAAmB,EACnB,UAAU,EACV,cAAc,EACd,QAAQ,EACT,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAMjF,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;CAClC;AAED,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB;AAMD,qBAAa,WAAY,SAAQ,YAAY;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IAGtD,OAAO,CAAC,KAAK,CAMX;IACF,OAAO,CAAC,UAAU,CAAa;IAG/B,OAAO,CAAC,gBAAgB,CAAC,CAA2B;gBAGlD,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,EACpC,aAAa,CAAC,EAAE,aAAa,EAC7B,SAAS,CAAC,EAAE,SAAS,EACrB,UAAU,CAAC,EAAE,UAAU,EACvB,iBAAiB,CAAC,EAAE,iBAAiB;IAcjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IASjC;;OAEG;IACH,mBAAmB,CAAC,MAAM,EAAE,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;IAQ3D;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAkC7C;;;;;;;OAOG;IACG,WAAW,CACf,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;IAsI3C;;OAEG;IACG,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAuB9D;;OAEG;IACG,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAkC/E;;OAEG;IACG,kBAAkB,IAAI,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAoB/D;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAoB7C;;OAEG;IACH,aAAa;IAIb;;OAEG;IACH,eAAe;IAIf;;OAEG;IACH,gBAAgB;IAQhB;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO;IAIpC;;OAEG;IACH,eAAe,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI;IAgBzC,aAAa,IAAI,UAAU;IAI3B,UAAU,IAAI,IAAI;IAelB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAgB3B,OAAO,CAAC,sBAAsB;IAgC9B,OAAO,CAAC,YAAY;IAIpB,gEAAgE;IAGhE,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,WAAW;CAyBpB;AAMD,wBAAgB,iBAAiB,CAC/B,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,EACpC,aAAa,CAAC,EAAE,aAAa,EAC7B,SAAS,CAAC,EAAE,SAAS,EACrB,UAAU,CAAC,EAAE,UAAU,EACvB,iBAAiB,CAAC,EAAE,iBAAiB,GACpC,WAAW,CAEb"}