@stackmemoryai/stackmemory 0.2.9 → 0.3.0

package/dist/src/core/database/batch-operations.js

@@ -0,0 +1,339 @@
+/**
+ * Batch Database Operations
+ * High-performance bulk operations with transaction management
+ */
+import { logger } from '../monitoring/logger.js';
+import { trace } from '../trace/index.js';
+/**
+ * High-performance batch operations manager
+ */
+export class BatchOperationsManager {
+    db;
+    preparedStatements = new Map();
+    batchQueue = [];
+    isProcessing = false;
+    constructor(db) {
+        if (db) {
+            this.db = db;
+            this.initializePreparedStatements();
+        }
+        else {
+            // Will be initialized when used with getConnectionPool().withConnection()
+            this.db = undefined;
+        }
+    }
+    /**
+     * Add events in bulk with optimized batching
+     */
+    async bulkInsertEvents(events, options = {}) {
+        const { batchSize = 100, onConflict = 'ignore', enableTransactions = true, } = options;
+        return this.performBulkInsert('events', events, {
+            batchSize,
+            onConflict,
+            enableTransactions,
+            preprocessor: (event) => ({
+                ...event,
+                event_id: `evt_${event.frame_id}_${event.seq}_${Date.now()}`,
+                payload: JSON.stringify(event.payload),
+            }),
+        });
+    }
+    /**
+     * Add anchors in bulk
+     */
+    async bulkInsertAnchors(anchors, options = {}) {
+        return this.performBulkInsert('anchors', anchors, {
+            ...options,
+            preprocessor: (anchor) => ({
+                ...anchor,
+                anchor_id: `anc_${anchor.frame_id}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+                metadata: JSON.stringify(anchor.metadata),
+                created_at: Date.now(),
+            }),
+        });
+    }
+    /**
+     * Bulk update frame digests
+     */
+    async bulkUpdateFrameDigests(updates, options = {}) {
+        const { batchSize = 50, enableTransactions = true, } = options;
+        return trace.traceAsync('function', 'bulkUpdateFrameDigests', { count: updates.length }, async () => {
+            const startTime = performance.now();
+            const stats = {
+                totalRecords: updates.length,
+                batchesProcessed: 0,
+                successfulInserts: 0,
+                failedInserts: 0,
+                totalTimeMs: 0,
+                avgBatchTimeMs: 0,
+            };
+            if (updates.length === 0)
+                return stats;
+            const stmt = this.db.prepare(`
+        UPDATE frames 
+        SET digest_text = ?, 
+            digest_json = ?, 
+            closed_at = COALESCE(?, closed_at),
+            state = CASE WHEN ? IS NOT NULL THEN 'closed' ELSE state END
+        WHERE frame_id = ?
+      `);
+            const updateFn = (batch) => {
+                for (const update of batch) {
+                    try {
+                        const result = stmt.run(update.digest_text, JSON.stringify(update.digest_json), update.closed_at, update.closed_at, update.frame_id);
+                        stats.successfulInserts += result.changes;
+                    }
+                    catch (error) {
+                        stats.failedInserts++;
+                        logger.warn('Failed to update frame digest', {
+                            frameId: update.frame_id,
+                            error: error.message,
+                        });
+                    }
+                }
+            };
+            if (enableTransactions) {
+                const transaction = this.db.transaction(updateFn);
+                await this.processBatches(updates, batchSize, transaction, stats);
+            }
+            else {
+                await this.processBatches(updates, batchSize, updateFn, stats);
+            }
+            stats.totalTimeMs = performance.now() - startTime;
+            stats.avgBatchTimeMs = stats.batchesProcessed > 0
+                ? stats.totalTimeMs / stats.batchesProcessed
+                : 0;
+            logger.info('Bulk frame digest update completed', stats);
+            return stats;
+        });
+    }
+    /**
+     * Generic bulk insert with preprocessing
+     */
+    async performBulkInsert(table, records, options = {}) {
+        const { batchSize = 100, onConflict = 'ignore', enableTransactions = true, preprocessor, } = options;
+        return trace.traceAsync('function', `bulkInsert${table}`, { count: records.length }, async () => {
+            const startTime = performance.now();
+            const stats = {
+                totalRecords: records.length,
+                batchesProcessed: 0,
+                successfulInserts: 0,
+                failedInserts: 0,
+                totalTimeMs: 0,
+                avgBatchTimeMs: 0,
+            };
+            if (records.length === 0)
+                return stats;
+            // Preprocess records if needed
+            const processedRecords = preprocessor
+                ? records.map(preprocessor)
+                : records;
+            // Build dynamic insert statement
+            const firstRecord = processedRecords[0];
+            const columns = Object.keys(firstRecord);
+            const placeholders = columns.map(() => '?').join(', ');
+            const conflictClause = this.getConflictClause(onConflict);
+            const insertSql = `INSERT ${conflictClause} INTO ${table} (${columns.join(', ')}) VALUES (${placeholders})`;
+            const stmt = this.db.prepare(insertSql);
+            const insertFn = (batch) => {
+                for (const record of batch) {
+                    try {
+                        const values = columns.map(col => record[col]);
+                        const result = stmt.run(...values);
+                        stats.successfulInserts += result.changes;
+                    }
+                    catch (error) {
+                        stats.failedInserts++;
+                        logger.warn(`Failed to insert ${table} record`, {
+                            record,
+                            error: error.message,
+                        });
+                    }
+                }
+            };
+            if (enableTransactions) {
+                const transaction = this.db.transaction(insertFn);
+                await this.processBatches(processedRecords, batchSize, transaction, stats);
+            }
+            else {
+                await this.processBatches(processedRecords, batchSize, insertFn, stats);
+            }
+            stats.totalTimeMs = performance.now() - startTime;
+            stats.avgBatchTimeMs = stats.batchesProcessed > 0
+                ? stats.totalTimeMs / stats.batchesProcessed
+                : 0;
+            logger.info(`Bulk ${table} insert completed`, stats);
+            return stats;
+        });
+    }
+    /**
+     * Process records in batches
+     */
+    async processBatches(records, batchSize, processFn, stats) {
+        for (let i = 0; i < records.length; i += batchSize) {
+            const batch = records.slice(i, i + batchSize);
+            const batchStart = performance.now();
+            try {
+                processFn(batch);
+                stats.batchesProcessed++;
+                const batchTime = performance.now() - batchStart;
+                logger.debug('Batch processed', {
+                    batchNumber: stats.batchesProcessed,
+                    records: batch.length,
+                    timeMs: batchTime.toFixed(2),
+                });
+                // Yield control to prevent blocking
+                if (stats.batchesProcessed % 10 === 0) {
+                    await new Promise(resolve => setImmediate(resolve));
+                }
+            }
+            catch (error) {
+                stats.failedInserts += batch.length;
+                logger.error('Batch processing failed', error, {
+                    batchNumber: stats.batchesProcessed + 1,
+                    batchSize: batch.length,
+                });
+            }
+        }
+    }
+    /**
+     * Queue batch operation for later processing
+     */
+    queueBatchOperation(operation) {
+        this.batchQueue.push(operation);
+        if (this.batchQueue.length >= 10 && !this.isProcessing) {
+            setImmediate(() => this.processBatchQueue());
+        }
+    }
+    /**
+     * Process queued batch operations
+     */
+    async processBatchQueue() {
+        if (this.isProcessing || this.batchQueue.length === 0) {
+            return;
+        }
+        this.isProcessing = true;
+        const operations = [...this.batchQueue];
+        this.batchQueue = [];
+        try {
+            const groupedOps = this.groupOperationsByTable(operations);
+            for (const [table, tableOps] of groupedOps) {
+                await this.processTableOperations(table, tableOps);
+            }
+            logger.info('Batch queue processed', {
+                operations: operations.length,
+                tables: groupedOps.size,
+            });
+        }
+        catch (error) {
+            logger.error('Batch queue processing failed', error);
+        }
+        finally {
+            this.isProcessing = false;
+        }
+    }
+    /**
+     * Flush any remaining queued operations
+     */
+    async flush() {
+        if (this.batchQueue.length > 0) {
+            await this.processBatchQueue();
+        }
+    }
+    /**
+     * Get SQL conflict clause
+     */
+    getConflictClause(onConflict) {
+        switch (onConflict) {
+            case 'ignore':
+                return 'OR IGNORE';
+            case 'replace':
+                return 'OR REPLACE';
+            case 'update':
+                return 'ON CONFLICT DO UPDATE SET';
+            default:
+                return '';
+        }
+    }
+    /**
+     * Group operations by table for efficient processing
+     */
+    groupOperationsByTable(operations) {
+        const grouped = new Map();
+        for (const op of operations) {
+            if (!grouped.has(op.table)) {
+                grouped.set(op.table, []);
+            }
+            grouped.get(op.table).push(op);
+        }
+        return grouped;
+    }
+    /**
+     * Process all operations for a specific table
+     */
+    async processTableOperations(table, operations) {
+        for (const op of operations) {
+            switch (op.operation) {
+                case 'insert':
+                    await this.performBulkInsert(table, op.data, {
+                        onConflict: op.onConflict,
+                    });
+                    break;
+                // Add update and delete operations as needed
+                default:
+                    logger.warn('Unsupported batch operation', { table, operation: op.operation });
+            }
+        }
+    }
+    /**
+     * Initialize commonly used prepared statements
+     */
+    initializePreparedStatements() {
+        // Event insertion
+        this.preparedStatements.set('insert_event', this.db.prepare(`
+        INSERT OR IGNORE INTO events 
+        (event_id, frame_id, run_id, seq, event_type, payload, ts) 
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+      `));
+        // Anchor insertion
+        this.preparedStatements.set('insert_anchor', this.db.prepare(`
+        INSERT OR IGNORE INTO anchors 
+        (anchor_id, frame_id, type, text, priority, metadata, created_at) 
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+      `));
+        logger.info('Batch operations prepared statements initialized');
+    }
+    /**
+     * Cleanup resources
+     */
+    cleanup() {
+        // Modern better-sqlite3 automatically handles cleanup
+        this.preparedStatements.clear();
+    }
+}
+// Global batch operations manager
+let globalBatchManager = null;
+/**
+ * Get or create global batch operations manager
+ */
+export function getBatchManager(db) {
+    if (!globalBatchManager) {
+        globalBatchManager = new BatchOperationsManager(db);
+    }
+    return globalBatchManager;
+}
+/**
+ * Convenience function for bulk event insertion
+ */
+export async function bulkInsertEvents(events, options) {
+    const manager = getBatchManager();
+    return manager.bulkInsertEvents(events, options);
+}
+/**
+ * Convenience function for bulk anchor insertion
+ */
+export async function bulkInsertAnchors(anchors, options) {
+    const manager = getBatchManager();
+    return manager.bulkInsertAnchors(anchors, options);
+}
+//# sourceMappingURL=batch-operations.js.map

package/dist/src/core/database/batch-operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-operations.js","sourceRoot":"","sources":["../../../../src/core/database/batch-operations.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAyB1C;;GAEG;AACH,MAAM,OAAO,sBAAsB;IACzB,EAAE,CAAoB;IACtB,kBAAkB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC3D,UAAU,GAAqB,EAAE,CAAC;IAClC,YAAY,GAAG,KAAK,CAAC;IAE7B,YAAY,EAAsB;QAChC,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,0EAA0E;YAC1E,IAAI,CAAC,EAAE,GAAG,SAAgB,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CACpB,MAOE,EACF,UAA6B,EAAE;QAE/B,MAAM,EACJ,SAAS,GAAG,GAAG,EACf,UAAU,GAAG,QAAQ,EACrB,kBAAkB,GAAG,IAAI,GAC1B,GAAG,OAAO,CAAC;QAEZ,OAAO,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE;YAC9C,SAAS;YACT,UAAU;YACV,kBAAkB;YAClB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACxB,GAAG,KAAK;gBACR,QAAQ,EAAE,OAAO,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;gBAC5D,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC;aACvC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CACrB,OAME,EACF,UAA6B,EAAE;QAE/B,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,OAAO,EAAE;YAChD,GAAG,OAAO;YACV,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACzB,GAAG,MAAM;gBACT,SAAS,EAAE,OAAO,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;gBAC5F,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;aACvB,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB,CAC1B,OAKE,EACF,UAA6B,EAAE;QAE/B,MAAM,EACJ,SAAS,GAAG,EAAE,EACd,kBAAkB,GAAG,IAAI,GAC1B,GAAG,OAAO,CAAC;QAEZ,OAAO,KAAK,CAAC,UAAU,CAAC,UAAU,EAAE,wBAAwB,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,KAAK,IAAI,EAAE;YAClG,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,KAAK,GAAe;gBACxB,YAAY,EAAE,OAAO,CAAC,MAAM;gBAC5B,gBAAgB,EAAE,CAAC;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,aAAa,EAAE,CAAC;gBAChB,WAAW,EAAE,CAAC;gBACd,cAAc,EAAE,CAAC;aAClB,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEvC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;OAO5B,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,CAAC,KAAqB,EAAE,EAAE;gBACzC,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;oBAC3B,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,MAAM,CAAC,WAAW,EAClB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,EAClC,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,CAChB,CAAC;wBACF,KAAK,CAAC,iBAAiB,IAAI,MAAM,CAAC,OAAO,CAAC;oBAC5C,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,KAAK,CAAC,aAAa,EAAE,CAAC;wBACtB,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;4BAC3C,OAAO,EAAE,MAAM,CAAC,QAAQ;4BACxB,KAAK,EAAG,KAAe,CAAC,OAAO;yBAChC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAClD,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YACpE,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjE,CAAC;YAED,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAClD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,gBAAgB,GAAG,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,gBAAgB;gBAC5C,CAAC,CAAC,CAAC,CAAC;YAEN,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAA2C,CAAC,CAAC;YAC/F,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,KAAa,EACb,OAAY,EACZ,UAEI,EAAE;QAEN,MAAM,EACJ,SAAS,GAAG,GAAG,EACf,UAAU,GAAG,QAAQ,EACrB,kBAAkB,GAAG,IAAI,EACzB,YAAY,GACb,GAAG,OAAO,CAAC;QAEZ,OAAO,KAAK,CAAC,UAAU,CAAC,UAAU,EAAE,aAAa,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,KAAK,IAAI,EAAE;YAC9F,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,KAAK,GAAe;gBACxB,YAAY,EAAE,OAAO,CAAC,MAAM;gBAC5B,gBAAgB,EAAE,CAAC;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,aAAa,EAAE,CAAC;gBAChB,WAAW,EAAE,CAAC;gBACd,cAAc,EAAE,CAAC;aAClB,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEvC,+BAA+B;YAC/B,MAAM,gBAAgB,GAAG,YAAY;gBACnC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;gBAC3B,CAAC,CAAC,OAAO,CAAC;YAEZ,iCAAiC;YACjC,MAAM,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvD,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAE1D,MAAM,SAAS,GAAG,UAAU,cAAc,SAAS,KAAK,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,YAAY,GAAG,CAAC;YAC5G,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAExC,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAE,EAAE;gBAClD,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;oBAC3B,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;wBAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;wBACnC,KAAK,CAAC,iBAAiB,IAAI,MAAM,CAAC,OAAO,CAAC;oBAC5C,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,KAAK,CAAC,aAAa,EAAE,CAAC;wBACtB,MAAM,CAAC,IAAI,CAAC,oBAAoB,KAAK,SAAS,EAAE;4BAC9C,MAAM;4BACN,KAAK,EAAG,KAAe,CAAC,OAAO;yBAChC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAClD,MAAM,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAC7E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC1E,CAAC;YAED,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAClD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,gBAAgB,GAAG,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,gBAAgB;gBAC5C,CAAC,CAAC,CAAC,CAAC;YAEN,MAAM,CAAC,IAAI,CAAC,QAAQ,KAAK,mBAAmB,EAAE,KAA2C,CAAC,CAAC;YAC3F,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CAC1B,OAAY,EACZ,SAAiB,EACjB,SAA+B,EAC/B,KAAiB;QAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAErC,IAAI,CAAC;gBACH,SAAS,CAAC,KAAK,CAAC,CAAC;gBACjB,KAAK,CAAC,gBAAgB,EAAE,CAAC;gBAEzB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;gBACjD,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE;oBAC9B,WAAW,EAAE,KAAK,CAAC,gBAAgB;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM;oBACrB,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;iBAC7B,CAAC,CAAC;gBAEH,oCAAoC;gBACpC,IAAI,KAAK,CAAC,gBAAgB,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;oBACtC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;gBACtD,CAAC;YAEH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAc,EAAE;oBACtD,WAAW,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC;oBACvC,SAAS,EAAE,KAAK,CAAC,MAAM;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,SAAyB;QAC3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvD,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QAErB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;YAE3D,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC3C,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE;gBACnC,UAAU,EAAE,UAAU,CAAC,MAAM;gBAC7B,MAAM,EAAE,UAAU,CAAC,IAAI;aACxB,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAc,CAAC,CAAC;QAChE,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,UAAkB;QAC1C,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,QAAQ;gBACX,OAAO,WAAW,CAAC;YACrB,KAAK,SAAS;gBACZ,OAAO,YAAY,CAAC;YACtB,KAAK,QAAQ;gBACX,OAAO,2BAA2B,CAAC;YACrC;gBACE,OAAO,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,UAA4B;QACzD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QAEpD,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC5B,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,sBAAsB,CAAC,KAAa,EAAE,UAA4B;QAC9E,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;YAC5B,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC;gBACrB,KAAK,QAAQ;oBACX,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,EAAE;wBAC3C,UAAU,EAAE,EAAE,CAAC,UAAU;qBAC1B,CAAC,CAAC;oBACH,MAAM;gBACR,6CAA6C;gBAC7C;oBACE,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,4BAA4B;QAClC,kBAAkB;QAClB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,cAAc,EACxC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAIf,CAAC,CACH,CAAC;QAEF,mBAAmB;QACnB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,eAAe,EACzC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAIf,CAAC,CACH,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,OAAO;QACL,sDAAsD;QACtD,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;CACF;AAED,kCAAkC;AAClC,IAAI,kBAAkB,GAAkC,IAAI,CAAC;AAE7D;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,EAAsB;IACpD,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,kBAAkB,GAAG,IAAI,sBAAsB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAa,EACb,OAA2B;IAE3B,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;IAClC,OAAO,OAAO,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAc,EACd,OAA2B;IAE3B,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;IAClC,OAAO,OAAO,CAAC,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC"}

package/dist/src/core/database/connection-pool.d.ts

@@ -0,0 +1,79 @@
+/**
+ * SQLite Connection Pool
+ * Manages a pool of SQLite database connections for performance optimization
+ */
+import Database from 'better-sqlite3';
+import { TracedDatabaseOptions } from '../trace/db-trace-wrapper.js';
+export interface ConnectionPoolOptions extends TracedDatabaseOptions {
+    maxConnections?: number;
+    acquireTimeoutMs?: number;
+    idleTimeoutMs?: number;
+    checkInterval?: number;
+}
+export interface PooledConnection {
+    db: Database.Database;
+    id: string;
+    createdAt: number;
+    lastUsed: number;
+    inUse: boolean;
+}
+export declare class SQLiteConnectionPool {
+    private connections;
+    private filename;
+    private options;
+    private maxConnections;
+    private acquireTimeoutMs;
+    private idleTimeoutMs;
+    private cleanupInterval;
+    private stats;
+    constructor(filename: string, options?: ConnectionPoolOptions);
+    /**
+     * Acquire a connection from the pool
+     */
+    acquire(): Promise<PooledConnection>;
+    /**
+     * Release a connection back to the pool
+     */
+    release(connection: PooledConnection): void;
+    /**
+     * Execute a function with a pooled connection
+     */
+    withConnection<T>(fn: (db: Database.Database) => T | Promise<T>): Promise<T>;
+    /**
+     * Execute a transaction with a pooled connection
+     */
+    withTransaction<T>(fn: (db: Database.Database) => T): Promise<T>;
+    /**
+     * Get pool statistics
+     */
+    getStats(): {
+        totalConnections: number;
+        maxConnections: number;
+        totalCreated: number;
+        totalAcquired: number;
+        totalReleased: number;
+        currentActive: number;
+        currentIdle: number;
+    };
+    /**
+     * Close all connections and cleanup
+     */
+    close(): Promise<void>;
+    /**
+     * Create a new connection
+     */
+    private createConnection;
+    /**
+     * Clean up idle connections
+     */
+    private cleanup;
+}
+/**
+ * Get or create the global connection pool
+ */
+export declare function getConnectionPool(filename?: string, options?: ConnectionPoolOptions): SQLiteConnectionPool;
+/**
+ * Close the global connection pool
+ */
+export declare function closeGlobalPool(): Promise<void>;
+//# sourceMappingURL=connection-pool.d.ts.map

package/dist/src/core/database/connection-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"connection-pool.d.ts","sourceRoot":"","sources":["../../../../src/core/database/connection-pool.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAwB,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAG3F,MAAM,WAAW,qBAAsB,SAAQ,qBAAqB;IAClE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,WAAW,CAA4C;IAC/D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAA+B;IAGtD,OAAO,CAAC,KAAK,CAMX;gBAEU,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,qBAA0B;IAuBjE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC;IA8B1C;;OAEG;IACH,OAAO,CAAC,UAAU,EAAE,gBAAgB,GAAG,IAAI;IAY3C;;OAEG;IACG,cAAc,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IASlF;;OAEG;IACG,eAAe,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAUtE;;OAEG;IACH,QAAQ;;;;;;;;;IAQR;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA6B5B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiCxB;;OAEG;IACH,OAAO,CAAC,OAAO;CAsChB;AAKD;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,oBAAoB,CAUtB;AAED;;GAEG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAKrD"}

package/dist/src/core/database/connection-pool.js

@@ -0,0 +1,236 @@
+/**
+ * SQLite Connection Pool
+ * Manages a pool of SQLite database connections for performance optimization
+ */
+import { createTracedDatabase } from '../trace/db-trace-wrapper.js';
+import { logger } from '../monitoring/logger.js';
+export class SQLiteConnectionPool {
+    connections = new Map();
+    filename;
+    options;
+    maxConnections;
+    acquireTimeoutMs;
+    idleTimeoutMs;
+    cleanupInterval = null;
+    // Statistics
+    stats = {
+        totalCreated: 0,
+        totalAcquired: 0,
+        totalReleased: 0,
+        currentActive: 0,
+        currentIdle: 0,
+    };
+    constructor(filename, options = {}) {
+        this.filename = filename;
+        this.options = options;
+        this.maxConnections = options.maxConnections ?? 5;
+        this.acquireTimeoutMs = options.acquireTimeoutMs ?? 5000;
+        this.idleTimeoutMs = options.idleTimeoutMs ?? 300000; // 5 minutes
+        // Start cleanup interval
+        if (options.checkInterval !== 0) {
+            this.cleanupInterval = setInterval(() => this.cleanup(), options.checkInterval ?? 60000 // 1 minute
+            );
+        }
+        logger.info('SQLite connection pool initialized', {
+            filename,
+            maxConnections: this.maxConnections,
+            acquireTimeoutMs: this.acquireTimeoutMs,
+            idleTimeoutMs: this.idleTimeoutMs,
+        });
+    }
+    /**
+     * Acquire a connection from the pool
+     */
+    async acquire() {
+        const startTime = Date.now();
+        while (Date.now() - startTime < this.acquireTimeoutMs) {
+            // Try to find an idle connection
+            for (const [id, connection] of this.connections) {
+                if (!connection.inUse) {
+                    connection.inUse = true;
+                    connection.lastUsed = Date.now();
+                    this.stats.totalAcquired++;
+                    this.stats.currentActive++;
+                    this.stats.currentIdle--;
+                    logger.debug('Reused connection from pool', { connectionId: id });
+                    return connection;
+                }
+            }
+            // Create new connection if under limit
+            if (this.connections.size < this.maxConnections) {
+                return this.createConnection();
+            }
+            // Wait briefly before retrying
+            await new Promise(resolve => setTimeout(resolve, 10));
+        }
+        throw new Error(`Failed to acquire connection within ${this.acquireTimeoutMs}ms timeout`);
+    }
+    /**
+     * Release a connection back to the pool
+     */
+    release(connection) {
+        if (this.connections.has(connection.id)) {
+            connection.inUse = false;
+            connection.lastUsed = Date.now();
+            this.stats.totalReleased++;
+            this.stats.currentActive--;
+            this.stats.currentIdle++;
+            logger.debug('Released connection to pool', { connectionId: connection.id });
+        }
+    }
+    /**
+     * Execute a function with a pooled connection
+     */
+    async withConnection(fn) {
+        const connection = await this.acquire();
+        try {
+            return await fn(connection.db);
+        }
+        finally {
+            this.release(connection);
+        }
+    }
+    /**
+     * Execute a transaction with a pooled connection
+     */
+    async withTransaction(fn) {
+        const connection = await this.acquire();
+        try {
+            const transaction = connection.db.transaction(() => fn(connection.db));
+            return transaction();
+        }
+        finally {
+            this.release(connection);
+        }
+    }
+    /**
+     * Get pool statistics
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            totalConnections: this.connections.size,
+            maxConnections: this.maxConnections,
+        };
+    }
+    /**
+     * Close all connections and cleanup
+     */
+    async close() {
+        if (this.cleanupInterval) {
+            clearInterval(this.cleanupInterval);
+            this.cleanupInterval = null;
+        }
+        for (const [id, connection] of this.connections) {
+            try {
+                connection.db.close();
+                logger.debug('Closed connection', { connectionId: id });
+            }
+            catch (error) {
+                logger.warn('Error closing connection', {
+                    connectionId: id,
+                    error: error.message,
+                });
+            }
+        }
+        this.connections.clear();
+        this.stats.currentActive = 0;
+        this.stats.currentIdle = 0;
+        logger.info('Connection pool closed', {
+            totalCreated: this.stats.totalCreated,
+            totalAcquired: this.stats.totalAcquired,
+            totalReleased: this.stats.totalReleased,
+        });
+    }
+    /**
+     * Create a new connection
+     */
+    createConnection() {
+        const id = `conn_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+        const now = Date.now();
+        const db = createTracedDatabase(this.filename, this.options);
+        // Configure for performance
+        db.pragma('journal_mode = WAL');
+        db.pragma('synchronous = NORMAL');
+        db.pragma('cache_size = 1000');
+        db.pragma('temp_store = MEMORY');
+        const connection = {
+            db,
+            id,
+            createdAt: now,
+            lastUsed: now,
+            inUse: true,
+        };
+        this.connections.set(id, connection);
+        this.stats.totalCreated++;
+        this.stats.totalAcquired++;
+        this.stats.currentActive++;
+        logger.debug('Created new connection', {
+            connectionId: id,
+            totalConnections: this.connections.size,
+        });
+        return connection;
+    }
+    /**
+     * Clean up idle connections
+     */
+    cleanup() {
+        const now = Date.now();
+        const toRemove = [];
+        for (const [id, connection] of this.connections) {
+            if (!connection.inUse && now - connection.lastUsed > this.idleTimeoutMs) {
+                toRemove.push(id);
+            }
+        }
+        for (const id of toRemove) {
+            const connection = this.connections.get(id);
+            if (connection) {
+                try {
+                    connection.db.close();
+                    this.connections.delete(id);
+                    this.stats.currentIdle--;
+                    logger.debug('Cleaned up idle connection', {
+                        connectionId: id,
+                        idleTime: now - connection.lastUsed,
+                    });
+                }
+                catch (error) {
+                    logger.warn('Error cleaning up connection', {
+                        connectionId: id,
+                        error: error.message,
+                    });
+                }
+            }
+        }
+        if (toRemove.length > 0) {
+            logger.info('Connection cleanup completed', {
+                removed: toRemove.length,
+                remaining: this.connections.size,
+            });
+        }
+    }
+}
+// Global pool instance
+let globalPool = null;
+/**
+ * Get or create the global connection pool
+ */
+export function getConnectionPool(filename, options) {
+    if (!globalPool && filename) {
+        globalPool = new SQLiteConnectionPool(filename, options);
+    }
+    if (!globalPool) {
+        throw new Error('Connection pool not initialized. Call with filename first.');
+    }
+    return globalPool;
+}
+/**
+ * Close the global connection pool
+ */
+export async function closeGlobalPool() {
+    if (globalPool) {
+        await globalPool.close();
+        globalPool = null;
+    }
+}
+//# sourceMappingURL=connection-pool.js.map