@pixagram/lacerta-db 0.9.0 → 0.9.2

package/index.js

@@ -1,5 +1,5 @@
 /**
- * LacertaDB V0.9.0 - Production Library with QuickStore (Optimized)
+ * LacertaDB V0.9.2 - Production Library with QuickStore (Optimized)
  * * A high-performance, browser-based document database with support for:
  * - IndexedDB storage with connection pooling
  * - Multiple caching strategies (LRU, LFU, TTL)
@@ -9,17 +9,35 @@
  * - MongoDB-like query syntax and aggregation pipeline
  * - Schema migrations and versioning
  * - QuickStore for fast localStorage-based operations
- * * Changelog V0.9.0:
+ * * Changelog V0.9.2:
+ * - CRITICAL: Fixed non-extractable CryptoKey in _importMasterKeys (changePin was broken).
+ * - CRITICAL: Fixed TransactionInactiveError in batchOperation (sync request queueing).
+ * - CRITICAL: Fixed batchAdd crash when called without options (from import()).
+ * - SECURITY: Constant-time comparison in _arrayEquals (timing attack prevention).
+ * - SECURITY: Unbiased PIN generation via rejection sampling.
+ * - SECURITY: changePin now verifies oldPin before allowing change.
+ * - SECURITY: Standardized AES-GCM IV to 12 bytes in encryptPrivateKey/decryptPrivateKey.
+ * - FIX: Environment-safe polyfills (no bare `window` at module scope).
+ * - FIX: BTreeIndex.remove only decrements _size when key/value actually existed.
+ * - FIX: LFUCache.has() now checks TTL expiration.
+ * - FIX: GeoIndex._size cannot go negative on redundant removePoint calls.
+ * - FIX: $group aggregation uses JSON.stringify for object keys (Map comparison).
+ * - FIX: TextIndex tokenizer minimum length normalized across code paths.
+ * - FIX: QuickStore properly cleans up beforeunload listener via destroy().
+ * - FIX: Consistent window.requestIdleCallback usage in _saveIndexMetadata.
+ * - FIX: Block-scoped const in $avg aggregation case.
+ * * Changelog V0.9.1:
  * - CRITICAL: Implemented Master Key Wrapping for encryption (fixes data loss on PIN change).
  * - CRITICAL: Fixed UI freezing in QuickStore using in-memory caching and async persistence.
  * - CRITICAL: Replaced O(N) GeoIndex with O(log N) QuadTree.
+ * - CRITICAL: Fixed TransactionInactiveError by implementing Batch Processing for Indexes.
  * - SECURITY: Increased PBKDF2 iterations to 600,000 (OWASP standard).
  * - OPTIMIZATION: Removed Global Mutex for read operations (concurrency fix).
  * - OPTIMIZATION: Implemented Cursor-based indexing (OOM fix).
  * - FIX: Added Magic Byte check for robust compression detection.
  * - FIX: Use Intl.Segmenter for proper CJK text tokenization.
  * * @module LacertaDB
- * @version 0.9.0
+ * @version 0.9.2
  * @license MIT
  * @author Pixagram SA
  */
@@ -30,7 +48,7 @@
 // Polyfills
 // ========================
 
-if (!window.requestIdleCallback) {
+if (typeof window !== 'undefined' && !window.requestIdleCallback) {
     window.requestIdleCallback = function(callback) {
         return setTimeout(callback, 0);
     };
@@ -76,8 +94,25 @@ class QuickStore {
         this._dirty = false;
 
         // Safety: Flush on unload to prevent data loss
+        this._flushHandler = () => this._flushSync();
         if (typeof window !== 'undefined') {
-            window.addEventListener('beforeunload', () => this._flushSync());
+            window.addEventListener('beforeunload', this._flushHandler);
+        }
+    }
+
+    destroy() {
+        this._flushSync();
+        if (typeof window !== 'undefined' && this._flushHandler) {
+            window.removeEventListener('beforeunload', this._flushHandler);
+            this._flushHandler = null;
+        }
+        if (this._saveIndexTimer) {
+            if (typeof window !== 'undefined' && window.cancelIdleCallback) {
+                window.cancelIdleCallback(this._saveIndexTimer);
+            } else {
+                clearTimeout(this._saveIndexTimer);
+            }
+            this._saveIndexTimer = null;
         }
     }
 
@@ -451,7 +486,7 @@ class LFUCache {
     }
 
     has(key) {
-        return this._cache.has(key);
+        return this.get(key) !== null;
     }
 
     get size() {
@@ -676,7 +711,7 @@ class BrowserEncryptionUtility {
             {
                 name: 'PBKDF2',
                 salt: salt,
-                iterations: 100000,
+                iterations: 600000,
                 hash: 'SHA-256'
             },
             keyMaterial,
@@ -718,7 +753,7 @@ class BrowserEncryptionUtility {
             {
                 name: 'PBKDF2',
                 salt: salt,
-                iterations: 100000,
+                iterations: 600000,
                 hash: 'SHA-256'
             },
             keyMaterial,
@@ -833,10 +868,10 @@ class SecureDatabaseEncryption {
         const hmacBytes = rawBytes.slice(32, 64);
 
         this._masterKey = await crypto.subtle.importKey(
-            'raw', encBytes, { name: 'AES-GCM', length: 256 }, false, ['encrypt', 'decrypt']
+            'raw', encBytes, { name: 'AES-GCM', length: 256 }, true, ['encrypt', 'decrypt']
         );
         this._hmacKey = await crypto.subtle.importKey(
-            'raw', hmacBytes, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign', 'verify']
+            'raw', hmacBytes, { name: 'HMAC', hash: 'SHA-256' }, true, ['sign', 'verify']
         );
     }
 
@@ -854,6 +889,17 @@ class SecureDatabaseEncryption {
             throw new Error('Database encryption not initialized');
         }
 
+        // Verify old PIN by attempting to unwrap current master key
+        const oldKek = await this._deriveKEK(new TextEncoder().encode(oldPin), this._salt);
+        const currentWrappedBytes = base64.decode(this._wrappedKeyBlob);
+        const currentIv = currentWrappedBytes.slice(0, 12);
+        const currentEncMK = currentWrappedBytes.slice(12);
+        try {
+            await crypto.subtle.decrypt({ name: 'AES-GCM', iv: currentIv }, oldKek, currentEncMK);
+        } catch (e) {
+            throw new Error('Invalid old PIN');
+        }
+
         // Derive new KEK
         const newSalt = crypto.getRandomValues(new Uint8Array(this._saltLength));
         const newKek = await this._deriveKEK(new TextEncoder().encode(newPin), newSalt);
@@ -973,7 +1019,7 @@ class SecureDatabaseEncryption {
             keyData = serializer.serialize(privateKey);
         }
 
-        const iv = crypto.getRandomValues(new Uint8Array(16));
+        const iv = crypto.getRandomValues(new Uint8Array(12));
 
         const encryptedKey = await crypto.subtle.encrypt(
             {
@@ -988,13 +1034,13 @@ class SecureDatabaseEncryption {
 
         const authLength = new Uint32Array([authData.length]);
         const result = new Uint8Array(
-            16 + 4 + authData.length + encryptedKey.byteLength
+            12 + 4 + authData.length + encryptedKey.byteLength
         );
 
         result.set(iv, 0);
-        result.set(new Uint8Array(authLength.buffer), 16);
-        result.set(authData, 20);
-        result.set(new Uint8Array(encryptedKey), 20 + authData.length);
+        result.set(new Uint8Array(authLength.buffer), 12);
+        result.set(authData, 16);
+        result.set(new Uint8Array(encryptedKey), 16 + authData.length);
 
         return base64.encode(result);
     }
@@ -1006,11 +1052,11 @@ class SecureDatabaseEncryption {
 
         const encryptedPackage = base64.decode(encryptedKeyString);
 
-        const iv = encryptedPackage.slice(0, 16);
-        const authLengthBytes = encryptedPackage.slice(16, 20);
+        const iv = encryptedPackage.slice(0, 12);
+        const authLengthBytes = encryptedPackage.slice(12, 16);
         const authLength = new Uint32Array(authLengthBytes.buffer)[0];
-        const authData = encryptedPackage.slice(20, 20 + authLength);
-        const encryptedKey = encryptedPackage.slice(20 + authLength);
+        const authData = encryptedPackage.slice(16, 16 + authLength);
+        const encryptedKey = encryptedPackage.slice(16 + authLength);
 
         const encoder = new TextEncoder();
         const expectedAuth = encoder.encode(additionalAuth);
@@ -1034,11 +1080,16 @@ class SecureDatabaseEncryption {
     }
 
     static generateSecurePIN(length = 6) {
-        const digits = new Uint8Array(length);
-        crypto.getRandomValues(digits);
-        return Array.from(digits)
-            .map(b => (b % 10).toString())
-            .join('');
+        const digits = [];
+        const buf = new Uint8Array(1);
+        while (digits.length < length) {
+            crypto.getRandomValues(buf);
+            // Rejection sampling: only accept values 0-249 to avoid modulo bias
+            if (buf[0] < 250) {
+                digits.push((buf[0] % 10).toString());
+            }
+        }
+        return digits.join('');
     }
 
     destroy() {
@@ -1049,10 +1100,12 @@ class SecureDatabaseEncryption {
 
     _arrayEquals(a, b) {
         if (a.length !== b.length) return false;
+        // Constant-time comparison to prevent timing attacks
+        let diff = 0;
         for (let i = 0; i < a.length; i++) {
-            if (a[i] !== b[i]) return false;
+            diff |= a[i] ^ b[i];
         }
-        return true;
+        return diff === 0;
     }
 
     exportMetadata() {
@@ -1385,13 +1438,14 @@ class BTreeIndex {
 
     remove(key, value) {
         if (!this._root) return;
-        this._root.remove(key, value);
-        if (this._root.n === 0) {
-            if (!this._root.leaf && this._root.children[0]) {
+        const existing = this._root.search(key);
+        if (existing && existing.has(value)) {
+            this._root.remove(key, value);
+            if (this._root.n === 0 && !this._root.leaf && this._root.children[0]) {
                 this._root = this._root.children[0];
             }
+            this._size--;
         }
-        this._size--;
     }
 
     verify() {
@@ -1492,7 +1546,7 @@ class TextIndex {
             return text.toLowerCase()
                 .replace(/[^\w\s]/g, ' ')
                 .split(/\s+/)
-                .filter(token => token.length > 2);
+                .filter(token => token.length > 1);
         }
     }
 
@@ -1521,7 +1575,7 @@ class GeoIndex {
 
     removePoint(docId) {
         this._tree.remove(docId);
-        this._size--;
+        if (this._size > 0) this._size--;
     }
 
     updatePoint(docId, newCoords) {
@@ -1636,36 +1690,58 @@ class IndexManager {
         const indexData = this._createIndexStructure(index.type);
         this._indexData.set(indexName, indexData);
 
-        // Optimization: Cursor-based iteration
-        await this._collection._indexedDB.iterate(this._collection._db, 'documents', async (docData) => {
-            let doc = docData;
+        // Optimization: Use Batched Processing instead of single Cursor
+        // This prevents transaction timeouts caused by async crypto operations inside the loop
+        let lastKey = null;
+        const batchSize = 100; // Keep batch small for responsiveness
+        
+        while (true) {
+            // 1. Fetch Batch (Transaction opens and closes here)
+            const batch = await this._collection._indexedDB.getBatch(
+                this._collection._db, 
+                'documents', 
+                lastKey, 
+                batchSize
+            );
             
-            if (docData.packedData) {
-                const d = new Document(docData, {
-                    compressed: docData._compressed, 
-                    encrypted: docData._encrypted
-                });
-                await d.unpack(this._collection.database.encryption);
-                doc = d.objectOutput();
-            }
+            if (batch.length === 0) break;
+            
+            // 2. Process Batch (Async crypto operations safe here)
+            for (const docData of batch) {
+                lastKey = docData._id; // Update for next batch
+                let doc = docData;
+                
+                if (docData.packedData) {
+                    const d = new Document(docData, {
+                        compressed: docData._compressed, 
+                        encrypted: docData._encrypted
+                    });
+                    // This await is what killed the transaction before
+                    await d.unpack(this._collection.database.encryption); 
+                    doc = d.objectOutput();
+                }
 
-            let value = this._getFieldValue(doc, index.fieldPath);
+                let value = this._getFieldValue(doc, index.fieldPath);
 
-            if (index.sparse && (value === null || value === undefined)) {
-                return;
-            }
+                if (index.sparse && (value === null || value === undefined)) {
+                    continue;
+                }
 
-            if (index.unique && indexData.has && indexData.has(value)) {
-                console.error(`Unique constraint violation on index '${indexName}'`);
-                return;
-            }
+                if (index.unique && indexData.has && indexData.has(value)) {
+                    console.error(`Unique constraint violation on index '${indexName}'`);
+                    continue;
+                }
 
-            if (index.hashed && index.type === 'btree') {
-                value = await this._hashVal(value);
-            }
+                if (index.hashed && index.type === 'btree') {
+                    value = await this._hashVal(value);
+                }
 
-            this._addToIndex(indexData, value, doc._id, index.type);
-        });
+                this._addToIndex(indexData, value, doc._id, index.type);
+            }
+            
+            // Optional: Yield to main thread briefly to prevent UI freeze
+            if (window.requestIdleCallback) await new Promise(r => window.requestIdleCallback(r));
+        }
     }
 
     _createIndexStructure(type) {
@@ -1883,7 +1959,7 @@ class IndexManager {
     async _saveIndexMetadata() {
         const key = `lacertadb_${this._collection.database.name}_${this._collection.name}_indexes`;
         return new Promise((resolve) => {
-            requestIdleCallback(() => {
+            const save = () => {
                 const metadata = {
                     indexes: Array.from(this._indexes.entries()).map(([name, index]) => ({
                         name,
@@ -1894,7 +1970,12 @@ class IndexManager {
                 const encoded = base64.encode(serialized);
                 localStorage.setItem(key, encoded);
                 resolve();
-            });
+            };
+            if (typeof window !== 'undefined' && window.requestIdleCallback) {
+                window.requestIdleCallback(save);
+            } else {
+                setTimeout(save, 0);
+            }
         });
     }
 
@@ -2081,7 +2162,7 @@ class OPFSUtility {
 }
 
 // ========================
-// IndexedDB Utility (Optimized with Cursors & Mutex Fix)
+// IndexedDB Utility (Optimized with Batches)
 // ========================
 
 class IndexedDBUtility {
@@ -2139,6 +2220,19 @@ class IndexedDBUtility {
         });
     }
 
+    // New: Batched Retrieval for processing large datasets efficiently
+    async getBatch(db, storeName, lastKey, limit) {
+        return this.performTransaction(db, [storeName], 'readonly', tx => {
+            const store = tx.objectStore(storeName);
+            let range;
+            if (lastKey !== null && lastKey !== undefined) {
+                range = IDBKeyRange.lowerBound(lastKey, true); // true = open range (skip lastKey)
+            }
+            // Use getAll which is faster than cursor for batches
+            return this._promisifyRequest(() => store.getAll(range, limit));
+        });
+    }
+
     add(db, storeName, value, key) {
         return this.performTransaction(db, [storeName], 'readwrite', tx => {
             const store = tx.objectStore(storeName);
@@ -2165,23 +2259,6 @@ class IndexedDBUtility {
         });
     }
 
-    // Cursor iteration for memory efficiency
-    async iterate(db, storeName, callback) {
-        return this.performTransaction(db, [storeName], 'readonly', tx => {
-            return new Promise((resolve, reject) => {
-                const req = tx.objectStore(storeName).openCursor();
-                req.onsuccess = async (e) => {
-                    const cursor = e.target.result;
-                    if (cursor) {
-                        await callback(cursor.value);
-                        cursor.continue();
-                    }
-                };
-                req.onerror = () => reject(req.error);
-            });
-        });
-    }
-
     delete(db, storeName, key) {
         return this.performTransaction(db, [storeName], 'readwrite', tx => {
             return this._promisifyRequest(() => tx.objectStore(storeName).delete(key));
@@ -2201,33 +2278,32 @@ class IndexedDBUtility {
     }
 
     async batchOperation(db, operations) {
-        return this.performTransaction(db, ['documents'], 'readwrite', async tx => {
+        return this.performTransaction(db, ['documents'], 'readwrite', tx => {
             const store = tx.objectStore('documents');
-            const results = [];
 
-            for (const op of operations) {
+            // CRITICAL: Queue ALL IDB requests synchronously to prevent
+            // TransactionInactiveError. Do NOT use await between requests.
+            const promises = operations.map(op => {
                 try {
-                    let result;
                     switch (op.type) {
                         case 'add':
-                            result = await this._promisifyRequest(() => store.add(op.data));
-                            break;
+                            return this._promisifyRequest(() => store.add(op.data))
+                                .then(result => ({ success: true, result }));
                         case 'put':
-                            result = await this._promisifyRequest(() => store.put(op.data));
-                            break;
+                            return this._promisifyRequest(() => store.put(op.data))
+                                .then(result => ({ success: true, result }));
                         case 'delete':
-                            result = await this._promisifyRequest(() => store.delete(op.key));
-                            break;
+                            return this._promisifyRequest(() => store.delete(op.key))
+                                .then(result => ({ success: true, result }));
                         default:
-                            throw new Error(`Unknown operation type: ${op.type}`);
+                            return Promise.resolve({ success: false, error: `Unknown operation type: ${op.type}` });
                     }
-                    results.push({ success: true, result });
                 } catch (error) {
-                    results.push({ success: false, error: error.message });
+                    return Promise.resolve({ success: false, error: error.message });
                 }
-            }
+            });
 
-            return results;
+            return Promise.all(promises);
         });
     }
 }
@@ -2654,7 +2730,7 @@ class AggregationPipeline {
                 for (const doc of docs) {
                     const groupKey = typeof idField === 'string' ?
                         queryEngine.getFieldValue(doc, idField.replace('$', '')) :
-                        serializer.serialize(idField);
+                        JSON.stringify(idField);
 
                     if (!groups.has(groupKey)) {
                         groups.set(groupKey, { _id: groupKey, docs: [] });
@@ -2675,10 +2751,11 @@ class AggregationPipeline {
                             case '$sum':
                                 result[fieldKey] = group.docs.reduce((sum, d) => sum + (queryEngine.getFieldValue(d, field) || 0), 0);
                                 break;
-                            case '$avg':
+                            case '$avg': {
                                 const sum = group.docs.reduce((s, d) => s + (queryEngine.getFieldValue(d, field) || 0), 0);
                                 result[fieldKey] = sum / group.docs.length;
                                 break;
+                            }
                             case '$count':
                                 result[fieldKey] = group.docs.length;
                                 break;
@@ -3243,7 +3320,7 @@ class Collection {
         return result;
     }
 
-    async batchAdd(documents, options) {
+    async batchAdd(documents, options = {}) {
         if (!this._initialized) await this.init();
 
         const startTime = performance.now();
@@ -3623,7 +3700,7 @@ class Database {
 
     async export(format = 'json', password = null) {
         const data = {
-            version: '0.9.0',
+            version: '0.9.2',
             database: this.name,
             timestamp: Date.now(),
             collections: {}
@@ -3704,7 +3781,7 @@ class Database {
 
         // Clear quickstore
         if (this._quickStore) {
-            this._quickStore.clear();
+            this._quickStore.destroy();
         }
 
         // Destroy encryption
@@ -3790,7 +3867,7 @@ class LacertaDB {
 
     async createBackup(password = null) {
         const backup = {
-            version: '0.9.0',
+            version: '0.9.2',
             timestamp: Date.now(),
             databases: {}
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pixagram/lacerta-db",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "Lacerta-DB is a Javascript IndexedDB Database for Web Browsers. Simple, Fast, Secure.",
   "devDependencies": {
     "@babel/core": "^7.23.6",