@danielsimonjr/memoryjs 1.0.0

package/dist/utils/compressedCache.js

@@ -0,0 +1,349 @@
+/**
+ * Compressed Cache Utility
+ *
+ * Provides an LRU cache with automatic compression of old entries.
+ * Reduces memory footprint for large knowledge graphs (50k+ entities).
+ *
+ * Uses synchronous brotli compression/decompression for cache operations
+ * to avoid async complexity in hot paths.
+ *
+ * Phase 3 Sprint 5: Archive & Cache Compression
+ *
+ * @module utils/compressedCache
+ */
+import { brotliCompressSync, brotliDecompressSync, constants } from 'zlib';
+import { COMPRESSION_CONFIG } from './constants.js';
+/**
+ * LRU cache with automatic compression of old entries.
+ *
+ * Reduces memory footprint by compressing infrequently accessed entries
+ * using brotli compression. Hot (recently accessed) entries stay
+ * uncompressed for fast access.
+ *
+ * @example
+ * ```typescript
+ * const cache = new CompressedCache({
+ *   maxUncompressed: 500,
+ *   compressionThresholdMs: 60000 // 1 minute
+ * });
+ *
+ * // Store entity
+ * cache.set('Alice', { name: 'Alice', entityType: 'person', observations: [] });
+ *
+ * // Retrieve entity (decompresses if needed)
+ * const entity = cache.get('Alice');
+ *
+ * // Check stats
+ * const stats = cache.getStats();
+ * console.log(`Memory saved: ${stats.memorySaved} bytes`);
+ * ```
+ */
+export class CompressedCache {
+    _entryMap = new Map();
+    maxUncompressed;
+    compressionThresholdMs;
+    autoCompress;
+    // Phase 12 Sprint 6: Adaptive compression options
+    minCompressionSize;
+    minCompressionRatio;
+    // Statistics
+    hits = 0;
+    misses = 0;
+    compressions = 0;
+    decompressions = 0;
+    // Phase 12 Sprint 6: Adaptive compression stats
+    skippedSmallEntries = 0;
+    skippedPoorRatio = 0;
+    compressionRatios = [];
+    constructor(options = {}) {
+        this.maxUncompressed = options.maxUncompressed ?? 1000;
+        this.compressionThresholdMs = options.compressionThresholdMs ?? 5 * 60 * 1000;
+        this.autoCompress = options.autoCompress ?? true;
+        // Phase 12 Sprint 6: Adaptive compression defaults
+        this.minCompressionSize = options.minCompressionSize ?? 256;
+        this.minCompressionRatio = options.minCompressionRatio ?? 0.7;
+    }
+    /**
+     * Get an entity from the cache.
+     *
+     * If the entity is compressed, it will be decompressed on access.
+     * The entry is marked as recently accessed to prevent re-compression.
+     *
+     * @param name - Entity name to retrieve
+     * @returns The entity if found, undefined otherwise
+     */
+    get(name) {
+        const entry = this._entryMap.get(name);
+        if (!entry) {
+            this.misses++;
+            return undefined;
+        }
+        this.hits++;
+        entry.lastAccessed = Date.now();
+        if (entry.compressed && entry.compressedData) {
+            // Decompress on access
+            try {
+                const decompressed = brotliDecompressSync(entry.compressedData);
+                entry.entity = JSON.parse(decompressed.toString('utf-8'));
+                entry.compressed = false;
+                entry.compressedData = undefined;
+                this.decompressions++;
+            }
+            catch {
+                // Decompression failed - remove corrupt entry
+                this._entryMap.delete(name);
+                return undefined;
+            }
+        }
+        return entry.entity ?? undefined;
+    }
+    /**
+     * Store an entity in the cache.
+     *
+     * Entries are stored uncompressed initially. Old entries may be
+     * compressed automatically based on cache settings.
+     *
+     * @param name - Entity name (key)
+     * @param entity - Entity to store
+     */
+    set(name, entity) {
+        const jsonStr = JSON.stringify(entity);
+        const originalSize = Buffer.byteLength(jsonStr, 'utf-8');
+        this._entryMap.set(name, {
+            entity,
+            compressed: false,
+            originalSize,
+            lastAccessed: Date.now(),
+        });
+        if (this.autoCompress) {
+            this.maybeCompressOldEntries();
+        }
+    }
+    /**
+     * Check if an entity exists in the cache.
+     *
+     * @param name - Entity name to check
+     * @returns True if entity exists in cache
+     */
+    has(name) {
+        return this._entryMap.has(name);
+    }
+    /**
+     * Delete an entity from the cache.
+     *
+     * @param name - Entity name to delete
+     * @returns True if entity was deleted, false if not found
+     */
+    delete(name) {
+        return this._entryMap.delete(name);
+    }
+    /**
+     * Clear all entries from the cache.
+     */
+    clear() {
+        this._entryMap.clear();
+        // Don't reset statistics - they track lifetime totals
+    }
+    /**
+     * Get the number of entries in the cache.
+     */
+    get size() {
+        return this._entryMap.size;
+    }
+    /**
+     * Get all entity names in the cache.
+     */
+    keys() {
+        return this._entryMap.keys();
+    }
+    /**
+     * Get comprehensive cache statistics.
+     *
+     * @returns Statistics about cache state and performance
+     */
+    getStats() {
+        let compressed = 0;
+        let uncompressed = 0;
+        let memorySaved = 0;
+        let totalOriginalSize = 0;
+        let totalCompressedSize = 0;
+        let estimatedMemoryBytes = 0;
+        for (const entry of this._entryMap.values()) {
+            totalOriginalSize += entry.originalSize;
+            if (entry.compressed && entry.compressedData) {
+                compressed++;
+                totalCompressedSize += entry.compressedData.length;
+                estimatedMemoryBytes += entry.compressedData.length;
+                // Memory saved = original size - compressed size
+                memorySaved += entry.originalSize - entry.compressedData.length;
+            }
+            else {
+                uncompressed++;
+                estimatedMemoryBytes += entry.originalSize;
+            }
+        }
+        // Calculate average compression ratio
+        const avgCompressionRatio = this.compressionRatios.length > 0
+            ? this.compressionRatios.reduce((a, b) => a + b, 0) / this.compressionRatios.length
+            : 0;
+        return {
+            total: this._entryMap.size,
+            compressed,
+            uncompressed,
+            memorySaved: Math.max(0, memorySaved),
+            totalOriginalSize,
+            totalCompressedSize,
+            hits: this.hits,
+            misses: this.misses,
+            compressions: this.compressions,
+            decompressions: this.decompressions,
+            // Phase 12 Sprint 6: Adaptive compression stats
+            skippedSmallEntries: this.skippedSmallEntries,
+            skippedPoorRatio: this.skippedPoorRatio,
+            avgCompressionRatio,
+            estimatedMemoryBytes,
+        };
+    }
+    /**
+     * Force compression of entries older than the threshold.
+     *
+     * Called automatically after set() when autoCompress is enabled.
+     * Can be called manually to trigger compression on demand.
+     *
+     * @returns Number of entries compressed
+     */
+    compressOldEntries() {
+        return this.maybeCompressOldEntries(true);
+    }
+    /**
+     * Compress old entries if we're over the uncompressed limit.
+     *
+     * @param force - If true, compress regardless of limit
+     * @returns Number of entries compressed
+     */
+    maybeCompressOldEntries(force = false) {
+        // Count uncompressed entries
+        let uncompressedCount = 0;
+        for (const entry of this._entryMap.values()) {
+            if (!entry.compressed) {
+                uncompressedCount++;
+            }
+        }
+        // Only compress if over limit (unless forced)
+        if (!force && uncompressedCount <= this.maxUncompressed) {
+            return 0;
+        }
+        const now = Date.now();
+        let compressedCount = 0;
+        // Sort uncompressed entries by last accessed time (oldest first)
+        const sortedEntries = [...this._entryMap.entries()]
+            .filter(([, e]) => !e.compressed && e.entity !== null)
+            .sort((a, b) => a[1].lastAccessed - b[1].lastAccessed);
+        // Compress oldest entries until we're under the limit
+        for (const [, entry] of sortedEntries) {
+            // Stop if we've compressed enough
+            if (!force && (uncompressedCount - compressedCount) <= this.maxUncompressed) {
+                break;
+            }
+            // Only compress if entry is old enough
+            if (now - entry.lastAccessed < this.compressionThresholdMs) {
+                continue;
+            }
+            // Compress the entry
+            if (entry.entity) {
+                // Phase 12 Sprint 6: Skip small entries (adaptive compression)
+                if (entry.originalSize < this.minCompressionSize) {
+                    this.skippedSmallEntries++;
+                    continue;
+                }
+                try {
+                    const jsonStr = JSON.stringify(entry.entity);
+                    const compressed = brotliCompressSync(Buffer.from(jsonStr, 'utf-8'), {
+                        params: {
+                            [constants.BROTLI_PARAM_QUALITY]: COMPRESSION_CONFIG.BROTLI_QUALITY_CACHE,
+                        },
+                    });
+                    // Phase 12 Sprint 6: Check compression ratio
+                    const ratio = compressed.length / entry.originalSize;
+                    if (ratio > this.minCompressionRatio) {
+                        // Compression didn't achieve enough reduction
+                        this.skippedPoorRatio++;
+                        continue;
+                    }
+                    // Track compression ratio
+                    this.compressionRatios.push(ratio);
+                    entry.compressedData = compressed;
+                    entry.compressed = true;
+                    entry.entity = null; // Free memory
+                    this.compressions++;
+                    compressedCount++;
+                }
+                catch {
+                    // Compression failed - leave entry uncompressed
+                    continue;
+                }
+            }
+        }
+        return compressedCount;
+    }
+    /**
+     * Decompress all entries in the cache.
+     *
+     * Useful when preparing for bulk operations or export.
+     *
+     * @returns Number of entries decompressed
+     */
+    decompressAll() {
+        let decompressedCount = 0;
+        for (const [name, entry] of this._entryMap) {
+            if (entry.compressed && entry.compressedData) {
+                try {
+                    const decompressed = brotliDecompressSync(entry.compressedData);
+                    entry.entity = JSON.parse(decompressed.toString('utf-8'));
+                    entry.compressed = false;
+                    entry.compressedData = undefined;
+                    this.decompressions++;
+                    decompressedCount++;
+                }
+                catch {
+                    // Decompression failed - remove corrupt entry
+                    this._entryMap.delete(name);
+                }
+            }
+        }
+        return decompressedCount;
+    }
+    /**
+     * Get all entities from the cache (decompressing as needed).
+     *
+     * @returns Array of all entities in the cache
+     */
+    getAllEntities() {
+        const entities = [];
+        for (const [name] of this._entryMap) {
+            const entity = this.get(name);
+            if (entity) {
+                entities.push(entity);
+            }
+        }
+        return entities;
+    }
+    /**
+     * Iterate over all entries with their compression status.
+     *
+     * Does not decompress entries - useful for inspection.
+     *
+     * @yields Entry information without decompressing
+     */
+    *entries() {
+        for (const [name, entry] of this._entryMap) {
+            yield {
+                name,
+                compressed: entry.compressed,
+                originalSize: entry.originalSize,
+                lastAccessed: entry.lastAccessed,
+            };
+        }
+    }
+}
+//# sourceMappingURL=compressedCache.js.map

package/dist/utils/compressedCache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compressedCache.js","sourceRoot":"","sources":["../../src/utils/compressedCache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAE3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AA8FpD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,OAAO,eAAe;IAClB,SAAS,GAA4B,IAAI,GAAG,EAAE,CAAC;IACtC,eAAe,CAAS;IACxB,sBAAsB,CAAS;IAC/B,YAAY,CAAU;IACvC,kDAAkD;IACjC,kBAAkB,CAAS;IAC3B,mBAAmB,CAAS;IAE7C,aAAa;IACL,IAAI,GAAW,CAAC,CAAC;IACjB,MAAM,GAAW,CAAC,CAAC;IACnB,YAAY,GAAW,CAAC,CAAC;IACzB,cAAc,GAAW,CAAC,CAAC;IACnC,gDAAgD;IACxC,mBAAmB,GAAW,CAAC,CAAC;IAChC,gBAAgB,GAAW,CAAC,CAAC;IAC7B,iBAAiB,GAAa,EAAE,CAAC;IAEzC,YAAY,UAAkC,EAAE;QAC9C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC;QACvD,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;QAC9E,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,mDAAmD;QACnD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,GAAG,CAAC;QAC5D,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;IAChE,CAAC;IAED;;;;;;;;OAQG;IACH,GAAG,CAAC,IAAY;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;YAC7C,uBAAuB;YACvB,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBAChE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC1D,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC;gBACzB,KAAK,CAAC,cAAc,GAAG,SAAS,CAAC;gBACjC,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;gBAC9C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5B,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,IAAI,SAAS,CAAC;IACnC,CAAC;IAED;;;;;;;;OAQG;IACH,GAAG,CAAC,IAAY,EAAE,MAAc;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;YACvB,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,YAAY;YACZ,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;SACzB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,GAAG,CAAC,IAAY;QACd,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAY;QACjB,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,sDAAsD;IACxD,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,IAAI;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,oBAAoB,GAAG,CAAC,CAAC;QAE7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,iBAAiB,IAAI,KAAK,CAAC,YAAY,CAAC;YAExC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC7C,UAAU,EAAE,CAAC;gBACb,mBAAmB,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC;gBACnD,oBAAoB,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC;gBACpD,iDAAiD;gBACjD,WAAW,IAAI,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC;YAClE,CAAC;iBAAM,CAAC;gBACN,YAAY,EAAE,CAAC;gBACf,oBAAoB,IAAI,KAAK,CAAC,YAAY,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,MAAM,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;YAC3D,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM;YACnF,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;YAC1B,UAAU;YACV,YAAY;YACZ,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;YACrC,iBAAiB;YACjB,mBAAmB;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,gDAAgD;YAChD,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,mBAAmB;YACnB,oBAAoB;SACrB,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,kBAAkB;QAChB,OAAO,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;OAKG;IACK,uBAAuB,CAAC,QAAiB,KAAK;QACpD,6BAA6B;QAC7B,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;gBACtB,iBAAiB,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QAED,8CAA8C;QAC9C,IAAI,CAAC,KAAK,IAAI,iBAAiB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACxD,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,eAAe,GAAG,CAAC,CAAC;QAExB,iEAAiE;QACjE,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;aAChD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;aACrD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAEzD,sDAAsD;QACtD,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;YACtC,kCAAkC;YAClC,IAAI,CAAC,KAAK,IAAI,CAAC,iBAAiB,GAAG,eAAe,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC5E,MAAM;YACR,CAAC;YAED,uCAAuC;YACvC,IAAI,GAAG,GAAG,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC3D,SAAS;YACX,CAAC;YAED,qBAAqB;YACrB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,+DAA+D;gBAC/D,IAAI,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBACjD,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBAC3B,SAAS;gBACX,CAAC;gBAED,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBAC7C,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;wBACnE,MAAM,EAAE;4BACN,CAAC,SAAS,CAAC,oBAAoB,CAAC,EAAE,kBAAkB,CAAC,oBAAoB;yBAC1E;qBACF,CAAC,CAAC;oBAEH,6CAA6C;oBAC7C,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC;oBACrD,IAAI,KAAK,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;wBACrC,8CAA8C;wBAC9C,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACxB,SAAS;oBACX,CAAC;oBAED,0BAA0B;oBAC1B,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAEnC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC;oBAClC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC;oBACxB,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,cAAc;oBACnC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpB,eAAe,EAAE,CAAC;gBACpB,CAAC;gBAAC,MAAM,CAAC;oBACP,gDAAgD;oBAChD,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,eAAe,CAAC;IACzB,CAAC;IAED;;;;;;OAMG;IACH,aAAa;QACX,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC7C,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;oBAChE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC1D,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC;oBACzB,KAAK,CAAC,cAAc,GAAG,SAAS,CAAC;oBACjC,IAAI,CAAC,cAAc,EAAE,CAAC;oBACtB,iBAAiB,EAAE,CAAC;gBACtB,CAAC;gBAAC,MAAM,CAAC;oBACP,8CAA8C;oBAC9C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,cAAc;QACZ,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,MAAM,EAAE,CAAC;gBACX,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;;;OAMG;IACH,CAAC,OAAO;QAMN,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3C,MAAM;gBACJ,IAAI;gBACJ,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/dist/utils/compressionUtil.d.ts

@@ -0,0 +1,214 @@
+/**
+ * Compression Utility Module
+ *
+ * Provides brotli compression and decompression utilities using Node.js
+ * built-in zlib module. No external dependencies required.
+ *
+ * Brotli offers 15-20% better compression than gzip, with 60-75%
+ * compression typical for JSON data.
+ *
+ * @module utils/compressionUtil
+ */
+/**
+ * Options for compression operations.
+ */
+export interface CompressionOptions {
+    /**
+     * Brotli quality level (0-11).
+     * Higher values = better compression but slower.
+     * @default 6
+     */
+    quality?: number;
+    /**
+     * Window size (10-24) for the LZ77 algorithm.
+     * Larger windows = better compression for large files.
+     * @default 22
+     */
+    lgwin?: number;
+    /**
+     * Compression mode hint.
+     * - 'text': Optimized for UTF-8 text
+     * - 'generic': General-purpose compression
+     * @default 'generic'
+     */
+    mode?: 'text' | 'generic';
+}
+/**
+ * Result of a compression operation.
+ */
+export interface CompressionResult {
+    /** The compressed data as a Buffer */
+    compressed: Buffer;
+    /** Original size in bytes */
+    originalSize: number;
+    /** Compressed size in bytes */
+    compressedSize: number;
+    /** Compression ratio (compressedSize / originalSize). Lower is better. */
+    ratio: number;
+}
+/**
+ * Metadata about a compressed file for storage alongside the compressed data.
+ */
+export interface CompressionMetadata {
+    /** Whether the data is compressed */
+    compressed: boolean;
+    /** Compression format used */
+    compressionFormat: 'brotli' | 'none';
+    /** Original size before compression in bytes */
+    originalSize: number;
+    /** Size after compression in bytes */
+    compressedSize: number;
+    /** Optional checksum of original data for integrity verification */
+    originalChecksum?: string;
+    /** ISO 8601 timestamp when compression was performed */
+    createdAt: string;
+}
+/**
+ * Check if a file path indicates brotli compression based on extension.
+ *
+ * Note: Brotli doesn't have reliable magic bytes for detection.
+ * Using file extension (.br) is the recommended detection method.
+ *
+ * @param filePath - The file path to check
+ * @returns True if the path ends with .br extension
+ *
+ * @example
+ * ```typescript
+ * hasBrotliExtension('backup.jsonl.br') // true
+ * hasBrotliExtension('backup.jsonl')    // false
+ * hasBrotliExtension('data.json')       // false
+ * ```
+ */
+export declare function hasBrotliExtension(filePath: string): boolean;
+/**
+ * Compress data using brotli algorithm.
+ *
+ * @param data - The data to compress (string or Buffer)
+ * @param options - Compression options
+ * @returns Compression result with compressed data and statistics
+ *
+ * @example
+ * ```typescript
+ * const jsonData = JSON.stringify({ entities: [...] });
+ * const result = await compress(jsonData, { quality: 11 });
+ * console.log(`Compressed from ${result.originalSize} to ${result.compressedSize} bytes`);
+ * console.log(`Compression ratio: ${(result.ratio * 100).toFixed(1)}%`);
+ * ```
+ */
+export declare function compress(data: Buffer | string, options?: CompressionOptions): Promise<CompressionResult>;
+/**
+ * Decompress brotli-compressed data.
+ *
+ * @param data - The compressed data as a Buffer
+ * @returns The decompressed data as a Buffer
+ * @throws Error if decompression fails (corrupt or invalid data)
+ *
+ * @example
+ * ```typescript
+ * const compressed = await fs.readFile('backup.jsonl.br');
+ * const decompressed = await decompress(compressed);
+ * const jsonData = decompressed.toString('utf-8');
+ * ```
+ */
+export declare function decompress(data: Buffer): Promise<Buffer>;
+/**
+ * Calculate compression ratio.
+ *
+ * @param originalSize - Original size in bytes
+ * @param compressedSize - Compressed size in bytes
+ * @returns Ratio as a decimal (e.g., 0.25 = 75% compression)
+ *
+ * @example
+ * ```typescript
+ * const ratio = getCompressionRatio(1000, 250);
+ * console.log(`Compression ratio: ${(1 - ratio) * 100}%`); // "Compression ratio: 75%"
+ * ```
+ */
+export declare function getCompressionRatio(originalSize: number, compressedSize: number): number;
+/**
+ * Compress a file and write the result to disk.
+ *
+ * @param inputPath - Path to the input file
+ * @param outputPath - Path for the compressed output file
+ * @param options - Compression options
+ * @returns Compression result with statistics
+ *
+ * @example
+ * ```typescript
+ * const result = await compressFile(
+ *   'memory.jsonl',
+ *   'memory.jsonl.br',
+ *   { quality: 11 }
+ * );
+ * console.log(`Saved ${result.originalSize - result.compressedSize} bytes`);
+ * ```
+ */
+export declare function compressFile(inputPath: string, outputPath: string, options?: CompressionOptions): Promise<CompressionResult>;
+/**
+ * Decompress a file and write the result to disk.
+ *
+ * @param inputPath - Path to the compressed input file
+ * @param outputPath - Path for the decompressed output file
+ * @throws Error if file not found or decompression fails
+ *
+ * @example
+ * ```typescript
+ * await decompressFile('backup.jsonl.br', 'restored.jsonl');
+ * ```
+ */
+export declare function decompressFile(inputPath: string, outputPath: string): Promise<void>;
+/**
+ * Create metadata object for a compression result.
+ * This metadata should be stored alongside compressed files for
+ * integrity verification and restoration.
+ *
+ * @param result - The compression result
+ * @param checksum - Optional checksum of the original data
+ * @returns Compression metadata object
+ *
+ * @example
+ * ```typescript
+ * const result = await compress(data);
+ * const metadata = createMetadata(result);
+ * await fs.writeFile('backup.meta.json', JSON.stringify(metadata, null, 2));
+ * ```
+ */
+export declare function createMetadata(result: CompressionResult, checksum?: string): CompressionMetadata;
+/**
+ * Create metadata for uncompressed data.
+ * Useful for consistent metadata format when compression is disabled.
+ *
+ * @param size - Size of the data in bytes
+ * @returns Compression metadata indicating no compression
+ */
+export declare function createUncompressedMetadata(size: number): CompressionMetadata;
+/**
+ * Compress a string and return base64-encoded result.
+ * Useful for embedding compressed data in JSON responses.
+ *
+ * @param data - String data to compress
+ * @param options - Compression options
+ * @returns Base64-encoded compressed data
+ *
+ * @example
+ * ```typescript
+ * const encoded = await compressToBase64(jsonString);
+ * // Send in response: { compressed: true, data: encoded }
+ * ```
+ */
+export declare function compressToBase64(data: string, options?: CompressionOptions): Promise<string>;
+/**
+ * Decompress base64-encoded compressed data.
+ * Counterpart to compressToBase64.
+ *
+ * @param base64Data - Base64-encoded compressed data
+ * @returns Decompressed string
+ *
+ * @example
+ * ```typescript
+ * const original = await decompressFromBase64(response.data);
+ * const parsed = JSON.parse(original);
+ * ```
+ */
+export declare function decompressFromBase64(base64Data: string): Promise<string>;
+//# sourceMappingURL=compressionUtil.d.ts.map

package/dist/utils/compressionUtil.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compressionUtil.d.ts","sourceRoot":"","sources":["../../src/utils/compressionUtil.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAWH;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,qCAAqC;IACrC,UAAU,EAAE,OAAO,CAAC;IACpB,8BAA8B;IAC9B,iBAAiB,EAAE,QAAQ,GAAG,MAAM,CAAC;IACrC,gDAAgD;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,wDAAwD;IACxD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,GAAG,MAAM,EACrB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CAoC5B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAe9D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,MAAM,EACpB,cAAc,EAAE,MAAM,GACrB,MAAM,CAGR;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,iBAAiB,CAAC,CAK5B;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,cAAc,CAClC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAIf;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,iBAAiB,EACzB,QAAQ,CAAC,EAAE,MAAM,GAChB,mBAAmB,CASrB;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAQ5E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAI9E"}