@soulcraft/brainy 0.53.0 → 0.54.1

package/dist/utils/requestCoalescer.js

@@ -0,0 +1,324 @@
+/**
+ * Request Coalescer
+ * Batches and deduplicates operations to reduce S3 API calls
+ * Automatically flushes based on size, time, or pressure
+ */
+import { createModuleLogger } from './logger.js';
+/**
+ * Coalesces multiple operations into efficient batches
+ */
+export class RequestCoalescer {
+    constructor(processor, options) {
+        this.logger = createModuleLogger('RequestCoalescer');
+        // Operation queues by type
+        this.writeQueue = new Map();
+        this.readQueue = new Map();
+        this.deleteQueue = new Map();
+        // Batch configuration
+        this.maxBatchSize = 100;
+        this.maxBatchAge = 100; // ms - flush quickly under load
+        this.minBatchSize = 10; // Don't flush until we have enough
+        // Flush timers
+        this.flushTimer = null;
+        this.lastFlush = Date.now();
+        // Statistics
+        this.stats = {
+            totalOperations: 0,
+            coalescedOperations: 0,
+            deduplicated: 0,
+            batchesProcessed: 0,
+            averageBatchSize: 0
+        };
+        this.processor = processor;
+        if (options) {
+            this.maxBatchSize = options.maxBatchSize || this.maxBatchSize;
+            this.maxBatchAge = options.maxBatchAge || this.maxBatchAge;
+            this.minBatchSize = options.minBatchSize || this.minBatchSize;
+        }
+    }
+    /**
+     * Add a write operation to be coalesced
+     */
+    async write(key, data) {
+        return new Promise((resolve, reject) => {
+            // Check if we already have a pending write for this key
+            const existing = this.writeQueue.get(key);
+            if (existing && existing.length > 0) {
+                // Replace the data but resolve all promises
+                const last = existing[existing.length - 1];
+                last.data = data; // Use latest data
+                // Add this promise to be resolved
+                existing.push({
+                    type: 'write',
+                    key,
+                    data,
+                    resolve,
+                    reject,
+                    timestamp: Date.now()
+                });
+                this.stats.deduplicated++;
+            }
+            else {
+                // New write operation
+                this.writeQueue.set(key, [{
+                        type: 'write',
+                        key,
+                        data,
+                        resolve,
+                        reject,
+                        timestamp: Date.now()
+                    }]);
+            }
+            this.stats.totalOperations++;
+            this.checkFlush();
+        });
+    }
+    /**
+     * Add a read operation to be coalesced
+     */
+    async read(key) {
+        return new Promise((resolve, reject) => {
+            // Check if we already have a pending read for this key
+            const existing = this.readQueue.get(key);
+            if (existing && existing.length > 0) {
+                // Coalesce with existing read
+                existing.push({
+                    type: 'read',
+                    key,
+                    resolve,
+                    reject,
+                    timestamp: Date.now()
+                });
+                this.stats.deduplicated++;
+            }
+            else {
+                // New read operation
+                this.readQueue.set(key, [{
+                        type: 'read',
+                        key,
+                        resolve,
+                        reject,
+                        timestamp: Date.now()
+                    }]);
+            }
+            this.stats.totalOperations++;
+            this.checkFlush();
+        });
+    }
+    /**
+     * Add a delete operation to be coalesced
+     */
+    async delete(key) {
+        return new Promise((resolve, reject) => {
+            // Cancel any pending writes for this key
+            if (this.writeQueue.has(key)) {
+                const writes = this.writeQueue.get(key);
+                writes.forEach(op => op.reject(new Error('Cancelled by delete')));
+                this.writeQueue.delete(key);
+                this.stats.deduplicated += writes.length;
+            }
+            // Cancel any pending reads for this key
+            if (this.readQueue.has(key)) {
+                const reads = this.readQueue.get(key);
+                reads.forEach(op => op.resolve(null)); // Return null for deleted items
+                this.readQueue.delete(key);
+                this.stats.deduplicated += reads.length;
+            }
+            // Check if we already have a pending delete
+            const existing = this.deleteQueue.get(key);
+            if (existing && existing.length > 0) {
+                // Coalesce with existing delete
+                existing.push({
+                    type: 'delete',
+                    key,
+                    resolve,
+                    reject,
+                    timestamp: Date.now()
+                });
+                this.stats.deduplicated++;
+            }
+            else {
+                // New delete operation
+                this.deleteQueue.set(key, [{
+                        type: 'delete',
+                        key,
+                        resolve,
+                        reject,
+                        timestamp: Date.now()
+                    }]);
+            }
+            this.stats.totalOperations++;
+            this.checkFlush();
+        });
+    }
+    /**
+     * Check if we should flush the queues
+     */
+    checkFlush() {
+        const totalSize = this.writeQueue.size + this.readQueue.size + this.deleteQueue.size;
+        const now = Date.now();
+        const age = now - this.lastFlush;
+        // Immediate flush conditions
+        if (totalSize >= this.maxBatchSize) {
+            this.flush('size_limit');
+            return;
+        }
+        // Age-based flush
+        if (age >= this.maxBatchAge && totalSize >= this.minBatchSize) {
+            this.flush('age_limit');
+            return;
+        }
+        // Schedule a flush if not already scheduled
+        if (!this.flushTimer && totalSize > 0) {
+            const delay = Math.max(10, this.maxBatchAge - age);
+            this.flushTimer = setTimeout(() => {
+                this.flush('timer');
+            }, delay);
+        }
+    }
+    /**
+     * Flush all queued operations
+     */
+    async flush(reason = 'manual') {
+        // Clear timer
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+            this.flushTimer = null;
+        }
+        // Collect all operations into a single batch
+        const batch = [];
+        // Process deletes first (highest priority)
+        this.deleteQueue.forEach((ops) => {
+            // Only take the first operation per key (others are duplicates)
+            if (ops.length > 0) {
+                batch.push(ops[0]);
+                this.stats.coalescedOperations += ops.length;
+            }
+        });
+        // Then writes
+        this.writeQueue.forEach((ops) => {
+            if (ops.length > 0) {
+                // Use the last write (most recent data)
+                const lastWrite = ops[ops.length - 1];
+                batch.push(lastWrite);
+                this.stats.coalescedOperations += ops.length;
+            }
+        });
+        // Then reads
+        this.readQueue.forEach((ops) => {
+            if (ops.length > 0) {
+                batch.push(ops[0]);
+                this.stats.coalescedOperations += ops.length;
+            }
+        });
+        // Clear queues
+        const allOps = [
+            ...Array.from(this.deleteQueue.values()).flat(),
+            ...Array.from(this.writeQueue.values()).flat(),
+            ...Array.from(this.readQueue.values()).flat()
+        ];
+        this.deleteQueue.clear();
+        this.writeQueue.clear();
+        this.readQueue.clear();
+        if (batch.length === 0) {
+            return;
+        }
+        // Update stats
+        this.stats.batchesProcessed++;
+        this.stats.averageBatchSize =
+            (this.stats.averageBatchSize * (this.stats.batchesProcessed - 1) + batch.length) /
+                this.stats.batchesProcessed;
+        this.logger.debug(`Flushing batch of ${batch.length} operations (${allOps.length} total) - reason: ${reason}`);
+        // Process the batch
+        try {
+            await this.processor(batch);
+            // Resolve all promises
+            allOps.forEach(op => {
+                if (op.type === 'read') {
+                    // Find the result for this read
+                    const result = batch.find(b => b.key === op.key && b.type === 'read');
+                    op.resolve(result?.data || null);
+                }
+                else {
+                    op.resolve(undefined);
+                }
+            });
+        }
+        catch (error) {
+            // Reject all promises
+            allOps.forEach(op => op.reject(error));
+            this.logger.error('Batch processing failed:', error);
+        }
+        this.lastFlush = Date.now();
+    }
+    /**
+     * Get current statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Get current queue sizes
+     */
+    getQueueSizes() {
+        return {
+            writes: this.writeQueue.size,
+            reads: this.readQueue.size,
+            deletes: this.deleteQueue.size,
+            total: this.writeQueue.size + this.readQueue.size + this.deleteQueue.size
+        };
+    }
+    /**
+     * Adjust batch parameters based on load
+     */
+    adjustParameters(pending) {
+        if (pending > 10000) {
+            // Extreme load - batch aggressively
+            this.maxBatchSize = 500;
+            this.maxBatchAge = 50;
+            this.minBatchSize = 50;
+        }
+        else if (pending > 1000) {
+            // High load - larger batches
+            this.maxBatchSize = 200;
+            this.maxBatchAge = 100;
+            this.minBatchSize = 20;
+        }
+        else if (pending > 100) {
+            // Moderate load
+            this.maxBatchSize = 100;
+            this.maxBatchAge = 200;
+            this.minBatchSize = 10;
+        }
+        else {
+            // Low load - optimize for latency
+            this.maxBatchSize = 50;
+            this.maxBatchAge = 500;
+            this.minBatchSize = 5;
+        }
+    }
+    /**
+     * Force immediate flush of all operations
+     */
+    async forceFlush() {
+        await this.flush('force');
+    }
+}
+// Global coalescer instances by storage type
+const coalescers = new Map();
+/**
+ * Get or create a coalescer for a storage instance
+ */
+export function getCoalescer(storageId, processor) {
+    if (!coalescers.has(storageId)) {
+        coalescers.set(storageId, new RequestCoalescer(processor));
+    }
+    return coalescers.get(storageId);
+}
+/**
+ * Clear all coalescers
+ */
+export function clearCoalescers() {
+    coalescers.clear();
+}
+//# sourceMappingURL=requestCoalescer.js.map

package/dist/utils/requestCoalescer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"requestCoalescer.js","sourceRoot":"","sources":["../../src/utils/requestCoalescer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAmBhD;;GAEG;AACH,MAAM,OAAO,gBAAgB;IA6B3B,YACE,SAAyD,EACzD,OAIC;QAlCK,WAAM,GAAG,kBAAkB,CAAC,kBAAkB,CAAC,CAAA;QAEvD,2BAA2B;QACnB,eAAU,GAAG,IAAI,GAAG,EAAgC,CAAA;QACpD,cAAS,GAAG,IAAI,GAAG,EAAgC,CAAA;QACnD,gBAAW,GAAG,IAAI,GAAG,EAAgC,CAAA;QAE7D,sBAAsB;QACd,iBAAY,GAAG,GAAG,CAAA;QAClB,gBAAW,GAAG,GAAG,CAAA,CAAE,gCAAgC;QACnD,iBAAY,GAAG,EAAE,CAAA,CAAG,mCAAmC;QAE/D,eAAe;QACP,eAAU,GAA0B,IAAI,CAAA;QACxC,cAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE9B,aAAa;QACL,UAAK,GAAe;YAC1B,eAAe,EAAE,CAAC;YAClB,mBAAmB,EAAE,CAAC;YACtB,YAAY,EAAE,CAAC;YACf,gBAAgB,EAAE,CAAC;YACnB,gBAAgB,EAAE,CAAC;SACpB,CAAA;QAaC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAE1B,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAA;YAC7D,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAA;YAC1D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAA;QAC/D,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,IAAS;QACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,wDAAwD;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzC,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,4CAA4C;gBAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBAC1C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA,CAAE,kBAAkB;gBAEpC,kCAAkC;gBAClC,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,OAAO;oBACb,GAAG;oBACH,IAAI;oBACJ,OAAO;oBACP,MAAM;oBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAA;gBAEF,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAA;YAC3B,CAAC;iBAAM,CAAC;gBACN,sBAAsB;gBACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;wBACxB,IAAI,EAAE,OAAO;wBACb,GAAG;wBACH,IAAI;wBACJ,OAAO;wBACP,MAAM;wBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC,CAAC,CAAA;YACL,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAA;YAC5B,IAAI,CAAC,UAAU,EAAE,CAAA;QACnB,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,IAAI,CAAC,GAAW;QAC3B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,uDAAuD;YACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAExC,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,8BAA8B;gBAC9B,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,MAAM;oBACZ,GAAG;oBACH,OAAO;oBACP,MAAM;oBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAA;gBAEF,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAA;YAC3B,CAAC;iBAAM,CAAC;gBACN,qBAAqB;gBACrB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;wBACvB,IAAI,EAAE,MAAM;wBACZ,GAAG;wBACH,OAAO;wBACP,MAAM;wBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC,CAAC,CAAA;YACL,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAA;YAC5B,IAAI,CAAC,UAAU,EAAE,CAAA;QACnB,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,MAAM,CAAC,GAAW;QAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,yCAAyC;YACzC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAA;gBACxC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;gBACjE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAC3B,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAA;YAC1C,CAAC;YAED,wCAAwC;YACxC,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAE,CAAA;gBACtC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA,CAAE,gCAAgC;gBACvE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAC1B,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,CAAA;YACzC,CAAC;YAED,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAE1C,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,gCAAgC;gBAChC,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,QAAQ;oBACd,GAAG;oBACH,OAAO;oBACP,MAAM;oBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAA;gBAEF,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAA;YAC3B,CAAC;iBAAM,CAAC;gBACN,uBAAuB;gBACvB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;wBACzB,IAAI,EAAE,QAAQ;wBACd,GAAG;wBACH,OAAO;wBACP,MAAM;wBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC,CAAC,CAAA;YACL,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAA;YAC5B,IAAI,CAAC,UAAU,EAAE,CAAA;QACnB,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACK,UAAU;QAChB,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAA;QACpF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,CAAA;QAEhC,6BAA6B;QAC7B,IAAI,SAAS,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YACxB,OAAM;QACR,CAAC;QAED,kBAAkB;QAClB,IAAI,GAAG,IAAI,IAAI,CAAC,WAAW,IAAI,SAAS,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9D,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;YACvB,OAAM;QACR,CAAC;QAED,4CAA4C;QAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,CAAA;YAClD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE;gBAChC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YACrB,CAAC,EAAE,KAAK,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,KAAK,CAAC,SAAiB,QAAQ;QAC1C,cAAc;QACd,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QACxB,CAAC;QAED,6CAA6C;QAC7C,MAAM,KAAK,GAAyB,EAAE,CAAA;QAEtC,2CAA2C;QAC3C,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/B,gEAAgE;YAChE,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClB,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,GAAG,CAAC,MAAM,CAAA;YAC9C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,cAAc;QACd,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC9B,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnB,wCAAwC;gBACxC,MAAM,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBACrC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACrB,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,GAAG,CAAC,MAAM,CAAA;YAC9C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,aAAa;QACb,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClB,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,GAAG,CAAC,MAAM,CAAA;YAC9C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,eAAe;QACf,MAAM,MAAM,GAAG;YACb,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE;YAC/C,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE;YAC9C,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE;SAC9C,CAAA;QAED,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QACxB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;QACvB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;QAEtB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAM;QACR,CAAC;QAED,eAAe;QACf,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAA;QAC7B,IAAI,CAAC,KAAK,CAAC,gBAAgB;YACzB,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;gBAChF,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAA;QAE7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,qBAAqB,MAAM,EAAE,CAAC,CAAA;QAE9G,oBAAoB;QACpB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YAE3B,uBAAuB;YACvB,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;gBAClB,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBACvB,gCAAgC;oBAChC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;oBACrE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,IAAI,CAAC,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACN,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;gBACvB,CAAC;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sBAAsB;YACtB,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;YAEtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAA;QACtD,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC7B,CAAC;IAED;;OAEG;IACI,QAAQ;QACb,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;IAC1B,CAAC;IAED;;OAEG;IACI,aAAa;QAMlB,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;YAC1B,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YAC9B,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI;SAC1E,CAAA;IACH,CAAC;IAED;;OAEG;IACI,gBAAgB,CAAC,OAAe;QACrC,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;YACpB,oCAAoC;YACpC,IAAI,CAAC,YAAY,GAAG,GAAG,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAA;YACrB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAA;QACxB,CAAC;aAAM,IAAI,OAAO,GAAG,IAAI,EAAE,CAAC;YAC1B,6BAA6B;YAC7B,IAAI,CAAC,YAAY,GAAG,GAAG,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,GAAG,CAAA;YACtB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAA;QACxB,CAAC;aAAM,IAAI,OAAO,GAAG,GAAG,EAAE,CAAC;YACzB,gBAAgB;YAChB,IAAI,CAAC,YAAY,GAAG,GAAG,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,GAAG,CAAA;YACtB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAA;QACxB,CAAC;aAAM,CAAC;YACN,kCAAkC;YAClC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAA;YACtB,IAAI,CAAC,WAAW,GAAG,GAAG,CAAA;YACtB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAA;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,UAAU;QACrB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAC3B,CAAC;CACF;AAED,6CAA6C;AAC7C,MAAM,UAAU,GAAG,IAAI,GAAG,EAA4B,CAAA;AAEtD;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAiB,EACjB,SAA0C;IAE1C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAA;IAC5D,CAAC;IACD,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,CAAE,CAAA;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,UAAU,CAAC,KAAK,EAAE,CAAA;AACpB,CAAC"}

package/dist/utils/writeBuffer.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Write Buffer
+ * Accumulates writes and flushes them in bulk to reduce S3 operations
+ * Implements intelligent deduplication and compression
+ */
+interface FlushResult {
+    successful: number;
+    failed: number;
+    duration: number;
+}
+/**
+ * High-performance write buffer for bulk operations
+ */
+export declare class WriteBuffer<T> {
+    private logger;
+    private buffer;
+    private maxBufferSize;
+    private flushInterval;
+    private minFlushSize;
+    private maxRetries;
+    private flushTimer;
+    private isFlushing;
+    private lastFlush;
+    private pendingFlush;
+    private totalWrites;
+    private totalFlushes;
+    private failedWrites;
+    private duplicatesRemoved;
+    private writeFunction;
+    private type;
+    private backpressure;
+    constructor(type: 'noun' | 'verb' | 'metadata', writeFunction: (items: Map<string, T>) => Promise<void>, options?: {
+        maxBufferSize?: number;
+        flushInterval?: number;
+        minFlushSize?: number;
+    });
+    /**
+     * Add item to buffer
+     */
+    add(id: string, data: T): Promise<void>;
+    /**
+     * Check if we should flush
+     */
+    private checkFlush;
+    /**
+     * Flush buffer to storage
+     */
+    flush(reason?: string): Promise<FlushResult>;
+    /**
+     * Perform the actual flush
+     */
+    private doFlush;
+    /**
+     * Start periodic flush timer
+     */
+    private startPeriodicFlush;
+    /**
+     * Stop periodic flush timer
+     */
+    stop(): void;
+    /**
+     * Force flush all pending writes
+     */
+    forceFlush(): Promise<FlushResult>;
+    /**
+     * Get buffer statistics
+     */
+    getStats(): {
+        bufferSize: number;
+        totalWrites: number;
+        totalFlushes: number;
+        failedWrites: number;
+        duplicatesRemoved: number;
+        avgFlushSize: number;
+    };
+    /**
+     * Adjust parameters based on load
+     */
+    adjustForLoad(pendingRequests: number): void;
+}
+/**
+ * Get or create a write buffer
+ */
+export declare function getWriteBuffer<T>(id: string, type: 'noun' | 'verb' | 'metadata', writeFunction: (items: Map<string, T>) => Promise<void>): WriteBuffer<T>;
+/**
+ * Flush all write buffers
+ */
+export declare function flushAllBuffers(): Promise<void>;
+/**
+ * Clear all write buffers
+ */
+export declare function clearWriteBuffers(): void;
+export {};