@soulcraft/brainy 2.1.0 → 2.3.0

package/dist/utils/periodicCleanup.js

@@ -0,0 +1,219 @@
+/**
+ * Periodic Cleanup for Soft-Deleted Items
+ *
+ * SAFETY-FIRST APPROACH:
+ * - Maintains durability guarantees (storage-first)
+ * - Coordinates HNSW and metadata index consistency
+ * - Isolated from live operations
+ * - Graceful failure handling
+ */
+import { prodLog } from './logger.js';
+import { isDeleted } from './metadataNamespace.js';
+/**
+ * Coordinates safe cleanup of old soft-deleted items across all indexes
+ *
+ * CRITICAL SAFETY FEATURES:
+ * 1. Storage-first deletion (durability)
+ * 2. Index consistency coordination
+ * 3. Batch processing with limits
+ * 4. Error isolation and recovery
+ */
+export class PeriodicCleanup {
+    constructor(storage, hnswIndex, metadataIndex, config = {}) {
+        this.cleanupTimer = null;
+        this.running = false;
+        this.storage = storage;
+        this.hnswIndex = hnswIndex;
+        this.metadataIndex = metadataIndex;
+        // Default: clean up items deleted more than 1 hour ago
+        this.config = {
+            maxAge: config.maxAge ?? 60 * 60 * 1000, // 1 hour
+            batchSize: config.batchSize ?? 100, // 100 items max per batch
+            cleanupInterval: config.cleanupInterval ?? 15 * 60 * 1000, // Every 15 minutes
+            enabled: config.enabled ?? true
+        };
+        this.stats = {
+            itemsProcessed: 0,
+            itemsDeleted: 0,
+            errors: 0,
+            lastRun: 0,
+            nextRun: 0
+        };
+    }
+    /**
+     * Start periodic cleanup
+     */
+    start() {
+        if (!this.config.enabled || this.cleanupTimer) {
+            return;
+        }
+        prodLog.info(`Starting periodic cleanup: maxAge=${this.config.maxAge}, batchSize=${this.config.batchSize}, interval=${this.config.cleanupInterval}`);
+        this.scheduleNext();
+    }
+    /**
+     * Stop periodic cleanup
+     */
+    stop() {
+        if (this.cleanupTimer) {
+            clearTimeout(this.cleanupTimer);
+            this.cleanupTimer = null;
+        }
+        prodLog.info('Stopped periodic cleanup');
+    }
+    /**
+     * Run cleanup manually
+     */
+    async runNow() {
+        if (this.running) {
+            throw new Error('Cleanup already running');
+        }
+        return this.performCleanup();
+    }
+    /**
+     * Get current cleanup statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    scheduleNext() {
+        const nextRun = Date.now() + this.config.cleanupInterval;
+        this.stats.nextRun = nextRun;
+        this.cleanupTimer = setTimeout(async () => {
+            await this.performCleanup();
+            this.scheduleNext();
+        }, this.config.cleanupInterval);
+    }
+    /**
+     * CRITICAL: Coordinated cleanup across all indexes
+     *
+     * SAFETY PROTOCOL:
+     * 1. Find eligible items (old + soft-deleted)
+     * 2. Remove from storage FIRST (durability)
+     * 3. Remove from HNSW (graph consistency)
+     * 4. Remove from metadata index (search consistency)
+     * 5. Track stats and errors
+     */
+    async performCleanup() {
+        if (this.running) {
+            prodLog.warn('Cleanup already running, skipping');
+            return this.stats;
+        }
+        this.running = true;
+        const startTime = Date.now();
+        this.stats.lastRun = startTime;
+        try {
+            prodLog.debug(`Starting cleanup run: maxAge=${this.config.maxAge}, cutoffTime=${startTime - this.config.maxAge}`);
+            // Step 1: Find eligible items for cleanup
+            const eligibleItems = await this.findEligibleItems(startTime);
+            if (eligibleItems.length === 0) {
+                prodLog.debug('No items eligible for cleanup');
+                return this.stats;
+            }
+            prodLog.info(`Found ${eligibleItems.length} items eligible for cleanup`);
+            // Step 2: Process in batches for safety
+            let processed = 0;
+            let deleted = 0;
+            let errors = 0;
+            for (let i = 0; i < eligibleItems.length; i += this.config.batchSize) {
+                const batch = eligibleItems.slice(i, i + this.config.batchSize);
+                const batchResult = await this.processBatch(batch);
+                processed += batchResult.processed;
+                deleted += batchResult.deleted;
+                errors += batchResult.errors;
+                // Small delay between batches to avoid overwhelming the system
+                if (i + this.config.batchSize < eligibleItems.length) {
+                    await new Promise(resolve => setTimeout(resolve, 10));
+                }
+            }
+            // Update stats
+            this.stats.itemsProcessed += processed;
+            this.stats.itemsDeleted += deleted;
+            this.stats.errors += errors;
+            prodLog.info(`Cleanup run completed: processed=${processed}, deleted=${deleted}, errors=${errors}, duration=${Date.now() - startTime}ms`);
+        }
+        catch (error) {
+            prodLog.error(`Cleanup run failed: ${error}`);
+            this.stats.errors++;
+        }
+        finally {
+            this.running = false;
+        }
+        return this.stats;
+    }
+    /**
+     * Find items eligible for cleanup (old + soft-deleted)
+     */
+    async findEligibleItems(currentTime) {
+        const cutoffTime = currentTime - this.config.maxAge;
+        const eligibleItems = [];
+        try {
+            // Get all nouns from storage (using pagination to avoid memory issues)
+            const nounsResult = await this.storage.getNouns({
+                pagination: { limit: 1000 } // Process in chunks
+            });
+            for (const noun of nounsResult.items) {
+                try {
+                    if (!noun.metadata || !isDeleted(noun.metadata)) {
+                        continue; // Not deleted, skip
+                    }
+                    // Check if old enough for cleanup
+                    const deletedTime = noun.metadata._brainy?.updated || 0;
+                    if (deletedTime && (currentTime - deletedTime) > this.config.maxAge) {
+                        eligibleItems.push(noun.id);
+                    }
+                }
+                catch (error) {
+                    prodLog.warn(`Failed to check item ${noun.id} for cleanup eligibility: ${error}`);
+                }
+            }
+        }
+        catch (error) {
+            prodLog.error(`Failed to find eligible items: ${error}`);
+            throw error;
+        }
+        return eligibleItems;
+    }
+    /**
+     * Process a batch of items for cleanup
+     *
+     * CRITICAL: This maintains the durability-first approach:
+     * Storage → HNSW → Metadata Index
+     */
+    async processBatch(itemIds) {
+        let processed = 0;
+        let deleted = 0;
+        let errors = 0;
+        for (const id of itemIds) {
+            processed++;
+            try {
+                // STEP 1: Remove from storage FIRST (durability guarantee)
+                try {
+                    await this.storage.deleteNoun(id);
+                }
+                catch (storageError) {
+                    prodLog.warn(`Failed to delete ${id} from storage: ${storageError}`);
+                    errors++;
+                    continue;
+                }
+                // STEP 2: Remove from HNSW index (vector search consistency)
+                const hnswResult = this.hnswIndex.removeItem(id);
+                if (!hnswResult) {
+                    prodLog.warn(`Failed to remove ${id} from HNSW index (may not have been indexed)`);
+                    // Not a critical error - item might not have been in vector index
+                }
+                // STEP 3: Remove from metadata index (faceted search consistency)  
+                if (this.metadataIndex) {
+                    await this.metadataIndex.removeFromIndex(id);
+                }
+                deleted++;
+                prodLog.debug(`Successfully cleaned up item ${id}`);
+            }
+            catch (error) {
+                errors++;
+                prodLog.error(`Failed to cleanup item ${id}: ${error}`);
+            }
+        }
+        return { processed, deleted, errors };
+    }
+}
+//# sourceMappingURL=periodicCleanup.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "2.1.0",
+  "version": "2.3.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -77,7 +77,12 @@
     "lint": "eslint --ext .ts,.js src/",
     "lint:fix": "eslint --ext .ts,.js src/ --fix",
     "format": "prettier --write \"src/**/*.{ts,js}\"",
-    "format:check": "prettier --check \"src/**/*.{ts,js}\""
+    "format:check": "prettier --check \"src/**/*.{ts,js}\"",
+    "release": "standard-version",
+    "release:patch": "standard-version --release-as patch",
+    "release:minor": "standard-version --release-as minor",
+    "release:major": "standard-version --release-as major",
+    "release:dry": "standard-version --dry-run"
   },
   "keywords": [
     "ai-database",
@@ -130,13 +135,14 @@
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
     "@vitest/coverage-v8": "^3.2.4",
+    "standard-version": "^9.5.0",
     "tsx": "^4.19.2",
     "typescript": "^5.4.5",
     "vitest": "^3.2.4"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.540.0",
-    "@huggingface/transformers": "^3.1.0",
+    "@huggingface/transformers": "^3.7.2",
     "boxen": "^8.0.1",
     "chalk": "^5.3.0",
     "cli-table3": "^0.6.5",