@soulcraft/brainy 2.1.0 → 3.0.0

package/dist/utils/metadataIndex.js

@@ -412,6 +412,36 @@ export class MetadataIndexManager {
             }
         }
     }
+    /**
+     * Get all IDs in the index
+     */
+    async getAllIds() {
+        // Collect all unique IDs from all index entries
+        const allIds = new Set();
+        // First, add all IDs from the in-memory cache
+        for (const entry of this.indexCache.values()) {
+            entry.ids.forEach(id => allIds.add(id));
+        }
+        // If storage has a method to get all nouns, use it as the source of truth
+        // This ensures we include items that might not be indexed yet
+        if (this.storage && typeof this.storage.getNouns === 'function') {
+            try {
+                const result = await this.storage.getNouns({
+                    pagination: { limit: 100000 }
+                });
+                if (result && result.items) {
+                    result.items.forEach((item) => {
+                        if (item.id)
+                            allIds.add(item.id);
+                    });
+                }
+            }
+            catch (e) {
+                // Fall back to using only indexed IDs
+            }
+        }
+        return Array.from(allIds);
+    }
     /**
      * Get IDs for a specific field-value combination with caching
      */
@@ -638,6 +668,21 @@ export class MetadataIndexManager {
                                 fieldResults = Array.from(allIds);
                             }
                             break;
+                        // Negation operators
+                        case 'notEquals':
+                        case 'isNot':
+                        case 'ne':
+                            // For notEquals, we need all IDs EXCEPT those matching the value
+                            // This is especially important for soft delete: deleted !== true
+                            // should include items without a deleted field
+                            // First, get all IDs in the database
+                            const allItemIds = await this.getAllIds();
+                            // Then get IDs that match the value we want to exclude
+                            const excludeIds = await this.getIds(field, operand);
+                            const excludeSet = new Set(excludeIds);
+                            // Return all IDs except those to exclude
+                            fieldResults = allItemIds.filter(id => !excludeSet.has(id));
+                            break;
                     }
                 }
             }

package/dist/utils/metadataNamespace.d.ts

@@ -0,0 +1,113 @@
+/**
+ * Clean Metadata Architecture for Brainy 2.2
+ * No backward compatibility - doing it RIGHT from the start!
+ */
+export declare const BRAINY_NS: "_brainy";
+export declare const AUG_NS: "_augmentations";
+export declare const AUDIT_NS: "_audit";
+export declare const DELETED_FIELD: "_brainy.deleted";
+export declare const INDEXED_FIELD: "_brainy.indexed";
+export declare const VERSION_FIELD: "_brainy.version";
+/**
+ * Internal Brainy metadata structure
+ * These fields are ALWAYS present and indexed for O(1) access
+ */
+export interface BrainyInternalMetadata {
+    deleted: boolean;
+    indexed: boolean;
+    version: number;
+    created: number;
+    updated: number;
+    partition?: number;
+    domain?: string;
+    priority?: number;
+    ttl?: number;
+}
+/**
+ * Complete metadata structure with namespaces
+ */
+export interface NamespacedMetadata<T = any> {
+    [key: string]: any;
+    [BRAINY_NS]: BrainyInternalMetadata;
+    [AUG_NS]?: {
+        [augmentationName: string]: any;
+    };
+    [AUDIT_NS]?: Array<{
+        timestamp: number;
+        augmentation: string;
+        field: string;
+        oldValue: any;
+        newValue: any;
+    }>;
+}
+/**
+ * Create properly namespaced metadata
+ * This is called for EVERY noun/verb creation
+ */
+export declare function createNamespacedMetadata<T = any>(userMetadata?: T): NamespacedMetadata<T>;
+/**
+ * Update metadata while preserving namespaces
+ */
+export declare function updateNamespacedMetadata<T = any>(existing: NamespacedMetadata<T>, updates: Partial<T>): NamespacedMetadata<T>;
+/**
+ * Soft delete a noun (O(1) operation)
+ */
+export declare function markDeleted<T = any>(metadata: NamespacedMetadata<T>): NamespacedMetadata<T>;
+/**
+ * Restore a soft-deleted noun (O(1) operation)
+ */
+export declare function markRestored<T = any>(metadata: NamespacedMetadata<T>): NamespacedMetadata<T>;
+/**
+ * Check if a noun is deleted (O(1) check)
+ */
+export declare function isDeleted<T = any>(metadata: NamespacedMetadata<T>): boolean;
+/**
+ * Get user metadata without internal fields
+ * Used by augmentations to get clean user data
+ */
+export declare function getUserMetadata<T = any>(metadata: NamespacedMetadata<T>): T;
+/**
+ * Set augmentation data in isolated namespace
+ */
+export declare function setAugmentationData<T = any>(metadata: NamespacedMetadata<T>, augmentationName: string, data: any): NamespacedMetadata<T>;
+/**
+ * Add audit entry for tracking
+ */
+export declare function addAuditEntry<T = any>(metadata: NamespacedMetadata<T>, entry: {
+    augmentation: string;
+    field: string;
+    oldValue: any;
+    newValue: any;
+}): NamespacedMetadata<T>;
+/**
+ * INDEXING EXPLANATION:
+ *
+ * The MetadataIndex flattens nested objects into dot-notation keys:
+ *
+ * Input metadata:
+ * {
+ *   name: "Django",
+ *   _brainy: {
+ *     deleted: false,
+ *     indexed: true
+ *   }
+ * }
+ *
+ * Creates index entries:
+ * - "name" -> "django" -> Set([id1, id2...])
+ * - "_brainy.deleted" -> "false" -> Set([id1, id2...])  // O(1) lookup!
+ * - "_brainy.indexed" -> "true" -> Set([id1, id2...])
+ *
+ * Query: { "_brainy.deleted": false }
+ * Lookup: index["_brainy.deleted"]["false"] -> Set of IDs in O(1)
+ *
+ * This is why namespacing doesn't hurt performance - it's all flattened!
+ */
+/**
+ * Fields that should ALWAYS be indexed for O(1) access
+ */
+export declare const ALWAYS_INDEXED_FIELDS: ("_brainy.deleted" | "_brainy.indexed" | "_brainy.version")[];
+/**
+ * Fields that should use sorted index for O(log n) range queries
+ */
+export declare const SORTED_INDEX_FIELDS: string[];

package/dist/utils/metadataNamespace.js

@@ -0,0 +1,162 @@
+/**
+ * Clean Metadata Architecture for Brainy 2.2
+ * No backward compatibility - doing it RIGHT from the start!
+ */
+// Namespace constants
+export const BRAINY_NS = '_brainy';
+export const AUG_NS = '_augmentations';
+export const AUDIT_NS = '_audit';
+// Field paths for O(1) indexing
+export const DELETED_FIELD = `${BRAINY_NS}.deleted`;
+export const INDEXED_FIELD = `${BRAINY_NS}.indexed`;
+export const VERSION_FIELD = `${BRAINY_NS}.version`;
+/**
+ * Create properly namespaced metadata
+ * This is called for EVERY noun/verb creation
+ */
+export function createNamespacedMetadata(userMetadata) {
+    const now = Date.now();
+    // Start with user metadata or empty object
+    const result = userMetadata ? { ...userMetadata } : {};
+    // ALWAYS add internal namespace with required fields
+    result[BRAINY_NS] = {
+        deleted: false, // CRITICAL: Always false for new items
+        indexed: true, // New items are indexed
+        version: 1, // Current schema version
+        created: now,
+        updated: now
+    };
+    return result;
+}
+/**
+ * Update metadata while preserving namespaces
+ */
+export function updateNamespacedMetadata(existing, updates) {
+    const now = Date.now();
+    // Merge user fields
+    const result = {
+        ...existing,
+        ...updates
+    };
+    // Preserve internal namespace but update timestamp
+    result[BRAINY_NS] = {
+        ...existing[BRAINY_NS],
+        updated: now
+    };
+    // Preserve augmentation namespace
+    if (existing[AUG_NS]) {
+        result[AUG_NS] = existing[AUG_NS];
+    }
+    // Preserve audit trail
+    if (existing[AUDIT_NS]) {
+        result[AUDIT_NS] = existing[AUDIT_NS];
+    }
+    return result;
+}
+/**
+ * Soft delete a noun (O(1) operation)
+ */
+export function markDeleted(metadata) {
+    return {
+        ...metadata,
+        [BRAINY_NS]: {
+            ...metadata[BRAINY_NS],
+            deleted: true,
+            updated: Date.now()
+        }
+    };
+}
+/**
+ * Restore a soft-deleted noun (O(1) operation)
+ */
+export function markRestored(metadata) {
+    return {
+        ...metadata,
+        [BRAINY_NS]: {
+            ...metadata[BRAINY_NS],
+            deleted: false,
+            updated: Date.now()
+        }
+    };
+}
+/**
+ * Check if a noun is deleted (O(1) check)
+ */
+export function isDeleted(metadata) {
+    return metadata[BRAINY_NS]?.deleted === true;
+}
+/**
+ * Get user metadata without internal fields
+ * Used by augmentations to get clean user data
+ */
+export function getUserMetadata(metadata) {
+    const { [BRAINY_NS]: _, [AUG_NS]: __, [AUDIT_NS]: ___, ...userMeta } = metadata;
+    return userMeta;
+}
+/**
+ * Set augmentation data in isolated namespace
+ */
+export function setAugmentationData(metadata, augmentationName, data) {
+    const result = { ...metadata };
+    if (!result[AUG_NS]) {
+        result[AUG_NS] = {};
+    }
+    result[AUG_NS][augmentationName] = data;
+    return result;
+}
+/**
+ * Add audit entry for tracking
+ */
+export function addAuditEntry(metadata, entry) {
+    const result = { ...metadata };
+    if (!result[AUDIT_NS]) {
+        result[AUDIT_NS] = [];
+    }
+    result[AUDIT_NS].push({
+        ...entry,
+        timestamp: Date.now()
+    });
+    return result;
+}
+/**
+ * INDEXING EXPLANATION:
+ *
+ * The MetadataIndex flattens nested objects into dot-notation keys:
+ *
+ * Input metadata:
+ * {
+ *   name: "Django",
+ *   _brainy: {
+ *     deleted: false,
+ *     indexed: true
+ *   }
+ * }
+ *
+ * Creates index entries:
+ * - "name" -> "django" -> Set([id1, id2...])
+ * - "_brainy.deleted" -> "false" -> Set([id1, id2...])  // O(1) lookup!
+ * - "_brainy.indexed" -> "true" -> Set([id1, id2...])
+ *
+ * Query: { "_brainy.deleted": false }
+ * Lookup: index["_brainy.deleted"]["false"] -> Set of IDs in O(1)
+ *
+ * This is why namespacing doesn't hurt performance - it's all flattened!
+ */
+/**
+ * Fields that should ALWAYS be indexed for O(1) access
+ */
+export const ALWAYS_INDEXED_FIELDS = [
+    DELETED_FIELD, // For soft delete filtering
+    INDEXED_FIELD, // For index management
+    VERSION_FIELD // For schema versioning
+];
+/**
+ * Fields that should use sorted index for O(log n) range queries
+ */
+export const SORTED_INDEX_FIELDS = [
+    `${BRAINY_NS}.created`,
+    `${BRAINY_NS}.updated`,
+    `${BRAINY_NS}.priority`,
+    `${BRAINY_NS}.ttl`
+];
+//# sourceMappingURL=metadataNamespace.js.map

package/dist/utils/periodicCleanup.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Periodic Cleanup for Soft-Deleted Items
+ *
+ * SAFETY-FIRST APPROACH:
+ * - Maintains durability guarantees (storage-first)
+ * - Coordinates HNSW and metadata index consistency
+ * - Isolated from live operations
+ * - Graceful failure handling
+ */
+import type { StorageAdapter } from '../coreTypes.js';
+import type { HNSWIndex } from '../hnsw/hnswIndex.js';
+import type { MetadataIndexManager } from './metadataIndex.js';
+export interface CleanupConfig {
+    /** Age in milliseconds after which soft-deleted items are eligible for cleanup */
+    maxAge: number;
+    /** Maximum number of items to clean up in one batch */
+    batchSize: number;
+    /** Interval between cleanup runs (milliseconds) */
+    cleanupInterval: number;
+    /** Whether to run cleanup automatically */
+    enabled: boolean;
+}
+export interface CleanupStats {
+    itemsProcessed: number;
+    itemsDeleted: number;
+    errors: number;
+    lastRun: number;
+    nextRun: number;
+}
+/**
+ * Coordinates safe cleanup of old soft-deleted items across all indexes
+ *
+ * CRITICAL SAFETY FEATURES:
+ * 1. Storage-first deletion (durability)
+ * 2. Index consistency coordination
+ * 3. Batch processing with limits
+ * 4. Error isolation and recovery
+ */
+export declare class PeriodicCleanup {
+    private storage;
+    private hnswIndex;
+    private metadataIndex;
+    private config;
+    private stats;
+    private cleanupTimer;
+    private running;
+    constructor(storage: StorageAdapter, hnswIndex: HNSWIndex, metadataIndex: MetadataIndexManager | null, config?: Partial<CleanupConfig>);
+    /**
+     * Start periodic cleanup
+     */
+    start(): void;
+    /**
+     * Stop periodic cleanup
+     */
+    stop(): void;
+    /**
+     * Run cleanup manually
+     */
+    runNow(): Promise<CleanupStats>;
+    /**
+     * Get current cleanup statistics
+     */
+    getStats(): CleanupStats;
+    private scheduleNext;
+    /**
+     * CRITICAL: Coordinated cleanup across all indexes
+     *
+     * SAFETY PROTOCOL:
+     * 1. Find eligible items (old + soft-deleted)
+     * 2. Remove from storage FIRST (durability)
+     * 3. Remove from HNSW (graph consistency)
+     * 4. Remove from metadata index (search consistency)
+     * 5. Track stats and errors
+     */
+    private performCleanup;
+    /**
+     * Find items eligible for cleanup (old + soft-deleted)
+     */
+    private findEligibleItems;
+    /**
+     * Process a batch of items for cleanup
+     *
+     * CRITICAL: This maintains the durability-first approach:
+     * Storage → HNSW → Metadata Index
+     */
+    private processBatch;
+}

package/dist/utils/periodicCleanup.js

@@ -0,0 +1,219 @@
+/**
+ * Periodic Cleanup for Soft-Deleted Items
+ *
+ * SAFETY-FIRST APPROACH:
+ * - Maintains durability guarantees (storage-first)
+ * - Coordinates HNSW and metadata index consistency
+ * - Isolated from live operations
+ * - Graceful failure handling
+ */
+import { prodLog } from './logger.js';
+import { isDeleted } from './metadataNamespace.js';
+/**
+ * Coordinates safe cleanup of old soft-deleted items across all indexes
+ *
+ * CRITICAL SAFETY FEATURES:
+ * 1. Storage-first deletion (durability)
+ * 2. Index consistency coordination
+ * 3. Batch processing with limits
+ * 4. Error isolation and recovery
+ */
+export class PeriodicCleanup {
+    constructor(storage, hnswIndex, metadataIndex, config = {}) {
+        this.cleanupTimer = null;
+        this.running = false;
+        this.storage = storage;
+        this.hnswIndex = hnswIndex;
+        this.metadataIndex = metadataIndex;
+        // Default: clean up items deleted more than 1 hour ago
+        this.config = {
+            maxAge: config.maxAge ?? 60 * 60 * 1000, // 1 hour
+            batchSize: config.batchSize ?? 100, // 100 items max per batch
+            cleanupInterval: config.cleanupInterval ?? 15 * 60 * 1000, // Every 15 minutes
+            enabled: config.enabled ?? true
+        };
+        this.stats = {
+            itemsProcessed: 0,
+            itemsDeleted: 0,
+            errors: 0,
+            lastRun: 0,
+            nextRun: 0
+        };
+    }
+    /**
+     * Start periodic cleanup
+     */
+    start() {
+        if (!this.config.enabled || this.cleanupTimer) {
+            return;
+        }
+        prodLog.info(`Starting periodic cleanup: maxAge=${this.config.maxAge}, batchSize=${this.config.batchSize}, interval=${this.config.cleanupInterval}`);
+        this.scheduleNext();
+    }
+    /**
+     * Stop periodic cleanup
+     */
+    stop() {
+        if (this.cleanupTimer) {
+            clearTimeout(this.cleanupTimer);
+            this.cleanupTimer = null;
+        }
+        prodLog.info('Stopped periodic cleanup');
+    }
+    /**
+     * Run cleanup manually
+     */
+    async runNow() {
+        if (this.running) {
+            throw new Error('Cleanup already running');
+        }
+        return this.performCleanup();
+    }
+    /**
+     * Get current cleanup statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    scheduleNext() {
+        const nextRun = Date.now() + this.config.cleanupInterval;
+        this.stats.nextRun = nextRun;
+        this.cleanupTimer = setTimeout(async () => {
+            await this.performCleanup();
+            this.scheduleNext();
+        }, this.config.cleanupInterval);
+    }
+    /**
+     * CRITICAL: Coordinated cleanup across all indexes
+     *
+     * SAFETY PROTOCOL:
+     * 1. Find eligible items (old + soft-deleted)
+     * 2. Remove from storage FIRST (durability)
+     * 3. Remove from HNSW (graph consistency)
+     * 4. Remove from metadata index (search consistency)
+     * 5. Track stats and errors
+     */
+    async performCleanup() {
+        if (this.running) {
+            prodLog.warn('Cleanup already running, skipping');
+            return this.stats;
+        }
+        this.running = true;
+        const startTime = Date.now();
+        this.stats.lastRun = startTime;
+        try {
+            prodLog.debug(`Starting cleanup run: maxAge=${this.config.maxAge}, cutoffTime=${startTime - this.config.maxAge}`);
+            // Step 1: Find eligible items for cleanup
+            const eligibleItems = await this.findEligibleItems(startTime);
+            if (eligibleItems.length === 0) {
+                prodLog.debug('No items eligible for cleanup');
+                return this.stats;
+            }
+            prodLog.info(`Found ${eligibleItems.length} items eligible for cleanup`);
+            // Step 2: Process in batches for safety
+            let processed = 0;
+            let deleted = 0;
+            let errors = 0;
+            for (let i = 0; i < eligibleItems.length; i += this.config.batchSize) {
+                const batch = eligibleItems.slice(i, i + this.config.batchSize);
+                const batchResult = await this.processBatch(batch);
+                processed += batchResult.processed;
+                deleted += batchResult.deleted;
+                errors += batchResult.errors;
+                // Small delay between batches to avoid overwhelming the system
+                if (i + this.config.batchSize < eligibleItems.length) {
+                    await new Promise(resolve => setTimeout(resolve, 10));
+                }
+            }
+            // Update stats
+            this.stats.itemsProcessed += processed;
+            this.stats.itemsDeleted += deleted;
+            this.stats.errors += errors;
+            prodLog.info(`Cleanup run completed: processed=${processed}, deleted=${deleted}, errors=${errors}, duration=${Date.now() - startTime}ms`);
+        }
+        catch (error) {
+            prodLog.error(`Cleanup run failed: ${error}`);
+            this.stats.errors++;
+        }
+        finally {
+            this.running = false;
+        }
+        return this.stats;
+    }
+    /**
+     * Find items eligible for cleanup (old + soft-deleted)
+     */
+    async findEligibleItems(currentTime) {
+        const cutoffTime = currentTime - this.config.maxAge;
+        const eligibleItems = [];
+        try {
+            // Get all nouns from storage (using pagination to avoid memory issues)
+            const nounsResult = await this.storage.getNouns({
+                pagination: { limit: 1000 } // Process in chunks
+            });
+            for (const noun of nounsResult.items) {
+                try {
+                    if (!noun.metadata || !isDeleted(noun.metadata)) {
+                        continue; // Not deleted, skip
+                    }
+                    // Check if old enough for cleanup
+                    const deletedTime = noun.metadata._brainy?.updated || 0;
+                    if (deletedTime && (currentTime - deletedTime) > this.config.maxAge) {
+                        eligibleItems.push(noun.id);
+                    }
+                }
+                catch (error) {
+                    prodLog.warn(`Failed to check item ${noun.id} for cleanup eligibility: ${error}`);
+                }
+            }
+        }
+        catch (error) {
+            prodLog.error(`Failed to find eligible items: ${error}`);
+            throw error;
+        }
+        return eligibleItems;
+    }
+    /**
+     * Process a batch of items for cleanup
+     *
+     * CRITICAL: This maintains the durability-first approach:
+     * Storage → HNSW → Metadata Index
+     */
+    async processBatch(itemIds) {
+        let processed = 0;
+        let deleted = 0;
+        let errors = 0;
+        for (const id of itemIds) {
+            processed++;
+            try {
+                // STEP 1: Remove from storage FIRST (durability guarantee)
+                try {
+                    await this.storage.deleteNoun(id);
+                }
+                catch (storageError) {
+                    prodLog.warn(`Failed to delete ${id} from storage: ${storageError}`);
+                    errors++;
+                    continue;
+                }
+                // STEP 2: Remove from HNSW index (vector search consistency)
+                const hnswResult = this.hnswIndex.removeItem(id);
+                if (!hnswResult) {
+                    prodLog.warn(`Failed to remove ${id} from HNSW index (may not have been indexed)`);
+                    // Not a critical error - item might not have been in vector index
+                }
+                // STEP 3: Remove from metadata index (faceted search consistency)  
+                if (this.metadataIndex) {
+                    await this.metadataIndex.removeFromIndex(id);
+                }
+                deleted++;
+                prodLog.debug(`Successfully cleaned up item ${id}`);
+            }
+            catch (error) {
+                errors++;
+                prodLog.error(`Failed to cleanup item ${id}: ${error}`);
+            }
+        }
+        return { processed, deleted, errors };
+    }
+}
+//# sourceMappingURL=periodicCleanup.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "2.1.0",
+  "version": "3.0.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -77,7 +77,12 @@
     "lint": "eslint --ext .ts,.js src/",
     "lint:fix": "eslint --ext .ts,.js src/ --fix",
     "format": "prettier --write \"src/**/*.{ts,js}\"",
-    "format:check": "prettier --check \"src/**/*.{ts,js}\""
+    "format:check": "prettier --check \"src/**/*.{ts,js}\"",
+    "release": "standard-version",
+    "release:patch": "standard-version --release-as patch",
+    "release:minor": "standard-version --release-as minor",
+    "release:major": "standard-version --release-as major",
+    "release:dry": "standard-version --dry-run"
   },
   "keywords": [
     "ai-database",
@@ -130,13 +135,14 @@
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
     "@vitest/coverage-v8": "^3.2.4",
+    "standard-version": "^9.5.0",
     "tsx": "^4.19.2",
     "typescript": "^5.4.5",
     "vitest": "^3.2.4"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.540.0",
-    "@huggingface/transformers": "^3.1.0",
+    "@huggingface/transformers": "^3.7.2",
     "boxen": "^8.0.1",
     "chalk": "^5.3.0",
     "cli-table3": "^0.6.5",