@soulcraft/brainy 2.1.0 → 2.3.0

package/dist/utils/deletedItemsIndex.js

@@ -0,0 +1,98 @@
+/**
+ * Dedicated index for tracking soft-deleted items
+ * This is MUCH more efficient than checking every item in the database
+ *
+ * Performance characteristics:
+ * - Add deleted item: O(1)
+ * - Remove deleted item: O(1)
+ * - Check if deleted: O(1)
+ * - Get all deleted: O(d) where d = number of deleted items << total items
+ */
+export class DeletedItemsIndex {
+    constructor() {
+        this.deletedIds = new Set();
+        this.deletedCount = 0;
+    }
+    /**
+     * Mark an item as deleted
+     */
+    markDeleted(id) {
+        if (!this.deletedIds.has(id)) {
+            this.deletedIds.add(id);
+            this.deletedCount++;
+        }
+    }
+    /**
+     * Mark an item as not deleted (restored)
+     */
+    markRestored(id) {
+        if (this.deletedIds.delete(id)) {
+            this.deletedCount--;
+        }
+    }
+    /**
+     * Check if an item is deleted - O(1)
+     */
+    isDeleted(id) {
+        return this.deletedIds.has(id);
+    }
+    /**
+     * Get all deleted item IDs - O(d)
+     */
+    getAllDeleted() {
+        return Array.from(this.deletedIds);
+    }
+    /**
+     * Filter out deleted items from results - O(k) where k = result count
+     */
+    filterDeleted(items) {
+        if (this.deletedCount === 0) {
+            // Fast path - no deleted items
+            return items;
+        }
+        return items.filter(item => {
+            const id = item.id;
+            return id ? !this.deletedIds.has(id) : true;
+        });
+    }
+    /**
+     * Get statistics
+     */
+    getStats() {
+        return {
+            deletedCount: this.deletedCount,
+            memoryUsage: this.deletedCount * 100 // Rough estimate: 100 bytes per ID
+        };
+    }
+    /**
+     * Clear all deleted items (for testing)
+     */
+    clear() {
+        this.deletedIds.clear();
+        this.deletedCount = 0;
+    }
+    /**
+     * Serialize for persistence
+     */
+    serialize() {
+        return JSON.stringify(Array.from(this.deletedIds));
+    }
+    /**
+     * Deserialize from persistence
+     */
+    deserialize(data) {
+        try {
+            const ids = JSON.parse(data);
+            this.deletedIds = new Set(ids);
+            this.deletedCount = this.deletedIds.size;
+        }
+        catch (e) {
+            console.warn('Failed to deserialize deleted items index');
+        }
+    }
+}
+/**
+ * Global singleton for deleted items tracking
+ */
+export const deletedItemsIndex = new DeletedItemsIndex();
+//# sourceMappingURL=deletedItemsIndex.js.map

package/dist/utils/ensureDeleted.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Utility to ensure all metadata has the deleted field set properly
+ * This is CRITICAL for O(1) soft delete filtering performance
+ *
+ * Uses _brainy namespace to avoid conflicts with user metadata
+ */
+/**
+ * Ensure metadata has internal Brainy fields set
+ * @param metadata The metadata object (could be null/undefined)
+ * @param preserveExisting If true, preserve existing deleted value
+ * @returns Metadata with internal fields guaranteed
+ */
+export declare function ensureDeletedField(metadata: any, preserveExisting?: boolean): any;
+/**
+ * Mark an item as soft deleted
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=true
+ */
+export declare function markAsDeleted(metadata: any): any;
+/**
+ * Mark an item as restored (not deleted)
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=false
+ */
+export declare function markAsRestored(metadata: any): any;
+/**
+ * Check if an item is deleted
+ * @param metadata The metadata object
+ * @returns true if deleted, false otherwise (including if field missing)
+ */
+export declare function isDeleted(metadata: any): boolean;
+/**
+ * Check if an item is active (not deleted)
+ * @param metadata The metadata object
+ * @returns true if not deleted (default), false if deleted
+ */
+export declare function isActive(metadata: any): boolean;
+export declare const BRAINY_DELETED_FIELD = "_brainy.deleted";

package/dist/utils/ensureDeleted.js

@@ -0,0 +1,79 @@
+/**
+ * Utility to ensure all metadata has the deleted field set properly
+ * This is CRITICAL for O(1) soft delete filtering performance
+ *
+ * Uses _brainy namespace to avoid conflicts with user metadata
+ */
+const BRAINY_NAMESPACE = '_brainy';
+/**
+ * Ensure metadata has internal Brainy fields set
+ * @param metadata The metadata object (could be null/undefined)
+ * @param preserveExisting If true, preserve existing deleted value
+ * @returns Metadata with internal fields guaranteed
+ */
+export function ensureDeletedField(metadata, preserveExisting = true) {
+    // Handle null/undefined metadata
+    if (!metadata) {
+        return {
+            [BRAINY_NAMESPACE]: {
+                deleted: false,
+                version: 1
+            }
+        };
+    }
+    // Clone to avoid mutation
+    const result = { ...metadata };
+    // Ensure _brainy namespace exists
+    if (!result[BRAINY_NAMESPACE]) {
+        result[BRAINY_NAMESPACE] = {};
+    }
+    // Set deleted field if not present
+    if (!('deleted' in result[BRAINY_NAMESPACE])) {
+        result[BRAINY_NAMESPACE].deleted = false;
+    }
+    else if (!preserveExisting) {
+        // Force to false if not preserving
+        result[BRAINY_NAMESPACE].deleted = false;
+    }
+    return result;
+}
+/**
+ * Mark an item as soft deleted
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=true
+ */
+export function markAsDeleted(metadata) {
+    const result = ensureDeletedField(metadata);
+    result[BRAINY_NAMESPACE].deleted = true;
+    return result;
+}
+/**
+ * Mark an item as restored (not deleted)
+ * @param metadata The metadata object
+ * @returns Metadata with _brainy.deleted=false
+ */
+export function markAsRestored(metadata) {
+    const result = ensureDeletedField(metadata);
+    result[BRAINY_NAMESPACE].deleted = false;
+    return result;
+}
+/**
+ * Check if an item is deleted
+ * @param metadata The metadata object
+ * @returns true if deleted, false otherwise (including if field missing)
+ */
+export function isDeleted(metadata) {
+    return metadata?.[BRAINY_NAMESPACE]?.deleted === true;
+}
+/**
+ * Check if an item is active (not deleted)
+ * @param metadata The metadata object
+ * @returns true if not deleted (default), false if deleted
+ */
+export function isActive(metadata) {
+    // If no deleted field or deleted=false, item is active
+    return !isDeleted(metadata);
+}
+// Export the namespace constant for use in queries
+export const BRAINY_DELETED_FIELD = `${BRAINY_NAMESPACE}.deleted`;
+//# sourceMappingURL=ensureDeleted.js.map

package/dist/utils/metadataFilter.js

@@ -24,8 +24,13 @@ function matchesQuery(value, query) {
             case 'notEquals':
             case 'isNot':
             case 'ne':
+                // Special handling: if value is undefined and operand is not undefined,
+                // they are not equal (so the condition passes)
+                // This ensures items without a 'deleted' field match 'deleted !== true'
                 if (value === operand)
                     return false;
+                // If value is undefined and operand is not, they're not equal (pass)
+                // If both are undefined, they're equal (fail, handled above)
                 break;
             // Comparison operators
             case 'greaterThan':

package/dist/utils/metadataIndex.d.ts

@@ -107,6 +107,10 @@ export declare class MetadataIndexManager {
      * Remove item from metadata indexes
      */
     removeFromIndex(id: string, metadata?: any): Promise<void>;
+    /**
+     * Get all IDs in the index
+     */
+    getAllIds(): Promise<string[]>;
     /**
      * Get IDs for a specific field-value combination with caching
      */

package/dist/utils/metadataIndex.js

@@ -412,6 +412,36 @@ export class MetadataIndexManager {
             }
         }
     }
+    /**
+     * Get all IDs in the index
+     */
+    async getAllIds() {
+        // Collect all unique IDs from all index entries
+        const allIds = new Set();
+        // First, add all IDs from the in-memory cache
+        for (const entry of this.indexCache.values()) {
+            entry.ids.forEach(id => allIds.add(id));
+        }
+        // If storage has a method to get all nouns, use it as the source of truth
+        // This ensures we include items that might not be indexed yet
+        if (this.storage && typeof this.storage.getNouns === 'function') {
+            try {
+                const result = await this.storage.getNouns({
+                    pagination: { limit: 100000 }
+                });
+                if (result && result.items) {
+                    result.items.forEach((item) => {
+                        if (item.id)
+                            allIds.add(item.id);
+                    });
+                }
+            }
+            catch (e) {
+                // Fall back to using only indexed IDs
+            }
+        }
+        return Array.from(allIds);
+    }
     /**
      * Get IDs for a specific field-value combination with caching
      */
@@ -638,6 +668,21 @@ export class MetadataIndexManager {
                                 fieldResults = Array.from(allIds);
                             }
                             break;
+                        // Negation operators
+                        case 'notEquals':
+                        case 'isNot':
+                        case 'ne':
+                            // For notEquals, we need all IDs EXCEPT those matching the value
+                            // This is especially important for soft delete: deleted !== true
+                            // should include items without a deleted field
+                            // First, get all IDs in the database
+                            const allItemIds = await this.getAllIds();
+                            // Then get IDs that match the value we want to exclude
+                            const excludeIds = await this.getIds(field, operand);
+                            const excludeSet = new Set(excludeIds);
+                            // Return all IDs except those to exclude
+                            fieldResults = allItemIds.filter(id => !excludeSet.has(id));
+                            break;
                     }
                 }
             }

package/dist/utils/metadataNamespace.d.ts

@@ -0,0 +1,113 @@
+/**
+ * Clean Metadata Architecture for Brainy 2.2
+ * No backward compatibility - doing it RIGHT from the start!
+ */
+export declare const BRAINY_NS: "_brainy";
+export declare const AUG_NS: "_augmentations";
+export declare const AUDIT_NS: "_audit";
+export declare const DELETED_FIELD: "_brainy.deleted";
+export declare const INDEXED_FIELD: "_brainy.indexed";
+export declare const VERSION_FIELD: "_brainy.version";
+/**
+ * Internal Brainy metadata structure
+ * These fields are ALWAYS present and indexed for O(1) access
+ */
+export interface BrainyInternalMetadata {
+    deleted: boolean;
+    indexed: boolean;
+    version: number;
+    created: number;
+    updated: number;
+    partition?: number;
+    domain?: string;
+    priority?: number;
+    ttl?: number;
+}
+/**
+ * Complete metadata structure with namespaces
+ */
+export interface NamespacedMetadata<T = any> {
+    [key: string]: any;
+    [BRAINY_NS]: BrainyInternalMetadata;
+    [AUG_NS]?: {
+        [augmentationName: string]: any;
+    };
+    [AUDIT_NS]?: Array<{
+        timestamp: number;
+        augmentation: string;
+        field: string;
+        oldValue: any;
+        newValue: any;
+    }>;
+}
+/**
+ * Create properly namespaced metadata
+ * This is called for EVERY noun/verb creation
+ */
+export declare function createNamespacedMetadata<T = any>(userMetadata?: T): NamespacedMetadata<T>;
+/**
+ * Update metadata while preserving namespaces
+ */
+export declare function updateNamespacedMetadata<T = any>(existing: NamespacedMetadata<T>, updates: Partial<T>): NamespacedMetadata<T>;
+/**
+ * Soft delete a noun (O(1) operation)
+ */
+export declare function markDeleted<T = any>(metadata: NamespacedMetadata<T>): NamespacedMetadata<T>;
+/**
+ * Restore a soft-deleted noun (O(1) operation)
+ */
+export declare function markRestored<T = any>(metadata: NamespacedMetadata<T>): NamespacedMetadata<T>;
+/**
+ * Check if a noun is deleted (O(1) check)
+ */
+export declare function isDeleted<T = any>(metadata: NamespacedMetadata<T>): boolean;
+/**
+ * Get user metadata without internal fields
+ * Used by augmentations to get clean user data
+ */
+export declare function getUserMetadata<T = any>(metadata: NamespacedMetadata<T>): T;
+/**
+ * Set augmentation data in isolated namespace
+ */
+export declare function setAugmentationData<T = any>(metadata: NamespacedMetadata<T>, augmentationName: string, data: any): NamespacedMetadata<T>;
+/**
+ * Add audit entry for tracking
+ */
+export declare function addAuditEntry<T = any>(metadata: NamespacedMetadata<T>, entry: {
+    augmentation: string;
+    field: string;
+    oldValue: any;
+    newValue: any;
+}): NamespacedMetadata<T>;
+/**
+ * INDEXING EXPLANATION:
+ *
+ * The MetadataIndex flattens nested objects into dot-notation keys:
+ *
+ * Input metadata:
+ * {
+ *   name: "Django",
+ *   _brainy: {
+ *     deleted: false,
+ *     indexed: true
+ *   }
+ * }
+ *
+ * Creates index entries:
+ * - "name" -> "django" -> Set([id1, id2...])
+ * - "_brainy.deleted" -> "false" -> Set([id1, id2...])  // O(1) lookup!
+ * - "_brainy.indexed" -> "true" -> Set([id1, id2...])
+ *
+ * Query: { "_brainy.deleted": false }
+ * Lookup: index["_brainy.deleted"]["false"] -> Set of IDs in O(1)
+ *
+ * This is why namespacing doesn't hurt performance - it's all flattened!
+ */
+/**
+ * Fields that should ALWAYS be indexed for O(1) access
+ */
+export declare const ALWAYS_INDEXED_FIELDS: ("_brainy.deleted" | "_brainy.indexed" | "_brainy.version")[];
+/**
+ * Fields that should use sorted index for O(log n) range queries
+ */
+export declare const SORTED_INDEX_FIELDS: string[];

package/dist/utils/metadataNamespace.js

@@ -0,0 +1,162 @@
+/**
+ * Clean Metadata Architecture for Brainy 2.2
+ * No backward compatibility - doing it RIGHT from the start!
+ */
+// Namespace constants
+export const BRAINY_NS = '_brainy';
+export const AUG_NS = '_augmentations';
+export const AUDIT_NS = '_audit';
+// Field paths for O(1) indexing
+export const DELETED_FIELD = `${BRAINY_NS}.deleted`;
+export const INDEXED_FIELD = `${BRAINY_NS}.indexed`;
+export const VERSION_FIELD = `${BRAINY_NS}.version`;
+/**
+ * Create properly namespaced metadata
+ * This is called for EVERY noun/verb creation
+ */
+export function createNamespacedMetadata(userMetadata) {
+    const now = Date.now();
+    // Start with user metadata or empty object
+    const result = userMetadata ? { ...userMetadata } : {};
+    // ALWAYS add internal namespace with required fields
+    result[BRAINY_NS] = {
+        deleted: false, // CRITICAL: Always false for new items
+        indexed: true, // New items are indexed
+        version: 1, // Current schema version
+        created: now,
+        updated: now
+    };
+    return result;
+}
+/**
+ * Update metadata while preserving namespaces
+ */
+export function updateNamespacedMetadata(existing, updates) {
+    const now = Date.now();
+    // Merge user fields
+    const result = {
+        ...existing,
+        ...updates
+    };
+    // Preserve internal namespace but update timestamp
+    result[BRAINY_NS] = {
+        ...existing[BRAINY_NS],
+        updated: now
+    };
+    // Preserve augmentation namespace
+    if (existing[AUG_NS]) {
+        result[AUG_NS] = existing[AUG_NS];
+    }
+    // Preserve audit trail
+    if (existing[AUDIT_NS]) {
+        result[AUDIT_NS] = existing[AUDIT_NS];
+    }
+    return result;
+}
+/**
+ * Soft delete a noun (O(1) operation)
+ */
+export function markDeleted(metadata) {
+    return {
+        ...metadata,
+        [BRAINY_NS]: {
+            ...metadata[BRAINY_NS],
+            deleted: true,
+            updated: Date.now()
+        }
+    };
+}
+/**
+ * Restore a soft-deleted noun (O(1) operation)
+ */
+export function markRestored(metadata) {
+    return {
+        ...metadata,
+        [BRAINY_NS]: {
+            ...metadata[BRAINY_NS],
+            deleted: false,
+            updated: Date.now()
+        }
+    };
+}
+/**
+ * Check if a noun is deleted (O(1) check)
+ */
+export function isDeleted(metadata) {
+    return metadata[BRAINY_NS]?.deleted === true;
+}
+/**
+ * Get user metadata without internal fields
+ * Used by augmentations to get clean user data
+ */
+export function getUserMetadata(metadata) {
+    const { [BRAINY_NS]: _, [AUG_NS]: __, [AUDIT_NS]: ___, ...userMeta } = metadata;
+    return userMeta;
+}
+/**
+ * Set augmentation data in isolated namespace
+ */
+export function setAugmentationData(metadata, augmentationName, data) {
+    const result = { ...metadata };
+    if (!result[AUG_NS]) {
+        result[AUG_NS] = {};
+    }
+    result[AUG_NS][augmentationName] = data;
+    return result;
+}
+/**
+ * Add audit entry for tracking
+ */
+export function addAuditEntry(metadata, entry) {
+    const result = { ...metadata };
+    if (!result[AUDIT_NS]) {
+        result[AUDIT_NS] = [];
+    }
+    result[AUDIT_NS].push({
+        ...entry,
+        timestamp: Date.now()
+    });
+    return result;
+}
+/**
+ * INDEXING EXPLANATION:
+ *
+ * The MetadataIndex flattens nested objects into dot-notation keys:
+ *
+ * Input metadata:
+ * {
+ *   name: "Django",
+ *   _brainy: {
+ *     deleted: false,
+ *     indexed: true
+ *   }
+ * }
+ *
+ * Creates index entries:
+ * - "name" -> "django" -> Set([id1, id2...])
+ * - "_brainy.deleted" -> "false" -> Set([id1, id2...])  // O(1) lookup!
+ * - "_brainy.indexed" -> "true" -> Set([id1, id2...])
+ *
+ * Query: { "_brainy.deleted": false }
+ * Lookup: index["_brainy.deleted"]["false"] -> Set of IDs in O(1)
+ *
+ * This is why namespacing doesn't hurt performance - it's all flattened!
+ */
+/**
+ * Fields that should ALWAYS be indexed for O(1) access
+ */
+export const ALWAYS_INDEXED_FIELDS = [
+    DELETED_FIELD, // For soft delete filtering
+    INDEXED_FIELD, // For index management
+    VERSION_FIELD // For schema versioning
+];
+/**
+ * Fields that should use sorted index for O(log n) range queries
+ */
+export const SORTED_INDEX_FIELDS = [
+    `${BRAINY_NS}.created`,
+    `${BRAINY_NS}.updated`,
+    `${BRAINY_NS}.priority`,
+    `${BRAINY_NS}.ttl`
+];
+//# sourceMappingURL=metadataNamespace.js.map

package/dist/utils/periodicCleanup.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Periodic Cleanup for Soft-Deleted Items
+ *
+ * SAFETY-FIRST APPROACH:
+ * - Maintains durability guarantees (storage-first)
+ * - Coordinates HNSW and metadata index consistency
+ * - Isolated from live operations
+ * - Graceful failure handling
+ */
+import type { StorageAdapter } from '../coreTypes.js';
+import type { HNSWIndex } from '../hnsw/hnswIndex.js';
+import type { MetadataIndexManager } from './metadataIndex.js';
+export interface CleanupConfig {
+    /** Age in milliseconds after which soft-deleted items are eligible for cleanup */
+    maxAge: number;
+    /** Maximum number of items to clean up in one batch */
+    batchSize: number;
+    /** Interval between cleanup runs (milliseconds) */
+    cleanupInterval: number;
+    /** Whether to run cleanup automatically */
+    enabled: boolean;
+}
+export interface CleanupStats {
+    itemsProcessed: number;
+    itemsDeleted: number;
+    errors: number;
+    lastRun: number;
+    nextRun: number;
+}
+/**
+ * Coordinates safe cleanup of old soft-deleted items across all indexes
+ *
+ * CRITICAL SAFETY FEATURES:
+ * 1. Storage-first deletion (durability)
+ * 2. Index consistency coordination
+ * 3. Batch processing with limits
+ * 4. Error isolation and recovery
+ */
+export declare class PeriodicCleanup {
+    private storage;
+    private hnswIndex;
+    private metadataIndex;
+    private config;
+    private stats;
+    private cleanupTimer;
+    private running;
+    constructor(storage: StorageAdapter, hnswIndex: HNSWIndex, metadataIndex: MetadataIndexManager | null, config?: Partial<CleanupConfig>);
+    /**
+     * Start periodic cleanup
+     */
+    start(): void;
+    /**
+     * Stop periodic cleanup
+     */
+    stop(): void;
+    /**
+     * Run cleanup manually
+     */
+    runNow(): Promise<CleanupStats>;
+    /**
+     * Get current cleanup statistics
+     */
+    getStats(): CleanupStats;
+    private scheduleNext;
+    /**
+     * CRITICAL: Coordinated cleanup across all indexes
+     *
+     * SAFETY PROTOCOL:
+     * 1. Find eligible items (old + soft-deleted)
+     * 2. Remove from storage FIRST (durability)
+     * 3. Remove from HNSW (graph consistency)
+     * 4. Remove from metadata index (search consistency)
+     * 5. Track stats and errors
+     */
+    private performCleanup;
+    /**
+     * Find items eligible for cleanup (old + soft-deleted)
+     */
+    private findEligibleItems;
+    /**
+     * Process a batch of items for cleanup
+     *
+     * CRITICAL: This maintains the durability-first approach:
+     * Storage → HNSW → Metadata Index
+     */
+    private processBatch;
+}