@soulcraft/brainy 3.20.1 → 3.20.3

package/dist/embeddings/worker-embedding.js

@@ -1,73 +0,0 @@
-/**
- * Worker process for embeddings - Workaround for transformers.js memory leak
- *
- * This worker can be killed and restarted to release memory completely.
- * Based on 2024 research: dispose() doesn't fully free memory in transformers.js
- */
-import { TransformerEmbedding } from '../utils/embedding.js';
-import { parentPort } from 'worker_threads';
-import { getModelPrecision } from '../config/modelPrecisionManager.js';
-let model = null;
-let requestCount = 0;
-const MAX_REQUESTS = 100; // Restart worker after 100 requests to prevent memory leak
-async function initModel() {
-    if (!model) {
-        model = new TransformerEmbedding({
-            verbose: false,
-            precision: getModelPrecision(), // Use centrally managed precision
-            localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
-        });
-        await model.init();
-        console.log('🔧 Worker: Model initialized');
-    }
-}
-if (parentPort) {
-    parentPort.on('message', async (message) => {
-        try {
-            const { id, type, data } = message;
-            switch (type) {
-                case 'embed':
-                    await initModel();
-                    const embeddings = await model.embed(data);
-                    parentPort.postMessage({ id, success: true, result: embeddings });
-                    requestCount++;
-                    // Proactively restart worker to prevent memory leak
-                    if (requestCount >= MAX_REQUESTS) {
-                        console.log(`🔄 Worker: Restarting after ${requestCount} requests (memory leak prevention)`);
-                        process.exit(0); // Parent will restart us
-                    }
-                    break;
-                case 'dispose':
-                    // SingletonModelManager persists - just acknowledge
-                    console.log('ℹ️ Worker: Singleton model persists');
-                    parentPort.postMessage({ id, success: true });
-                    break;
-                case 'restart':
-                    // Force restart to clear memory
-                    console.log('🔄 Worker: Force restart requested');
-                    process.exit(0);
-                    break;
-                default:
-                    parentPort.postMessage({
-                        id,
-                        success: false,
-                        error: `Unknown message type: ${type}`
-                    });
-            }
-        }
-        catch (error) {
-            parentPort.postMessage({
-                id: message.id,
-                success: false,
-                error: error instanceof Error ? error.message : String(error)
-            });
-        }
-    });
-    console.log('🚀 Embedding worker started');
-    parentPort.postMessage({ type: 'ready' });
-}
-else {
-    console.error('❌ Worker: parentPort is null, cannot communicate with main thread');
-    process.exit(1);
-}
-//# sourceMappingURL=worker-embedding.js.map

package/dist/embeddings/worker-manager.d.ts

@@ -1,28 +0,0 @@
-/**
- * Worker Manager for Memory-Safe Embeddings
- *
- * Manages worker lifecycle to prevent transformers.js memory leaks
- * Workers are automatically restarted when memory usage grows too high
- */
-import { Vector, EmbeddingFunction } from '../coreTypes.js';
-export declare class WorkerEmbeddingManager {
-    private worker;
-    private requestId;
-    private pendingRequests;
-    private isRestarting;
-    private totalRequests;
-    getEmbeddingFunction(): Promise<EmbeddingFunction>;
-    embed(data: string | string[]): Promise<Vector>;
-    private ensureWorker;
-    private createWorker;
-    dispose(): Promise<void>;
-    forceRestart(): Promise<void>;
-    getStats(): {
-        totalRequests: number;
-        pendingRequests: number;
-        workerActive: boolean;
-        isRestarting: boolean;
-    };
-}
-export declare const workerEmbeddingManager: WorkerEmbeddingManager;
-export declare function getWorkerEmbeddingFunction(): Promise<EmbeddingFunction>;

package/dist/embeddings/worker-manager.js

@@ -1,162 +0,0 @@
-/**
- * Worker Manager for Memory-Safe Embeddings
- *
- * Manages worker lifecycle to prevent transformers.js memory leaks
- * Workers are automatically restarted when memory usage grows too high
- */
-import { Worker } from 'worker_threads';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-// Get current directory for worker path
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-export class WorkerEmbeddingManager {
-    constructor() {
-        this.worker = null;
-        this.requestId = 0;
-        this.pendingRequests = new Map();
-        this.isRestarting = false;
-        this.totalRequests = 0;
-    }
-    async getEmbeddingFunction() {
-        return async (data) => {
-            return this.embed(data);
-        };
-    }
-    async embed(data) {
-        await this.ensureWorker();
-        const id = ++this.requestId;
-        this.totalRequests++;
-        return new Promise((resolve, reject) => {
-            const timeout = setTimeout(() => {
-                this.pendingRequests.delete(id);
-                reject(new Error('Embedding request timed out (120s)'));
-            }, 120000);
-            this.pendingRequests.set(id, { resolve, reject, timeout });
-            this.worker.postMessage({
-                id,
-                type: 'embed',
-                data
-            });
-        });
-    }
-    async ensureWorker() {
-        if (this.worker && !this.isRestarting) {
-            return;
-        }
-        if (this.isRestarting) {
-            // Wait for restart to complete
-            return new Promise((resolve) => {
-                const checkRestart = () => {
-                    if (!this.isRestarting) {
-                        resolve();
-                    }
-                    else {
-                        setTimeout(checkRestart, 100);
-                    }
-                };
-                checkRestart();
-            });
-        }
-        await this.createWorker();
-    }
-    async createWorker() {
-        this.isRestarting = true;
-        // Kill existing worker if any
-        if (this.worker) {
-            this.worker.terminate();
-            this.worker = null;
-        }
-        // Clear pending requests
-        for (const [id, request] of this.pendingRequests) {
-            if (request.timeout) {
-                clearTimeout(request.timeout);
-            }
-            request.reject(new Error('Worker restarted'));
-        }
-        this.pendingRequests.clear();
-        console.log('🔄 Starting embedding worker...');
-        // Create new worker
-        const workerPath = join(__dirname, 'worker-embedding.js');
-        this.worker = new Worker(workerPath);
-        // Handle worker messages
-        this.worker.on('message', (message) => {
-            if (message.type === 'ready') {
-                console.log('✅ Embedding worker ready');
-                this.isRestarting = false;
-                return;
-            }
-            const { id, success, result, error } = message;
-            const request = this.pendingRequests.get(id);
-            if (request) {
-                if (request.timeout) {
-                    clearTimeout(request.timeout);
-                }
-                this.pendingRequests.delete(id);
-                if (success) {
-                    request.resolve(result);
-                }
-                else {
-                    request.reject(new Error(error));
-                }
-            }
-        });
-        // Handle worker exit
-        this.worker.on('exit', (code) => {
-            console.log(`🔄 Embedding worker exited with code ${code}`);
-            if (code !== 0 && !this.isRestarting) {
-                console.log('🔄 Worker crashed, will restart on next request');
-            }
-            this.worker = null;
-        });
-        // Wait for worker to be ready
-        return new Promise((resolve, reject) => {
-            const timeout = setTimeout(() => {
-                reject(new Error('Worker startup timeout'));
-            }, 30000);
-            const checkReady = () => {
-                if (!this.isRestarting) {
-                    clearTimeout(timeout);
-                    resolve();
-                }
-                else {
-                    setTimeout(checkReady, 100);
-                }
-            };
-            checkReady();
-        });
-    }
-    async dispose() {
-        if (this.worker) {
-            this.worker.terminate();
-            this.worker = null;
-        }
-        // Clear pending requests
-        for (const [id, request] of this.pendingRequests) {
-            if (request.timeout) {
-                clearTimeout(request.timeout);
-            }
-            request.reject(new Error('Manager disposed'));
-        }
-        this.pendingRequests.clear();
-    }
-    async forceRestart() {
-        console.log('🔄 Force restarting embedding worker (memory cleanup)');
-        await this.createWorker();
-    }
-    getStats() {
-        return {
-            totalRequests: this.totalRequests,
-            pendingRequests: this.pendingRequests.size,
-            workerActive: this.worker !== null,
-            isRestarting: this.isRestarting
-        };
-    }
-}
-// Export singleton instance
-export const workerEmbeddingManager = new WorkerEmbeddingManager();
-// Export convenience function
-export async function getWorkerEmbeddingFunction() {
-    return workerEmbeddingManager.getEmbeddingFunction();
-}
-//# sourceMappingURL=worker-manager.js.map

package/dist/examples/basicUsage.d.ts

@@ -1,4 +0,0 @@
-/**
- * Basic usage example for the Soulcraft Brainy database
- */
-export {};

package/dist/examples/basicUsage.js

@@ -1,121 +0,0 @@
-/**
- * Basic usage example for the Soulcraft Brainy database
- */
-import { Brainy } from '../brainy.js';
-// Example data - word embeddings
-const wordEmbeddings = {
-    cat: [0.2, 0.3, 0.4, 0.1],
-    dog: [0.3, 0.2, 0.4, 0.2],
-    fish: [0.1, 0.1, 0.8, 0.2],
-    bird: [0.1, 0.4, 0.2, 0.5],
-    tiger: [0.3, 0.4, 0.3, 0.1],
-    lion: [0.4, 0.3, 0.2, 0.1],
-    shark: [0.2, 0.1, 0.7, 0.3],
-    eagle: [0.2, 0.5, 0.1, 0.4]
-};
-// Example metadata
-const metadata = {
-    cat: { type: 'mammal', domesticated: true },
-    dog: { type: 'mammal', domesticated: true },
-    fish: { type: 'fish', domesticated: false },
-    bird: { type: 'bird', domesticated: false },
-    tiger: { type: 'mammal', domesticated: false },
-    lion: { type: 'mammal', domesticated: false },
-    shark: { type: 'fish', domesticated: false },
-    eagle: { type: 'bird', domesticated: false }
-};
-/**
- * Run the example
- */
-async function runExample() {
-    console.log('Initializing vector database...');
-    // Create a new vector database
-    const db = new Brainy();
-    await db.init();
-    console.log('Adding vectors to the database...');
-    // Add vectors to the database
-    const ids = {};
-    for (const [word, vector] of Object.entries(wordEmbeddings)) {
-        // Determine noun type based on the metadata
-        const meta = metadata[word];
-        const nounType = meta.type === 'mammal' || meta.type === 'bird' || meta.type === 'fish' ? 'Thing' : 'Content';
-        ids[word] = await db.addNoun(vector, nounType, meta);
-        console.log(`Added "${word}" with ID: ${ids[word]}`);
-    }
-    console.log('\nDatabase size:', db.size());
-    // Search for similar vectors
-    console.log('\nSearching for vectors similar to "cat"...');
-    const catResults = await db.search(wordEmbeddings['cat'], { limit: 3 });
-    console.log('Results:');
-    for (const result of catResults) {
-        const word = Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown';
-        console.log(`- ${word} (score: ${result.score.toFixed(4)}, metadata:`, result.metadata, ')');
-    }
-    // Search for similar vectors
-    console.log('\nSearching for vectors similar to "fish"...');
-    const fishResults = await db.search(wordEmbeddings['fish'], { limit: 3 });
-    console.log('Results:');
-    for (const result of fishResults) {
-        const word = Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown';
-        console.log(`- ${word} (score: ${result.score.toFixed(4)}, metadata:`, result.metadata, ')');
-    }
-    // Update metadata
-    console.log('\nUpdating metadata for "bird"...');
-    await db.updateNounMetadata(ids['bird'], {
-        ...metadata['bird'],
-        notes: 'Can fly'
-    });
-    // Get the updated document
-    const birdDoc = await db.getNoun(ids['bird']);
-    console.log('Updated bird document:', birdDoc);
-    // Delete a vector
-    console.log('\nDeleting "shark"...');
-    await db.deleteNoun(ids['shark']);
-    console.log('Database size after deletion:', db.size());
-    // Search again to verify shark is gone
-    console.log('\nSearching for vectors similar to "fish" after deletion...');
-    const fishResultsAfterDeletion = await db.search(wordEmbeddings['fish'], { limit: 3 });
-    console.log('Results:');
-    for (const result of fishResultsAfterDeletion) {
-        const word = Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown';
-        console.log(`- ${word} (score: ${result.score.toFixed(4)}, metadata:`, result.metadata, ')');
-    }
-    console.log('\nExample completed successfully!');
-}
-// Check if we're in a browser or Node.js environment
-if (typeof window !== 'undefined') {
-    // Browser environment
-    document.addEventListener('DOMContentLoaded', () => {
-        const button = document.createElement('button');
-        button.textContent = 'Run Brainy Example';
-        button.addEventListener('click', async () => {
-            const output = document.createElement('pre');
-            document.body.appendChild(output);
-            // Redirect console.log to the output element
-            const originalLog = console.log;
-            console.log = (...args) => {
-                originalLog(...args);
-                output.textContent +=
-                    args
-                        .map((arg) => typeof arg === 'object' ? JSON.stringify(arg, null, 2) : arg)
-                        .join(' ') + '\n';
-            };
-            try {
-                await runExample();
-            }
-            catch (error) {
-                console.error('Error running example:', error);
-            }
-            // Restore console.log
-            console.log = originalLog;
-        });
-        document.body.appendChild(button);
-    });
-}
-else {
-    // Node.js environment
-    runExample().catch((error) => {
-        console.error('Error running example:', error);
-    });
-}
-//# sourceMappingURL=basicUsage.js.map

package/dist/indices/fieldIndex.d.ts

@@ -1,76 +0,0 @@
-/**
- * Field Index for efficient field-based queries
- * Provides O(log n) lookups for field values and range queries
- */
-interface RangeQueryOptions {
-    field: string;
-    min?: any;
-    max?: any;
-    includeMin?: boolean;
-    includeMax?: boolean;
-}
-export declare class FieldIndex {
-    private indices;
-    private sortedIndices;
-    private indexedFields;
-    /**
-     * Add a document to the field index
-     */
-    add(id: string, metadata: Record<string, any>): void;
-    /**
-     * Remove a document from the field index
-     */
-    remove(id: string, metadata: Record<string, any>): void;
-    /**
-     * Query for exact field value match
-     * O(1) hash lookup
-     */
-    queryExact(field: string, value: any): string[];
-    /**
-     * Query for multiple values (IN operator)
-     * O(k) where k is number of values
-     */
-    queryIn(field: string, values: any[]): string[];
-    /**
-     * Query for range of values
-     * O(log n + m) where m is number of results
-     */
-    queryRange(options: RangeQueryOptions): string[];
-    /**
-     * Query with complex where clause
-     */
-    query(where: Record<string, any>): string[];
-    /**
-     * Mark sorted index as needing rebuild
-     */
-    private markSortedIndexDirty;
-    /**
-     * Ensure sorted index is up to date for a field
-     */
-    private ensureSortedIndex;
-    /**
-     * Binary search for start position (inclusive)
-     */
-    private binarySearch;
-    /**
-     * Binary search for end position (inclusive)
-     */
-    private binarySearchEnd;
-    /**
-     * Debug method to inspect index contents
-     */
-    debugIndex(field?: string): any;
-    /**
-     * Get statistics about the index
-     */
-    getStats(): {
-        indexedFields: number;
-        totalValues: number;
-        totalMappings: number;
-    };
-    /**
-     * Clear all indices
-     */
-    clear(): void;
-}
-export {};