@soulcraft/brainy 3.0.0 → 3.0.1

package/dist/config/index.js

@@ -0,0 +1,35 @@
+/**
+ * Zero-Configuration System
+ * Main entry point for all auto-configuration features
+ */
+// Model configuration
+export { autoSelectModelPrecision, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
+// Model precision - Always Q8 now (99% accuracy, 75% smaller)
+export const getModelPrecision = () => 'q8';
+// Storage configuration
+export { autoDetectStorage, StorageType, StoragePreset, logStorageConfig } from './storageAutoConfig.js';
+// Shared configuration for multi-instance
+export { SharedConfigManager } from './sharedConfigManager.js';
+// Main zero-config processor
+export { processZeroConfig, createEmbeddingFunctionWithPrecision } from './zeroConfig.js';
+// Strongly-typed presets and enums
+export { PresetName, PresetCategory, ModelPrecision, StorageOption, FeatureSet, DistributedRole, PRESET_CONFIGS, getPreset, isValidPreset, getPresetsByCategory, getAllPresetNames, getPresetDescription, presetToBrainyConfig } from './distributedPresets.js';
+// Extensible configuration
+export { registerStorageAugmentation, registerPresetAugmentation, getConfigRegistry } from './extensibleConfig.js';
+/**
+ * Main zero-config processor
+ * This is what Brainy will call
+ */
+export async function applyZeroConfig(input) {
+    // Handle legacy config (full object) by detecting known legacy properties
+    if (input && typeof input === 'object' &&
+        (input.storage?.forceMemoryStorage || input.storage?.forceFileSystemStorage || input.storage?.s3Storage)) {
+        // This is a legacy config object - pass through unchanged
+        console.log('📦 Using legacy configuration format');
+        return input;
+    }
+    // Process as zero-config (includes new object format)
+    const { processZeroConfig } = await import('./zeroConfig.js');
+    return processZeroConfig(input);
+}
+//# sourceMappingURL=index.js.map

package/dist/config/modelAutoConfig.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Model Configuration Auto-Selection
+ * Always uses Q8 for optimal size/performance balance (99% accuracy, 75% smaller)
+ */
+export type ModelPrecision = 'q8';
+export type ModelPreset = 'small' | 'auto';
+interface ModelConfigResult {
+    precision: ModelPrecision;
+    reason: string;
+    autoSelected: boolean;
+}
+/**
+ * Auto-select model precision - Always returns Q8
+ * Q8 provides 99% accuracy with 75% smaller size
+ * @param override - For backward compatibility, ignored
+ */
+export declare function autoSelectModelPrecision(override?: ModelPrecision | ModelPreset): ModelConfigResult;
+/**
+ * Convenience function to check if models need to be downloaded
+ * This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
+ */
+export declare function shouldAutoDownloadModels(): boolean;
+/**
+ * Get the model path with intelligent defaults
+ * This replaces the need for BRAINY_MODELS_PATH env var
+ */
+export declare function getModelPath(): string;
+/**
+ * Log model configuration decision (only in verbose mode)
+ */
+export declare function logModelConfig(config: ModelConfigResult, verbose?: boolean): void;
+export {};

package/dist/config/modelAutoConfig.js

@@ -0,0 +1,139 @@
+/**
+ * Model Configuration Auto-Selection
+ * Always uses Q8 for optimal size/performance balance (99% accuracy, 75% smaller)
+ */
+import { isBrowser, isNode } from '../utils/environment.js';
+/**
+ * Auto-select model precision - Always returns Q8
+ * Q8 provides 99% accuracy with 75% smaller size
+ * @param override - For backward compatibility, ignored
+ */
+export function autoSelectModelPrecision(override) {
+    // Always use Q8 regardless of override for simplicity
+    // Q8 is optimal: 33MB vs 130MB, 99% accuracy retained
+    // Log deprecation notice if FP32 was requested
+    if (typeof override === 'string' && override.toLowerCase().includes('fp32')) {
+        console.log('Note: FP32 precision is deprecated. Using Q8 (99% accuracy, 75% smaller).');
+    }
+    return {
+        precision: 'q8',
+        reason: 'Q8 precision (99% accuracy, 75% smaller)',
+        autoSelected: true
+    };
+}
+/**
+ * Automatically detect the best model precision for the environment
+ * DEPRECATED: Always returns Q8 now
+ */
+function autoDetectBestPrecision() {
+    // Always return Q8 - deprecated function kept for backward compatibility
+    return {
+        precision: 'q8',
+        reason: 'Q8 precision (99% accuracy, 75% smaller)',
+        autoSelected: true
+    };
+}
+/**
+ * Check if running in a serverless environment
+ */
+function isServerlessEnvironment() {
+    if (!isNode())
+        return false;
+    return !!(process.env.AWS_LAMBDA_FUNCTION_NAME ||
+        process.env.VERCEL ||
+        process.env.NETLIFY ||
+        process.env.CLOUDFLARE_WORKERS ||
+        process.env.FUNCTIONS_WORKER_RUNTIME ||
+        process.env.K_SERVICE // Google Cloud Run
+    );
+}
+/**
+ * Get available memory in MB
+ */
+function getAvailableMemoryMB() {
+    if (isBrowser()) {
+        // @ts-ignore - navigator.deviceMemory is experimental
+        if (navigator.deviceMemory) {
+            // @ts-ignore
+            return navigator.deviceMemory * 1024; // Device memory in GB
+        }
+        return 256; // Conservative default for browsers
+    }
+    if (isNode()) {
+        try {
+            // Try to get memory info synchronously for Node.js
+            // This will be available in Node.js environments
+            if (typeof process !== 'undefined' && process.memoryUsage) {
+                // Use RSS (Resident Set Size) as a proxy for available memory
+                const rss = process.memoryUsage().rss;
+                // Assume we can use up to 4GB or 50% more than current usage
+                return Math.min(4096, Math.floor(rss / (1024 * 1024) * 1.5));
+            }
+        }
+        catch {
+            // Fall through to default
+        }
+        return 1024; // Default 1GB for Node.js
+    }
+    return 512; // Conservative default
+}
+/**
+ * Convenience function to check if models need to be downloaded
+ * This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
+ */
+export function shouldAutoDownloadModels() {
+    // Always allow downloads unless explicitly disabled
+    // This eliminates the need for BRAINY_ALLOW_REMOTE_MODELS
+    const explicitlyDisabled = process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false';
+    if (explicitlyDisabled) {
+        console.warn('Model downloads disabled via BRAINY_ALLOW_REMOTE_MODELS=false');
+        return false;
+    }
+    // In production, always allow downloads for seamless operation
+    if (process.env.NODE_ENV === 'production') {
+        return true;
+    }
+    // In development, allow downloads with a one-time notice
+    if (process.env.NODE_ENV === 'development') {
+        return true;
+    }
+    // Default: allow downloads
+    return true;
+}
+/**
+ * Get the model path with intelligent defaults
+ * This replaces the need for BRAINY_MODELS_PATH env var
+ */
+export function getModelPath() {
+    // Check if user explicitly set a path (keeping this for advanced users)
+    if (process.env.BRAINY_MODELS_PATH) {
+        return process.env.BRAINY_MODELS_PATH;
+    }
+    // Browser - use cache API or IndexedDB (handled by transformers.js)
+    if (isBrowser()) {
+        return 'browser-cache';
+    }
+    // Serverless - use /tmp for ephemeral storage
+    if (isServerlessEnvironment()) {
+        return '/tmp/.brainy/models';
+    }
+    // Node.js - use home directory for persistent storage
+    if (isNode()) {
+        // Use process.env.HOME as a fallback
+        const homeDir = process.env.HOME || process.env.USERPROFILE || '~';
+        return `${homeDir}/.brainy/models`;
+    }
+    // Fallback
+    return './.brainy/models';
+}
+/**
+ * Log model configuration decision (only in verbose mode)
+ */
+export function logModelConfig(config, verbose = false) {
+    if (!verbose && process.env.NODE_ENV === 'production') {
+        return; // Silent in production unless verbose
+    }
+    const icon = config.autoSelected ? '🤖' : '👤';
+    console.log(`${icon} Model: ${config.precision.toUpperCase()} - ${config.reason}`);
+}
+//# sourceMappingURL=modelAutoConfig.js.map

package/dist/config/modelPrecisionManager.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Central Model Precision Manager
+ *
+ * Single source of truth for model precision configuration.
+ * Ensures consistent usage of Q8 or FP32 models throughout the system.
+ */
+import { ModelPrecision } from './modelAutoConfig.js';
+export declare class ModelPrecisionManager {
+    private static instance;
+    private precision;
+    private isLocked;
+    private constructor();
+    static getInstance(): ModelPrecisionManager;
+    /**
+     * Get the current model precision
+     */
+    getPrecision(): ModelPrecision;
+    /**
+     * Set the model precision (can only be done before first model load)
+     */
+    setPrecision(precision: ModelPrecision): void;
+    /**
+     * Lock the precision (called after first model load)
+     */
+    lock(): void;
+    /**
+     * Check if precision is locked
+     */
+    isConfigLocked(): boolean;
+    /**
+     * Get precision info for logging
+     */
+    getInfo(): string;
+    /**
+     * Validate that a given precision matches the configured one
+     */
+    validatePrecision(precision: ModelPrecision): boolean;
+}
+export declare const getModelPrecision: () => ModelPrecision;
+export declare const setModelPrecision: (precision: ModelPrecision) => void;
+export declare const lockModelPrecision: () => void;
+export declare const validateModelPrecision: (precision: ModelPrecision) => boolean;

package/dist/config/modelPrecisionManager.js

@@ -0,0 +1,98 @@
+/**
+ * Central Model Precision Manager
+ *
+ * Single source of truth for model precision configuration.
+ * Ensures consistent usage of Q8 or FP32 models throughout the system.
+ */
+export class ModelPrecisionManager {
+    constructor() {
+        this.precision = 'q8'; // DEFAULT TO Q8
+        this.isLocked = false;
+        // Check environment variable override
+        const envPrecision = process.env.BRAINY_MODEL_PRECISION;
+        if (envPrecision === 'fp32' || envPrecision === 'q8') {
+            this.precision = envPrecision;
+            console.log(`Model precision set from environment: ${envPrecision.toUpperCase()}`);
+        }
+        else {
+            console.log('Using default model precision: Q8 (75% smaller, 99% accuracy)');
+        }
+    }
+    static getInstance() {
+        if (!ModelPrecisionManager.instance) {
+            ModelPrecisionManager.instance = new ModelPrecisionManager();
+        }
+        return ModelPrecisionManager.instance;
+    }
+    /**
+     * Get the current model precision
+     */
+    getPrecision() {
+        return this.precision;
+    }
+    /**
+     * Set the model precision (can only be done before first model load)
+     */
+    setPrecision(precision) {
+        if (this.isLocked) {
+            console.warn(`⚠️ Cannot change precision after model initialization. Current: ${this.precision.toUpperCase()}`);
+            return;
+        }
+        if (precision !== this.precision) {
+            console.log(`Model precision changed: ${this.precision.toUpperCase()} → ${precision.toUpperCase()}`);
+            this.precision = precision;
+        }
+    }
+    /**
+     * Lock the precision (called after first model load)
+     */
+    lock() {
+        if (!this.isLocked) {
+            this.isLocked = true;
+            console.log(`Model precision locked: ${this.precision.toUpperCase()}`);
+        }
+    }
+    /**
+     * Check if precision is locked
+     */
+    isConfigLocked() {
+        return this.isLocked;
+    }
+    /**
+     * Get precision info for logging
+     */
+    getInfo() {
+        const info = this.precision === 'q8'
+            ? 'Q8 (quantized, 23MB, 99% accuracy)'
+            : 'FP32 (full precision, 90MB, 100% accuracy)';
+        return `${info}${this.isLocked ? ' [LOCKED]' : ''}`;
+    }
+    /**
+     * Validate that a given precision matches the configured one
+     */
+    validatePrecision(precision) {
+        if (precision !== this.precision) {
+            console.error(`❌ Precision mismatch! Expected: ${this.precision.toUpperCase()}, Got: ${precision.toUpperCase()}`);
+            console.error('This will cause incompatible embeddings!');
+            return false;
+        }
+        return true;
+    }
+}
+// Export singleton instance getter
+export const getModelPrecision = () => {
+    return ModelPrecisionManager.getInstance().getPrecision();
+};
+// Export setter (for configuration phase)
+export const setModelPrecision = (precision) => {
+    ModelPrecisionManager.getInstance().setPrecision(precision);
+};
+// Export lock function (for after model initialization)
+export const lockModelPrecision = () => {
+    ModelPrecisionManager.getInstance().lock();
+};
+// Export validation function
+export const validateModelPrecision = (precision) => {
+    return ModelPrecisionManager.getInstance().validatePrecision(precision);
+};
+//# sourceMappingURL=modelPrecisionManager.js.map

package/dist/config/sharedConfigManager.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Shared Configuration Manager
+ * Ensures configuration consistency across multiple instances using shared storage
+ */
+import { ModelPrecision } from './modelAutoConfig.js';
+export interface SharedConfig {
+    version: string;
+    precision: ModelPrecision;
+    dimensions: number;
+    hnswM: number;
+    hnswEfConstruction: number;
+    distanceFunction: string;
+    createdAt: string;
+    lastUpdated: string;
+    instanceCount?: number;
+    lastAccessedBy?: string;
+}
+/**
+ * Manages configuration consistency for shared storage
+ */
+export declare class SharedConfigManager {
+    private static CONFIG_FILE;
+    /**
+     * Load or create shared configuration
+     * When connecting to existing data, this OVERRIDES auto-configuration!
+     */
+    static loadOrCreateSharedConfig(storage: any, localConfig: any): Promise<{
+        config: any;
+        warnings: string[];
+    }>;
+    /**
+     * Check if storage type is shared (multi-instance)
+     */
+    private static isSharedStorage;
+    /**
+     * Load configuration from shared storage
+     */
+    private static loadConfigFromStorage;
+    /**
+     * Save configuration to shared storage
+     */
+    private static saveConfigToStorage;
+    /**
+     * Create shared configuration from local config
+     */
+    private static createSharedConfig;
+    /**
+     * Check for critical configuration mismatches
+     */
+    private static checkCriticalMismatches;
+    /**
+     * Merge local config with shared config (shared takes precedence for critical params)
+     */
+    private static mergeWithSharedConfig;
+    /**
+     * Update access information in shared config
+     */
+    private static updateAccessInfo;
+    /**
+     * Get unique identifier for this instance
+     */
+    private static getInstanceIdentifier;
+    /**
+     * Validate that a configuration is compatible with shared data
+     */
+    static validateCompatibility(config1: SharedConfig, config2: SharedConfig): boolean;
+}

package/dist/config/sharedConfigManager.js

@@ -0,0 +1,215 @@
+/**
+ * Shared Configuration Manager
+ * Ensures configuration consistency across multiple instances using shared storage
+ */
+/**
+ * Manages configuration consistency for shared storage
+ */
+export class SharedConfigManager {
+    /**
+     * Load or create shared configuration
+     * When connecting to existing data, this OVERRIDES auto-configuration!
+     */
+    static async loadOrCreateSharedConfig(storage, localConfig) {
+        const warnings = [];
+        try {
+            // Check if we're using shared storage
+            if (!this.isSharedStorage(localConfig.storageType)) {
+                // Local storage - use local config
+                return { config: localConfig, warnings: [] };
+            }
+            // Try to load existing configuration from shared storage
+            const existingConfig = await this.loadConfigFromStorage(storage);
+            if (existingConfig) {
+                // EXISTING SHARED DATA - Must use its configuration!
+                console.log('📁 Found existing shared data configuration');
+                // Check for critical mismatches
+                const mismatches = this.checkCriticalMismatches(localConfig, existingConfig);
+                if (mismatches.length > 0) {
+                    console.warn('⚠️  Configuration override required for shared storage:');
+                    mismatches.forEach(m => {
+                        console.warn(`   - ${m.param}: ${m.local} → ${m.shared} (using shared)`);
+                        warnings.push(`${m.param} overridden: ${m.local} → ${m.shared}`);
+                    });
+                }
+                // Override critical parameters with shared values
+                const mergedConfig = this.mergeWithSharedConfig(localConfig, existingConfig);
+                // Update last accessed
+                await this.updateAccessInfo(storage, existingConfig);
+                return { config: mergedConfig, warnings };
+            }
+            else {
+                // NEW SHARED STORAGE - Save our configuration
+                console.log('📝 Initializing new shared storage with configuration');
+                const sharedConfig = this.createSharedConfig(localConfig);
+                await this.saveConfigToStorage(storage, sharedConfig);
+                return { config: localConfig, warnings: [] };
+            }
+        }
+        catch (error) {
+            console.error('Failed to manage shared configuration:', error);
+            warnings.push('Could not verify shared configuration - proceeding with caution');
+            return { config: localConfig, warnings };
+        }
+    }
+    /**
+     * Check if storage type is shared (multi-instance)
+     */
+    static isSharedStorage(storageType) {
+        return ['s3', 'gcs', 'r2'].includes(storageType);
+    }
+    /**
+     * Load configuration from shared storage
+     */
+    static async loadConfigFromStorage(storage) {
+        try {
+            const configData = await storage.get(this.CONFIG_FILE);
+            if (!configData)
+                return null;
+            const config = JSON.parse(configData);
+            return config;
+        }
+        catch (error) {
+            // Config doesn't exist yet
+            return null;
+        }
+    }
+    /**
+     * Save configuration to shared storage
+     */
+    static async saveConfigToStorage(storage, config) {
+        try {
+            await storage.set(this.CONFIG_FILE, JSON.stringify(config, null, 2));
+        }
+        catch (error) {
+            console.error('Failed to save shared configuration:', error);
+            throw new Error('Cannot initialize shared storage without saving configuration');
+        }
+    }
+    /**
+     * Create shared configuration from local config
+     */
+    static createSharedConfig(localConfig) {
+        return {
+            version: '2.10.0', // Brainy version
+            precision: localConfig.embeddingOptions?.precision || 'fp32',
+            dimensions: 384, // Fixed for all-MiniLM-L6-v2
+            hnswM: localConfig.hnsw?.M || 16,
+            hnswEfConstruction: localConfig.hnsw?.efConstruction || 200,
+            distanceFunction: localConfig.distanceFunction || 'cosine',
+            createdAt: new Date().toISOString(),
+            lastUpdated: new Date().toISOString(),
+            instanceCount: 1,
+            lastAccessedBy: this.getInstanceIdentifier()
+        };
+    }
+    /**
+     * Check for critical configuration mismatches
+     */
+    static checkCriticalMismatches(localConfig, sharedConfig) {
+        const mismatches = [];
+        // Model precision - CRITICAL!
+        const localPrecision = localConfig.embeddingOptions?.precision || 'fp32';
+        if (localPrecision !== sharedConfig.precision) {
+            mismatches.push({
+                param: 'Model Precision',
+                local: localPrecision,
+                shared: sharedConfig.precision
+            });
+        }
+        // HNSW parameters - Important for index consistency
+        const localM = localConfig.hnsw?.M || 16;
+        if (localM !== sharedConfig.hnswM) {
+            mismatches.push({
+                param: 'HNSW M',
+                local: localM,
+                shared: sharedConfig.hnswM
+            });
+        }
+        const localEf = localConfig.hnsw?.efConstruction || 200;
+        if (localEf !== sharedConfig.hnswEfConstruction) {
+            mismatches.push({
+                param: 'HNSW efConstruction',
+                local: localEf,
+                shared: sharedConfig.hnswEfConstruction
+            });
+        }
+        // Distance function
+        const localDistance = localConfig.distanceFunction || 'cosine';
+        if (localDistance !== sharedConfig.distanceFunction) {
+            mismatches.push({
+                param: 'Distance Function',
+                local: localDistance,
+                shared: sharedConfig.distanceFunction
+            });
+        }
+        return mismatches;
+    }
+    /**
+     * Merge local config with shared config (shared takes precedence for critical params)
+     */
+    static mergeWithSharedConfig(localConfig, sharedConfig) {
+        return {
+            ...localConfig,
+            // Override critical parameters with shared values
+            embeddingOptions: {
+                ...localConfig.embeddingOptions,
+                precision: sharedConfig.precision // MUST use shared precision!
+            },
+            hnsw: {
+                ...localConfig.hnsw,
+                M: sharedConfig.hnswM,
+                efConstruction: sharedConfig.hnswEfConstruction
+            },
+            distanceFunction: sharedConfig.distanceFunction,
+            // Add metadata about shared configuration
+            _sharedConfig: {
+                loaded: true,
+                version: sharedConfig.version,
+                createdAt: sharedConfig.createdAt,
+                precision: sharedConfig.precision
+            }
+        };
+    }
+    /**
+     * Update access information in shared config
+     */
+    static async updateAccessInfo(storage, config) {
+        try {
+            config.lastUpdated = new Date().toISOString();
+            config.instanceCount = (config.instanceCount || 0) + 1;
+            config.lastAccessedBy = this.getInstanceIdentifier();
+            await this.saveConfigToStorage(storage, config);
+        }
+        catch {
+            // Non-critical - don't fail if we can't update access info
+        }
+    }
+    /**
+     * Get unique identifier for this instance
+     */
+    static getInstanceIdentifier() {
+        if (process.env.HOSTNAME)
+            return process.env.HOSTNAME;
+        if (process.env.CONTAINER_ID)
+            return process.env.CONTAINER_ID;
+        if (process.env.K_SERVICE)
+            return process.env.K_SERVICE;
+        if (process.env.AWS_LAMBDA_FUNCTION_NAME)
+            return process.env.AWS_LAMBDA_FUNCTION_NAME;
+        // Generate a random identifier
+        return `instance-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    }
+    /**
+     * Validate that a configuration is compatible with shared data
+     */
+    static validateCompatibility(config1, config2) {
+        return (config1.precision === config2.precision &&
+            config1.dimensions === config2.dimensions &&
+            config1.hnswM === config2.hnswM &&
+            config1.hnswEfConstruction === config2.hnswEfConstruction &&
+            config1.distanceFunction === config2.distanceFunction);
+    }
+}
+SharedConfigManager.CONFIG_FILE = '.brainy/config.json';
+//# sourceMappingURL=sharedConfigManager.js.map

package/dist/config/storageAutoConfig.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Storage Configuration Auto-Detection
+ * Intelligently selects storage based on environment and available services
+ */
+/**
+ * Low-level storage implementation types
+ */
+export declare enum StorageType {
+    MEMORY = "memory",
+    FILESYSTEM = "filesystem",
+    OPFS = "opfs",
+    S3 = "s3",
+    GCS = "gcs",
+    R2 = "r2"
+}
+/**
+ * High-level storage presets (maps to StorageType)
+ */
+export declare enum StoragePreset {
+    AUTO = "auto",
+    MEMORY = "memory",
+    DISK = "disk",
+    CLOUD = "cloud"
+}
+export type StorageTypeString = 'memory' | 'filesystem' | 'opfs' | 's3' | 'gcs' | 'r2';
+export type StoragePresetString = 'auto' | 'memory' | 'disk' | 'cloud';
+export interface StorageConfigResult {
+    type: StorageType | StorageTypeString;
+    config: any;
+    reason: string;
+    autoSelected: boolean;
+}
+/**
+ * Auto-detect the best storage configuration
+ * @param override - Manual override: specific type or preset
+ */
+export declare function autoDetectStorage(override?: StorageType | StoragePreset | StorageTypeString | StoragePresetString | any): Promise<StorageConfigResult>;
+/**
+ * Log storage configuration decision
+ */
+export declare function logStorageConfig(config: StorageConfigResult, verbose?: boolean): void;