@soulcraft/brainy 2.9.0 → 2.10.1

package/dist/config/index.js

@@ -0,0 +1,33 @@
+/**
+ * Zero-Configuration System
+ * Main entry point for all auto-configuration features
+ */
+// Model configuration
+export { autoSelectModelPrecision, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
+// Storage configuration
+export { autoDetectStorage, StorageType, StoragePreset, logStorageConfig } from './storageAutoConfig.js';
+// Shared configuration for multi-instance
+export { SharedConfigManager } from './sharedConfigManager.js';
+// Main zero-config processor
+export { processZeroConfig, createEmbeddingFunctionWithPrecision } from './zeroConfig.js';
+// Strongly-typed presets and enums
+export { PresetName, PresetCategory, ModelPrecision, StorageOption, FeatureSet, DistributedRole, PRESET_CONFIGS, getPreset, isValidPreset, getPresetsByCategory, getAllPresetNames, getPresetDescription, presetToBrainyConfig } from './distributedPresets.js';
+// Extensible configuration
+export { registerStorageAugmentation, registerPresetAugmentation, getConfigRegistry } from './extensibleConfig.js';
+/**
+ * Main zero-config processor
+ * This is what BrainyData will call
+ */
+export async function applyZeroConfig(input) {
+    // Handle legacy config (full object) by detecting known legacy properties
+    if (input && typeof input === 'object' &&
+        (input.storage?.forceMemoryStorage || input.storage?.forceFileSystemStorage || input.storage?.s3Storage)) {
+        // This is a legacy config object - pass through unchanged
+        console.log('📦 Using legacy configuration format');
+        return input;
+    }
+    // Process as zero-config (includes new object format)
+    const { processZeroConfig } = await import('./zeroConfig.js');
+    return processZeroConfig(input);
+}
+//# sourceMappingURL=index.js.map

package/dist/config/modelAutoConfig.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Model Configuration Auto-Selection
+ * Intelligently selects model precision based on environment
+ * while allowing manual override
+ */
+export type ModelPrecision = 'fp32' | 'q8';
+export type ModelPreset = 'fast' | 'small' | 'auto';
+interface ModelConfigResult {
+    precision: ModelPrecision;
+    reason: string;
+    autoSelected: boolean;
+}
+/**
+ * Auto-select model precision based on environment and resources
+ * @param override - Manual override: 'fp32', 'q8', 'fast' (fp32), 'small' (q8), or 'auto'
+ */
+export declare function autoSelectModelPrecision(override?: ModelPrecision | ModelPreset): ModelConfigResult;
+/**
+ * Convenience function to check if models need to be downloaded
+ * This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
+ */
+export declare function shouldAutoDownloadModels(): boolean;
+/**
+ * Get the model path with intelligent defaults
+ * This replaces the need for BRAINY_MODELS_PATH env var
+ */
+export declare function getModelPath(): string;
+/**
+ * Log model configuration decision (only in verbose mode)
+ */
+export declare function logModelConfig(config: ModelConfigResult, verbose?: boolean): void;
+export {};

package/dist/config/modelAutoConfig.js

@@ -0,0 +1,193 @@
+/**
+ * Model Configuration Auto-Selection
+ * Intelligently selects model precision based on environment
+ * while allowing manual override
+ */
+import { isBrowser, isNode } from '../utils/environment.js';
+/**
+ * Auto-select model precision based on environment and resources
+ * @param override - Manual override: 'fp32', 'q8', 'fast' (fp32), 'small' (q8), or 'auto'
+ */
+export function autoSelectModelPrecision(override) {
+    // Handle direct precision override
+    if (override === 'fp32' || override === 'q8') {
+        return {
+            precision: override,
+            reason: `Manually specified: ${override}`,
+            autoSelected: false
+        };
+    }
+    // Handle preset overrides
+    if (override === 'fast') {
+        return {
+            precision: 'fp32',
+            reason: 'Preset: fast (fp32 for best quality)',
+            autoSelected: false
+        };
+    }
+    if (override === 'small') {
+        return {
+            precision: 'q8',
+            reason: 'Preset: small (q8 for reduced size)',
+            autoSelected: false
+        };
+    }
+    // Auto-selection logic
+    return autoDetectBestPrecision();
+}
+/**
+ * Automatically detect the best model precision for the environment
+ */
+function autoDetectBestPrecision() {
+    // Browser environment - use Q8 for smaller download/memory
+    if (isBrowser()) {
+        return {
+            precision: 'q8',
+            reason: 'Browser environment detected - using Q8 for smaller size',
+            autoSelected: true
+        };
+    }
+    // Serverless environments - use Q8 for faster cold starts
+    if (isServerlessEnvironment()) {
+        return {
+            precision: 'q8',
+            reason: 'Serverless environment detected - using Q8 for faster cold starts',
+            autoSelected: true
+        };
+    }
+    // Check available memory
+    const memoryMB = getAvailableMemoryMB();
+    if (memoryMB < 512) {
+        return {
+            precision: 'q8',
+            reason: `Low memory detected (${memoryMB}MB) - using Q8`,
+            autoSelected: true
+        };
+    }
+    // Development environment - use FP32 for best quality
+    if (process.env.NODE_ENV === 'development') {
+        return {
+            precision: 'fp32',
+            reason: 'Development environment - using FP32 for best quality',
+            autoSelected: true
+        };
+    }
+    // Production with adequate memory - use FP32
+    if (memoryMB >= 2048) {
+        return {
+            precision: 'fp32',
+            reason: `Adequate memory (${memoryMB}MB) - using FP32 for best quality`,
+            autoSelected: true
+        };
+    }
+    // Default to Q8 for moderate memory environments
+    return {
+        precision: 'q8',
+        reason: `Moderate memory (${memoryMB}MB) - using Q8 for balance`,
+        autoSelected: true
+    };
+}
+/**
+ * Check if running in a serverless environment
+ */
+function isServerlessEnvironment() {
+    if (!isNode())
+        return false;
+    return !!(process.env.AWS_LAMBDA_FUNCTION_NAME ||
+        process.env.VERCEL ||
+        process.env.NETLIFY ||
+        process.env.CLOUDFLARE_WORKERS ||
+        process.env.FUNCTIONS_WORKER_RUNTIME ||
+        process.env.K_SERVICE // Google Cloud Run
+    );
+}
+/**
+ * Get available memory in MB
+ */
+function getAvailableMemoryMB() {
+    if (isBrowser()) {
+        // @ts-ignore - navigator.deviceMemory is experimental
+        if (navigator.deviceMemory) {
+            // @ts-ignore
+            return navigator.deviceMemory * 1024; // Device memory in GB
+        }
+        return 256; // Conservative default for browsers
+    }
+    if (isNode()) {
+        try {
+            // Try to get memory info synchronously for Node.js
+            // This will be available in Node.js environments
+            if (typeof process !== 'undefined' && process.memoryUsage) {
+                // Use RSS (Resident Set Size) as a proxy for available memory
+                const rss = process.memoryUsage().rss;
+                // Assume we can use up to 4GB or 50% more than current usage
+                return Math.min(4096, Math.floor(rss / (1024 * 1024) * 1.5));
+            }
+        }
+        catch {
+            // Fall through to default
+        }
+        return 1024; // Default 1GB for Node.js
+    }
+    return 512; // Conservative default
+}
+/**
+ * Convenience function to check if models need to be downloaded
+ * This replaces the need for BRAINY_ALLOW_REMOTE_MODELS
+ */
+export function shouldAutoDownloadModels() {
+    // Always allow downloads unless explicitly disabled
+    // This eliminates the need for BRAINY_ALLOW_REMOTE_MODELS
+    const explicitlyDisabled = process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false';
+    if (explicitlyDisabled) {
+        console.warn('Model downloads disabled via BRAINY_ALLOW_REMOTE_MODELS=false');
+        return false;
+    }
+    // In production, always allow downloads for seamless operation
+    if (process.env.NODE_ENV === 'production') {
+        return true;
+    }
+    // In development, allow downloads with a one-time notice
+    if (process.env.NODE_ENV === 'development') {
+        return true;
+    }
+    // Default: allow downloads
+    return true;
+}
+/**
+ * Get the model path with intelligent defaults
+ * This replaces the need for BRAINY_MODELS_PATH env var
+ */
+export function getModelPath() {
+    // Check if user explicitly set a path (keeping this for advanced users)
+    if (process.env.BRAINY_MODELS_PATH) {
+        return process.env.BRAINY_MODELS_PATH;
+    }
+    // Browser - use cache API or IndexedDB (handled by transformers.js)
+    if (isBrowser()) {
+        return 'browser-cache';
+    }
+    // Serverless - use /tmp for ephemeral storage
+    if (isServerlessEnvironment()) {
+        return '/tmp/.brainy/models';
+    }
+    // Node.js - use home directory for persistent storage
+    if (isNode()) {
+        // Use process.env.HOME as a fallback
+        const homeDir = process.env.HOME || process.env.USERPROFILE || '~';
+        return `${homeDir}/.brainy/models`;
+    }
+    // Fallback
+    return './.brainy/models';
+}
+/**
+ * Log model configuration decision (only in verbose mode)
+ */
+export function logModelConfig(config, verbose = false) {
+    if (!verbose && process.env.NODE_ENV === 'production') {
+        return; // Silent in production unless verbose
+    }
+    const icon = config.autoSelected ? '🤖' : '👤';
+    console.log(`${icon} Model: ${config.precision.toUpperCase()} - ${config.reason}`);
+}
+//# sourceMappingURL=modelAutoConfig.js.map

package/dist/config/sharedConfigManager.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Shared Configuration Manager
+ * Ensures configuration consistency across multiple instances using shared storage
+ */
+import { ModelPrecision } from './modelAutoConfig.js';
+export interface SharedConfig {
+    version: string;
+    precision: ModelPrecision;
+    dimensions: number;
+    hnswM: number;
+    hnswEfConstruction: number;
+    distanceFunction: string;
+    createdAt: string;
+    lastUpdated: string;
+    instanceCount?: number;
+    lastAccessedBy?: string;
+}
+/**
+ * Manages configuration consistency for shared storage
+ */
+export declare class SharedConfigManager {
+    private static CONFIG_FILE;
+    /**
+     * Load or create shared configuration
+     * When connecting to existing data, this OVERRIDES auto-configuration!
+     */
+    static loadOrCreateSharedConfig(storage: any, localConfig: any): Promise<{
+        config: any;
+        warnings: string[];
+    }>;
+    /**
+     * Check if storage type is shared (multi-instance)
+     */
+    private static isSharedStorage;
+    /**
+     * Load configuration from shared storage
+     */
+    private static loadConfigFromStorage;
+    /**
+     * Save configuration to shared storage
+     */
+    private static saveConfigToStorage;
+    /**
+     * Create shared configuration from local config
+     */
+    private static createSharedConfig;
+    /**
+     * Check for critical configuration mismatches
+     */
+    private static checkCriticalMismatches;
+    /**
+     * Merge local config with shared config (shared takes precedence for critical params)
+     */
+    private static mergeWithSharedConfig;
+    /**
+     * Update access information in shared config
+     */
+    private static updateAccessInfo;
+    /**
+     * Get unique identifier for this instance
+     */
+    private static getInstanceIdentifier;
+    /**
+     * Validate that a configuration is compatible with shared data
+     */
+    static validateCompatibility(config1: SharedConfig, config2: SharedConfig): boolean;
+}

package/dist/config/sharedConfigManager.js

@@ -0,0 +1,215 @@
+/**
+ * Shared Configuration Manager
+ * Ensures configuration consistency across multiple instances using shared storage
+ */
+/**
+ * Manages configuration consistency for shared storage
+ */
+export class SharedConfigManager {
+    /**
+     * Load or create shared configuration
+     * When connecting to existing data, this OVERRIDES auto-configuration!
+     */
+    static async loadOrCreateSharedConfig(storage, localConfig) {
+        const warnings = [];
+        try {
+            // Check if we're using shared storage
+            if (!this.isSharedStorage(localConfig.storageType)) {
+                // Local storage - use local config
+                return { config: localConfig, warnings: [] };
+            }
+            // Try to load existing configuration from shared storage
+            const existingConfig = await this.loadConfigFromStorage(storage);
+            if (existingConfig) {
+                // EXISTING SHARED DATA - Must use its configuration!
+                console.log('📁 Found existing shared data configuration');
+                // Check for critical mismatches
+                const mismatches = this.checkCriticalMismatches(localConfig, existingConfig);
+                if (mismatches.length > 0) {
+                    console.warn('⚠️  Configuration override required for shared storage:');
+                    mismatches.forEach(m => {
+                        console.warn(`   - ${m.param}: ${m.local} → ${m.shared} (using shared)`);
+                        warnings.push(`${m.param} overridden: ${m.local} → ${m.shared}`);
+                    });
+                }
+                // Override critical parameters with shared values
+                const mergedConfig = this.mergeWithSharedConfig(localConfig, existingConfig);
+                // Update last accessed
+                await this.updateAccessInfo(storage, existingConfig);
+                return { config: mergedConfig, warnings };
+            }
+            else {
+                // NEW SHARED STORAGE - Save our configuration
+                console.log('📝 Initializing new shared storage with configuration');
+                const sharedConfig = this.createSharedConfig(localConfig);
+                await this.saveConfigToStorage(storage, sharedConfig);
+                return { config: localConfig, warnings: [] };
+            }
+        }
+        catch (error) {
+            console.error('Failed to manage shared configuration:', error);
+            warnings.push('Could not verify shared configuration - proceeding with caution');
+            return { config: localConfig, warnings };
+        }
+    }
+    /**
+     * Check if storage type is shared (multi-instance)
+     */
+    static isSharedStorage(storageType) {
+        return ['s3', 'gcs', 'r2'].includes(storageType);
+    }
+    /**
+     * Load configuration from shared storage
+     */
+    static async loadConfigFromStorage(storage) {
+        try {
+            const configData = await storage.get(this.CONFIG_FILE);
+            if (!configData)
+                return null;
+            const config = JSON.parse(configData);
+            return config;
+        }
+        catch (error) {
+            // Config doesn't exist yet
+            return null;
+        }
+    }
+    /**
+     * Save configuration to shared storage
+     */
+    static async saveConfigToStorage(storage, config) {
+        try {
+            await storage.set(this.CONFIG_FILE, JSON.stringify(config, null, 2));
+        }
+        catch (error) {
+            console.error('Failed to save shared configuration:', error);
+            throw new Error('Cannot initialize shared storage without saving configuration');
+        }
+    }
+    /**
+     * Create shared configuration from local config
+     */
+    static createSharedConfig(localConfig) {
+        return {
+            version: '2.10.0', // Brainy version
+            precision: localConfig.embeddingOptions?.precision || 'fp32',
+            dimensions: 384, // Fixed for all-MiniLM-L6-v2
+            hnswM: localConfig.hnsw?.M || 16,
+            hnswEfConstruction: localConfig.hnsw?.efConstruction || 200,
+            distanceFunction: localConfig.distanceFunction || 'cosine',
+            createdAt: new Date().toISOString(),
+            lastUpdated: new Date().toISOString(),
+            instanceCount: 1,
+            lastAccessedBy: this.getInstanceIdentifier()
+        };
+    }
+    /**
+     * Check for critical configuration mismatches
+     */
+    static checkCriticalMismatches(localConfig, sharedConfig) {
+        const mismatches = [];
+        // Model precision - CRITICAL!
+        const localPrecision = localConfig.embeddingOptions?.precision || 'fp32';
+        if (localPrecision !== sharedConfig.precision) {
+            mismatches.push({
+                param: 'Model Precision',
+                local: localPrecision,
+                shared: sharedConfig.precision
+            });
+        }
+        // HNSW parameters - Important for index consistency
+        const localM = localConfig.hnsw?.M || 16;
+        if (localM !== sharedConfig.hnswM) {
+            mismatches.push({
+                param: 'HNSW M',
+                local: localM,
+                shared: sharedConfig.hnswM
+            });
+        }
+        const localEf = localConfig.hnsw?.efConstruction || 200;
+        if (localEf !== sharedConfig.hnswEfConstruction) {
+            mismatches.push({
+                param: 'HNSW efConstruction',
+                local: localEf,
+                shared: sharedConfig.hnswEfConstruction
+            });
+        }
+        // Distance function
+        const localDistance = localConfig.distanceFunction || 'cosine';
+        if (localDistance !== sharedConfig.distanceFunction) {
+            mismatches.push({
+                param: 'Distance Function',
+                local: localDistance,
+                shared: sharedConfig.distanceFunction
+            });
+        }
+        return mismatches;
+    }
+    /**
+     * Merge local config with shared config (shared takes precedence for critical params)
+     */
+    static mergeWithSharedConfig(localConfig, sharedConfig) {
+        return {
+            ...localConfig,
+            // Override critical parameters with shared values
+            embeddingOptions: {
+                ...localConfig.embeddingOptions,
+                precision: sharedConfig.precision // MUST use shared precision!
+            },
+            hnsw: {
+                ...localConfig.hnsw,
+                M: sharedConfig.hnswM,
+                efConstruction: sharedConfig.hnswEfConstruction
+            },
+            distanceFunction: sharedConfig.distanceFunction,
+            // Add metadata about shared configuration
+            _sharedConfig: {
+                loaded: true,
+                version: sharedConfig.version,
+                createdAt: sharedConfig.createdAt,
+                precision: sharedConfig.precision
+            }
+        };
+    }
+    /**
+     * Update access information in shared config
+     */
+    static async updateAccessInfo(storage, config) {
+        try {
+            config.lastUpdated = new Date().toISOString();
+            config.instanceCount = (config.instanceCount || 0) + 1;
+            config.lastAccessedBy = this.getInstanceIdentifier();
+            await this.saveConfigToStorage(storage, config);
+        }
+        catch {
+            // Non-critical - don't fail if we can't update access info
+        }
+    }
+    /**
+     * Get unique identifier for this instance
+     */
+    static getInstanceIdentifier() {
+        if (process.env.HOSTNAME)
+            return process.env.HOSTNAME;
+        if (process.env.CONTAINER_ID)
+            return process.env.CONTAINER_ID;
+        if (process.env.K_SERVICE)
+            return process.env.K_SERVICE;
+        if (process.env.AWS_LAMBDA_FUNCTION_NAME)
+            return process.env.AWS_LAMBDA_FUNCTION_NAME;
+        // Generate a random identifier
+        return `instance-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    }
+    /**
+     * Validate that a configuration is compatible with shared data
+     */
+    static validateCompatibility(config1, config2) {
+        return (config1.precision === config2.precision &&
+            config1.dimensions === config2.dimensions &&
+            config1.hnswM === config2.hnswM &&
+            config1.hnswEfConstruction === config2.hnswEfConstruction &&
+            config1.distanceFunction === config2.distanceFunction);
+    }
+}
+SharedConfigManager.CONFIG_FILE = '.brainy/config.json';
+//# sourceMappingURL=sharedConfigManager.js.map

package/dist/config/storageAutoConfig.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Storage Configuration Auto-Detection
+ * Intelligently selects storage based on environment and available services
+ */
+/**
+ * Low-level storage implementation types
+ */
+export declare enum StorageType {
+    MEMORY = "memory",
+    FILESYSTEM = "filesystem",
+    OPFS = "opfs",
+    S3 = "s3",
+    GCS = "gcs",
+    R2 = "r2"
+}
+/**
+ * High-level storage presets (maps to StorageType)
+ */
+export declare enum StoragePreset {
+    AUTO = "auto",
+    MEMORY = "memory",
+    DISK = "disk",
+    CLOUD = "cloud"
+}
+export type StorageTypeString = 'memory' | 'filesystem' | 'opfs' | 's3' | 'gcs' | 'r2';
+export type StoragePresetString = 'auto' | 'memory' | 'disk' | 'cloud';
+export interface StorageConfigResult {
+    type: StorageType | StorageTypeString;
+    config: any;
+    reason: string;
+    autoSelected: boolean;
+}
+/**
+ * Auto-detect the best storage configuration
+ * @param override - Manual override: specific type or preset
+ */
+export declare function autoDetectStorage(override?: StorageType | StoragePreset | StorageTypeString | StoragePresetString | any): Promise<StorageConfigResult>;
+/**
+ * Log storage configuration decision
+ */
+export declare function logStorageConfig(config: StorageConfigResult, verbose?: boolean): void;