@soulcraft/brainy 3.0.0 → 3.0.1

package/dist/config/storageAutoConfig.js

@@ -0,0 +1,328 @@
+/**
+ * Storage Configuration Auto-Detection
+ * Intelligently selects storage based on environment and available services
+ */
+import { isBrowser, isNode } from '../utils/environment.js';
+/**
+ * Low-level storage implementation types
+ */
+export var StorageType;
+(function (StorageType) {
+    StorageType["MEMORY"] = "memory";
+    StorageType["FILESYSTEM"] = "filesystem";
+    StorageType["OPFS"] = "opfs";
+    StorageType["S3"] = "s3";
+    StorageType["GCS"] = "gcs";
+    StorageType["R2"] = "r2";
+})(StorageType || (StorageType = {}));
+/**
+ * High-level storage presets (maps to StorageType)
+ */
+export var StoragePreset;
+(function (StoragePreset) {
+    StoragePreset["AUTO"] = "auto";
+    StoragePreset["MEMORY"] = "memory";
+    StoragePreset["DISK"] = "disk";
+    StoragePreset["CLOUD"] = "cloud";
+})(StoragePreset || (StoragePreset = {}));
+/**
+ * Auto-detect the best storage configuration
+ * @param override - Manual override: specific type or preset
+ */
+export async function autoDetectStorage(override) {
+    // Handle direct storage config object
+    if (override && typeof override === 'object') {
+        return {
+            type: override.type || 'memory',
+            config: override,
+            reason: 'Manually configured storage',
+            autoSelected: false
+        };
+    }
+    // Handle storage type override (enum values or strings)
+    if (override && Object.values(StorageType).includes(override)) {
+        return {
+            type: override,
+            config: override,
+            reason: `Manually specified: ${override}`,
+            autoSelected: false
+        };
+    }
+    // Handle presets (both enum and string values)
+    if (override === StoragePreset.MEMORY || override === 'memory') {
+        return {
+            type: StorageType.MEMORY,
+            config: StorageType.MEMORY,
+            reason: 'Preset: memory storage',
+            autoSelected: false
+        };
+    }
+    if (override === StoragePreset.DISK || override === 'disk') {
+        const diskType = isBrowser() ? StorageType.OPFS : StorageType.FILESYSTEM;
+        return {
+            type: diskType,
+            config: diskType,
+            reason: `Preset: disk storage (${diskType})`,
+            autoSelected: false
+        };
+    }
+    if (override === StoragePreset.CLOUD || override === 'cloud') {
+        const cloudStorage = await detectCloudStorage();
+        if (cloudStorage) {
+            return {
+                ...cloudStorage,
+                reason: `Preset: cloud storage (${cloudStorage.type})`,
+                autoSelected: false
+            };
+        }
+        // Fallback to disk if no cloud storage detected
+        const diskType = isBrowser() ? StorageType.OPFS : StorageType.FILESYSTEM;
+        return {
+            type: diskType,
+            config: diskType,
+            reason: 'Preset: cloud (none detected, using disk)',
+            autoSelected: false
+        };
+    }
+    // Auto-detection logic
+    return await autoDetectBestStorage();
+}
+/**
+ * Automatically detect the best storage option
+ */
+async function autoDetectBestStorage() {
+    // Check for cloud storage first (highest priority in production)
+    const cloudStorage = await detectCloudStorage();
+    if (cloudStorage && process.env.NODE_ENV === 'production') {
+        return {
+            ...cloudStorage,
+            reason: `Auto-detected ${cloudStorage.type} in production`,
+            autoSelected: true
+        };
+    }
+    // Browser environment
+    if (isBrowser()) {
+        // Check for OPFS support
+        if (await hasOPFSSupport()) {
+            return {
+                type: StorageType.OPFS,
+                config: { requestPersistentStorage: true },
+                reason: 'Browser with OPFS support detected',
+                autoSelected: true
+            };
+        }
+        // Fallback to memory for browsers without OPFS
+        return {
+            type: StorageType.MEMORY,
+            config: StorageType.MEMORY,
+            reason: 'Browser without OPFS - using memory',
+            autoSelected: true
+        };
+    }
+    // Serverless environment - prefer memory or cloud
+    if (isServerlessEnvironment()) {
+        if (cloudStorage) {
+            return {
+                ...cloudStorage,
+                reason: `Serverless with ${cloudStorage.type} detected`,
+                autoSelected: true
+            };
+        }
+        return {
+            type: StorageType.MEMORY,
+            config: StorageType.MEMORY,
+            reason: 'Serverless environment - using memory',
+            autoSelected: true
+        };
+    }
+    // Node.js environment - use filesystem
+    if (isNode()) {
+        const dataPath = await findBestDataPath();
+        return {
+            type: StorageType.FILESYSTEM,
+            config: StorageType.FILESYSTEM,
+            reason: `Node.js environment - using filesystem at ${dataPath}`,
+            autoSelected: true
+        };
+    }
+    // Fallback to memory
+    return {
+        type: StorageType.MEMORY,
+        config: StorageType.MEMORY,
+        reason: 'Default fallback - using memory',
+        autoSelected: true
+    };
+}
+/**
+ * Detect cloud storage from environment variables
+ */
+async function detectCloudStorage() {
+    // AWS S3 Detection
+    if (hasAWSConfig()) {
+        return {
+            type: StorageType.S3,
+            config: {
+                s3Storage: {
+                    bucketName: process.env.AWS_BUCKET || process.env.S3_BUCKET_NAME || 'brainy-data',
+                    region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
+                    // Credentials will be picked up by AWS SDK automatically
+                }
+            }
+        };
+    }
+    // Google Cloud Storage Detection
+    if (hasGCPConfig()) {
+        return {
+            type: StorageType.GCS,
+            config: {
+                gcsStorage: {
+                    bucketName: process.env.GCS_BUCKET || process.env.GOOGLE_STORAGE_BUCKET || 'brainy-data',
+                    // Credentials will be picked up by GCP SDK automatically
+                }
+            }
+        };
+    }
+    // Cloudflare R2 Detection
+    if (hasR2Config()) {
+        return {
+            type: StorageType.R2,
+            config: {
+                r2Storage: {
+                    bucketName: process.env.R2_BUCKET || 'brainy-data',
+                    accountId: process.env.CLOUDFLARE_ACCOUNT_ID,
+                    accessKeyId: process.env.R2_ACCESS_KEY_ID,
+                    secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
+                }
+            }
+        };
+    }
+    return null;
+}
+/**
+ * Check if AWS S3 is configured
+ */
+function hasAWSConfig() {
+    return !!(
+    // Explicit S3 bucket
+    process.env.AWS_BUCKET ||
+        process.env.S3_BUCKET_NAME ||
+        // AWS credentials (SDK will find them)
+        process.env.AWS_ACCESS_KEY_ID ||
+        process.env.AWS_PROFILE ||
+        // AWS environment indicators
+        process.env.AWS_EXECUTION_ENV ||
+        process.env.AWS_LAMBDA_FUNCTION_NAME ||
+        process.env.ECS_CONTAINER_METADATA_URI);
+}
+/**
+ * Check if Google Cloud Storage is configured
+ */
+function hasGCPConfig() {
+    return !!(
+    // Explicit GCS bucket
+    process.env.GCS_BUCKET ||
+        process.env.GOOGLE_STORAGE_BUCKET ||
+        // GCP credentials
+        process.env.GOOGLE_APPLICATION_CREDENTIALS ||
+        // GCP environment indicators
+        process.env.GOOGLE_CLOUD_PROJECT ||
+        process.env.K_SERVICE ||
+        process.env.GAE_SERVICE);
+}
+/**
+ * Check if Cloudflare R2 is configured
+ */
+function hasR2Config() {
+    return !!(process.env.R2_BUCKET ||
+        (process.env.CLOUDFLARE_ACCOUNT_ID && process.env.R2_ACCESS_KEY_ID));
+}
+/**
+ * Check if running in serverless environment
+ */
+function isServerlessEnvironment() {
+    if (!isNode())
+        return false;
+    return !!(process.env.AWS_LAMBDA_FUNCTION_NAME ||
+        process.env.VERCEL ||
+        process.env.NETLIFY ||
+        process.env.CLOUDFLARE_WORKERS ||
+        process.env.FUNCTIONS_WORKER_RUNTIME ||
+        process.env.K_SERVICE ||
+        process.env.RAILWAY_ENVIRONMENT ||
+        process.env.FLY_APP_NAME);
+}
+/**
+ * Check for OPFS support in browser
+ */
+async function hasOPFSSupport() {
+    if (!isBrowser())
+        return false;
+    try {
+        return 'storage' in navigator &&
+            'getDirectory' in navigator.storage;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Find the best path for filesystem storage
+ */
+async function findBestDataPath() {
+    if (!isNode())
+        return './brainy-data';
+    const homeDir = process.env.HOME || process.env.USERPROFILE || '~';
+    const tempDir = process.env.TMPDIR || process.env.TEMP || '/tmp';
+    const candidates = [
+        // User-specified path
+        process.env.BRAINY_DATA_PATH,
+        // Current directory
+        './brainy-data',
+        // Home directory
+        `${homeDir}/.brainy/data`,
+        // Temp directory (last resort)
+        `${tempDir}/brainy-data`
+    ].filter(Boolean);
+    // Find first writable directory
+    for (const candidate of candidates) {
+        if (await isWritable(candidate)) {
+            return candidate;
+        }
+    }
+    // Default fallback
+    return candidates[1]; // ./brainy-data
+}
+/**
+ * Check if a directory is writable
+ */
+async function isWritable(dirPath) {
+    if (!isNode())
+        return false;
+    try {
+        // Dynamic import fs for Node.js
+        const { promises: fs } = await import('fs');
+        const path = await import('path');
+        // Try to create directory if it doesn't exist
+        await fs.mkdir(dirPath, { recursive: true });
+        // Try to write a test file
+        const testFile = path.join(dirPath, '.write-test');
+        await fs.writeFile(testFile, 'test');
+        await fs.unlink(testFile);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// Legacy getStorageConfig function removed - now using simple string types
+/**
+ * Log storage configuration decision
+ */
+export function logStorageConfig(config, verbose = false) {
+    if (!verbose && process.env.NODE_ENV === 'production') {
+        return; // Silent in production unless verbose
+    }
+    const icon = config.autoSelected ? '🤖' : '👤';
+    console.log(`${icon} Storage: ${config.type.toUpperCase()} - ${config.reason}`);
+}
+//# sourceMappingURL=storageAutoConfig.js.map

package/dist/config/zeroConfig.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Zero-Configuration System for Brainy
+ * Provides intelligent defaults while preserving full control
+ */
+import { ModelPrecision, ModelPreset } from './modelAutoConfig.js';
+import { StorageType, StoragePreset } from './storageAutoConfig.js';
+/**
+ * Simplified configuration interface
+ * Everything is optional - zero config by default!
+ */
+export interface BrainyZeroConfig {
+    /**
+     * Configuration preset for common scenarios
+     * - 'production': Optimized for production (disk storage, auto model, default features)
+     * - 'development': Optimized for development (memory storage, fp32, verbose logging)
+     * - 'minimal': Minimal footprint (memory storage, q8, minimal features)
+     * - 'zero': True zero config (all auto-detected)
+     * - 'writer': Write-only instance for distributed setups (no index loading)
+     * - 'reader': Read-only instance for distributed setups (no write operations)
+     */
+    mode?: 'production' | 'development' | 'minimal' | 'zero' | 'writer' | 'reader';
+    /**
+     * Model precision configuration
+     * - 'fp32': Full precision (best quality, larger size)
+     * - 'q8': Quantized 8-bit (smaller size, slightly lower quality)
+     * - 'fast': Alias for fp32
+     * - 'small': Alias for q8
+     * - 'auto': Auto-detect based on environment (default)
+     */
+    model?: ModelPrecision | ModelPreset;
+    /**
+     * Storage configuration
+     * - 'memory': In-memory only (no persistence)
+     * - 'disk': Local disk (filesystem or OPFS)
+     * - 'cloud': Cloud storage (S3/GCS/R2 if configured)
+     * - 'auto': Auto-detect best option (default)
+     * - Object: Custom storage configuration
+     */
+    storage?: StorageType | StoragePreset | any;
+    /**
+     * Feature set configuration
+     * - 'minimal': Core features only (fastest startup)
+     * - 'default': Standard features (balanced)
+     * - 'full': All features enabled (most capable)
+     * - Array: Specific features to enable
+     */
+    features?: 'minimal' | 'default' | 'full' | string[];
+    /**
+     * Logging verbosity
+     * - true: Show configuration decisions and progress
+     * - false: Silent operation (default in production)
+     */
+    verbose?: boolean;
+    /**
+     * Advanced configuration (escape hatch for power users)
+     * Any additional configuration can be passed here
+     */
+    advanced?: any;
+}
+/**
+ * Process zero-config input into full configuration
+ */
+export declare function processZeroConfig(input?: string | BrainyZeroConfig): Promise<any>;
+/**
+ * Create embedding function with specified precision
+ * This ensures the model precision is respected
+ */
+export declare function createEmbeddingFunctionWithPrecision(precision: ModelPrecision): Promise<any>;

package/dist/config/zeroConfig.js

@@ -0,0 +1,301 @@
+/**
+ * Zero-Configuration System for Brainy
+ * Provides intelligent defaults while preserving full control
+ */
+import { autoSelectModelPrecision, getModelPath, shouldAutoDownloadModels } from './modelAutoConfig.js';
+import { autoDetectStorage } from './storageAutoConfig.js';
+import { AutoConfiguration } from '../utils/autoConfiguration.js';
+/**
+ * Configuration presets for common scenarios
+ */
+const PRESETS = {
+    production: {
+        storage: 'disk',
+        model: 'auto',
+        features: 'default',
+        verbose: false
+    },
+    development: {
+        storage: 'memory',
+        model: 'q8', // Q8 is now the default for all presets
+        features: 'full',
+        verbose: true
+    },
+    minimal: {
+        storage: 'memory',
+        model: 'q8',
+        features: 'minimal',
+        verbose: false
+    },
+    zero: {
+        storage: 'auto',
+        model: 'auto',
+        features: 'default',
+        verbose: false
+    },
+    writer: {
+        storage: 'auto',
+        model: 'auto',
+        features: 'minimal',
+        verbose: false,
+        // Writer-specific settings
+        distributed: true,
+        role: 'writer',
+        writeOnly: true,
+        allowDirectReads: true // Allow deduplication checks
+    },
+    reader: {
+        storage: 'auto',
+        model: 'auto',
+        features: 'default',
+        verbose: false,
+        // Reader-specific settings
+        distributed: true,
+        role: 'reader',
+        readOnly: true,
+        lazyLoadInReadOnlyMode: true // Optimize for search
+    }
+};
+/**
+ * Feature sets configuration
+ */
+const FEATURE_SETS = {
+    minimal: [
+        'core',
+        'search',
+        'storage'
+    ],
+    default: [
+        'core',
+        'search',
+        'storage',
+        'cache',
+        'metadata-index',
+        'batch-processing',
+        'entity-registry',
+        'request-deduplicator'
+    ],
+    full: [
+        'core',
+        'search',
+        'storage',
+        'cache',
+        'metadata-index',
+        'batch-processing',
+        'entity-registry',
+        'request-deduplicator',
+        'connection-pool',
+        'wal',
+        'monitoring',
+        'metrics',
+        'intelligent-verb-scoring',
+        'triple-intelligence',
+        'neural-api'
+    ]
+};
+/**
+ * Process zero-config input into full configuration
+ */
+export async function processZeroConfig(input) {
+    let config = {};
+    // Handle string shorthand (preset name)
+    if (typeof input === 'string') {
+        if (input in PRESETS) {
+            config = { mode: input };
+        }
+        else {
+            throw new Error(`Unknown preset: ${input}. Valid presets: ${Object.keys(PRESETS).join(', ')}`);
+        }
+    }
+    else if (input) {
+        config = input;
+    }
+    // Apply preset if specified
+    if (config.mode && config.mode in PRESETS) {
+        const preset = PRESETS[config.mode];
+        config = {
+            ...preset,
+            ...config,
+            // Preserve explicit overrides
+            model: config.model ?? preset.model,
+            storage: config.storage ?? preset.storage,
+            features: config.features ?? preset.features,
+            verbose: config.verbose ?? preset.verbose
+        };
+    }
+    // Auto-detect environment if not in preset mode
+    const environment = detectEnvironmentMode();
+    // Process model configuration
+    const modelConfig = autoSelectModelPrecision(config.model);
+    // Process storage configuration
+    const storageConfig = await autoDetectStorage(config.storage);
+    // Process features configuration
+    const features = processFeatures(config.features);
+    // Get auto-configuration recommendations
+    const autoConfig = await AutoConfiguration.getInstance().detectAndConfigure({
+        expectedDataSize: estimateDataSize(environment),
+        s3Available: storageConfig.type === 's3',
+        memoryBudget: undefined // Let it auto-detect
+    });
+    // Determine verbosity
+    const verbose = config.verbose ?? (process.env.NODE_ENV === 'development');
+    // Log configuration decisions if verbose
+    if (verbose) {
+        logConfigurationSummary({
+            mode: config.mode || 'auto',
+            model: modelConfig,
+            storage: storageConfig,
+            features: features,
+            environment: environment,
+            autoConfig: autoConfig
+        });
+    }
+    // Build final configuration
+    const finalConfig = {
+        // Model configuration
+        embeddingFunction: undefined, // Will be created with correct precision
+        embeddingOptions: {
+            precision: modelConfig.precision,
+            modelPath: getModelPath(),
+            allowRemoteDownload: shouldAutoDownloadModels()
+        },
+        // Storage configuration
+        storage: storageConfig.config,
+        storageType: storageConfig.type,
+        // HNSW configuration from auto-config
+        hnsw: {
+            M: autoConfig.recommendedConfig.enablePartitioning ? 32 : 16,
+            efConstruction: autoConfig.recommendedConfig.enablePartitioning ? 400 : 200,
+            maxDatasetSize: autoConfig.recommendedConfig.expectedDatasetSize,
+            partitioning: autoConfig.recommendedConfig.enablePartitioning,
+            maxNodesPerPartition: autoConfig.recommendedConfig.maxNodesPerPartition
+        },
+        // Cache configuration from auto-config
+        cache: {
+            autoTune: true,
+            hotCacheMaxSize: Math.floor(autoConfig.recommendedConfig.maxMemoryUsage / (1024 * 1024 * 10)), // 10% of memory budget
+            batchSize: autoConfig.recommendedConfig.enablePartitioning ? 100 : 50
+        },
+        // Features configuration
+        enabledFeatures: features,
+        // Metadata index configuration
+        metadataIndex: features.includes('metadata-index') ? {
+            enabled: true,
+            autoRebuild: true
+        } : undefined,
+        // Intelligent verb scoring
+        intelligentVerbScoring: features.includes('intelligent-verb-scoring') ? {
+            enabled: true
+        } : undefined,
+        // Logging configuration
+        logging: {
+            verbose: verbose
+        },
+        // Performance flags from auto-config
+        optimizations: autoConfig.optimizationFlags,
+        // Advanced overrides (if any)
+        ...config.advanced
+    };
+    // Apply distributed preset settings if applicable
+    if (config.mode === 'writer' || config.mode === 'reader') {
+        const presetSettings = PRESETS[config.mode]; // Cast to any since we know these presets have additional properties
+        // Apply distributed-specific settings
+        finalConfig.distributed = presetSettings.distributed;
+        finalConfig.readOnly = presetSettings.readOnly || false;
+        finalConfig.writeOnly = presetSettings.writeOnly || false;
+        finalConfig.allowDirectReads = presetSettings.allowDirectReads || false;
+        finalConfig.lazyLoadInReadOnlyMode = presetSettings.lazyLoadInReadOnlyMode || false;
+        // Set distributed role in distributed config
+        if (finalConfig.distributed) {
+            finalConfig.distributed = {
+                enabled: true,
+                role: presetSettings.role
+            };
+        }
+        // Log distributed mode if verbose
+        if (verbose) {
+            console.log(`📡 Distributed mode: ${config.mode.toUpperCase()}`);
+            console.log(`   Role: ${presetSettings.role}`);
+            console.log(`   Read-only: ${finalConfig.readOnly}`);
+            console.log(`   Write-only: ${finalConfig.writeOnly}`);
+        }
+    }
+    return finalConfig;
+}
+/**
+ * Detect environment mode if not specified
+ */
+function detectEnvironmentMode() {
+    if (process.env.NODE_ENV === 'production')
+        return 'production';
+    if (process.env.NODE_ENV === 'development')
+        return 'development';
+    if (process.env.NODE_ENV === 'test')
+        return 'development';
+    // Check for CI environments
+    if (process.env.CI || process.env.GITHUB_ACTIONS)
+        return 'production';
+    // Check for production indicators
+    if (process.env.VERCEL_ENV === 'production' ||
+        process.env.NETLIFY_ENV === 'production' ||
+        process.env.RAILWAY_ENVIRONMENT === 'production') {
+        return 'production';
+    }
+    return 'unknown';
+}
+/**
+ * Process features configuration
+ */
+function processFeatures(features) {
+    if (Array.isArray(features)) {
+        return features;
+    }
+    if (features && features in FEATURE_SETS) {
+        return FEATURE_SETS[features];
+    }
+    // Default based on environment
+    const env = detectEnvironmentMode();
+    if (env === 'production')
+        return FEATURE_SETS.default;
+    if (env === 'development')
+        return FEATURE_SETS.full;
+    return FEATURE_SETS.default;
+}
+/**
+ * Estimate dataset size based on environment
+ */
+function estimateDataSize(environment) {
+    switch (environment) {
+        case 'production': return 100000;
+        case 'development': return 10000;
+        default: return 50000;
+    }
+}
+/**
+ * Log configuration summary
+ */
+function logConfigurationSummary(config) {
+    console.log('\n🧠 Brainy Zero-Config Summary');
+    console.log('================================');
+    console.log(`Mode: ${config.mode}`);
+    console.log(`Environment: ${config.environment}`);
+    console.log(`Model: ${config.model.precision.toUpperCase()} (${config.model.reason})`);
+    console.log(`Storage: ${config.storage.type.toUpperCase()} (${config.storage.reason})`);
+    console.log(`Features: ${config.features.length} enabled`);
+    console.log(`Memory Budget: ${Math.floor(config.autoConfig.recommendedConfig.maxMemoryUsage / (1024 * 1024))}MB`);
+    console.log(`Expected Dataset: ${config.autoConfig.recommendedConfig.expectedDatasetSize.toLocaleString()} items`);
+    console.log('================================\n');
+}
+/**
+ * Create embedding function with specified precision
+ * This ensures the model precision is respected
+ */
+export async function createEmbeddingFunctionWithPrecision(precision) {
+    const { createEmbeddingFunction } = await import('../utils/embedding.js');
+    // Create embedding function with specified precision
+    return createEmbeddingFunction({
+        precision: precision,
+        verbose: false // Silent by default in zero-config
+    });
+}
+//# sourceMappingURL=zeroConfig.js.map