@soulcraft/brainy 3.0.0 → 3.0.1

package/dist/config/distributedPresets.js

@@ -0,0 +1,318 @@
+/**
+ * Extended Distributed Configuration Presets
+ * Common patterns for distributed and multi-service architectures
+ * All strongly typed with enums for compile-time safety
+ */
+/**
+ * Strongly typed enum for preset names
+ */
+export var PresetName;
+(function (PresetName) {
+    // Basic presets
+    PresetName["PRODUCTION"] = "production";
+    PresetName["DEVELOPMENT"] = "development";
+    PresetName["MINIMAL"] = "minimal";
+    PresetName["ZERO"] = "zero";
+    // Distributed presets
+    PresetName["WRITER"] = "writer";
+    PresetName["READER"] = "reader";
+    // Service-specific presets
+    PresetName["INGESTION_SERVICE"] = "ingestion-service";
+    PresetName["SEARCH_API"] = "search-api";
+    PresetName["ANALYTICS_SERVICE"] = "analytics-service";
+    PresetName["EDGE_CACHE"] = "edge-cache";
+    PresetName["BATCH_PROCESSOR"] = "batch-processor";
+    PresetName["STREAMING_SERVICE"] = "streaming-service";
+    PresetName["ML_TRAINING"] = "ml-training";
+    PresetName["SIDECAR"] = "sidecar";
+})(PresetName || (PresetName = {}));
+/**
+ * Preset categories for organization
+ */
+export var PresetCategory;
+(function (PresetCategory) {
+    PresetCategory["BASIC"] = "basic";
+    PresetCategory["DISTRIBUTED"] = "distributed";
+    PresetCategory["SERVICE"] = "service";
+})(PresetCategory || (PresetCategory = {}));
+/**
+ * Model precision options
+ */
+export var ModelPrecision;
+(function (ModelPrecision) {
+    ModelPrecision["FP32"] = "fp32";
+    ModelPrecision["Q8"] = "q8";
+    ModelPrecision["AUTO"] = "auto";
+    ModelPrecision["FAST"] = "fast";
+    ModelPrecision["SMALL"] = "small";
+})(ModelPrecision || (ModelPrecision = {}));
+/**
+ * Storage options
+ */
+export var StorageOption;
+(function (StorageOption) {
+    StorageOption["AUTO"] = "auto";
+    StorageOption["MEMORY"] = "memory";
+    StorageOption["DISK"] = "disk";
+    StorageOption["CLOUD"] = "cloud";
+})(StorageOption || (StorageOption = {}));
+/**
+ * Feature set options
+ */
+export var FeatureSet;
+(function (FeatureSet) {
+    FeatureSet["MINIMAL"] = "minimal";
+    FeatureSet["DEFAULT"] = "default";
+    FeatureSet["FULL"] = "full";
+    FeatureSet["CUSTOM"] = "custom"; // For custom feature arrays
+})(FeatureSet || (FeatureSet = {}));
+/**
+ * Distributed role options
+ */
+export var DistributedRole;
+(function (DistributedRole) {
+    DistributedRole["WRITER"] = "writer";
+    DistributedRole["READER"] = "reader";
+    DistributedRole["HYBRID"] = "hybrid";
+})(DistributedRole || (DistributedRole = {}));
+/**
+ * Strongly typed preset configurations
+ */
+export const PRESET_CONFIGS = {
+    // Basic presets
+    [PresetName.PRODUCTION]: {
+        storage: StorageOption.DISK,
+        model: ModelPrecision.AUTO,
+        features: FeatureSet.DEFAULT,
+        distributed: false,
+        verbose: false,
+        description: 'Optimized for production use',
+        category: PresetCategory.BASIC
+    },
+    [PresetName.DEVELOPMENT]: {
+        storage: StorageOption.MEMORY,
+        model: ModelPrecision.FP32,
+        features: FeatureSet.FULL,
+        distributed: false,
+        verbose: true,
+        description: 'Optimized for development with verbose logging',
+        category: PresetCategory.BASIC
+    },
+    [PresetName.MINIMAL]: {
+        storage: StorageOption.MEMORY,
+        model: ModelPrecision.Q8,
+        features: FeatureSet.MINIMAL,
+        distributed: false,
+        verbose: false,
+        description: 'Minimal footprint configuration',
+        category: PresetCategory.BASIC
+    },
+    [PresetName.ZERO]: {
+        storage: StorageOption.AUTO,
+        model: ModelPrecision.AUTO,
+        features: FeatureSet.DEFAULT,
+        distributed: false,
+        verbose: false,
+        description: 'True zero configuration with auto-detection',
+        category: PresetCategory.BASIC
+    },
+    // Distributed basic presets
+    [PresetName.WRITER]: {
+        storage: StorageOption.AUTO,
+        model: ModelPrecision.AUTO,
+        features: FeatureSet.MINIMAL,
+        distributed: true,
+        role: DistributedRole.WRITER,
+        writeOnly: true,
+        allowDirectReads: true,
+        verbose: false,
+        description: 'Write-only instance for distributed setups',
+        category: PresetCategory.DISTRIBUTED
+    },
+    [PresetName.READER]: {
+        storage: StorageOption.AUTO,
+        model: ModelPrecision.AUTO,
+        features: FeatureSet.DEFAULT,
+        distributed: true,
+        role: DistributedRole.READER,
+        readOnly: true,
+        lazyLoadInReadOnlyMode: true,
+        verbose: false,
+        description: 'Read-only instance for distributed setups',
+        category: PresetCategory.DISTRIBUTED
+    },
+    // Service-specific presets
+    [PresetName.INGESTION_SERVICE]: {
+        storage: StorageOption.CLOUD,
+        model: ModelPrecision.Q8,
+        features: ['core', 'batch-processing', 'entity-registry'],
+        distributed: true,
+        role: DistributedRole.WRITER,
+        writeOnly: true,
+        allowDirectReads: true,
+        verbose: false,
+        description: 'High-throughput data ingestion service',
+        category: PresetCategory.SERVICE
+    },
+    [PresetName.SEARCH_API]: {
+        storage: StorageOption.CLOUD,
+        model: ModelPrecision.FP32,
+        features: ['core', 'search', 'cache', 'triple-intelligence'],
+        distributed: true,
+        role: DistributedRole.READER,
+        readOnly: true,
+        lazyLoadInReadOnlyMode: true,
+        cache: {
+            hotCacheMaxSize: 10000,
+            autoTune: true
+        },
+        verbose: false,
+        description: 'Low-latency search API service',
+        category: PresetCategory.SERVICE
+    },
+    [PresetName.ANALYTICS_SERVICE]: {
+        storage: StorageOption.CLOUD,
+        model: ModelPrecision.AUTO,
+        features: ['core', 'search', 'metrics', 'monitoring'],
+        distributed: true,
+        role: DistributedRole.HYBRID,
+        verbose: false,
+        description: 'Analytics and data processing service',
+        category: PresetCategory.SERVICE
+    },
+    [PresetName.EDGE_CACHE]: {
+        storage: StorageOption.AUTO,
+        model: ModelPrecision.Q8,
+        features: ['core', 'search', 'cache'],
+        distributed: true,
+        role: DistributedRole.READER,
+        readOnly: true,
+        lazyLoadInReadOnlyMode: true,
+        cache: {
+            hotCacheMaxSize: 1000,
+            autoTune: false
+        },
+        verbose: false,
+        description: 'Edge location cache with minimal footprint',
+        category: PresetCategory.SERVICE
+    },
+    [PresetName.BATCH_PROCESSOR]: {
+        storage: StorageOption.CLOUD,
+        model: ModelPrecision.Q8,
+        features: ['core', 'batch-processing', 'neural-api'],
+        distributed: true,
+        role: DistributedRole.HYBRID,
+        cache: {
+            hotCacheMaxSize: 5000,
+            batchSize: 500
+        },
+        verbose: false,
+        description: 'Batch processing and bulk operations',
+        category: PresetCategory.SERVICE
+    },
+    [PresetName.STREAMING_SERVICE]: {
+        storage: StorageOption.CLOUD,
+        model: ModelPrecision.Q8,
+        features: ['core', 'batch-processing', 'wal'],
+        distributed: true,
+        role: DistributedRole.WRITER,
+        writeOnly: true,
+        allowDirectReads: false,
+        verbose: false,
+        description: 'Real-time data streaming service',
+        category: PresetCategory.SERVICE
+    },
+    [PresetName.ML_TRAINING]: {
+        storage: StorageOption.CLOUD,
+        model: ModelPrecision.FP32,
+        features: FeatureSet.FULL,
+        distributed: true,
+        role: DistributedRole.HYBRID,
+        cache: {
+            hotCacheMaxSize: 20000,
+            autoTune: true
+        },
+        verbose: true,
+        description: 'Machine learning training service',
+        category: PresetCategory.SERVICE
+    },
+    [PresetName.SIDECAR]: {
+        storage: StorageOption.AUTO,
+        model: ModelPrecision.Q8,
+        features: FeatureSet.MINIMAL,
+        distributed: false,
+        verbose: false,
+        description: 'Lightweight sidecar for microservices',
+        category: PresetCategory.SERVICE
+    }
+};
+/**
+ * Type-safe preset getter
+ */
+export function getPreset(name) {
+    return PRESET_CONFIGS[name];
+}
+/**
+ * Check if a string is a valid preset name
+ */
+export function isValidPreset(name) {
+    return Object.values(PresetName).includes(name);
+}
+/**
+ * Get presets by category
+ */
+export function getPresetsByCategory(category) {
+    return Object.entries(PRESET_CONFIGS)
+        .filter(([_, config]) => config.category === category)
+        .map(([name]) => name);
+}
+/**
+ * Get all preset names
+ */
+export function getAllPresetNames() {
+    return Object.values(PresetName);
+}
+/**
+ * Get preset description
+ */
+export function getPresetDescription(name) {
+    return PRESET_CONFIGS[name].description;
+}
+/**
+ * Convert preset config to Brainy config
+ */
+export function presetToBrainyConfig(preset) {
+    const config = {
+        storage: preset.storage,
+        model: preset.model,
+        verbose: preset.verbose
+    };
+    // Handle features
+    if (Array.isArray(preset.features)) {
+        config.features = preset.features;
+    }
+    else {
+        config.features = preset.features; // Will be expanded by processZeroConfig
+    }
+    // Handle distributed settings
+    if (preset.distributed) {
+        config.distributed = {
+            enabled: true,
+            role: preset.role
+        };
+        if (preset.readOnly)
+            config.readOnly = true;
+        if (preset.writeOnly)
+            config.writeOnly = true;
+        if (preset.allowDirectReads)
+            config.allowDirectReads = true;
+        if (preset.lazyLoadInReadOnlyMode)
+            config.lazyLoadInReadOnlyMode = true;
+    }
+    // Handle cache settings
+    if (preset.cache) {
+        config.cache = preset.cache;
+    }
+    return config;
+}
+//# sourceMappingURL=distributedPresets.js.map

package/dist/config/extensibleConfig.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Extensible Configuration System
+ * Allows augmentations to register new storage types, presets, and configurations
+ */
+import { PresetConfig } from './distributedPresets.js';
+import { StorageConfigResult } from './storageAutoConfig.js';
+/**
+ * Storage provider registration interface
+ */
+export interface StorageProvider {
+    type: string;
+    name: string;
+    description: string;
+    detect: () => Promise<boolean>;
+    getConfig: () => Promise<any>;
+    priority?: number;
+    requirements?: {
+        env?: string[];
+        packages?: string[];
+    };
+}
+/**
+ * Preset extension interface
+ */
+export interface PresetExtension {
+    name: string;
+    config: PresetConfig;
+    override?: boolean;
+}
+/**
+ * Global registry for extensions
+ */
+declare class ConfigurationRegistry {
+    private static instance;
+    private storageProviders;
+    private presetExtensions;
+    private autoDetectHooks;
+    private constructor();
+    static getInstance(): ConfigurationRegistry;
+    /**
+     * Register a new storage provider
+     * This is how augmentations add new storage types
+     */
+    registerStorageProvider(provider: StorageProvider): void;
+    /**
+     * Register a new preset
+     */
+    registerPreset(name: string, extension: PresetExtension): void;
+    /**
+     * Register an auto-detection hook
+     */
+    registerAutoDetectHook(hook: () => Promise<any>): void;
+    /**
+     * Get all registered storage providers
+     */
+    getStorageProviders(): StorageProvider[];
+    /**
+     * Get all registered presets (built-in + extensions)
+     */
+    getAllPresets(): Map<string, PresetConfig>;
+    /**
+     * Auto-detect storage including extensions
+     */
+    autoDetectStorage(): Promise<StorageConfigResult>;
+    /**
+     * Register built-in providers
+     */
+    private registerBuiltInProviders;
+}
+/**
+ * Example: Redis storage provider registration
+ * This would be in the redis augmentation package
+ */
+export declare const redisStorageProvider: StorageProvider;
+/**
+ * Example: MongoDB storage provider
+ */
+export declare const mongoStorageProvider: StorageProvider;
+/**
+ * Example: PostgreSQL with pgvector extension
+ */
+export declare const postgresStorageProvider: StorageProvider;
+/**
+ * How an augmentation would register its storage provider
+ */
+export declare function registerStorageAugmentation(provider: StorageProvider): void;
+/**
+ * How to register a new preset
+ */
+export declare function registerPresetAugmentation(name: string, config: PresetConfig): void;
+/**
+ * Example preset for Redis-based caching service
+ */
+export declare const redisCachePreset: PresetConfig;
+/**
+ * Get the configuration registry
+ */
+export declare function getConfigRegistry(): ConfigurationRegistry;
+export {};

package/dist/config/extensibleConfig.js

@@ -0,0 +1,268 @@
+/**
+ * Extensible Configuration System
+ * Allows augmentations to register new storage types, presets, and configurations
+ */
+import { ModelPrecision, DistributedRole, PresetCategory } from './distributedPresets.js';
+/**
+ * Global registry for extensions
+ */
+class ConfigurationRegistry {
+    constructor() {
+        // Registered storage providers
+        this.storageProviders = new Map();
+        // Registered preset extensions
+        this.presetExtensions = new Map();
+        // Custom auto-detection hooks
+        this.autoDetectHooks = [];
+        // Initialize with built-in providers
+        this.registerBuiltInProviders();
+    }
+    static getInstance() {
+        if (!ConfigurationRegistry.instance) {
+            ConfigurationRegistry.instance = new ConfigurationRegistry();
+        }
+        return ConfigurationRegistry.instance;
+    }
+    /**
+     * Register a new storage provider
+     * This is how augmentations add new storage types
+     */
+    registerStorageProvider(provider) {
+        console.log(`📦 Registering storage provider: ${provider.type} (${provider.name})`);
+        this.storageProviders.set(provider.type, provider);
+    }
+    /**
+     * Register a new preset
+     */
+    registerPreset(name, extension) {
+        console.log(`🎨 Registering preset: ${name}`);
+        this.presetExtensions.set(name, extension);
+    }
+    /**
+     * Register an auto-detection hook
+     */
+    registerAutoDetectHook(hook) {
+        this.autoDetectHooks.push(hook);
+    }
+    /**
+     * Get all registered storage providers
+     */
+    getStorageProviders() {
+        return Array.from(this.storageProviders.values())
+            .sort((a, b) => (b.priority || 0) - (a.priority || 0));
+    }
+    /**
+     * Get all registered presets (built-in + extensions)
+     */
+    getAllPresets() {
+        // Start with built-in presets
+        const allPresets = new Map();
+        // Note: Would import from distributedPresets-new.ts
+        // Add extended presets
+        for (const [name, extension] of this.presetExtensions) {
+            if (extension.override || !allPresets.has(name)) {
+                allPresets.set(name, extension.config);
+            }
+        }
+        return allPresets;
+    }
+    /**
+     * Auto-detect storage including extensions
+     */
+    async autoDetectStorage() {
+        // Check registered providers first (in priority order)
+        for (const provider of this.getStorageProviders()) {
+            try {
+                if (await provider.detect()) {
+                    const config = await provider.getConfig();
+                    return {
+                        type: provider.type,
+                        config,
+                        reason: `Auto-detected ${provider.name}`,
+                        autoSelected: true
+                    };
+                }
+            }
+            catch (error) {
+                console.warn(`Failed to detect ${provider.type}:`, error);
+            }
+        }
+        // Fallback to built-in detection
+        const { autoDetectStorage } = await import('./storageAutoConfig.js');
+        return autoDetectStorage();
+    }
+    /**
+     * Register built-in providers
+     */
+    registerBuiltInProviders() {
+        // These would be the built-in ones, but could be overridden
+    }
+}
+/**
+ * Example: Redis storage provider registration
+ * This would be in the redis augmentation package
+ */
+export const redisStorageProvider = {
+    type: 'redis',
+    name: 'Redis Storage',
+    description: 'High-performance in-memory data store',
+    priority: 10, // Check before filesystem
+    requirements: {
+        env: ['REDIS_URL', 'REDIS_HOST'],
+        packages: ['redis', 'ioredis']
+    },
+    async detect() {
+        // Check for Redis connection info
+        if (process.env.REDIS_URL || process.env.REDIS_HOST) {
+            try {
+                // Try to connect to Redis (dynamic import for optional dependency)
+                const redis = await new Function('return import("ioredis")')().catch(() => null);
+                if (!redis)
+                    return false;
+                const client = new redis.default(process.env.REDIS_URL);
+                await client.ping();
+                await client.quit();
+                return true;
+            }
+            catch {
+                // Redis not available
+            }
+        }
+        return false;
+    },
+    async getConfig() {
+        return {
+            type: 'redis',
+            redisStorage: {
+                url: process.env.REDIS_URL || `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}`,
+                prefix: process.env.REDIS_PREFIX || 'brainy:',
+                ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined
+            }
+        };
+    }
+};
+/**
+ * Example: MongoDB storage provider
+ */
+export const mongoStorageProvider = {
+    type: 'mongodb',
+    name: 'MongoDB Storage',
+    description: 'Document database for complex data',
+    priority: 8,
+    requirements: {
+        env: ['MONGODB_URI', 'MONGO_URL'],
+        packages: ['mongodb']
+    },
+    async detect() {
+        if (process.env.MONGODB_URI || process.env.MONGO_URL) {
+            try {
+                const mongodb = await new Function('return import("mongodb")')().catch(() => null);
+                if (!mongodb)
+                    return false;
+                const client = new mongodb.MongoClient(process.env.MONGODB_URI || process.env.MONGO_URL);
+                await client.connect();
+                await client.close();
+                return true;
+            }
+            catch {
+                // MongoDB not available
+            }
+        }
+        return false;
+    },
+    async getConfig() {
+        return {
+            type: 'mongodb',
+            mongoStorage: {
+                uri: process.env.MONGODB_URI || process.env.MONGO_URL,
+                database: process.env.MONGO_DATABASE || 'brainy',
+                collection: process.env.MONGO_COLLECTION || 'vectors'
+            }
+        };
+    }
+};
+/**
+ * Example: PostgreSQL with pgvector extension
+ */
+export const postgresStorageProvider = {
+    type: 'postgres',
+    name: 'PostgreSQL Storage',
+    description: 'PostgreSQL with pgvector for scalable vector search',
+    priority: 9,
+    requirements: {
+        env: ['DATABASE_URL', 'POSTGRES_URL'],
+        packages: ['pg', 'pgvector']
+    },
+    async detect() {
+        const url = process.env.DATABASE_URL || process.env.POSTGRES_URL;
+        if (url && url.includes('postgres')) {
+            try {
+                const pg = await new Function('return import("pg")')().catch(() => null);
+                if (!pg)
+                    return false;
+                const client = new pg.Client({ connectionString: url });
+                await client.connect();
+                // Check for pgvector extension
+                const result = await client.query("SELECT * FROM pg_extension WHERE extname = 'vector'");
+                await client.end();
+                return result.rows.length > 0;
+            }
+            catch {
+                // PostgreSQL not available or pgvector not installed
+            }
+        }
+        return false;
+    },
+    async getConfig() {
+        return {
+            type: 'postgres',
+            postgresStorage: {
+                connectionString: process.env.DATABASE_URL || process.env.POSTGRES_URL,
+                table: process.env.POSTGRES_TABLE || 'brainy_vectors',
+                schema: process.env.POSTGRES_SCHEMA || 'public'
+            }
+        };
+    }
+};
+/**
+ * How an augmentation would register its storage provider
+ */
+export function registerStorageAugmentation(provider) {
+    const registry = ConfigurationRegistry.getInstance();
+    registry.registerStorageProvider(provider);
+}
+/**
+ * How to register a new preset
+ */
+export function registerPresetAugmentation(name, config) {
+    const registry = ConfigurationRegistry.getInstance();
+    registry.registerPreset(name, {
+        name,
+        config,
+        override: false
+    });
+}
+/**
+ * Example preset for Redis-based caching service
+ */
+export const redisCachePreset = {
+    storage: 'redis', // Extended storage type
+    model: ModelPrecision.Q8,
+    features: ['core', 'cache', 'search'],
+    distributed: true,
+    role: DistributedRole.READER,
+    readOnly: true,
+    cache: {
+        hotCacheMaxSize: 50000, // Large Redis cache
+        autoTune: true
+    },
+    description: 'Redis-backed caching layer',
+    category: PresetCategory.SERVICE
+};
+/**
+ * Get the configuration registry
+ */
+export function getConfigRegistry() {
+    return ConfigurationRegistry.getInstance();
+}
+//# sourceMappingURL=extensibleConfig.js.map

package/dist/config/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Zero-Configuration System
+ * Main entry point for all auto-configuration features
+ */
+export { autoSelectModelPrecision, ModelPrecision as ModelPrecisionType, // Avoid conflict
+ModelPreset, shouldAutoDownloadModels, getModelPath, logModelConfig } from './modelAutoConfig.js';
+export declare const getModelPrecision: () => "q8";
+export { autoDetectStorage, StorageType, StoragePreset, StorageConfigResult, logStorageConfig, type StorageTypeString, type StoragePresetString } from './storageAutoConfig.js';
+export { SharedConfig, SharedConfigManager } from './sharedConfigManager.js';
+export { BrainyZeroConfig, processZeroConfig, createEmbeddingFunctionWithPrecision } from './zeroConfig.js';
+export { PresetName, PresetCategory, ModelPrecision, StorageOption, FeatureSet, DistributedRole, PresetConfig, PRESET_CONFIGS, getPreset, isValidPreset, getPresetsByCategory, getAllPresetNames, getPresetDescription, presetToBrainyConfig } from './distributedPresets.js';
+export { StorageProvider, registerStorageAugmentation, registerPresetAugmentation, getConfigRegistry } from './extensibleConfig.js';
+/**
+ * Main zero-config processor
+ * This is what Brainy will call
+ */
+export declare function applyZeroConfig(input?: string | any): Promise<any>;