@soulcraft/brainy 1.2.0 → 1.4.0

package/dist/embeddings/model-manager.js

@@ -0,0 +1,189 @@
+/**
+ * Model Manager - Ensures transformer models are available at runtime
+ *
+ * Strategy:
+ * 1. Check local cache first
+ * 2. Try GitHub releases (our backup)
+ * 3. Fall back to Hugging Face
+ * 4. Future: CDN at models.soulcraft.com
+ */
+import { existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { env } from '@huggingface/transformers';
+// Model sources in order of preference
+const MODEL_SOURCES = {
+    // GitHub Release - our controlled backup
+    github: 'https://github.com/soulcraftlabs/brainy/releases/download/models-v1/all-MiniLM-L6-v2.tar.gz',
+    // Future CDN - fastest option when available
+    cdn: 'https://models.soulcraft.com/brainy/all-MiniLM-L6-v2.tar.gz',
+    // Original Hugging Face - fallback
+    huggingface: 'default' // Uses transformers.js default
+};
+// Expected model files and their hashes
+const MODEL_MANIFEST = {
+    'Xenova/all-MiniLM-L6-v2': {
+        files: {
+            'onnx/model.onnx': {
+                size: 90555481,
+                sha256: null // Will be computed from actual model
+            },
+            'tokenizer.json': {
+                size: 711661,
+                sha256: null
+            },
+            'config.json': {
+                size: 650,
+                sha256: null
+            },
+            'tokenizer_config.json': {
+                size: 366,
+                sha256: null
+            }
+        }
+    }
+};
+export class ModelManager {
+    constructor() {
+        this.isInitialized = false;
+        // Determine models path
+        this.modelsPath = this.getModelsPath();
+    }
+    static getInstance() {
+        if (!ModelManager.instance) {
+            ModelManager.instance = new ModelManager();
+        }
+        return ModelManager.instance;
+    }
+    getModelsPath() {
+        // Check various possible locations
+        const paths = [
+            process.env.BRAINY_MODELS_PATH,
+            './models',
+            join(process.cwd(), 'models'),
+            join(process.env.HOME || '', '.brainy', 'models'),
+            env.cacheDir
+        ];
+        // Find first existing path or use default
+        for (const path of paths) {
+            if (path && existsSync(path)) {
+                return path;
+            }
+        }
+        // Default to local models directory
+        return join(process.cwd(), 'models');
+    }
+    async ensureModels(modelName = 'Xenova/all-MiniLM-L6-v2') {
+        if (this.isInitialized) {
+            return true;
+        }
+        const modelPath = join(this.modelsPath, ...modelName.split('/'));
+        // Check if model already exists locally
+        if (await this.verifyModelFiles(modelPath, modelName)) {
+            console.log('✅ Models found in cache:', modelPath);
+            this.configureTransformers(modelPath);
+            this.isInitialized = true;
+            return true;
+        }
+        // Try to download from our sources
+        console.log('📥 Downloading transformer models...');
+        // Try GitHub first (our backup)
+        if (await this.downloadFromGitHub(modelName)) {
+            this.isInitialized = true;
+            return true;
+        }
+        // Try CDN (when available)
+        if (await this.downloadFromCDN(modelName)) {
+            this.isInitialized = true;
+            return true;
+        }
+        // Fall back to Hugging Face (default transformers.js behavior)
+        console.log('⚠️ Using Hugging Face fallback for models');
+        env.allowRemoteModels = true;
+        this.isInitialized = true;
+        return true;
+    }
+    async verifyModelFiles(modelPath, modelName) {
+        const manifest = MODEL_MANIFEST[modelName];
+        if (!manifest)
+            return false;
+        for (const [filePath, info] of Object.entries(manifest.files)) {
+            const fullPath = join(modelPath, filePath);
+            if (!existsSync(fullPath)) {
+                return false;
+            }
+            // Optionally verify size
+            if (process.env.VERIFY_MODEL_SIZE === 'true') {
+                const stats = await import('fs').then(fs => fs.promises.stat(fullPath));
+                if (stats.size !== info.size) {
+                    console.warn(`⚠️ Model file size mismatch: ${filePath}`);
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    async downloadFromGitHub(modelName) {
+        try {
+            const url = MODEL_SOURCES.github;
+            console.log('📥 Downloading from GitHub releases...');
+            // Download tar.gz file
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`GitHub download failed: ${response.status}`);
+            }
+            const buffer = await response.arrayBuffer();
+            // Extract tar.gz (would need tar library in production)
+            // For now, return false to fall back to other methods
+            console.log('⚠️ GitHub model extraction not yet implemented');
+            return false;
+        }
+        catch (error) {
+            console.log('⚠️ GitHub download failed:', error.message);
+            return false;
+        }
+    }
+    async downloadFromCDN(modelName) {
+        try {
+            const url = MODEL_SOURCES.cdn;
+            console.log('📥 Downloading from Soulcraft CDN...');
+            // Try to fetch from CDN
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`CDN download failed: ${response.status}`);
+            }
+            // Would extract files here
+            console.log('⚠️ CDN not yet available');
+            return false;
+        }
+        catch (error) {
+            console.log('⚠️ CDN download failed:', error.message);
+            return false;
+        }
+    }
+    configureTransformers(modelPath) {
+        // Configure transformers.js to use our local models
+        env.localModelPath = dirname(modelPath);
+        env.allowRemoteModels = false;
+        console.log('🔧 Configured transformers.js to use local models');
+    }
+    /**
+     * Pre-download models for deployment
+     * This is what npm run download-models calls
+     */
+    static async predownload() {
+        const manager = ModelManager.getInstance();
+        const success = await manager.ensureModels();
+        if (!success) {
+            throw new Error('Failed to download models');
+        }
+        console.log('✅ Models downloaded successfully');
+    }
+}
+// Auto-initialize on import in production
+if (process.env.NODE_ENV === 'production' && process.env.SKIP_MODEL_CHECK !== 'true') {
+    ModelManager.getInstance().ensureModels().catch(error => {
+        console.error('⚠️ Model initialization failed:', error);
+        // Don't throw - allow app to start and try downloading on first use
+    });
+}
+//# sourceMappingURL=model-manager.js.map

package/dist/intelligence/neuralEngine.d.ts

@@ -0,0 +1,207 @@
+/**
+ * Neural Detection Engine - Shared AI Intelligence Infrastructure
+ *
+ * 🧠 Consolidates all neural analysis capabilities for consistent intelligence across Brainy
+ * ⚛️ Replaces basic heuristics with sophisticated semantic analysis
+ *
+ * This engine provides the core AI capabilities that power:
+ * - Smart entity type detection
+ * - Intelligent relationship inference
+ * - Data quality assessment
+ * - Impact analysis for operations
+ * - Pattern recognition and insights
+ */
+import { BrainyData } from '../brainyData.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+export interface EntityAnalysis {
+    detectedType: NounType;
+    confidence: number;
+    reasoning: string;
+    alternativeTypes: Array<{
+        type: NounType;
+        confidence: number;
+    }>;
+    suggestedId?: string;
+    qualityScore: number;
+}
+export interface RelationshipAnalysis {
+    verbType: VerbType;
+    confidence: number;
+    weight: number;
+    reasoning: string;
+    context: string;
+    metadata?: Record<string, any>;
+}
+export interface DeleteAnalysis {
+    strategy: 'soft-delete' | 'hard-delete' | 'cascade-delete' | 'soft-delete-preserve';
+    impactedEntities: string[];
+    orphanedRelationships: string[];
+    riskLevel: 'low' | 'medium' | 'high';
+    recommendation: string;
+    preserveData: boolean;
+}
+export interface DataInsights {
+    entityDistribution: Record<string, number>;
+    relationshipPatterns: Array<{
+        pattern: string;
+        frequency: number;
+    }>;
+    hierarchies: Array<{
+        type: string;
+        levels: number;
+        entities: string[];
+    }>;
+    clusters: Array<{
+        type: string;
+        size: number;
+        entities: string[];
+    }>;
+    anomalies: Array<{
+        type: string;
+        description: string;
+        entities: string[];
+    }>;
+    qualityMetrics: QualityAssessment;
+}
+export interface QualityAssessment {
+    completeness: number;
+    consistency: number;
+    accuracy: number;
+    richness: number;
+    recommendations: string[];
+}
+export interface DetectionOptions {
+    confidenceThreshold?: number;
+    includeAlternatives?: boolean;
+    generateReasoning?: boolean;
+    enableCaching?: boolean;
+}
+export interface ImpactOptions {
+    analyzeCascade?: boolean;
+    checkOrphans?: boolean;
+    riskAssessment?: boolean;
+}
+/**
+ * Neural Detection Engine - The Brainy AI Brain
+ */
+export declare class NeuralDetectionEngine {
+    private brainy;
+    private analysisCache;
+    private config;
+    constructor(brainy: BrainyData, config?: Partial<typeof NeuralDetectionEngine.prototype.config>);
+    /**
+     * Detect Entity Type - Replaces basic detectNounType() with neural analysis
+     */
+    detectEntityType(data: any, options?: DetectionOptions): Promise<EntityAnalysis>;
+    /**
+     * Detect Relationship Types - Intelligent relationship inference
+     */
+    detectRelationshipTypes(sourceData: any, targetData: any, options?: DetectionOptions): Promise<RelationshipAnalysis[]>;
+    /**
+     * Analyze Delete Impact - Intelligent cascade and orphan analysis
+     */
+    analyzeDeleteImpact(nounId: string, options?: ImpactOptions): Promise<DeleteAnalysis>;
+    /**
+     * Generate Data Insights - Pattern recognition and analysis
+     */
+    generateDataInsights(entities: any[], relationships: any[]): Promise<DataInsights>;
+    /**
+     * Assess Data Quality
+     */
+    assessDataQuality(data: any[]): Promise<QualityAssessment>;
+    /**
+     * Calculate entity type confidence using multi-factor analysis
+     */
+    private calculateEntityTypeConfidence;
+    /**
+     * Field-based confidence calculation
+     */
+    private calculateFieldBasedConfidence;
+    /**
+     * Pattern-based confidence calculation
+     */
+    private calculatePatternBasedConfidence;
+    /**
+     * Generate reasoning for entity type selection
+     */
+    private generateEntityReasoning;
+    /**
+     * Calculate relationship confidence
+     */
+    private calculateRelationshipConfidence;
+    /**
+     * Calculate relationship weight/strength
+     */
+    private calculateRelationshipWeight;
+    /**
+     * Calculate type compatibility for relationships
+     */
+    private calculateTypeCompatibility;
+    /**
+     * Generate relationship reasoning
+     */
+    private generateRelationshipReasoning;
+    /**
+     * Extract main text from data object
+     */
+    private extractMainText;
+    /**
+     * Extract relationship context
+     */
+    private extractRelationshipContext;
+    /**
+     * Generate smart ID for entities
+     */
+    private generateSmartId;
+    /**
+     * Assess entity quality
+     */
+    private assessEntityQuality;
+    /**
+     * Basic type detection fallback
+     */
+    private basicTypeDetection;
+    /**
+     * Check if relationship is critical for cascade analysis
+     */
+    private isRelationshipCritical;
+    /**
+     * Get verb specificity score
+     */
+    private getVerbSpecificity;
+    /**
+     * Get relevant fields for entity type
+     */
+    private getRelevantFields;
+    /**
+     * Get matched patterns for entity type
+     */
+    private getMatchedPatterns;
+    /**
+     * Detect hierarchies in relationships
+     */
+    private detectHierarchies;
+    /**
+     * Detect clusters in entities
+     */
+    private detectClusters;
+    /**
+     * Extract relationship metadata
+     */
+    private extractRelationshipMetadata;
+    /**
+     * Generate cache key
+     */
+    private generateCacheKey;
+    /**
+     * Clear analysis cache
+     */
+    clearCache(): void;
+    /**
+     * Get cache statistics
+     */
+    getCacheStats(): {
+        size: number;
+        keys: string[];
+    };
+}