@soulcraft/brainy 1.5.0 → 2.0.1

package/dist/augmentations/neuralImport.js

@@ -4,260 +4,110 @@
  * 🧠 Built-in AI augmentation for intelligent data processing
  * ⚛️ Always free, always included, always enabled
  *
- * This is the default AI-powered augmentation that comes with every Brainy installation.
- * It provides intelligent data understanding, entity detection, and relationship analysis.
+ * Now using the unified BrainyAugmentation interface!
  */
-import { NounType, VerbType } from '../types/graphTypes.js';
-import * as fs from '../universal/fs.js';
+import { BaseAugmentation } from './brainyAugmentation.js';
 import * as path from '../universal/path.js';
 /**
- * Neural Import SENSE Augmentation - The Brain's Perceptual System
+ * Neural Import Augmentation - Unified Implementation
+ * Processes data with AI before storage operations
  */
-export class NeuralImportAugmentation {
-    constructor(brainy, config = {}) {
+export class NeuralImportAugmentation extends BaseAugmentation {
+    constructor(config = {}) {
+        super();
         this.name = 'neural-import';
-        this.description = 'Built-in AI-powered data understanding and entity detection';
-        this.enabled = true;
-        this.brainy = brainy;
+        this.timing = 'before'; // Process data before storage
+        this.operations = ['add', 'addNoun', 'addVerb', 'all']; // Use 'all' to catch batch operations
+        this.priority = 80; // High priority for data processing
+        this.analysisCache = new Map();
         this.config = {
             confidenceThreshold: 0.7,
             enableWeights: true,
             skipDuplicates: true,
+            dataType: 'json',
             ...config
         };
     }
-    async initialize() {
-        // Initialize the cortex analysis system
-        console.log('🧠 Neural Import augmentation initialized');
+    async onInitialize() {
+        this.log('🧠 Neural Import augmentation initialized');
     }
-    async shutDown() {
-        console.log('🧠 Neural Import SENSE augmentation shut down');
-    }
-    async getStatus() {
-        return this.enabled ? 'active' : 'inactive';
+    async onShutdown() {
+        this.analysisCache.clear();
+        this.log('🧠 Neural Import augmentation shut down');
     }
     /**
-     * Process raw data into structured nouns and verbs using neural analysis
+     * Execute augmentation - process data with AI before storage
      */
-    async processRawData(rawData, dataType, options) {
+    async execute(operation, params, next) {
+        // Only process on add operations
+        if (!this.operations.includes(operation)) {
+            return next();
+        }
         try {
-            // Merge options with config
-            const mergedConfig = { ...this.config, ...options };
-            // Parse the raw data based on type
-            const parsedData = await this.parseRawData(rawData, dataType);
+            // Extract data from params based on operation
+            const rawData = this.extractRawData(operation, params);
+            if (!rawData) {
+                return next();
+            }
             // Perform neural analysis
-            const analysis = await this.performNeuralAnalysis(parsedData, mergedConfig);
-            // Extract nouns and verbs for the ISenseAugmentation interface
-            const nouns = analysis.detectedEntities.map(entity => entity.suggestedId);
-            const verbs = analysis.detectedRelationships.map(rel => `${rel.sourceId}->${rel.verbType}->${rel.targetId}`);
-            // Store the full analysis for later retrieval
+            const analysis = await this.performNeuralAnalysis(rawData, this.config);
+            // Enhance params with neural insights
+            if (params.metadata) {
+                params.metadata._neuralProcessed = true;
+                params.metadata._neuralConfidence = analysis.confidence;
+                params.metadata._detectedEntities = analysis.detectedEntities.length;
+                params.metadata._detectedRelationships = analysis.detectedRelationships.length;
+                params.metadata._neuralInsights = analysis.insights;
+            }
+            else if (typeof params === 'object') {
+                params.metadata = {
+                    _neuralProcessed: true,
+                    _neuralConfidence: analysis.confidence,
+                    _detectedEntities: analysis.detectedEntities.length,
+                    _detectedRelationships: analysis.detectedRelationships.length,
+                    _neuralInsights: analysis.insights
+                };
+            }
+            // Store neural analysis for later retrieval
             await this.storeNeuralAnalysis(analysis);
-            return {
-                success: true,
-                data: {
-                    nouns,
-                    verbs,
-                    confidence: analysis.confidence,
-                    insights: analysis.insights.map((insight) => ({
-                        type: insight.type,
-                        description: insight.description,
-                        confidence: insight.confidence
-                    })),
-                    metadata: {
-                        detectedEntities: analysis.detectedEntities.length,
-                        detectedRelationships: analysis.detectedRelationships.length,
-                        timestamp: new Date().toISOString(),
-                        augmentation: 'neural-import-sense'
-                    }
-                }
-            };
-        }
-        catch (error) {
-            return {
-                success: false,
-                data: { nouns: [], verbs: [] },
-                error: error instanceof Error ? error.message : 'Neural analysis failed'
-            };
-        }
-    }
-    /**
-     * Listen to real-time data feeds and process them
-     */
-    async listenToFeed(feedUrl, callback) {
-        // For file-based feeds, watch for changes
-        if (feedUrl.startsWith('file://')) {
-            const filePath = feedUrl.replace('file://', '');
-            // Watch file for changes using Node.js fs.watch
-            const fsWatch = require('fs');
-            const watcher = fsWatch.watch(filePath, async (eventType) => {
-                if (eventType === 'change') {
-                    try {
-                        const fileContent = await fs.readFile(filePath);
-                        const result = await this.processRawData(fileContent, this.getDataTypeFromPath(filePath));
-                        if (result.success) {
-                            callback({
-                                nouns: result.data.nouns,
-                                verbs: result.data.verbs,
-                                confidence: result.data.confidence
-                            });
-                        }
-                    }
-                    catch (error) {
-                        console.error('Neural Import feed error:', error);
-                    }
-                }
-            });
-            return;
-        }
-        // For other feed types, implement appropriate listeners
-        console.log(`🧠 Neural Import listening to feed: ${feedUrl}`);
-    }
-    /**
-     * Analyze data structure without processing (preview mode)
-     */
-    async analyzeStructure(rawData, dataType, options) {
-        try {
-            // Parse the raw data
-            const parsedData = await this.parseRawData(rawData, dataType);
-            // Perform lightweight analysis for structure detection
-            const analysis = await this.performNeuralAnalysis(parsedData, { ...this.config, ...options });
-            // Summarize entity types
-            const entityTypeCounts = new Map();
-            analysis.detectedEntities.forEach(entity => {
-                const existing = entityTypeCounts.get(entity.nounType) || { count: 0, totalConfidence: 0 };
-                entityTypeCounts.set(entity.nounType, {
-                    count: existing.count + 1,
-                    totalConfidence: existing.totalConfidence + entity.confidence
-                });
-            });
-            const entityTypes = Array.from(entityTypeCounts.entries()).map(([type, stats]) => ({
-                type,
-                count: stats.count,
-                confidence: stats.totalConfidence / stats.count
-            }));
-            // Summarize relationship types
-            const relationshipTypeCounts = new Map();
-            analysis.detectedRelationships.forEach(rel => {
-                const existing = relationshipTypeCounts.get(rel.verbType) || { count: 0, totalConfidence: 0 };
-                relationshipTypeCounts.set(rel.verbType, {
-                    count: existing.count + 1,
-                    totalConfidence: existing.totalConfidence + rel.confidence
-                });
-            });
-            const relationshipTypes = Array.from(relationshipTypeCounts.entries()).map(([type, stats]) => ({
-                type,
-                count: stats.count,
-                confidence: stats.totalConfidence / stats.count
-            }));
-            // Assess data quality
-            const dataQuality = this.assessDataQuality(parsedData, analysis);
-            // Generate recommendations
-            const recommendations = this.generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes);
-            return {
-                success: true,
-                data: {
-                    entityTypes,
-                    relationshipTypes,
-                    dataQuality,
-                    recommendations
-                }
-            };
+            // If we detected entities/relationships, potentially add them
+            if (this.context?.brain && analysis.detectedEntities.length > 0) {
+                // This could automatically create entities/relationships
+                // But for now, just enhance the metadata
+                this.log(`Detected ${analysis.detectedEntities.length} entities and ${analysis.detectedRelationships.length} relationships`);
+            }
+            // Continue with enhanced data
+            return next();
         }
         catch (error) {
-            return {
-                success: false,
-                data: {
-                    entityTypes: [],
-                    relationshipTypes: [],
-                    dataQuality: { completeness: 0, consistency: 0, accuracy: 0 },
-                    recommendations: []
-                },
-                error: error instanceof Error ? error.message : 'Structure analysis failed'
-            };
+            this.log(`Neural analysis failed: ${error}`, 'warn');
+            // Continue without neural processing
+            return next();
         }
     }
     /**
-     * Validate data compatibility with current knowledge base
+     * Extract raw data from operation params
      */
-    async validateCompatibility(rawData, dataType) {
-        try {
-            // Parse the raw data
-            const parsedData = await this.parseRawData(rawData, dataType);
-            // Perform neural analysis
-            const analysis = await this.performNeuralAnalysis(parsedData);
-            const issues = [];
-            const suggestions = [];
-            // Check for low confidence entities
-            const lowConfidenceEntities = analysis.detectedEntities.filter((e) => e.confidence < 0.5);
-            if (lowConfidenceEntities.length > 0) {
-                issues.push({
-                    type: 'confidence',
-                    description: `${lowConfidenceEntities.length} entities have low confidence scores`,
-                    severity: 'medium'
-                });
-                suggestions.push('Consider reviewing field names and data structure for better entity detection');
-            }
-            // Check for missing relationships
-            if (analysis.detectedRelationships.length === 0 && analysis.detectedEntities.length > 1) {
-                issues.push({
-                    type: 'relationships',
-                    description: 'No relationships detected between entities',
-                    severity: 'low'
-                });
-                suggestions.push('Consider adding contextual fields that describe entity relationships');
-            }
-            // Check for data type compatibility
-            const supportedTypes = ['json', 'csv', 'yaml', 'text'];
-            if (!supportedTypes.includes(dataType.toLowerCase())) {
-                issues.push({
-                    type: 'format',
-                    description: `Data type '${dataType}' may not be fully supported`,
-                    severity: 'high'
-                });
-                suggestions.push(`Convert data to one of: ${supportedTypes.join(', ')}`);
-            }
-            // Check for data completeness
-            const incompleteEntities = analysis.detectedEntities.filter((e) => !e.originalData || Object.keys(e.originalData).length < 2);
-            if (incompleteEntities.length > 0) {
-                issues.push({
-                    type: 'completeness',
-                    description: `${incompleteEntities.length} entities have insufficient data`,
-                    severity: 'medium'
-                });
-                suggestions.push('Ensure each entity has multiple descriptive fields');
-            }
-            const compatible = issues.filter(i => i.severity === 'high').length === 0;
-            return {
-                success: true,
-                data: {
-                    compatible,
-                    issues,
-                    suggestions
-                }
-            };
-        }
-        catch (error) {
-            return {
-                success: false,
-                data: {
-                    compatible: false,
-                    issues: [{
-                            type: 'error',
-                            description: error instanceof Error ? error.message : 'Validation failed',
-                            severity: 'high'
-                        }],
-                    suggestions: []
-                },
-                error: error instanceof Error ? error.message : 'Compatibility validation failed'
-            };
+    extractRawData(operation, params) {
+        switch (operation) {
+            case 'add':
+                return params.content || params.data || params;
+            case 'addNoun':
+                return params.noun || params.data || params;
+            case 'addVerb':
+                return params.verb || params;
+            case 'addBatch':
+                return params.items || params.batch || params;
+            default:
+                return null;
         }
     }
     /**
-     * Get the full neural analysis result (custom method for Cortex integration)
+     * Get the full neural analysis result (for external use)
      */
     async getNeuralAnalysis(rawData, dataType) {
-        const parsedData = await this.parseRawData(rawData, dataType);
-        return await this.performNeuralAnalysis(parsedData);
+        const parsedData = await this.parseRawData(rawData, dataType || this.config.dataType || 'json');
+        return await this.performNeuralAnalysis(parsedData, this.config);
     }
     /**
      * Parse raw data based on type
@@ -266,33 +116,45 @@ export class NeuralImportAugmentation {
         const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8');
         switch (dataType.toLowerCase()) {
             case 'json':
-                const jsonData = JSON.parse(content);
-                return Array.isArray(jsonData) ? jsonData : [jsonData];
+                try {
+                    const jsonData = JSON.parse(content);
+                    return Array.isArray(jsonData) ? jsonData : [jsonData];
+                }
+                catch {
+                    // If JSON parse fails, treat as text
+                    return [{ text: content }];
+                }
             case 'csv':
                 return this.parseCSV(content);
             case 'yaml':
             case 'yml':
                 // For now, basic YAML support - in full implementation would use yaml parser
-                return JSON.parse(content); // Placeholder
+                try {
+                    return JSON.parse(content); // Placeholder
+                }
+                catch {
+                    return [{ text: content }];
+                }
             case 'txt':
             case 'text':
                 // Split text into sentences/paragraphs for analysis
                 return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line }));
             default:
-                throw new Error(`Unsupported data type: ${dataType}`);
+                // Unknown type, treat as text
+                return [{ text: content }];
         }
     }
     /**
-     * Basic CSV parser
+     * Parse CSV data
      */
     parseCSV(content) {
         const lines = content.split('\n').filter(line => line.trim());
-        if (lines.length < 2)
+        if (lines.length === 0)
             return [];
-        const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, ''));
+        const headers = lines[0].split(',').map(h => h.trim());
         const data = [];
         for (let i = 1; i < lines.length; i++) {
-            const values = lines[i].split(',').map(v => v.trim().replace(/"/g, ''));
+            const values = lines[i].split(',').map(v => v.trim());
             const row = {};
             headers.forEach((header, index) => {
                 row[header] = values[index] || '';
@@ -304,447 +166,211 @@ export class NeuralImportAugmentation {
     /**
      * Perform neural analysis on parsed data
      */
-    async performNeuralAnalysis(parsedData, config = this.config) {
-        // Phase 1: Neural Entity Detection
-        const detectedEntities = await this.detectEntitiesWithNeuralAnalysis(parsedData, config);
-        // Phase 2: Neural Relationship Detection  
-        const detectedRelationships = await this.detectRelationshipsWithNeuralAnalysis(detectedEntities, parsedData, config);
-        // Phase 3: Neural Insights Generation
-        const insights = await this.generateNeuralInsights(detectedEntities, detectedRelationships);
-        // Phase 4: Confidence Scoring
-        const overallConfidence = this.calculateOverallConfidence(detectedEntities, detectedRelationships);
+    async performNeuralAnalysis(data, config) {
+        const detectedEntities = [];
+        const detectedRelationships = [];
+        const insights = [];
+        // Simple entity detection (in real implementation, would use ML)
+        for (const item of data) {
+            if (typeof item === 'object') {
+                // Detect entities from object properties
+                const entityId = item.id || item.name || item.title || `entity_${Date.now()}_${Math.random()}`;
+                detectedEntities.push({
+                    originalData: item,
+                    nounType: this.inferNounType(item),
+                    confidence: 0.85,
+                    suggestedId: String(entityId),
+                    reasoning: 'Detected from structured data',
+                    alternativeTypes: []
+                });
+                // Detect relationships from references
+                this.detectRelationships(item, entityId, detectedRelationships);
+            }
+        }
+        // Generate insights
+        if (detectedEntities.length > 10) {
+            insights.push({
+                type: 'pattern',
+                description: `Large dataset with ${detectedEntities.length} entities detected`,
+                confidence: 0.9,
+                affectedEntities: detectedEntities.slice(0, 5).map(e => e.suggestedId),
+                recommendation: 'Consider batch processing for optimal performance'
+            });
+        }
+        // Look for clusters
+        const typeGroups = this.groupByType(detectedEntities);
+        if (Object.keys(typeGroups).length > 1) {
+            insights.push({
+                type: 'cluster',
+                description: `Multiple entity types detected: ${Object.keys(typeGroups).join(', ')}`,
+                confidence: 0.8,
+                affectedEntities: [],
+                recommendation: 'Data contains diverse entity types suitable for graph analysis'
+            });
+        }
         return {
             detectedEntities,
             detectedRelationships,
-            confidence: overallConfidence,
+            confidence: detectedEntities.length > 0 ? 0.85 : 0.5,
             insights
         };
     }
     /**
-     * Neural Entity Detection - The Core AI Engine
-     */
-    async detectEntitiesWithNeuralAnalysis(rawData, config = this.config) {
-        const entities = [];
-        const nounTypes = Object.values(NounType);
-        for (const [index, dataItem] of rawData.entries()) {
-            const mainText = this.extractMainText(dataItem);
-            const detections = [];
-            // Test against all noun types using semantic similarity
-            for (const nounType of nounTypes) {
-                const confidence = await this.calculateEntityTypeConfidence(mainText, dataItem, nounType);
-                if (confidence >= config.confidenceThreshold - 0.2) { // Allow slightly lower for alternatives
-                    const reasoning = await this.generateEntityReasoning(mainText, dataItem, nounType);
-                    detections.push({ type: nounType, confidence, reasoning });
-                }
-            }
-            if (detections.length > 0) {
-                // Sort by confidence
-                detections.sort((a, b) => b.confidence - a.confidence);
-                const primaryType = detections[0];
-                const alternatives = detections.slice(1, 3); // Top 2 alternatives
-                entities.push({
-                    originalData: dataItem,
-                    nounType: primaryType.type,
-                    confidence: primaryType.confidence,
-                    suggestedId: this.generateSmartId(dataItem, primaryType.type, index),
-                    reasoning: primaryType.reasoning,
-                    alternativeTypes: alternatives
-                });
-            }
-        }
-        return entities;
-    }
-    /**
-     * Calculate entity type confidence using AI
-     */
-    async calculateEntityTypeConfidence(text, data, nounType) {
-        // Base semantic similarity using search
-        const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
-        const textSimilarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
-        // Field-based confidence boost
-        const fieldBoost = this.calculateFieldBasedConfidence(data, nounType);
-        // Pattern-based confidence boost  
-        const patternBoost = this.calculatePatternBasedConfidence(text, data, nounType);
-        // Combine confidences with weights
-        const combined = (textSimilarity * 0.5) + (fieldBoost * 0.3) + (patternBoost * 0.2);
-        return Math.min(combined, 1.0);
-    }
-    /**
-     * Field-based confidence calculation
+     * Infer noun type from object structure
      */
-    calculateFieldBasedConfidence(data, nounType) {
-        const fields = Object.keys(data);
-        let boost = 0;
-        // Field patterns that boost confidence for specific noun types
-        const fieldPatterns = {
-            [NounType.Person]: ['name', 'email', 'phone', 'age', 'firstname', 'lastname', 'employee'],
-            [NounType.Organization]: ['company', 'organization', 'corp', 'inc', 'ltd', 'department', 'team'],
-            [NounType.Project]: ['project', 'task', 'deadline', 'status', 'milestone', 'deliverable'],
-            [NounType.Location]: ['address', 'city', 'country', 'state', 'zip', 'location', 'coordinates'],
-            [NounType.Product]: ['product', 'price', 'sku', 'inventory', 'category', 'brand'],
-            [NounType.Event]: ['date', 'time', 'venue', 'event', 'meeting', 'conference', 'schedule']
-        };
-        const relevantPatterns = fieldPatterns[nounType] || [];
-        for (const field of fields) {
-            for (const pattern of relevantPatterns) {
-                if (field.toLowerCase().includes(pattern)) {
-                    boost += 0.1;
-                }
-            }
-        }
-        return Math.min(boost, 0.5);
+    inferNounType(obj) {
+        // Simple heuristics for type detection
+        if (obj.email || obj.username)
+            return 'Person';
+        if (obj.title && obj.content)
+            return 'Document';
+        if (obj.price || obj.product)
+            return 'Product';
+        if (obj.date || obj.timestamp)
+            return 'Event';
+        if (obj.url || obj.link)
+            return 'Resource';
+        if (obj.lat || obj.longitude)
+            return 'Location';
+        // Default fallback
+        return 'Entity';
     }
     /**
-     * Pattern-based confidence calculation
+     * Detect relationships from object references
      */
-    calculatePatternBasedConfidence(text, data, nounType) {
-        let boost = 0;
-        // Content patterns that indicate entity types
-        const patterns = {
-            [NounType.Person]: [
-                /@.*\.com/i, // Email pattern
-                /\b[A-Z][a-z]+ [A-Z][a-z]+\b/, // Name pattern
-                /Mr\.|Mrs\.|Dr\.|Prof\./i // Title pattern
-            ],
-            [NounType.Organization]: [
-                /\bInc\.|Corp\.|LLC\.|Ltd\./i, // Corporate suffixes
-                /Company|Corporation|Enterprise/i
-            ],
-            [NounType.Location]: [
-                /\b\d{5}(-\d{4})?\b/, // ZIP code
-                /Street|Ave|Road|Blvd/i
-            ]
-        };
-        const relevantPatterns = patterns[nounType] || [];
-        for (const pattern of relevantPatterns) {
-            if (pattern.test(text)) {
-                boost += 0.15;
+    detectRelationships(obj, sourceId, relationships) {
+        // Look for reference patterns
+        for (const [key, value] of Object.entries(obj)) {
+            if (key.endsWith('Id') || key.endsWith('_id') || key === 'parentId' || key === 'userId') {
+                relationships.push({
+                    sourceId,
+                    targetId: String(value),
+                    verbType: this.inferVerbType(key),
+                    confidence: 0.75,
+                    weight: 1,
+                    reasoning: `Reference detected in field: ${key}`,
+                    context: key
+                });
             }
-        }
-        return Math.min(boost, 0.3);
-    }
-    /**
-     * Generate reasoning for entity type selection
-     */
-    async generateEntityReasoning(text, data, nounType) {
-        const reasons = [];
-        // Semantic similarity reason
-        const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
-        const similarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
-        if (similarity > 0.7) {
-            reasons.push(`High semantic similarity (${(similarity * 100).toFixed(1)}%)`);
-        }
-        // Field-based reasons
-        const relevantFields = this.getRelevantFields(data, nounType);
-        if (relevantFields.length > 0) {
-            reasons.push(`Contains ${nounType}-specific fields: ${relevantFields.join(', ')}`);
-        }
-        // Pattern-based reasons
-        const matchedPatterns = this.getMatchedPatterns(text, data, nounType);
-        if (matchedPatterns.length > 0) {
-            reasons.push(`Matches ${nounType} patterns: ${matchedPatterns.join(', ')}`);
-        }
-        return reasons.length > 0 ? reasons.join('; ') : 'General semantic match';
-    }
-    /**
-     * Neural Relationship Detection
-     */
-    async detectRelationshipsWithNeuralAnalysis(entities, rawData, config = this.config) {
-        const relationships = [];
-        const verbTypes = Object.values(VerbType);
-        // For each pair of entities, test relationship possibilities
-        for (let i = 0; i < entities.length; i++) {
-            for (let j = i + 1; j < entities.length; j++) {
-                const sourceEntity = entities[i];
-                const targetEntity = entities[j];
-                // Extract context for relationship detection
-                const context = this.extractRelationshipContext(sourceEntity.originalData, targetEntity.originalData, rawData);
-                // Test all verb types
-                for (const verbType of verbTypes) {
-                    const confidence = await this.calculateRelationshipConfidence(sourceEntity, targetEntity, verbType, context);
-                    if (confidence >= config.confidenceThreshold - 0.1) { // Slightly lower threshold for relationships
-                        const weight = config.enableWeights ?
-                            this.calculateRelationshipWeight(sourceEntity, targetEntity, verbType, context) :
-                            0.5;
-                        const reasoning = await this.generateRelationshipReasoning(sourceEntity, targetEntity, verbType, context);
+            // Array of IDs
+            if (Array.isArray(value) && value.length > 0 && typeof value[0] === 'string') {
+                if (key.endsWith('Ids') || key.endsWith('_ids')) {
+                    for (const targetId of value) {
                         relationships.push({
-                            sourceId: sourceEntity.suggestedId,
-                            targetId: targetEntity.suggestedId,
-                            verbType,
-                            confidence,
-                            weight,
-                            reasoning,
-                            context,
-                            metadata: this.extractRelationshipMetadata(sourceEntity.originalData, targetEntity.originalData, verbType)
+                            sourceId,
+                            targetId: String(targetId),
+                            verbType: this.inferVerbType(key),
+                            confidence: 0.7,
+                            weight: 1,
+                            reasoning: `Array reference in field: ${key}`,
+                            context: key
                         });
                     }
                 }
             }
         }
-        // Sort by confidence and remove duplicates/conflicts
-        return this.pruneRelationships(relationships);
-    }
-    /**
-     * Calculate relationship confidence
-     */
-    async calculateRelationshipConfidence(source, target, verbType, context) {
-        // Semantic similarity between entities and verb type
-        const relationshipText = `${this.extractMainText(source.originalData)} ${verbType} ${this.extractMainText(target.originalData)}`;
-        const directResults = await this.brainy.search(relationshipText, 1);
-        const directSimilarity = directResults.length > 0 ? directResults[0].score : 0.5;
-        // Context-based similarity
-        const contextResults = await this.brainy.search(context + ' ' + verbType, 1);
-        const contextSimilarity = contextResults.length > 0 ? contextResults[0].score : 0.5;
-        // Entity type compatibility
-        const typeCompatibility = this.calculateTypeCompatibility(source.nounType, target.nounType, verbType);
-        // Combine with weights
-        return (directSimilarity * 0.4) + (contextSimilarity * 0.4) + (typeCompatibility * 0.2);
-    }
-    /**
-     * Calculate relationship weight/strength
-     */
-    calculateRelationshipWeight(source, target, verbType, context) {
-        let weight = 0.5; // Base weight
-        // Context richness (more descriptive = stronger)
-        const contextWords = context.split(' ').length;
-        weight += Math.min(contextWords / 20, 0.2);
-        // Entity importance (higher confidence entities = stronger relationships)
-        const avgEntityConfidence = (source.confidence + target.confidence) / 2;
-        weight += avgEntityConfidence * 0.2;
-        // Verb type specificity (more specific verbs = stronger)
-        const verbSpecificity = this.getVerbSpecificity(verbType);
-        weight += verbSpecificity * 0.1;
-        return Math.min(weight, 1.0);
     }
     /**
-     * Generate Neural Insights - The Intelligence Layer
+     * Infer verb type from field name
      */
-    async generateNeuralInsights(entities, relationships) {
-        const insights = [];
-        // Detect hierarchies
-        const hierarchies = this.detectHierarchies(relationships);
-        hierarchies.forEach(hierarchy => {
-            insights.push({
-                type: 'hierarchy',
-                description: `Detected ${hierarchy.type} hierarchy with ${hierarchy.levels} levels`,
-                confidence: hierarchy.confidence,
-                affectedEntities: hierarchy.entities,
-                recommendation: `Consider visualizing the ${hierarchy.type} structure`
-            });
-        });
-        // Detect clusters  
-        const clusters = this.detectClusters(entities, relationships);
-        clusters.forEach(cluster => {
-            insights.push({
-                type: 'cluster',
-                description: `Found cluster of ${cluster.size} ${cluster.primaryType} entities`,
-                confidence: cluster.confidence,
-                affectedEntities: cluster.entities,
-                recommendation: `These ${cluster.primaryType}s might form a natural grouping`
-            });
-        });
-        // Detect patterns
-        const patterns = this.detectPatterns(relationships);
-        patterns.forEach(pattern => {
-            insights.push({
-                type: 'pattern',
-                description: `Common relationship pattern: ${pattern.description}`,
-                confidence: pattern.confidence,
-                affectedEntities: pattern.entities,
-                recommendation: pattern.recommendation
-            });
-        });
-        return insights;
+    inferVerbType(fieldName) {
+        const normalized = fieldName.toLowerCase();
+        if (normalized.includes('parent'))
+            return 'childOf';
+        if (normalized.includes('user'))
+            return 'belongsTo';
+        if (normalized.includes('author'))
+            return 'authoredBy';
+        if (normalized.includes('owner'))
+            return 'ownedBy';
+        if (normalized.includes('creator'))
+            return 'createdBy';
+        if (normalized.includes('member'))
+            return 'memberOf';
+        if (normalized.includes('tag'))
+            return 'taggedWith';
+        if (normalized.includes('category'))
+            return 'categorizedAs';
+        return 'relatedTo';
     }
     /**
-     * Helper methods for the neural system
+     * Group entities by type
      */
-    extractMainText(data) {
-        // Extract the most relevant text from a data object
-        const textFields = ['name', 'title', 'description', 'content', 'text', 'label'];
-        for (const field of textFields) {
-            if (data[field] && typeof data[field] === 'string') {
-                return data[field];
+    groupByType(entities) {
+        const groups = {};
+        for (const entity of entities) {
+            if (!groups[entity.nounType]) {
+                groups[entity.nounType] = [];
             }
+            groups[entity.nounType].push(entity);
         }
-        // Fallback: concatenate all string values
-        return Object.values(data)
-            .filter(v => typeof v === 'string')
-            .join(' ')
-            .substring(0, 200); // Limit length
-    }
-    generateSmartId(data, nounType, index) {
-        const mainText = this.extractMainText(data);
-        const cleanText = mainText.toLowerCase().replace(/[^a-z0-9]/g, '_').substring(0, 20);
-        return `${nounType}_${cleanText}_${index}`;
+        return groups;
     }
-    extractRelationshipContext(source, target, allData) {
-        // Extract context for relationship detection
-        return [
-            this.extractMainText(source),
-            this.extractMainText(target),
-            // Add more contextual information
-        ].join(' ');
-    }
-    calculateTypeCompatibility(sourceType, targetType, verbType) {
-        // Define type compatibility matrix for relationships
-        const compatibilityMatrix = {
-            [NounType.Person]: {
-                [NounType.Organization]: [VerbType.MemberOf, VerbType.WorksWith],
-                [NounType.Project]: [VerbType.WorksWith, VerbType.Creates],
-                [NounType.Person]: [VerbType.WorksWith, VerbType.Mentors, VerbType.ReportsTo]
+    /**
+     * Store neural analysis results
+     */
+    async storeNeuralAnalysis(analysis) {
+        // Cache the analysis for potential later use
+        const key = `analysis_${Date.now()}`;
+        this.analysisCache.set(key, analysis);
+        // Limit cache size
+        if (this.analysisCache.size > 100) {
+            const firstKey = this.analysisCache.keys().next().value;
+            if (firstKey) {
+                this.analysisCache.delete(firstKey);
             }
-            // Add more compatibility rules
-        };
-        const sourceCompatibility = compatibilityMatrix[sourceType];
-        if (sourceCompatibility && sourceCompatibility[targetType]) {
-            return sourceCompatibility[targetType].includes(verbType) ? 1.0 : 0.3;
         }
-        return 0.5; // Default compatibility
-    }
-    getVerbSpecificity(verbType) {
-        // More specific verbs get higher scores
-        const specificityScores = {
-            [VerbType.RelatedTo]: 0.1, // Very generic
-            [VerbType.WorksWith]: 0.7, // Specific
-            [VerbType.Mentors]: 0.9, // Very specific
-            [VerbType.ReportsTo]: 0.9, // Very specific
-            [VerbType.Supervises]: 0.9 // Very specific
-        };
-        return specificityScores[verbType] || 0.5;
-    }
-    getRelevantFields(data, nounType) {
-        // Implementation for finding relevant fields
-        return [];
-    }
-    getMatchedPatterns(text, data, nounType) {
-        // Implementation for finding matched patterns
-        return [];
-    }
-    pruneRelationships(relationships) {
-        // Remove duplicates and low-confidence relationships
-        return relationships
-            .sort((a, b) => b.confidence - a.confidence)
-            .slice(0, 1000); // Limit to top 1000 relationships
-    }
-    detectHierarchies(relationships) {
-        // Detect hierarchical structures
-        return [];
-    }
-    detectClusters(entities, relationships) {
-        // Detect entity clusters
-        return [];
-    }
-    detectPatterns(relationships) {
-        // Detect relationship patterns
-        return [];
-    }
-    calculateOverallConfidence(entities, relationships) {
-        if (entities.length === 0)
-            return 0;
-        const entityConfidence = entities.reduce((sum, e) => sum + e.confidence, 0) / entities.length;
-        if (relationships.length === 0)
-            return entityConfidence;
-        const relationshipConfidence = relationships.reduce((sum, r) => sum + r.confidence, 0) / relationships.length;
-        return (entityConfidence + relationshipConfidence) / 2;
-    }
-    async storeNeuralAnalysis(analysis) {
-        // Store the full analysis result for later retrieval by Neural Import or other systems
-        // This could be stored in the brainy instance metadata or a separate analysis store
     }
+    /**
+     * Helper to get data type from file path
+     */
     getDataTypeFromPath(filePath) {
         const ext = path.extname(filePath).toLowerCase();
         switch (ext) {
             case '.json': return 'json';
             case '.csv': return 'csv';
+            case '.txt': return 'text';
             case '.yaml':
             case '.yml': return 'yaml';
-            case '.txt': return 'text';
             default: return 'text';
         }
     }
-    async generateRelationshipReasoning(source, target, verbType, context) {
-        return `Neural analysis detected ${verbType} relationship based on semantic context`;
-    }
-    extractRelationshipMetadata(sourceData, targetData, verbType) {
-        return {
-            sourceType: typeof sourceData,
-            targetType: typeof targetData,
-            detectedBy: 'neural-import-sense',
-            timestamp: new Date().toISOString()
-        };
-    }
-    /**
-     * Assess data quality metrics
-     */
-    assessDataQuality(parsedData, analysis) {
-        // Completeness: ratio of fields with data
-        let totalFields = 0;
-        let filledFields = 0;
-        parsedData.forEach(item => {
-            const fields = Object.keys(item);
-            totalFields += fields.length;
-            filledFields += fields.filter(field => item[field] !== null &&
-                item[field] !== undefined &&
-                item[field] !== '').length;
-        });
-        const completeness = totalFields > 0 ? filledFields / totalFields : 0;
-        // Consistency: variance in field structure
-        const fieldSets = parsedData.map(item => new Set(Object.keys(item)));
-        const allFields = new Set(fieldSets.flatMap(set => Array.from(set)));
-        let consistencyScore = 0;
-        if (fieldSets.length > 0) {
-            consistencyScore = Array.from(allFields).reduce((score, field) => {
-                const hasField = fieldSets.filter(set => set.has(field)).length;
-                return score + (hasField / fieldSets.length);
-            }, 0) / allFields.size;
-        }
-        // Accuracy: average confidence of detected entities
-        const accuracy = analysis.detectedEntities.length > 0 ?
-            analysis.detectedEntities.reduce((sum, e) => sum + e.confidence, 0) / analysis.detectedEntities.length :
-            0;
-        return {
-            completeness,
-            consistency: consistencyScore,
-            accuracy
-        };
-    }
     /**
-     * Generate recommendations based on analysis
+     * PUBLIC API: Process raw data (for external use, like Synapses)
+     * This maintains compatibility with code that wants to use Neural Import directly
      */
-    generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes) {
-        const recommendations = [];
-        // Low entity confidence recommendations
-        const lowConfidenceEntities = entityTypes.filter(et => et.confidence < 0.7);
-        if (lowConfidenceEntities.length > 0) {
-            recommendations.push(`Consider improving field names for ${lowConfidenceEntities.map(e => e.type).join(', ')} entities`);
-        }
-        // Missing relationships recommendations
-        if (relationshipTypes.length === 0 && entityTypes.length > 1) {
-            recommendations.push('Add fields that describe how entities relate to each other');
-        }
-        // Data structure recommendations
-        if (parsedData.length > 0) {
-            const firstItem = parsedData[0];
-            const fieldCount = Object.keys(firstItem).length;
-            if (fieldCount < 3) {
-                recommendations.push('Consider adding more descriptive fields to each entity');
-            }
-            if (fieldCount > 20) {
-                recommendations.push('Consider grouping related fields or splitting complex entities');
-            }
-        }
-        // Entity distribution recommendations
-        const dominantEntityType = entityTypes.reduce((max, current) => current.count > max.count ? current : max, entityTypes[0] || { count: 0 });
-        if (dominantEntityType && dominantEntityType.count > parsedData.length * 0.8) {
-            recommendations.push(`Consider diversifying entity types - ${dominantEntityType.type} dominates the dataset`);
+    async processRawData(rawData, dataType, options) {
+        try {
+            const analysis = await this.getNeuralAnalysis(rawData, dataType);
+            // Convert to legacy format for compatibility
+            const nouns = analysis.detectedEntities.map(e => e.suggestedId);
+            const verbs = analysis.detectedRelationships.map(r => `${r.sourceId}->${r.verbType}->${r.targetId}`);
+            return {
+                success: true,
+                data: {
+                    nouns,
+                    verbs,
+                    confidence: analysis.confidence,
+                    insights: analysis.insights.map(i => ({
+                        type: i.type,
+                        description: i.description,
+                        confidence: i.confidence
+                    })),
+                    metadata: {
+                        detectedEntities: analysis.detectedEntities.length,
+                        detectedRelationships: analysis.detectedRelationships.length,
+                        timestamp: new Date().toISOString()
+                    }
+                }
+            };
         }
-        // Relationship quality recommendations
-        const lowWeightRelationships = relationshipTypes.filter(rt => rt.confidence < 0.6);
-        if (lowWeightRelationships.length > 0) {
-            recommendations.push('Consider adding more contextual information to strengthen relationship detection');
+        catch (error) {
+            return {
+                success: false,
+                data: { nouns: [], verbs: [] },
+                error: error instanceof Error ? error.message : 'Neural analysis failed'
+            };
         }
-        return recommendations;
     }
 }
 //# sourceMappingURL=neuralImport.js.map