@soulcraft/brainy 3.0.0 → 3.1.0

package/dist/neural/naturalLanguageProcessor.js

@@ -9,29 +9,369 @@
  * - Progressive learning from usage
  */
 import { PatternLibrary } from './patternLibrary.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
 export class NaturalLanguageProcessor {
     constructor(brain) {
         this.initialized = false;
+        this.embeddingCache = new Map();
+        // Field discovery with semantic matching
+        this.fieldEmbeddings = new Map();
+        this.fieldNames = [];
+        this.lastFieldRefresh = 0;
+        this.FIELD_REFRESH_INTERVAL = 60000; // Refresh every minute
+        // Type embeddings for NounType and VerbType matching
+        this.nounTypeEmbeddings = new Map();
+        this.verbTypeEmbeddings = new Map();
+        this.typeEmbeddingsInitialized = false;
         this.brain = brain;
         this.patternLibrary = new PatternLibrary(brain);
         this.queryHistory = [];
     }
+    /**
+     * Get embedding directly using brain's embed method
+     */
+    async getEmbedding(text) {
+        // Check cache first
+        if (this.embeddingCache.has(text)) {
+            return this.embeddingCache.get(text);
+        }
+        // Use brain's embed method directly to avoid recursion
+        const embedding = await this.brain.embed(text);
+        // Cache the embedding
+        this.embeddingCache.set(text, embedding);
+        return embedding;
+    }
     /**
      * Initialize the pattern library (lazy loading)
      */
     async ensureInitialized() {
         if (!this.initialized) {
             await this.patternLibrary.init();
+            await this.initializeTypeEmbeddings(); // Embed all noun/verb types
+            await this.refreshFieldEmbeddings(); // Load field embeddings
             this.initialized = true;
         }
     }
+    /**
+     * Initialize embeddings for all NounTypes and VerbTypes
+     * These are fixed types that never change - perfect for caching
+     */
+    async initializeTypeEmbeddings() {
+        if (this.typeEmbeddingsInitialized)
+            return;
+        // Embed all NounTypes (30+ types)
+        for (const [key, value] of Object.entries(NounType)) {
+            if (typeof value === 'string') {
+                // Embed both the key (Person) and value (person)
+                const keyEmbedding = await this.getEmbedding(key);
+                const valueEmbedding = await this.getEmbedding(value);
+                this.nounTypeEmbeddings.set(key, keyEmbedding);
+                this.nounTypeEmbeddings.set(value, valueEmbedding);
+                // Also embed common variations
+                const spaceSeparated = key.replace(/([A-Z])/g, ' $1').trim().toLowerCase();
+                if (spaceSeparated !== value) {
+                    const variantEmbedding = await this.getEmbedding(spaceSeparated);
+                    this.nounTypeEmbeddings.set(spaceSeparated, variantEmbedding);
+                }
+            }
+        }
+        // Embed all VerbTypes (40+ types)
+        for (const [key, value] of Object.entries(VerbType)) {
+            if (typeof value === 'string') {
+                const keyEmbedding = await this.getEmbedding(key);
+                const valueEmbedding = await this.getEmbedding(value);
+                this.verbTypeEmbeddings.set(key, keyEmbedding);
+                this.verbTypeEmbeddings.set(value, valueEmbedding);
+                // Common variations for verbs
+                const spaceSeparated = key.replace(/([A-Z])/g, ' $1').trim().toLowerCase();
+                if (spaceSeparated !== value) {
+                    const variantEmbedding = await this.getEmbedding(spaceSeparated);
+                    this.verbTypeEmbeddings.set(spaceSeparated, variantEmbedding);
+                }
+            }
+        }
+        this.typeEmbeddingsInitialized = true;
+    }
+    /**
+     * Find best matching NounType using semantic similarity
+     */
+    async findBestNounType(term) {
+        const termEmbedding = await this.getEmbedding(term);
+        let bestMatch = null;
+        let bestScore = 0;
+        for (const [typeName, typeEmbedding] of this.nounTypeEmbeddings) {
+            const similarity = this.cosineSimilarity(termEmbedding, typeEmbedding);
+            if (similarity > bestScore && similarity > 0.75) { // Higher threshold for types
+                bestScore = similarity;
+                bestMatch = typeName;
+            }
+        }
+        // Map back to the actual NounType value
+        if (bestMatch) {
+            for (const [key, value] of Object.entries(NounType)) {
+                if (key === bestMatch || value === bestMatch ||
+                    key.toLowerCase() === bestMatch.toLowerCase()) {
+                    return { type: value, confidence: bestScore };
+                }
+            }
+        }
+        return { type: null, confidence: 0 };
+    }
+    /**
+     * Find best matching VerbType using semantic similarity
+     */
+    async findBestVerbType(term) {
+        const termEmbedding = await this.getEmbedding(term);
+        let bestMatch = null;
+        let bestScore = 0;
+        for (const [typeName, typeEmbedding] of this.verbTypeEmbeddings) {
+            const similarity = this.cosineSimilarity(termEmbedding, typeEmbedding);
+            if (similarity > bestScore && similarity > 0.75) {
+                bestScore = similarity;
+                bestMatch = typeName;
+            }
+        }
+        // Map back to the actual VerbType value
+        if (bestMatch) {
+            for (const [key, value] of Object.entries(VerbType)) {
+                if (key === bestMatch || value === bestMatch ||
+                    key.toLowerCase() === bestMatch.toLowerCase()) {
+                    return { type: value, confidence: bestScore };
+                }
+            }
+        }
+        return { type: null, confidence: 0 };
+    }
+    /**
+     * Refresh field embeddings from metadata index
+     * Creates embeddings for all indexed fields for semantic matching
+     */
+    async refreshFieldEmbeddings() {
+        const now = Date.now();
+        if (now - this.lastFieldRefresh < this.FIELD_REFRESH_INTERVAL) {
+            return; // Skip if recently refreshed
+        }
+        try {
+            // Get actual indexed fields from metadata
+            this.fieldNames = await this.brain.getAvailableFields();
+            // Create embeddings for each field name for semantic matching
+            for (const field of this.fieldNames) {
+                if (!this.fieldEmbeddings.has(field)) {
+                    // Embed the field name itself
+                    const fieldEmbedding = await this.getEmbedding(field);
+                    this.fieldEmbeddings.set(field, fieldEmbedding);
+                    // Also embed common variations
+                    const variations = this.getFieldVariations(field);
+                    for (const variant of variations) {
+                        const variantEmbedding = await this.getEmbedding(variant);
+                        this.fieldEmbeddings.set(variant, variantEmbedding);
+                    }
+                }
+            }
+            this.lastFieldRefresh = now;
+        }
+        catch (error) {
+            console.warn('Failed to refresh field embeddings:', error);
+        }
+    }
+    /**
+     * Generate linguistic variations of field names - NO HARDCODED TERMS
+     * Uses algorithmic patterns to create natural variations
+     */
+    getFieldVariations(field) {
+        const variations = [];
+        // camelCase to space separated: publishDate -> publish date
+        const spaceSeparated = field.replace(/([A-Z])/g, ' $1').toLowerCase().trim();
+        if (spaceSeparated !== field.toLowerCase()) {
+            variations.push(spaceSeparated);
+        }
+        // snake_case to space separated: created_at -> created at
+        const underscoreRemoved = field.replace(/_/g, ' ').toLowerCase();
+        if (underscoreRemoved !== field.toLowerCase()) {
+            variations.push(underscoreRemoved);
+        }
+        // kebab-case to space separated: publish-date -> publish date
+        const dashRemoved = field.replace(/-/g, ' ').toLowerCase();
+        if (dashRemoved !== field.toLowerCase()) {
+            variations.push(dashRemoved);
+        }
+        // Generate suffix variations (remove common suffixes)
+        const suffixes = ['At', 'Date', 'Time', 'Id', 'Ref', 'Name', 'Value', 'Count', 'Number'];
+        for (const suffix of suffixes) {
+            if (field.endsWith(suffix) && field.length > suffix.length) {
+                const withoutSuffix = field.slice(0, -suffix.length).toLowerCase();
+                variations.push(withoutSuffix);
+            }
+        }
+        // Generate prefix variations (remove common prefixes)
+        const prefixes = ['is', 'has', 'can', 'get', 'set'];
+        for (const prefix of prefixes) {
+            if (field.toLowerCase().startsWith(prefix) && field.length > prefix.length) {
+                const withoutPrefix = field.slice(prefix.length).toLowerCase();
+                variations.push(withoutPrefix);
+            }
+        }
+        return [...new Set(variations)]; // Remove duplicates
+    }
+    /**
+     * Find best matching field using semantic similarity
+     * Returns field name and confidence score
+     */
+    async findBestMatchingField(term) {
+        // Ensure fields are loaded
+        await this.refreshFieldEmbeddings();
+        if (this.fieldNames.length === 0) {
+            return { field: null, confidence: 0 };
+        }
+        // Get embedding for the search term
+        const termEmbedding = await this.getEmbedding(term);
+        // Find most similar field using cosine similarity
+        let bestMatch = null;
+        let bestScore = 0;
+        for (const [fieldName, fieldEmbedding] of this.fieldEmbeddings) {
+            const similarity = this.cosineSimilarity(termEmbedding, fieldEmbedding);
+            if (similarity > bestScore && similarity > 0.7) { // 0.7 threshold for semantic match
+                bestScore = similarity;
+                bestMatch = fieldName;
+            }
+        }
+        // Map back to actual field name if it was a variation
+        if (bestMatch && !this.fieldNames.includes(bestMatch)) {
+            // Find the original field this variation belongs to
+            for (const field of this.fieldNames) {
+                const variations = this.getFieldVariations(field);
+                if (variations.includes(bestMatch)) {
+                    bestMatch = field;
+                    break;
+                }
+            }
+        }
+        return { field: bestMatch, confidence: bestScore };
+    }
+    /**
+     * Find best matching field with type context prioritization
+     * Fields with high type affinity get boosted scores
+     */
+    async findBestMatchingFieldWithTypeContext(term, typeSpecificFields) {
+        // First do normal field matching
+        const normalMatch = await this.findBestMatchingField(term);
+        if (!normalMatch.field || typeSpecificFields.length === 0) {
+            return normalMatch;
+        }
+        // Check if the matched field has type affinity
+        const typeField = typeSpecificFields.find(tf => tf.field === normalMatch.field);
+        if (typeField) {
+            // Boost confidence based on type affinity
+            // High affinity (0.8+) gets 20% boost, medium affinity (0.5+) gets 10% boost
+            let boost = 0;
+            if (typeField.affinity >= 0.8) {
+                boost = 0.2;
+            }
+            else if (typeField.affinity >= 0.5) {
+                boost = 0.1;
+            }
+            const boostedConfidence = Math.min(1.0, normalMatch.confidence + boost);
+            return { field: normalMatch.field, confidence: boostedConfidence };
+        }
+        return normalMatch;
+    }
+    /**
+     * Validate field-type compatibility
+     * Returns validation result with suggestions for invalid combinations
+     */
+    async validateFieldForType(field, nounType) {
+        // Get fields that actually appear with this type
+        const typeFields = await this.brain.getFieldsForType(nounType);
+        // Check if this field appears with this type
+        const fieldInfo = typeFields.find(tf => tf.field === field);
+        if (fieldInfo) {
+            return {
+                isValid: true,
+                affinity: fieldInfo.affinity
+            };
+        }
+        // Field doesn't appear with this type - provide suggestions
+        const suggestions = typeFields
+            .filter(tf => tf.affinity > 0.1) // Only suggest common fields
+            .slice(0, 3) // Top 3 suggestions
+            .map(tf => tf.field);
+        return {
+            isValid: false,
+            affinity: 0,
+            suggestions,
+            reason: `Field '${field}' rarely appears with ${nounType} entities. Common fields: ${suggestions.join(', ')}`
+        };
+    }
+    /**
+     * Enhanced intelligent parse with validation
+     */
+    async validateAndOptimizeQuery(tripleQuery, detectedNounType, fieldMatches) {
+        if (!detectedNounType || !tripleQuery.where) {
+            return tripleQuery;
+        }
+        const validationErrors = [];
+        const optimizedWhere = {};
+        // Validate each field in the where clause
+        for (const [field, value] of Object.entries(tripleQuery.where)) {
+            if (field === 'noun') {
+                optimizedWhere[field] = value; // Always valid
+                continue;
+            }
+            const validation = await this.validateFieldForType(field, detectedNounType);
+            if (validation.isValid || validation.affinity > 0.05) {
+                // Valid or has some affinity - include in query
+                optimizedWhere[field] = value;
+            }
+            else {
+                // Invalid field for this type
+                validationErrors.push(validation.reason || `Invalid field: ${field}`);
+                // Try to find a better field match from suggestions
+                if (validation.suggestions && validation.suggestions.length > 0) {
+                    const bestSuggestion = validation.suggestions[0];
+                    optimizedWhere[bestSuggestion] = value;
+                }
+            }
+        }
+        // Log validation errors for debugging (in production, could throw or return errors)
+        if (validationErrors.length > 0) {
+            console.warn('Field validation warnings:', validationErrors);
+        }
+        return {
+            ...tripleQuery,
+            where: optimizedWhere
+        };
+    }
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length)
+            return 0;
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        if (normA === 0 || normB === 0)
+            return 0;
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+    /**
+     * Public initialization method for external callers
+     */
+    async init() {
+        await this.ensureInitialized();
+    }
     /**
      * 🎯 MAIN METHOD: Convert natural language to Triple Intelligence query
      */
     async processNaturalQuery(naturalQuery) {
         await this.ensureInitialized();
-        // Step 1: Embed the query for semantic matching
-        const queryEmbedding = await this.brain.embed(naturalQuery);
+        // Step 1: Get embedding via add/get/delete pattern
+        const queryEmbedding = await this.getEmbedding(naturalQuery);
         // Step 2: Find best matching patterns from our library
         const matches = await this.patternLibrary.findBestPatterns(queryEmbedding, 3);
         // Step 3: Try each pattern until we get a good match
@@ -54,63 +394,198 @@ export class NaturalLanguageProcessor {
                 return query;
             }
         }
-        // Step 4: Fall back to hybrid approach if no pattern matches well
-        return this.hybridParse(naturalQuery, queryEmbedding);
+        // Step 4: Use intelligent field-aware parsing instead of fallback
+        return this.intelligentParse(naturalQuery, queryEmbedding);
     }
     /**
-     * Hybrid parse when pattern matching fails
+     * Intelligent parse using type-aware field discovery and semantic matching
+     * NO FALLBACKS - uses actual indexed fields, entities, and type context
      */
-    async hybridParse(query, queryEmbedding) {
-        // Analyze intent using embeddings and keywords
+    async intelligentParse(query, queryEmbedding) {
+        // Step 1: Analyze intent and extract structure
         const intent = await this.analyzeIntent(query);
-        // Find similar successful queries from history
-        // TODO: Implement findSimilarQueries method
-        // const similar = await this.findSimilarQueries(queryEmbedding)
-        // if (similar.length > 0 && similar[0].similarity > 0.9) {
-        //   // Adapt a very similar previous query
-        //   return this.adaptQuery(query, similar[0].result)
-        // }
-        // Extract entities using Brainy's search
-        // TODO: Implement extractEntities method
-        // const entities = await this.extractEntities(query)
-        // Build query based on intent and entities
-        // TODO: Implement buildQuery method
-        // return this.buildQuery(query, intent, entities)
-        // Return a basic query for now
-        return {
-            like: query,
-            limit: 10
-        };
+        // Step 2: Extract query terms
+        const queryTerms = query.split(/\s+/).filter(term => term.length > 2);
+        // Step 3: Detect NounType first for context
+        let detectedNounType = null;
+        let typeConfidence = 0;
+        for (const term of queryTerms) {
+            const nounTypeMatch = await this.findBestNounType(term);
+            if (nounTypeMatch.type && nounTypeMatch.confidence > typeConfidence) {
+                detectedNounType = nounTypeMatch.type;
+                typeConfidence = nounTypeMatch.confidence;
+            }
+        }
+        // Step 4: Get type-specific fields if we detected a type
+        let typeSpecificFields = [];
+        if (detectedNounType && typeConfidence > 0.75) {
+            const fieldsForType = await this.brain.getFieldsForType(detectedNounType);
+            typeSpecificFields = fieldsForType.map(f => ({ field: f.field, affinity: f.affinity }));
+        }
+        // Step 5: Find matching fields and entities with type context
+        const entityMatches = await this.findEntityMatchesWithTypeContext(queryTerms, typeSpecificFields);
+        // Step 6: Build structured query from matches
+        const tripleQuery = {};
+        // Separate fields from entities
+        const fieldMatches = entityMatches.filter(m => m.type === 'field');
+        const entityRefs = entityMatches.filter(m => m.type === 'entity');
+        // Add detected type constraint if confident
+        if (detectedNounType && typeConfidence > 0.75) {
+            tripleQuery.where = { ...tripleQuery.where, noun: detectedNounType };
+        }
+        // Build metadata filters from field matches
+        if (fieldMatches.length > 0) {
+            // Use field cardinality to optimize query order
+            fieldMatches.sort((a, b) => (a.cardinality || 0) - (b.cardinality || 0));
+            tripleQuery.where = {};
+            for (const match of fieldMatches) {
+                // Extract value for this field from query
+                const valuePattern = new RegExp(`${match.term}\\s*(?:is|=|:)?\\s*(\\S+)`, 'i');
+                const valueMatch = query.match(valuePattern);
+                if (valueMatch) {
+                    tripleQuery.where[match.field] = valueMatch[1];
+                }
+            }
+        }
+        // Build graph connections from entity references
+        if (entityRefs.length > 0) {
+            tripleQuery.connected = {
+                to: entityRefs[0].id
+            };
+        }
+        // Use remaining terms for vector search
+        const usedTerms = new Set([...fieldMatches.map(m => m.term), ...entityRefs.map(m => m.term)]);
+        const searchTerms = queryTerms.filter(term => !usedTerms.has(term));
+        if (searchTerms.length > 0) {
+            tripleQuery.like = searchTerms.join(' ');
+        }
+        else if (!tripleQuery.where || Object.keys(tripleQuery.where).length === 0) {
+            // If no specific filters, use the full query for vector search
+            tripleQuery.like = query;
+        }
+        // Validate and optimize query based on type context
+        const validatedQuery = await this.validateAndOptimizeQuery(tripleQuery, detectedNounType, fieldMatches);
+        // Add query optimization hints based on field statistics
+        if (fieldMatches.length > 0 && validatedQuery.where) {
+            const queryPlan = await this.brain.getOptimalQueryPlan(validatedQuery.where);
+            // Attach optimization hints as a separate property
+            const hints = validatedQuery;
+            hints.optimizationHints = {
+                detectedType: detectedNounType,
+                typeConfidence,
+                fieldOrder: queryPlan.fieldOrder,
+                strategy: queryPlan.strategy,
+                estimatedCost: queryPlan.estimatedCost,
+                typeSpecificFieldCount: typeSpecificFields.length
+            };
+        }
+        return validatedQuery;
     }
     /**
-     * Analyze intent using keywords and structure
+     * Analyze intent using keywords and structure with enhanced classification
      */
     async analyzeIntent(query) {
-        // Use Brainy's embedding function to get semantic representation
-        const queryEmbedding = await this.brain.embed(query);
-        // Search for similar queries in history (if available)
-        let confidence = 0.7; // Base confidence
-        let type = 'vector'; // Default
         // Analyze query structure patterns
         const lowerQuery = query.toLowerCase();
+        // Determine primary intent
+        let primaryIntent = 'search';
+        let confidence = 0.7; // Base confidence
+        let type = 'vector'; // Default
+        // Intent detection patterns
+        if (lowerQuery.match(/\b(filter|where|with|having)\b/)) {
+            primaryIntent = 'filter';
+            confidence += 0.15;
+        }
+        else if (lowerQuery.match(/\b(count|sum|average|total|group by)\b/)) {
+            primaryIntent = 'aggregate';
+            confidence += 0.2;
+        }
+        else if (lowerQuery.match(/\b(compare|versus|vs|difference|between)\b/)) {
+            primaryIntent = 'compare';
+            confidence += 0.15;
+        }
+        else if (lowerQuery.match(/\b(explain|why|how|what causes)\b/)) {
+            primaryIntent = 'explain';
+            confidence += 0.1;
+        }
+        else if (lowerQuery.match(/\b(connected|related|linked|from.*to)\b/)) {
+            primaryIntent = 'navigate';
+            type = 'graph';
+            confidence += 0.15;
+        }
         // Detect field queries
         if (this.hasFieldPatterns(lowerQuery)) {
-            type = 'field';
-            confidence += 0.2;
+            type = type === 'graph' ? 'combined' : 'field';
+            confidence += 0.1;
         }
         // Detect connection queries  
         if (this.hasConnectionPatterns(lowerQuery)) {
             type = type === 'field' ? 'combined' : 'graph';
             confidence += 0.1;
         }
-        // Extract basic terms
+        // Extract context
+        const context = {
+            domain: this.detectDomain(query),
+            temporalScope: this.detectTemporalScope(query),
+            complexity: this.assessComplexity(query)
+        };
+        // Extract basic terms with enhanced modifiers
         const extractedTerms = this.extractTerms(query);
         return {
             type,
-            confidence: Math.min(confidence, 1.0),
-            extractedTerms
+            primaryIntent,
+            confidence,
+            extractedTerms,
+            context
         };
     }
+    /**
+     * Detect the domain of the query
+     */
+    detectDomain(query) {
+        const lowerQuery = query.toLowerCase();
+        if (lowerQuery.match(/\b(code|function|api|bug|error|debug)\b/)) {
+            return 'technical';
+        }
+        else if (lowerQuery.match(/\b(revenue|sales|profit|customer|market)\b/)) {
+            return 'business';
+        }
+        else if (lowerQuery.match(/\b(research|study|paper|theory|hypothesis)\b/)) {
+            return 'academic';
+        }
+        return 'general';
+    }
+    /**
+     * Detect temporal scope in query
+     */
+    detectTemporalScope(query) {
+        const lowerQuery = query.toLowerCase();
+        if (lowerQuery.match(/\b(was|were|did|had|yesterday|last|previous|ago)\b/)) {
+            return 'past';
+        }
+        else if (lowerQuery.match(/\b(will|going to|tomorrow|next|future|upcoming)\b/)) {
+            return 'future';
+        }
+        else if (lowerQuery.match(/\b(is|are|currently|now|today|present)\b/)) {
+            return 'present';
+        }
+        return 'all';
+    }
+    /**
+     * Assess query complexity
+     */
+    assessComplexity(query) {
+        const words = query.split(/\s+/).length;
+        const hasMultipleClauses = query.match(/\b(and|or|but|with|where)\b/g)?.length || 0;
+        const hasNesting = query.includes('(') || query.includes('[');
+        if (words < 5 && hasMultipleClauses === 0) {
+            return 'simple';
+        }
+        else if (words > 15 || hasMultipleClauses > 2 || hasNesting) {
+            return 'complex';
+        }
+        return 'moderate';
+    }
     /**
      * Step 2: Use neural analysis to decompose complex queries
      */
@@ -182,8 +657,15 @@ export class NaturalLanguageProcessor {
             const mods = intent.extractedTerms.modifiers;
             if (mods.limit)
                 query.limit = mods.limit;
-            if (mods.boost)
-                query.boost = mods.boost;
+            // Convert string boost to proper boost object
+            if (mods.boost) {
+                if (mods.boost === 'recent') {
+                    query.boost = { field: 2.0, vector: 1.0, graph: 1.0 };
+                }
+                else if (mods.boost === 'popular') {
+                    query.boost = { graph: 2.0, vector: 1.0, field: 1.0 };
+                }
+            }
         }
         return query;
     }
@@ -200,7 +682,7 @@ export class NaturalLanguageProcessor {
         // "Show me recent posts by John"
         patterns.set(/show\\s+me\\s+recent\\s+(.+?)\\s+by\\s+(.+)/i, (match) => ({
             like: match[1],
-            boost: 'recent',
+            boost: { field: 2.0, vector: 1.0, graph: 1.0 },
             connected: { from: match[2] }
         }));
         // "Papers with more than 100 citations"  
@@ -255,14 +737,48 @@ export class NaturalLanguageProcessor {
         return extracted;
     }
     /**
-     * Find entity matches using Brainy's search capabilities
+     * Find entity matches using type context and semantic similarity
      */
     async findEntityMatches(terms) {
+        return this.findEntityMatchesWithTypeContext(terms, []);
+    }
+    /**
+     * Find entity matches with type context for better field prioritization
+     */
+    async findEntityMatchesWithTypeContext(terms, typeSpecificFields) {
         const matches = [];
+        // Get field statistics for optimization hints
+        const fieldStats = await this.brain.getFieldStatistics();
+        // Create field priority map based on type affinity
+        const fieldPriorityMap = new Map();
+        for (const { field, affinity } of typeSpecificFields) {
+            fieldPriorityMap.set(field, affinity);
+        }
         for (const term of terms) {
             try {
+                // First, check if term matches a field using semantic similarity
+                const fieldMatch = await this.findBestMatchingFieldWithTypeContext(term, typeSpecificFields);
+                if (fieldMatch.field && fieldMatch.confidence > 0.7) {
+                    const stats = fieldStats.get(fieldMatch.field);
+                    const typeAffinity = fieldPriorityMap.get(fieldMatch.field) || 0;
+                    matches.push({
+                        term,
+                        type: 'field',
+                        field: fieldMatch.field,
+                        confidence: fieldMatch.confidence,
+                        typeAffinity, // NEW: How likely this field appears with this type
+                        cardinality: stats?.cardinality.uniqueValues,
+                        distribution: stats?.cardinality.distribution,
+                        indexType: stats?.indexType
+                    });
+                    // Skip entity search if we found a field match
+                    continue;
+                }
                 // Search for similar entities in the knowledge base
-                const results = await this.brain.search(term, { limit: 5 });
+                const results = await this.brain.find({
+                    query: term,
+                    limit: 5
+                });
                 for (const result of results) {
                     if (result.score > 0.8) { // High similarity threshold
                         matches.push({
@@ -270,19 +786,10 @@ export class NaturalLanguageProcessor {
                             id: result.id,
                             type: 'entity',
                             confidence: result.score,
-                            metadata: result.metadata
+                            metadata: result.entity?.metadata
                         });
                     }
                 }
-                // Check if term matches known field names
-                if (this.isKnownField(term)) {
-                    matches.push({
-                        term,
-                        type: 'field',
-                        field: this.mapToFieldName(term),
-                        confidence: 0.9
-                    });
-                }
             }
             catch (error) {
                 // If search fails, continue with other terms
@@ -291,27 +798,395 @@ export class NaturalLanguageProcessor {
         }
         return matches;
     }
+    // REMOVED: isKnownField and mapToFieldName - now using semantic field matching
+    // The findBestMatchingField method with embeddings replaces these hardcoded approaches
     /**
-     * Check if term is a known field name
+     * Find similar successful queries from history
+     * Uses Brainy's vector search to find semantically similar previous queries
      */
-    isKnownField(term) {
-        const knownFields = [
-            'year', 'date', 'created', 'published', 'author', 'title',
-            'citations', 'views', 'score', 'rating', 'category', 'type'
-        ];
-        return knownFields.includes(term.toLowerCase());
+    async findSimilarQueries(queryEmbedding) {
+        try {
+            // Search for similar queries in a hypothetical query history
+            // For now, return empty array since we don't have query history storage yet
+            // This would integrate with Brainy's search to find similar query patterns
+            // Future implementation could search a query_history noun type:
+            // const similarQueries = await this.brainy.search(queryEmbedding, {
+            //   limit: 5,
+            //   metadata: { type: 'successful_query' },
+            //   nounTypes: ['query_history']
+            // })
+            return [];
+        }
+        catch (error) {
+            console.debug('Failed to find similar queries:', error);
+            return [];
+        }
+    }
+    /**
+     * Extract entities from query using Brainy's semantic search
+     * Identifies known entities, concepts, and relationships in the query text
+     */
+    async extractEntities(query) {
+        try {
+            // Split query into potential entity terms
+            const terms = query.toLowerCase()
+                .split(/[\s,\.;!?]+/)
+                .filter(term => term.length > 2);
+            const entities = [];
+            // Search for each term in Brainy to see if it matches known entities
+            for (const term of terms) {
+                try {
+                    const results = await this.brain.find(term);
+                    if (results && results.length > 0) {
+                        // Found matching entities
+                        entities.push({
+                            term,
+                            matches: results,
+                            confidence: results[0].score || 0.7
+                        });
+                    }
+                }
+                catch (searchError) {
+                    // Continue if individual term search fails
+                    console.debug(`Entity search failed for term: ${term}`, searchError);
+                }
+            }
+            return entities;
+        }
+        catch (error) {
+            console.debug('Failed to extract entities:', error);
+            return [];
+        }
     }
     /**
-     * Map colloquial terms to actual field names
+     * Build final TripleQuery based on intent, entities, and query analysis
+     * Constructs optimized query combining vector, graph, and field searches
      */
-    mapToFieldName(term) {
-        const fieldMappings = {
-            'published': 'publishDate',
-            'created': 'createdAt',
-            'author': 'authorId',
-            'citations': 'citationCount'
+    async buildQuery(query, intent, entities) {
+        try {
+            const tripleQuery = {
+                like: query, // Default to semantic search
+                limit: 10
+            };
+            // Add field filters based on intent
+            if (intent.hasFieldPatterns) {
+                // Extract field-based constraints from the query
+                const whereClause = {};
+                // Look for date/year patterns
+                const yearMatch = query.match(/(\d{4})/g);
+                if (yearMatch) {
+                    whereClause.year = parseInt(yearMatch[0]);
+                }
+                // Look for numeric constraints
+                const moreThanMatch = query.match(/more than (\d+)/i);
+                if (moreThanMatch) {
+                    whereClause.count = { greaterThan: parseInt(moreThanMatch[1]) };
+                }
+                if (Object.keys(whereClause).length > 0) {
+                    tripleQuery.where = whereClause;
+                }
+            }
+            // Add connection-based searches
+            if (intent.hasConnectionPatterns) {
+                // Look for relationship patterns in the query
+                const connectedMatch = query.match(/connected to (.+?)$/i) ||
+                    query.match(/related to (.+?)$/i);
+                if (connectedMatch) {
+                    tripleQuery.connected = {
+                        to: connectedMatch[1].trim()
+                    };
+                }
+            }
+            // Add entity-specific filters
+            if (entities && entities.length > 0) {
+                const highConfidenceEntities = entities.filter(e => e.confidence > 0.8);
+                if (highConfidenceEntities.length > 0) {
+                    // Use the highest confidence entity to refine search
+                    const topEntity = highConfidenceEntities[0];
+                    if (topEntity.matches && topEntity.matches.length > 0) {
+                        // Add entity-specific metadata or connection
+                        const entityData = topEntity.matches[0].metadata;
+                        if (entityData && entityData.category) {
+                            tripleQuery.where = {
+                                ...tripleQuery.where,
+                                category: entityData.category
+                            };
+                        }
+                    }
+                }
+            }
+            return tripleQuery;
+        }
+        catch (error) {
+            console.debug('Failed to build query:', error);
+            // Return simple query as fallback
+            return {
+                like: query,
+                limit: 10
+            };
+        }
+    }
+    /**
+     * Extract entities from text using NEURAL matching to strict NounTypes
+     * ALWAYS uses neural matching, NEVER falls back to patterns
+     */
+    async extract(text, options) {
+        await this.ensureInitialized();
+        // ALWAYS use NeuralEntityExtractor for proper type matching
+        const { NeuralEntityExtractor } = await import('./entityExtractor.js');
+        const extractor = new NeuralEntityExtractor(this.brain);
+        // Convert string types to NounTypes if provided
+        const nounTypes = options?.types ?
+            options.types.map(t => t) :
+            undefined;
+        // Extract using neural matching
+        const entities = await extractor.extract(text, {
+            types: nounTypes,
+            confidence: options?.confidence || 0.0, // Accept ALL matches
+            includeVectors: false,
+            neuralMatching: true // ALWAYS use neural matching
+        });
+        // Convert to expected format
+        return entities.map(entity => ({
+            text: entity.text,
+            type: entity.type,
+            position: entity.position,
+            confidence: entity.confidence,
+            metadata: options?.includeMetadata ? {
+                ...entity.metadata,
+                neuralMatch: true,
+                extractedAt: Date.now()
+            } : undefined
+        }));
+    }
+    /**
+     * DEPRECATED - Old pattern-based extraction
+     * This should NEVER be used - kept only for reference
+     */
+    async extractWithPatterns_DEPRECATED(text, options) {
+        const extracted = [];
+        // Common entity patterns
+        const patterns = {
+            // People (names with capitals)
+            person: /\b([A-Z][a-z]+ [A-Z][a-z]+)\b/g,
+            // Organizations (capitals, Inc, LLC, etc)
+            organization: /\b([A-Z][a-zA-Z&]+(?: [A-Z][a-zA-Z&]+)*(?:,? (?:Inc|LLC|Corp|Ltd|Co|Group|Foundation|Institute|University|College|School|Hospital|Bank|Agency)\.?))\b/g,
+            // Locations (capitals, common place words)
+            location: /\b([A-Z][a-z]+(?: [A-Z][a-z]+)*(?:,? (?:[A-Z][a-z]+))?)(?= (?:City|County|State|Country|Street|Road|Avenue|Boulevard|Drive|Park|Square|Place|Island|Mountain|River|Lake|Ocean|Sea))\b/g,
+            // Dates
+            date: /\b(\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}|\d{4}[\/\-]\d{1,2}[\/\-]\d{1,2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}|\d{1,2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{4})\b/gi,
+            // Times
+            time: /\b(\d{1,2}:\d{2}(?::\d{2})?(?:\s?[AP]M)?)\b/gi,
+            // Emails
+            email: /\b([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})\b/g,
+            // URLs
+            url: /\b(https?:\/\/[^\s]+)\b/g,
+            // Phone numbers
+            phone: /\b(\+?\d{1,3}?[- .]?\(?\d{1,4}\)?[- .]?\d{1,4}[- .]?\d{1,4})\b/g,
+            // Money
+            money: /\b(\$[\d,]+(?:\.\d{2})?|[\d,]+(?:\.\d{2})?\s*(?:USD|EUR|GBP|JPY|CNY))\b/gi,
+            // Percentages
+            percentage: /\b(\d+(?:\.\d+)?%)\b/g,
+            // Products/versions
+            product: /\b([A-Z][a-zA-Z0-9]*(?: [A-Z][a-zA-Z0-9]*)*\s+v?\d+(?:\.\d+)*)\b/g,
+            // Hashtags
+            hashtag: /#[a-zA-Z0-9_]+/g,
+            // Mentions
+            mention: /@[a-zA-Z0-9_]+/g
+        };
+        const minConfidence = options?.confidence || 0.5;
+        const targetTypes = options?.types || Object.keys(patterns);
+        // Apply each pattern
+        for (const [type, pattern] of Object.entries(patterns)) {
+            if (!targetTypes.includes(type))
+                continue;
+            let match;
+            while ((match = pattern.exec(text)) !== null) {
+                const extractedText = match[1] || match[0];
+                const confidence = this.calculateConfidence(extractedText, type);
+                if (confidence >= minConfidence) {
+                    const entity = {
+                        text: extractedText,
+                        type,
+                        position: {
+                            start: match.index,
+                            end: match.index + match[0].length
+                        },
+                        confidence
+                    };
+                    if (options?.includeMetadata) {
+                        ;
+                        entity.metadata = {
+                            pattern: pattern.source,
+                            contextBefore: text.substring(Math.max(0, match.index - 20), match.index),
+                            contextAfter: text.substring(match.index + match[0].length, Math.min(text.length, match.index + match[0].length + 20))
+                        };
+                    }
+                    extracted.push(entity);
+                }
+            }
+        }
+        // Sort by position
+        extracted.sort((a, b) => a.position.start - b.position.start);
+        // Remove overlapping entities (keep higher confidence)
+        const filtered = [];
+        for (const entity of extracted) {
+            const overlapping = filtered.find(e => (entity.position.start >= e.position.start && entity.position.start < e.position.end) ||
+                (entity.position.end > e.position.start && entity.position.end <= e.position.end));
+            if (!overlapping) {
+                filtered.push(entity);
+            }
+            else if (entity.confidence > overlapping.confidence) {
+                const index = filtered.indexOf(overlapping);
+                filtered[index] = entity;
+            }
+        }
+        return filtered;
+    }
+    /**
+     * Analyze sentiment of text
+     */
+    async sentiment(text, options) {
+        // Sentiment words with scores
+        const positiveWords = new Set(['good', 'great', 'excellent', 'amazing', 'wonderful', 'fantastic', 'love', 'like', 'best', 'happy', 'joy', 'brilliant', 'outstanding', 'perfect', 'beautiful', 'awesome', 'super', 'nice', 'fun', 'exciting', 'impressive', 'incredible', 'remarkable', 'delightful', 'pleased', 'satisfied', 'successful', 'effective', 'helpful']);
+        const negativeWords = new Set(['bad', 'terrible', 'awful', 'horrible', 'hate', 'dislike', 'worst', 'sad', 'angry', 'poor', 'disappointing', 'failed', 'broken', 'useless', 'waste', 'sucks', 'disgusting', 'ugly', 'boring', 'annoying', 'frustrating', 'difficult', 'complicated', 'confusing', 'slow', 'expensive', 'unfair', 'wrong', 'mistake', 'problem', 'issue']);
+        const intensifiers = new Set(['very', 'extremely', 'really', 'absolutely', 'completely', 'totally', 'quite', 'rather', 'so']);
+        const negations = new Set(['not', 'no', 'never', 'neither', 'none', 'nobody', 'nothing', 'nowhere', 'hardly', 'barely', 'scarcely']);
+        const normalizedText = text.toLowerCase();
+        const words = normalizedText.split(/\s+/);
+        // Calculate overall sentiment
+        let positiveCount = 0;
+        let negativeCount = 0;
+        let intensifierBoost = 1;
+        for (let i = 0; i < words.length; i++) {
+            const word = words[i].replace(/[^a-z]/g, '');
+            const prevWord = i > 0 ? words[i - 1].replace(/[^a-z]/g, '') : '';
+            // Check for intensifiers
+            if (intensifiers.has(prevWord)) {
+                intensifierBoost = 1.5;
+            }
+            else {
+                intensifierBoost = 1;
+            }
+            // Check for negation
+            const isNegated = negations.has(prevWord);
+            if (positiveWords.has(word)) {
+                if (isNegated) {
+                    negativeCount += intensifierBoost;
+                }
+                else {
+                    positiveCount += intensifierBoost;
+                }
+            }
+            else if (negativeWords.has(word)) {
+                if (isNegated) {
+                    positiveCount += intensifierBoost;
+                }
+                else {
+                    negativeCount += intensifierBoost;
+                }
+            }
+        }
+        const total = positiveCount + negativeCount;
+        const score = total > 0 ? (positiveCount - negativeCount) / total : 0;
+        const magnitude = Math.min(1, total / words.length);
+        let label;
+        if (score > 0.2)
+            label = 'positive';
+        else if (score < -0.2)
+            label = 'negative';
+        else if (magnitude > 0.3)
+            label = 'mixed';
+        else
+            label = 'neutral';
+        const result = {
+            overall: {
+                score,
+                magnitude,
+                label
+            }
         };
-        return fieldMappings[term.toLowerCase()] || term.toLowerCase();
+        // Sentence-level analysis
+        if (options?.granularity === 'sentence' || options?.granularity === 'aspect') {
+            const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
+            result.sentences = [];
+            for (const sentence of sentences) {
+                const sentenceResult = await this.sentiment(sentence);
+                result.sentences.push({
+                    text: sentence.trim(),
+                    score: sentenceResult.overall.score,
+                    magnitude: sentenceResult.overall.magnitude,
+                    label: sentenceResult.overall.label
+                });
+            }
+        }
+        // Aspect-based analysis
+        if (options?.granularity === 'aspect' && options?.aspects) {
+            result.aspects = {};
+            for (const aspect of options.aspects) {
+                const aspectRegex = new RegExp(`[^.!?]*\\b${aspect}\\b[^.!?]*[.!?]?`, 'gi');
+                const aspectSentences = text.match(aspectRegex) || [];
+                if (aspectSentences.length > 0) {
+                    let aspectScore = 0;
+                    let aspectMagnitude = 0;
+                    for (const sentence of aspectSentences) {
+                        const sentimentResult = await this.sentiment(sentence);
+                        aspectScore += sentimentResult.overall.score;
+                        aspectMagnitude += sentimentResult.overall.magnitude;
+                    }
+                    result.aspects[aspect] = {
+                        score: aspectScore / aspectSentences.length,
+                        magnitude: aspectMagnitude / aspectSentences.length,
+                        mentions: aspectSentences.length
+                    };
+                }
+            }
+        }
+        return result;
+    }
+    /**
+     * Calculate confidence for entity extraction
+     */
+    calculateConfidence(text, type) {
+        let confidence = 0.5; // Base confidence
+        // Adjust based on type-specific rules
+        switch (type) {
+            case 'person':
+                // Names with 2-3 capitalized words are more confident
+                const nameWords = text.split(' ');
+                if (nameWords.length >= 2 && nameWords.length <= 3) {
+                    confidence += 0.3;
+                }
+                if (nameWords.every(w => /^[A-Z]/.test(w))) {
+                    confidence += 0.2;
+                }
+                break;
+            case 'organization':
+                // Presence of corporate suffixes increases confidence
+                if (/\b(Inc|LLC|Corp|Ltd|Co|Group)\.?$/.test(text)) {
+                    confidence += 0.4;
+                }
+                break;
+            case 'email':
+            case 'url':
+                // These patterns are very specific, high confidence
+                confidence = 0.95;
+                break;
+            case 'date':
+            case 'time':
+            case 'money':
+            case 'percentage':
+                // Numeric patterns are reliable
+                confidence = 0.9;
+                break;
+            case 'location':
+                // Geographic terms increase confidence
+                if (/\b(City|State|Country|Street|Road|Avenue)$/.test(text)) {
+                    confidence += 0.3;
+                }
+                break;
+        }
+        return Math.min(1, confidence);
     }
 }
 //# sourceMappingURL=naturalLanguageProcessor.js.map