@soulcraft/brainy 4.1.4 → 4.2.1

package/dist/neural/SmartExtractor.js

@@ -0,0 +1,592 @@
+/**
+ * SmartExtractor - Unified entity type extraction using ensemble of neural signals
+ *
+ * PRODUCTION-READY: Single orchestration class for all entity type classification
+ *
+ * Design Philosophy:
+ * - Simplicity over complexity (KISS principle)
+ * - One class instead of multiple strategy layers
+ * - Clear execution path for debugging
+ * - Comprehensive format intelligence built-in
+ *
+ * Ensemble Architecture:
+ * - ExactMatchSignal (40%) - Explicit patterns and exact keywords
+ * - EmbeddingSignal (35%) - Neural similarity with type embeddings
+ * - PatternSignal (20%) - Regex patterns and naming conventions
+ * - ContextSignal (5%) - Relationship-based inference
+ *
+ * Format Intelligence:
+ * Supports 7 major formats with automatic hint extraction:
+ * - Excel (.xlsx): Column headers, sheet names, "Related Terms" detection
+ * - CSV (.csv): Header row patterns, naming conventions
+ * - PDF (.pdf): Form field names and labels
+ * - YAML (.yaml, .yml): Semantic key names
+ * - DOCX (.docx): Heading levels and structure
+ * - JSON (.json): Field name patterns
+ * - Markdown (.md): Heading hierarchy
+ *
+ * Performance:
+ * - Parallel signal execution (~15ms total)
+ * - LRU caching for hot entities
+ * - Confidence boosting when signals agree
+ * - Graceful degradation on errors
+ */
+import { ExactMatchSignal } from './signals/ExactMatchSignal.js';
+import { PatternSignal } from './signals/PatternSignal.js';
+import { EmbeddingSignal } from './signals/EmbeddingSignal.js';
+import { ContextSignal } from './signals/ContextSignal.js';
+/**
+ * SmartExtractor - Unified entity type classification
+ *
+ * This is the single entry point for all entity type extraction.
+ * It orchestrates all 4 signals, applies format intelligence,
+ * and combines results using ensemble weighting.
+ *
+ * Production features:
+ * - Parallel signal execution for performance
+ * - Format-specific hint extraction
+ * - Ensemble voting with confidence boosting
+ * - Comprehensive statistics and observability
+ * - LRU caching for hot paths
+ * - Graceful error handling
+ */
+export class SmartExtractor {
+    constructor(brain, options) {
+        // LRU cache
+        this.cache = new Map();
+        this.cacheOrder = [];
+        // Statistics
+        this.stats = {
+            calls: 0,
+            cacheHits: 0,
+            exactMatchWins: 0,
+            patternWins: 0,
+            embeddingWins: 0,
+            contextWins: 0,
+            ensembleWins: 0,
+            agreementBoosts: 0,
+            formatHintsUsed: 0,
+            averageConfidence: 0,
+            averageSignalsUsed: 0
+        };
+        this.brain = brain;
+        // Set default options
+        this.options = {
+            minConfidence: options?.minConfidence ?? 0.60,
+            enableFormatHints: options?.enableFormatHints ?? true,
+            enableEnsemble: options?.enableEnsemble ?? true,
+            cacheSize: options?.cacheSize ?? 2000,
+            weights: {
+                exactMatch: options?.weights?.exactMatch ?? 0.40,
+                embedding: options?.weights?.embedding ?? 0.35,
+                pattern: options?.weights?.pattern ?? 0.20,
+                context: options?.weights?.context ?? 0.05
+            }
+        };
+        // Validate weights sum to 1.0
+        const weightSum = Object.values(this.options.weights).reduce((a, b) => a + b, 0);
+        if (Math.abs(weightSum - 1.0) > 0.01) {
+            throw new Error(`Signal weights must sum to 1.0, got ${weightSum}`);
+        }
+        // Initialize signals
+        this.exactMatchSignal = new ExactMatchSignal(brain, {
+            minConfidence: 0.50, // Lower threshold, ensemble will filter
+            cacheSize: Math.floor(this.options.cacheSize / 4)
+        });
+        this.patternSignal = new PatternSignal(brain, {
+            minConfidence: 0.50,
+            cacheSize: Math.floor(this.options.cacheSize / 4)
+        });
+        this.embeddingSignal = new EmbeddingSignal(brain, {
+            minConfidence: 0.50,
+            checkGraph: true,
+            checkHistory: true,
+            cacheSize: Math.floor(this.options.cacheSize / 4)
+        });
+        this.contextSignal = new ContextSignal(brain, {
+            minConfidence: 0.50,
+            cacheSize: Math.floor(this.options.cacheSize / 4)
+        });
+    }
+    /**
+     * Extract entity type using ensemble of signals
+     *
+     * Main entry point - orchestrates all signals and combines results
+     *
+     * @param candidate Entity text to classify
+     * @param context Classification context with format hints
+     * @returns ExtractionResult with type and confidence
+     */
+    async extract(candidate, context) {
+        this.stats.calls++;
+        // Check cache first
+        const cacheKey = this.getCacheKey(candidate, context);
+        const cached = this.getFromCache(cacheKey);
+        if (cached !== undefined) {
+            this.stats.cacheHits++;
+            return cached;
+        }
+        try {
+            // Extract format hints if enabled
+            const formatHints = this.options.enableFormatHints && context?.formatContext
+                ? this.extractFormatHints(context.formatContext)
+                : [];
+            if (formatHints.length > 0) {
+                this.stats.formatHintsUsed++;
+            }
+            // Build enriched context with format hints
+            const enrichedContext = {
+                definition: context?.definition,
+                allTerms: [...(context?.allTerms || []), ...formatHints],
+                metadata: context?.metadata
+            };
+            // Execute all signals in parallel
+            const [exactMatch, patternMatch, embeddingMatch, contextMatch] = await Promise.all([
+                this.exactMatchSignal.classify(candidate, enrichedContext).catch(() => null),
+                this.patternSignal.classify(candidate, enrichedContext).catch(() => null),
+                this.embeddingSignal.classify(candidate, enrichedContext).catch(() => null),
+                this.contextSignal.classify(candidate, enrichedContext).catch(() => null)
+            ]);
+            // Wrap results with weights
+            const signalResults = [
+                {
+                    signal: 'exact-match',
+                    type: exactMatch?.type || null,
+                    confidence: exactMatch?.confidence || 0,
+                    weight: this.options.weights.exactMatch,
+                    evidence: exactMatch?.evidence || ''
+                },
+                {
+                    signal: 'pattern',
+                    type: patternMatch?.type || null,
+                    confidence: patternMatch?.confidence || 0,
+                    weight: this.options.weights.pattern,
+                    evidence: patternMatch?.evidence || ''
+                },
+                {
+                    signal: 'embedding',
+                    type: embeddingMatch?.type || null,
+                    confidence: embeddingMatch?.confidence || 0,
+                    weight: this.options.weights.embedding,
+                    evidence: embeddingMatch?.evidence || ''
+                },
+                {
+                    signal: 'context',
+                    type: contextMatch?.type || null,
+                    confidence: contextMatch?.confidence || 0,
+                    weight: this.options.weights.context,
+                    evidence: contextMatch?.evidence || ''
+                }
+            ];
+            // Combine using ensemble or best signal
+            const result = this.options.enableEnsemble
+                ? this.combineEnsemble(signalResults, formatHints, context?.formatContext)
+                : this.selectBestSignal(signalResults, formatHints, context?.formatContext);
+            // Cache result (including nulls to avoid recomputation)
+            this.addToCache(cacheKey, result);
+            // Update statistics
+            if (result) {
+                this.updateStatistics(result);
+            }
+            return result;
+        }
+        catch (error) {
+            // Graceful degradation
+            console.warn(`SmartExtractor error for "${candidate}":`, error);
+            return null;
+        }
+    }
+    /**
+     * Extract format-specific hints from context
+     *
+     * Returns array of hint strings that can help with classification
+     */
+    extractFormatHints(formatContext) {
+        const hints = [];
+        switch (formatContext.format) {
+            case 'excel':
+                hints.push(...this.extractExcelHints(formatContext));
+                break;
+            case 'csv':
+                hints.push(...this.extractCsvHints(formatContext));
+                break;
+            case 'pdf':
+                hints.push(...this.extractPdfHints(formatContext));
+                break;
+            case 'yaml':
+                hints.push(...this.extractYamlHints(formatContext));
+                break;
+            case 'docx':
+                hints.push(...this.extractDocxHints(formatContext));
+                break;
+            case 'json':
+                hints.push(...this.extractJsonHints(formatContext));
+                break;
+            case 'markdown':
+                hints.push(...this.extractMarkdownHints(formatContext));
+                break;
+        }
+        return hints.filter(h => h && h.trim().length > 0);
+    }
+    /**
+     * Extract Excel-specific hints
+     */
+    extractExcelHints(context) {
+        const hints = [];
+        if (context.columnHeader) {
+            hints.push(context.columnHeader);
+            // Extract type keywords from header
+            const headerLower = context.columnHeader.toLowerCase();
+            const typeKeywords = [
+                'person', 'people', 'user', 'author', 'creator', 'employee', 'member',
+                'organization', 'company', 'org', 'business',
+                'location', 'place', 'city', 'country', 'address',
+                'event', 'meeting', 'conference', 'workshop',
+                'concept', 'idea', 'term', 'definition',
+                'document', 'file', 'report', 'paper',
+                'project', 'initiative', 'program',
+                'product', 'service', 'offering',
+                'date', 'time', 'timestamp', 'when'
+            ];
+            for (const keyword of typeKeywords) {
+                if (headerLower.includes(keyword)) {
+                    hints.push(keyword);
+                }
+            }
+        }
+        if (context.sheetName) {
+            hints.push(context.sheetName);
+        }
+        return hints;
+    }
+    /**
+     * Extract CSV-specific hints
+     */
+    extractCsvHints(context) {
+        const hints = [];
+        if (context.columnHeader) {
+            hints.push(context.columnHeader);
+            // Parse underscore/hyphen patterns
+            const headerLower = context.columnHeader.toLowerCase();
+            if (headerLower.includes('_') || headerLower.includes('-')) {
+                const parts = headerLower.split(/[_-]/);
+                hints.push(...parts);
+            }
+        }
+        return hints;
+    }
+    /**
+     * Extract PDF-specific hints
+     */
+    extractPdfHints(context) {
+        const hints = [];
+        if (context.fieldName) {
+            hints.push(context.fieldName);
+            // Convert snake_case or camelCase to words
+            const words = context.fieldName
+                .replace(/([A-Z])/g, ' $1')
+                .replace(/[_-]/g, ' ')
+                .trim()
+                .split(/\s+/);
+            hints.push(...words);
+        }
+        return hints;
+    }
+    /**
+     * Extract YAML-specific hints
+     */
+    extractYamlHints(context) {
+        const hints = [];
+        if (context.yamlKey) {
+            hints.push(context.yamlKey);
+            // Parse key structure
+            const keyWords = context.yamlKey
+                .replace(/([A-Z])/g, ' $1')
+                .replace(/[-_]/g, ' ')
+                .trim()
+                .split(/\s+/);
+            hints.push(...keyWords);
+        }
+        return hints;
+    }
+    /**
+     * Extract DOCX-specific hints
+     */
+    extractDocxHints(context) {
+        const hints = [];
+        if (context.headingLevel !== undefined) {
+            // Heading 1 = major entities (organizations, projects)
+            // Heading 2-3 = sub-entities (people, concepts)
+            if (context.headingLevel === 1) {
+                hints.push('major entity', 'organization', 'project');
+            }
+            else if (context.headingLevel === 2) {
+                hints.push('sub entity', 'person', 'concept');
+            }
+        }
+        return hints;
+    }
+    /**
+     * Extract JSON-specific hints
+     */
+    extractJsonHints(context) {
+        const hints = [];
+        if (context.fieldName) {
+            hints.push(context.fieldName);
+            // Parse camelCase or snake_case
+            const words = context.fieldName
+                .replace(/([A-Z])/g, ' $1')
+                .replace(/[_-]/g, ' ')
+                .trim()
+                .split(/\s+/);
+            hints.push(...words);
+        }
+        return hints;
+    }
+    /**
+     * Extract Markdown-specific hints
+     */
+    extractMarkdownHints(context) {
+        const hints = [];
+        if (context.headingLevel !== undefined) {
+            if (context.headingLevel === 1) {
+                hints.push('major entity');
+            }
+            else if (context.headingLevel === 2) {
+                hints.push('sub entity');
+            }
+        }
+        return hints;
+    }
+    /**
+     * Combine signal results using ensemble voting
+     *
+     * Applies weighted voting with confidence boosting when signals agree
+     */
+    combineEnsemble(signalResults, formatHints, formatContext) {
+        // Filter out null results
+        const validResults = signalResults.filter(r => r.type !== null);
+        if (validResults.length === 0) {
+            return null;
+        }
+        // Count votes by type with weighted confidence
+        const typeScores = new Map();
+        for (const result of validResults) {
+            if (!result.type)
+                continue;
+            const weighted = result.confidence * result.weight;
+            const existing = typeScores.get(result.type);
+            if (existing) {
+                existing.score += weighted;
+                existing.signals.push(result);
+            }
+            else {
+                typeScores.set(result.type, { score: weighted, signals: [result] });
+            }
+        }
+        // Find best type
+        let bestType = null;
+        let bestScore = 0;
+        let bestSignals = [];
+        for (const [type, data] of typeScores.entries()) {
+            // Apply agreement boost (multiple signals agree)
+            let finalScore = data.score;
+            if (data.signals.length > 1) {
+                const agreementBoost = 0.05 * (data.signals.length - 1);
+                finalScore += agreementBoost;
+                this.stats.agreementBoosts++;
+            }
+            if (finalScore > bestScore) {
+                bestScore = finalScore;
+                bestType = type;
+                bestSignals = data.signals;
+            }
+        }
+        // Check minimum confidence threshold
+        if (!bestType || bestScore < this.options.minConfidence) {
+            return null;
+        }
+        // Track signal contributions
+        const usedSignals = bestSignals.length;
+        this.stats.averageSignalsUsed =
+            (this.stats.averageSignalsUsed * (this.stats.calls - 1) + usedSignals) / this.stats.calls;
+        // Build evidence string
+        const signalNames = bestSignals.map(s => s.signal).join(' + ');
+        const evidence = `Ensemble: ${signalNames} (${bestSignals.length} signal${bestSignals.length > 1 ? 's' : ''} agree)`;
+        return {
+            type: bestType,
+            confidence: Math.min(bestScore, 1.0), // Cap at 1.0
+            source: 'ensemble',
+            evidence,
+            metadata: {
+                signalResults: bestSignals.map(s => ({
+                    signal: s.signal,
+                    type: s.type,
+                    confidence: s.confidence,
+                    weight: s.weight
+                })),
+                agreementBoost: bestSignals.length > 1 ? 0.05 * (bestSignals.length - 1) : 0,
+                formatHints: formatHints.length > 0 ? formatHints : undefined,
+                formatContext
+            }
+        };
+    }
+    /**
+     * Select best single signal (when ensemble is disabled)
+     */
+    selectBestSignal(signalResults, formatHints, formatContext) {
+        // Filter valid results and sort by weighted confidence
+        const validResults = signalResults
+            .filter(r => r.type !== null)
+            .map(r => ({ ...r, weightedScore: r.confidence * r.weight }))
+            .sort((a, b) => b.weightedScore - a.weightedScore);
+        if (validResults.length === 0) {
+            return null;
+        }
+        const best = validResults[0];
+        if (best.weightedScore < this.options.minConfidence) {
+            return null;
+        }
+        return {
+            type: best.type,
+            confidence: best.confidence,
+            source: best.signal,
+            evidence: best.evidence,
+            metadata: {
+                formatHints: formatHints.length > 0 ? formatHints : undefined,
+                formatContext
+            }
+        };
+    }
+    /**
+     * Update statistics based on result
+     */
+    updateStatistics(result) {
+        // Track win counts
+        if (result.source === 'ensemble') {
+            this.stats.ensembleWins++;
+        }
+        else if (result.source === 'exact-match') {
+            this.stats.exactMatchWins++;
+        }
+        else if (result.source === 'pattern') {
+            this.stats.patternWins++;
+        }
+        else if (result.source === 'embedding') {
+            this.stats.embeddingWins++;
+        }
+        else if (result.source === 'context') {
+            this.stats.contextWins++;
+        }
+        // Update rolling average confidence
+        this.stats.averageConfidence =
+            (this.stats.averageConfidence * (this.stats.calls - 1) + result.confidence) / this.stats.calls;
+    }
+    /**
+     * Get cache key from candidate and context
+     */
+    getCacheKey(candidate, context) {
+        const normalized = candidate.toLowerCase().trim();
+        const defSnippet = context?.definition?.substring(0, 50) || '';
+        const format = context?.formatContext?.format || '';
+        return `${normalized}:${defSnippet}:${format}`;
+    }
+    /**
+     * Get from LRU cache
+     */
+    getFromCache(key) {
+        if (!this.cache.has(key))
+            return undefined;
+        const cached = this.cache.get(key);
+        // Move to end (most recently used)
+        this.cacheOrder = this.cacheOrder.filter(k => k !== key);
+        this.cacheOrder.push(key);
+        return cached ?? null;
+    }
+    /**
+     * Add to LRU cache with eviction
+     */
+    addToCache(key, value) {
+        this.cache.set(key, value);
+        this.cacheOrder.push(key);
+        // Evict oldest if over limit
+        if (this.cache.size > this.options.cacheSize) {
+            const oldest = this.cacheOrder.shift();
+            if (oldest) {
+                this.cache.delete(oldest);
+            }
+        }
+    }
+    /**
+     * Get comprehensive statistics
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            cacheSize: this.cache.size,
+            cacheHitRate: this.stats.calls > 0 ? this.stats.cacheHits / this.stats.calls : 0,
+            ensembleRate: this.stats.calls > 0 ? this.stats.ensembleWins / this.stats.calls : 0,
+            formatHintRate: this.stats.calls > 0 ? this.stats.formatHintsUsed / this.stats.calls : 0,
+            signalStats: {
+                exactMatch: this.exactMatchSignal.getStats(),
+                pattern: this.patternSignal.getStats(),
+                embedding: this.embeddingSignal.getStats(),
+                context: this.contextSignal.getStats()
+            }
+        };
+    }
+    /**
+     * Reset all statistics
+     */
+    resetStats() {
+        this.stats = {
+            calls: 0,
+            cacheHits: 0,
+            exactMatchWins: 0,
+            patternWins: 0,
+            embeddingWins: 0,
+            contextWins: 0,
+            ensembleWins: 0,
+            agreementBoosts: 0,
+            formatHintsUsed: 0,
+            averageConfidence: 0,
+            averageSignalsUsed: 0
+        };
+        this.exactMatchSignal.resetStats();
+        this.patternSignal.resetStats();
+        this.embeddingSignal.resetStats();
+        this.contextSignal.resetStats();
+    }
+    /**
+     * Clear all caches
+     */
+    clearCache() {
+        this.cache.clear();
+        this.cacheOrder = [];
+        this.exactMatchSignal.clearCache();
+        this.patternSignal.clearCache();
+        this.embeddingSignal.clearCache();
+        this.contextSignal.clearCache();
+    }
+    /**
+     * Add entity to historical data (for embedding signal temporal boosting)
+     */
+    addToHistory(text, type, vector) {
+        this.embeddingSignal.addToHistory(text, type, vector);
+    }
+    /**
+     * Clear historical data
+     */
+    clearHistory() {
+        this.embeddingSignal.clearHistory();
+    }
+}
+/**
+ * Create a new SmartExtractor instance
+ *
+ * Convenience factory function
+ */
+export function createSmartExtractor(brain, options) {
+    return new SmartExtractor(brain, options);
+}
+//# sourceMappingURL=SmartExtractor.js.map