@soulcraft/brainy 3.47.1 → 3.48.0

package/dist/query/typeAwareQueryPlanner.js

@@ -0,0 +1,297 @@
+/**
+ * Type-Aware Query Planner - Phase 3: Type-First Query Optimization
+ *
+ * Generates optimized query execution plans by inferring entity types from
+ * natural language queries using semantic similarity and routing to specific
+ * TypeAwareHNSWIndex graphs.
+ *
+ * Performance Impact:
+ * - Single-type queries: 31x speedup (search 1/31 graphs)
+ * - Multi-type queries: 6-15x speedup (search 2-5/31 graphs)
+ * - Overall: 40% latency reduction @ 1B scale
+ *
+ * Examples:
+ * - "Find engineers" → single-type → [Person] → 31x speedup
+ * - "People at Tesla" → multi-type → [Person, Organization] → 15.5x speedup
+ * - "Everything about AI" → all-types → [all 31 types] → no speedup
+ */
+import { NounType, NOUN_TYPE_COUNT } from '../types/graphTypes.js';
+import { inferNouns } from './semanticTypeInference.js';
+import { prodLog } from '../utils/logger.js';
+/**
+ * Type-Aware Query Planner
+ *
+ * Generates optimized query plans using semantic type inference to route queries
+ * to specific TypeAwareHNSWIndex graphs for billion-scale performance.
+ */
+export class TypeAwareQueryPlanner {
+    constructor(config) {
+        this.config = {
+            singleTypeThreshold: config?.singleTypeThreshold ?? 0.8,
+            multiTypeThreshold: config?.multiTypeThreshold ?? 0.6,
+            maxMultiTypes: config?.maxMultiTypes ?? 5,
+            debug: config?.debug ?? false
+        };
+        this.stats = {
+            totalQueries: 0,
+            singleTypeQueries: 0,
+            multiTypeQueries: 0,
+            allTypesQueries: 0,
+            avgConfidence: 0
+        };
+        prodLog.info(`TypeAwareQueryPlanner initialized: thresholds single=${this.config.singleTypeThreshold}, multi=${this.config.multiTypeThreshold}`);
+    }
+    /**
+     * Plan an optimized query execution strategy using semantic type inference
+     *
+     * @param query - Natural language query string
+     * @returns Promise resolving to optimized query plan with routing strategy
+     */
+    async planQuery(query) {
+        const startTime = performance.now();
+        if (!query || query.trim().length === 0) {
+            return this.createAllTypesPlan(query, 'Empty query');
+        }
+        // Infer noun types for graph routing (nouns only, verbs not used for routing)
+        const inferences = await inferNouns(query, {
+            maxResults: this.config.maxMultiTypes,
+            minConfidence: this.config.multiTypeThreshold
+        });
+        if (inferences.length === 0) {
+            return this.createAllTypesPlan(query, 'No types inferred from query');
+        }
+        // Determine routing strategy based on inference confidence
+        const plan = this.selectRoutingStrategy(query, inferences);
+        // Update statistics
+        this.updateStats(plan);
+        const elapsed = performance.now() - startTime;
+        if (this.config.debug) {
+            prodLog.debug(`Query plan: ${plan.routing} with ${plan.targetTypes.length} types (${elapsed.toFixed(2)}ms)`);
+        }
+        // Performance assertion
+        if (elapsed > 10) {
+            prodLog.warn(`Query planning slow: ${elapsed.toFixed(2)}ms (target: < 10ms)`);
+        }
+        return plan;
+    }
+    /**
+     * Select routing strategy based on semantic inference results
+     */
+    selectRoutingStrategy(query, inferences) {
+        const topInference = inferences[0];
+        // Strategy 1: Single-type routing (highest confidence)
+        if (topInference.confidence >= this.config.singleTypeThreshold &&
+            (inferences.length === 1 ||
+                inferences[1].confidence < this.config.multiTypeThreshold)) {
+            return {
+                originalQuery: query,
+                inferredTypes: inferences,
+                routing: 'single-type',
+                targetTypes: [topInference.type],
+                estimatedSpeedup: NOUN_TYPE_COUNT / 1,
+                confidence: topInference.confidence,
+                reasoning: `High confidence (${(topInference.confidence * 100).toFixed(0)}%) for single type: ${topInference.type}`
+            };
+        }
+        // Strategy 2: Multi-type routing (moderate confidence, multiple types)
+        if (topInference.confidence >= this.config.multiTypeThreshold) {
+            const relevantTypes = inferences
+                .filter(inf => inf.confidence >= this.config.multiTypeThreshold)
+                .slice(0, this.config.maxMultiTypes)
+                .map(inf => inf.type);
+            const avgConfidence = relevantTypes.reduce((sum, type) => {
+                const inf = inferences.find(i => i.type === type);
+                return sum + (inf?.confidence || 0);
+            }, 0) / relevantTypes.length;
+            return {
+                originalQuery: query,
+                inferredTypes: inferences,
+                routing: 'multi-type',
+                targetTypes: relevantTypes,
+                estimatedSpeedup: NOUN_TYPE_COUNT / relevantTypes.length,
+                confidence: avgConfidence,
+                reasoning: `Multiple types detected with moderate confidence (avg ${(avgConfidence * 100).toFixed(0)}%): ${relevantTypes.join(', ')}`
+            };
+        }
+        // Strategy 3: All-types fallback (low confidence)
+        return this.createAllTypesPlan(query, `Low confidence (${(topInference.confidence * 100).toFixed(0)}%) - searching all types for safety`);
+    }
+    /**
+     * Create an all-types plan (fallback strategy)
+     */
+    createAllTypesPlan(query, reasoning) {
+        return {
+            originalQuery: query,
+            inferredTypes: [],
+            routing: 'all-types',
+            targetTypes: this.getAllNounTypes(),
+            estimatedSpeedup: 1.0,
+            confidence: 0.0,
+            reasoning
+        };
+    }
+    /**
+     * Get all noun types (for all-types routing)
+     */
+    getAllNounTypes() {
+        return [
+            NounType.Person,
+            NounType.Organization,
+            NounType.Location,
+            NounType.Thing,
+            NounType.Concept,
+            NounType.Event,
+            NounType.Document,
+            NounType.Media,
+            NounType.File,
+            NounType.Message,
+            NounType.Content,
+            NounType.Collection,
+            NounType.Dataset,
+            NounType.Product,
+            NounType.Service,
+            NounType.User,
+            NounType.Task,
+            NounType.Project,
+            NounType.Process,
+            NounType.State,
+            NounType.Role,
+            NounType.Topic,
+            NounType.Language,
+            NounType.Currency,
+            NounType.Measurement,
+            NounType.Hypothesis,
+            NounType.Experiment,
+            NounType.Contract,
+            NounType.Regulation,
+            NounType.Interface,
+            NounType.Resource
+        ];
+    }
+    /**
+     * Update query statistics
+     */
+    updateStats(plan) {
+        this.stats.totalQueries++;
+        switch (plan.routing) {
+            case 'single-type':
+                this.stats.singleTypeQueries++;
+                break;
+            case 'multi-type':
+                this.stats.multiTypeQueries++;
+                break;
+            case 'all-types':
+                this.stats.allTypesQueries++;
+                break;
+        }
+        // Update rolling average confidence
+        this.stats.avgConfidence =
+            (this.stats.avgConfidence * (this.stats.totalQueries - 1) + plan.confidence) /
+                this.stats.totalQueries;
+    }
+    /**
+     * Get query statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Get detailed statistics report
+     */
+    getStatsReport() {
+        const total = this.stats.totalQueries;
+        if (total === 0) {
+            return 'No queries processed yet';
+        }
+        const singlePct = ((this.stats.singleTypeQueries / total) * 100).toFixed(1);
+        const multiPct = ((this.stats.multiTypeQueries / total) * 100).toFixed(1);
+        const allPct = ((this.stats.allTypesQueries / total) * 100).toFixed(1);
+        const avgConf = (this.stats.avgConfidence * 100).toFixed(1);
+        // Calculate weighted average speedup
+        const avgSpeedup = ((this.stats.singleTypeQueries * 31.0 +
+            this.stats.multiTypeQueries * 10.0 +
+            this.stats.allTypesQueries * 1.0) /
+            total).toFixed(1);
+        return `
+Query Statistics (${total} total):
+- Single-type: ${this.stats.singleTypeQueries} (${singlePct}%) - 31x speedup
+- Multi-type:  ${this.stats.multiTypeQueries} (${multiPct}%) - ~10x speedup
+- All-types:   ${this.stats.allTypesQueries} (${allPct}%) - 1x speedup
+- Avg confidence: ${avgConf}%
+- Avg speedup: ${avgSpeedup}x
+`.trim();
+    }
+    /**
+     * Reset statistics
+     */
+    resetStats() {
+        this.stats = {
+            totalQueries: 0,
+            singleTypeQueries: 0,
+            multiTypeQueries: 0,
+            allTypesQueries: 0,
+            avgConfidence: 0
+        };
+    }
+    /**
+     * Analyze a batch of queries to understand distribution
+     *
+     * Useful for optimizing thresholds and understanding usage patterns
+     */
+    async analyzeQueries(queries) {
+        const distribution = {
+            'single-type': 0,
+            'multi-type': 0,
+            'all-types': 0
+        };
+        let totalSpeedup = 0;
+        for (const query of queries) {
+            const plan = await this.planQuery(query);
+            distribution[plan.routing]++;
+            totalSpeedup += plan.estimatedSpeedup;
+        }
+        const avgSpeedup = totalSpeedup / queries.length;
+        // Generate recommendations
+        const recommendations = [];
+        const singlePct = (distribution['single-type'] / queries.length) * 100;
+        const multiPct = (distribution['multi-type'] / queries.length) * 100;
+        const allPct = (distribution['all-types'] / queries.length) * 100;
+        if (allPct > 30) {
+            recommendations.push(`High all-types usage (${allPct.toFixed(0)}%) - consider lowering multiTypeThreshold or expanding keyword dictionary`);
+        }
+        if (singlePct > 70) {
+            recommendations.push(`High single-type usage (${singlePct.toFixed(0)}%) - excellent! Type inference is working well`);
+        }
+        if (avgSpeedup < 5) {
+            recommendations.push(`Low average speedup (${avgSpeedup.toFixed(1)}x) - consider adjusting confidence thresholds`);
+        }
+        else if (avgSpeedup > 15) {
+            recommendations.push(`Excellent average speedup (${avgSpeedup.toFixed(1)}x) - type-first routing is highly effective`);
+        }
+        return {
+            distribution,
+            avgSpeedup,
+            recommendations
+        };
+    }
+}
+/**
+ * Global singleton instance for convenience
+ */
+let globalPlanner = null;
+/**
+ * Get or create the global TypeAwareQueryPlanner instance
+ */
+export function getQueryPlanner(config) {
+    if (!globalPlanner) {
+        globalPlanner = new TypeAwareQueryPlanner(config);
+    }
+    return globalPlanner;
+}
+/**
+ * Convenience function to plan a query
+ */
+export async function planQuery(query, config) {
+    return getQueryPlanner(config).planQuery(query);
+}
+//# sourceMappingURL=typeAwareQueryPlanner.js.map

package/dist/query/typeInference.d.ts

@@ -0,0 +1,158 @@
+/**
+ * Type Inference System - Phase 3: Type-First Query Optimization
+ *
+ * Automatically infers NounTypes from natural language queries using keyword-based
+ * heuristics for fast O(1) type detection.
+ *
+ * Performance Guarantee: < 1ms per query
+ * Accuracy Target: > 80%
+ *
+ * Examples:
+ * - "Find engineers in San Francisco" → [Person, Location]
+ * - "Show documents about AI" → [Document, Concept]
+ * - "List companies in tech sector" → [Organization, Topic]
+ */
+import { NounType } from '../types/graphTypes.js';
+/**
+ * Result of type inference with confidence score
+ */
+export interface TypeInference {
+    type: NounType;
+    confidence: number;
+    matchedKeywords: string[];
+}
+/**
+ * Configuration for type inference behavior
+ */
+export interface TypeInferenceConfig {
+    /**
+     * Minimum confidence threshold to include a type (default: 0.4)
+     */
+    minConfidence?: number;
+    /**
+     * Maximum number of types to return (default: 5)
+     */
+    maxTypes?: number;
+    /**
+     * Enable debug logging (default: false)
+     */
+    debug?: boolean;
+    /**
+     * Enable vector similarity fallback for unknown words (default: false)
+     * When enabled, queries with low keyword confidence trigger vector-based type inference
+     */
+    enableVectorFallback?: boolean;
+    /**
+     * Minimum confidence threshold to trigger vector fallback (default: 0.7)
+     * If keyword matching produces confidence below this, vector fallback is used
+     */
+    fallbackConfidenceThreshold?: number;
+    /**
+     * Minimum similarity score for vector-based type matches (default: 0.5)
+     */
+    vectorThreshold?: number;
+}
+/**
+ * Type Inference System
+ *
+ * Uses keyword matching for fast type detection from natural language.
+ * Designed for billion-scale performance with minimal latency.
+ */
+export declare class TypeInferenceSystem {
+    private keywordMap;
+    private phraseMap;
+    private config;
+    private typeEmbeddings;
+    private embedder;
+    constructor(config?: TypeInferenceConfig);
+    /**
+     * Infer noun types from a natural language query (synchronous keyword matching only)
+     * For hybrid mode with vector fallback, use inferTypesAsync()
+     *
+     * @param query - Natural language query string
+     * @returns Array of type inferences sorted by confidence (highest first)
+     */
+    inferTypes(query: string): TypeInference[];
+    /**
+     * Infer noun types with hybrid approach: keyword matching + optional vector fallback
+     * This is the async version that supports vector similarity fallback
+     *
+     * @param query - Natural language query string
+     * @returns Promise resolving to array of type inferences
+     */
+    inferTypesAsync(query: string): Promise<TypeInference[]>;
+    /**
+     * Internal: Keyword-based type inference (synchronous, fast)
+     */
+    private inferTypesViaKeywords;
+    /**
+     * Internal: Hybrid inference with vector fallback (asynchronous)
+     */
+    private inferTypesWithFallback;
+    /**
+     * Match multi-word phrases in query
+     */
+    private matchPhrases;
+    /**
+     * Match individual keywords in query
+     */
+    private matchKeywords;
+    /**
+     * Find closest keyword using edit distance (for typo correction)
+     * Allows edit distance 1-2 depending on word length
+     */
+    private findFuzzyKeywordMatch;
+    /**
+     * Calculate Levenshtein (edit) distance between two strings
+     */
+    private levenshteinDistance;
+    /**
+     * Update type score with new match
+     */
+    private updateTypeScore;
+    /**
+     * Load pre-compiled type embeddings from embeddedTypeEmbeddings.ts
+     */
+    private loadTypeEmbeddings;
+    /**
+     * Lazy-load TransformerEmbedding model (only when vector fallback is triggered)
+     */
+    private loadEmbedder;
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    private cosineSimilarity;
+    /**
+     * Infer types using vector similarity against pre-compiled type embeddings
+     */
+    private inferTypesViaVectorSimilarity;
+    /**
+     * Merge keyword-based and vector-based results
+     * Prioritizes keyword results (explicit matches) over vector results (semantic matches)
+     */
+    private mergeResults;
+    /**
+     * Build keyword dictionary for single-word matching
+     */
+    private buildKeywordMap;
+    /**
+     * Build phrase dictionary for multi-word matching
+     */
+    private buildPhraseMap;
+    /**
+     * Get statistics about the inference system
+     */
+    getStats(): {
+        keywordCount: number;
+        phraseCount: number;
+        config: Required<TypeInferenceConfig>;
+    };
+}
+/**
+ * Get or create the global TypeInferenceSystem instance
+ */
+export declare function getTypeInferenceSystem(config?: TypeInferenceConfig): TypeInferenceSystem;
+/**
+ * Convenience function to infer types from a query
+ */
+export declare function inferTypes(query: string, config?: TypeInferenceConfig): TypeInference[];