@soulcraft/brainy 1.5.0 → 2.0.0

package/dist/neural/patternLibrary.js

@@ -0,0 +1,313 @@
+/**
+ * 🧠 Pattern Library for Natural Language Processing
+ * Manages pre-computed pattern embeddings and smart matching
+ *
+ * Uses Brainy's own features for self-leveraging intelligence:
+ * - Embeddings for semantic similarity
+ * - Pattern caching for performance
+ * - Progressive learning from usage
+ */
+import { EMBEDDED_PATTERNS, getPatternEmbeddings, PATTERNS_METADATA } from './embeddedPatterns.js';
+export class PatternLibrary {
+    constructor(brain) {
+        this.brain = brain;
+        this.patterns = new Map();
+        this.patternEmbeddings = new Map();
+        this.embeddingCache = new Map();
+        this.successMetrics = new Map();
+    }
+    /**
+     * Initialize pattern library with pre-computed embeddings
+     */
+    async init() {
+        // Try to load pre-computed embeddings first
+        const precomputedEmbeddings = getPatternEmbeddings();
+        if (precomputedEmbeddings.size > 0) {
+            // Use pre-computed embeddings (instant!)
+            console.debug(`Loading ${precomputedEmbeddings.size} pre-computed pattern embeddings`);
+            for (const pattern of EMBEDDED_PATTERNS) {
+                this.patterns.set(pattern.id, pattern);
+                this.successMetrics.set(pattern.id, pattern.confidence);
+                const embedding = precomputedEmbeddings.get(pattern.id);
+                if (embedding) {
+                    this.patternEmbeddings.set(pattern.id, Array.from(embedding));
+                }
+            }
+            console.debug(`Pattern library ready: ${PATTERNS_METADATA.totalPatterns} patterns loaded instantly`);
+        }
+        else {
+            // Fall back to runtime computation
+            console.debug('No pre-computed embeddings found, computing at runtime...');
+            for (const pattern of EMBEDDED_PATTERNS) {
+                this.patterns.set(pattern.id, pattern);
+                this.successMetrics.set(pattern.id, pattern.confidence);
+            }
+            // Compute embeddings for all patterns
+            await this.precomputeEmbeddings();
+        }
+    }
+    /**
+     * Pre-compute embeddings for all patterns for fast matching
+     */
+    async precomputeEmbeddings() {
+        for (const [id, pattern] of this.patterns) {
+            // Average embeddings of all examples for robust representation
+            const embeddings = [];
+            for (const example of pattern.examples) {
+                const embedding = await this.getEmbedding(example);
+                embeddings.push(embedding);
+            }
+            // Average the embeddings
+            const avgEmbedding = this.averageVectors(embeddings);
+            this.patternEmbeddings.set(id, avgEmbedding);
+        }
+    }
+    /**
+     * Get embedding with caching
+     */
+    async getEmbedding(text) {
+        if (this.embeddingCache.has(text)) {
+            return this.embeddingCache.get(text);
+        }
+        const embedding = await this.brain.embed(text);
+        this.embeddingCache.set(text, embedding);
+        return embedding;
+    }
+    /**
+     * Find best matching patterns for a query
+     */
+    async findBestPatterns(queryEmbedding, k = 3) {
+        const matches = [];
+        // Calculate similarity with all patterns
+        for (const [id, patternEmbedding] of this.patternEmbeddings) {
+            const similarity = this.cosineSimilarity(queryEmbedding, patternEmbedding);
+            const pattern = this.patterns.get(id);
+            // Apply success metric boost
+            const successBoost = this.successMetrics.get(id) || 0.5;
+            const adjustedSimilarity = similarity * (0.7 + 0.3 * successBoost);
+            matches.push({
+                pattern,
+                similarity: adjustedSimilarity
+            });
+        }
+        // Sort by similarity and return top k
+        matches.sort((a, b) => b.similarity - a.similarity);
+        return matches.slice(0, k);
+    }
+    /**
+     * Extract slots from query based on pattern
+     */
+    extractSlots(query, pattern) {
+        const slots = {};
+        let confidence = pattern.confidence;
+        // Try regex extraction first
+        const regex = new RegExp(pattern.pattern, 'i');
+        const match = query.match(regex);
+        if (match) {
+            // Extract captured groups as slots
+            for (let i = 1; i < match.length; i++) {
+                slots[`$${i}`] = match[i];
+            }
+            // High confidence if regex matches
+            confidence = Math.min(confidence * 1.2, 1.0);
+        }
+        else {
+            // Fall back to token-based extraction
+            const tokens = this.tokenize(query);
+            const exampleTokens = this.tokenize(pattern.examples[0]);
+            // Simple alignment-based extraction
+            for (let i = 0; i < tokens.length; i++) {
+                if (i < exampleTokens.length && exampleTokens[i].startsWith('$')) {
+                    slots[exampleTokens[i]] = tokens[i];
+                }
+            }
+            // Lower confidence for fuzzy matching
+            confidence *= 0.7;
+        }
+        // Post-process slots
+        this.postProcessSlots(slots, pattern);
+        return { slots, confidence };
+    }
+    /**
+     * Fill template with extracted slots
+     */
+    fillTemplate(template, slots) {
+        const filled = JSON.parse(JSON.stringify(template));
+        // Recursively replace slot placeholders
+        const replacePlaceholders = (obj) => {
+            if (typeof obj === 'string') {
+                // Replace ${1}, ${2}, etc. with slot values
+                return obj.replace(/\$\{(\d+)\}/g, (_, num) => {
+                    return slots[`$${num}`] || '';
+                });
+            }
+            else if (Array.isArray(obj)) {
+                return obj.map(item => replacePlaceholders(item));
+            }
+            else if (typeof obj === 'object' && obj !== null) {
+                const result = {};
+                for (const [key, value] of Object.entries(obj)) {
+                    const newKey = replacePlaceholders(key);
+                    result[newKey] = replacePlaceholders(value);
+                }
+                return result;
+            }
+            return obj;
+        };
+        return replacePlaceholders(filled);
+    }
+    /**
+     * Update pattern success metrics based on usage
+     */
+    updateSuccessMetric(patternId, success) {
+        const current = this.successMetrics.get(patternId) || 0.5;
+        // Exponential moving average
+        const alpha = 0.1;
+        const newMetric = success
+            ? current + alpha * (1 - current)
+            : current - alpha * current;
+        this.successMetrics.set(patternId, newMetric);
+    }
+    /**
+     * Learn new pattern from successful query
+     */
+    async learnPattern(query, result) {
+        // Find similar existing patterns
+        const queryEmbedding = await this.getEmbedding(query);
+        const similar = await this.findBestPatterns(queryEmbedding, 1);
+        if (similar[0]?.similarity < 0.7) {
+            // This is a new pattern type - add it
+            const newPattern = {
+                id: `learned_${Date.now()}`,
+                category: 'learned',
+                examples: [query],
+                pattern: this.generateRegexFromQuery(query),
+                template: result,
+                confidence: 0.6 // Start with moderate confidence
+            };
+            this.patterns.set(newPattern.id, newPattern);
+            this.patternEmbeddings.set(newPattern.id, queryEmbedding);
+            this.successMetrics.set(newPattern.id, 0.6);
+        }
+        else {
+            // Similar pattern exists - add as example
+            const pattern = similar[0].pattern;
+            if (!pattern.examples.includes(query)) {
+                pattern.examples.push(query);
+                // Update pattern embedding with new example
+                const embeddings = await Promise.all(pattern.examples.map(ex => this.getEmbedding(ex)));
+                const newEmbedding = this.averageVectors(embeddings);
+                this.patternEmbeddings.set(pattern.id, newEmbedding);
+            }
+        }
+    }
+    /**
+     * Helper: Average multiple vectors
+     */
+    averageVectors(vectors) {
+        if (vectors.length === 0)
+            return [];
+        const dim = vectors[0].length;
+        const avg = new Array(dim).fill(0);
+        for (const vec of vectors) {
+            for (let i = 0; i < dim; i++) {
+                avg[i] += vec[i];
+            }
+        }
+        for (let i = 0; i < dim; i++) {
+            avg[i] /= vectors.length;
+        }
+        return avg;
+    }
+    /**
+     * Helper: Calculate cosine similarity
+     */
+    cosineSimilarity(a, b) {
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        normA = Math.sqrt(normA);
+        normB = Math.sqrt(normB);
+        if (normA === 0 || normB === 0)
+            return 0;
+        return dotProduct / (normA * normB);
+    }
+    /**
+     * Helper: Simple tokenization
+     */
+    tokenize(text) {
+        return text.toLowerCase().split(/\s+/).filter(t => t.length > 0);
+    }
+    /**
+     * Helper: Post-process extracted slots
+     */
+    postProcessSlots(slots, pattern) {
+        // Convert string numbers to actual numbers
+        for (const [key, value] of Object.entries(slots)) {
+            if (typeof value === 'string') {
+                // Check if it's a number
+                const num = parseFloat(value);
+                if (!isNaN(num) && value.match(/^\d+(\.\d+)?$/)) {
+                    slots[key] = num;
+                }
+                // Parse dates
+                if (value.match(/\d{4}/) || value.match(/(january|february|march|april|may|june|july|august|september|october|november|december)/i)) {
+                    // Simple year extraction
+                    const year = value.match(/\d{4}/);
+                    if (year) {
+                        slots[key] = parseInt(year[0]);
+                    }
+                }
+                // Clean up captured values
+                slots[key] = value.trim();
+            }
+        }
+    }
+    /**
+     * Helper: Generate regex pattern from query
+     */
+    generateRegexFromQuery(query) {
+        // Simple pattern generation - replace variable parts with capture groups
+        let pattern = query.toLowerCase();
+        // Replace numbers with \d+ capture
+        pattern = pattern.replace(/\d+/g, '(\\d+)');
+        // Replace quoted strings with .+ capture
+        pattern = pattern.replace(/"[^"]+"/g, '(.+)');
+        // Replace proper nouns (capitalized words) with capture
+        pattern = pattern.replace(/\b[A-Z]\w+\b/g, '([A-Z][\\w]+)');
+        return pattern;
+    }
+    /**
+     * Get pattern statistics for monitoring
+     */
+    getStatistics() {
+        const stats = {
+            totalPatterns: this.patterns.size,
+            categories: {},
+            averageConfidence: 0,
+            topPatterns: []
+        };
+        // Count by category
+        for (const pattern of this.patterns.values()) {
+            stats.categories[pattern.category] = (stats.categories[pattern.category] || 0) + 1;
+        }
+        // Calculate average confidence
+        let totalConfidence = 0;
+        for (const confidence of this.successMetrics.values()) {
+            totalConfidence += confidence;
+        }
+        stats.averageConfidence = totalConfidence / this.successMetrics.size;
+        // Get top patterns by success
+        const sortedPatterns = Array.from(this.successMetrics.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 10);
+        stats.topPatterns = sortedPatterns.map(([id, success]) => ({ id, success }));
+        return stats;
+    }
+}
+//# sourceMappingURL=patternLibrary.js.map

package/dist/neural/patterns.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Core Pattern Library with Pre-computed Embeddings
+ *
+ * This file is auto-generated by scripts/buildPatterns.ts
+ * DO NOT EDIT MANUALLY - edit src/patterns/comprehensive-library.json instead
+ *
+ * Storage strategy:
+ * - Patterns are bundled directly into Brainy for zero-latency access
+ * - Embeddings are pre-computed and stored as binary Float32Array
+ * - Total size: ~140KB (negligible for a neural library)
+ * - No external files needed, works in all environments
+ */
+import type { Pattern } from './patternLibrary.js';
+export declare const CORE_PATTERNS: Pattern[];
+export declare const PATTERN_EMBEDDINGS_BINARY: Uint8Array | null;
+export declare function getPatternEmbeddings(): Map<string, Float32Array>;
+export declare const PATTERNS_VERSION = "2.0.0";
+export declare const PATTERNS_METADATA: {
+    totalPatterns: number;
+    categories: string[];
+    embeddingDimensions: number;
+    storageSize: {
+        patterns: string;
+        embeddings: string;
+        total: string;
+    };
+};

package/dist/neural/patterns.js

@@ -0,0 +1,68 @@
+/**
+ * Core Pattern Library with Pre-computed Embeddings
+ *
+ * This file is auto-generated by scripts/buildPatterns.ts
+ * DO NOT EDIT MANUALLY - edit src/patterns/comprehensive-library.json instead
+ *
+ * Storage strategy:
+ * - Patterns are bundled directly into Brainy for zero-latency access
+ * - Embeddings are pre-computed and stored as binary Float32Array
+ * - Total size: ~140KB (negligible for a neural library)
+ * - No external files needed, works in all environments
+ */
+// Pattern data embedded directly for reliability
+export const CORE_PATTERNS = [
+    // Informational queries
+    {
+        id: "info_what_is",
+        category: "informational",
+        examples: ["what is artificial intelligence", "what is machine learning"],
+        pattern: "what is (.+)",
+        template: { like: "${1}" },
+        confidence: 0.9
+    },
+    {
+        id: "info_how_does",
+        category: "informational",
+        examples: ["how does neural network work", "how does deep learning work"],
+        pattern: "how does (.+) work",
+        template: { like: "${1}" },
+        confidence: 0.85
+    },
+    // ... more patterns loaded from library.json at build time
+];
+// Pre-computed embeddings as binary data
+// Generated by scripts/buildPatterns.ts using Brainy's embedding model
+export const PATTERN_EMBEDDINGS_BINARY = null; // Will be populated at build
+// Helper to decode embeddings
+export function getPatternEmbeddings() {
+    if (!PATTERN_EMBEDDINGS_BINARY) {
+        return new Map(); // Will compute at runtime if not pre-built
+    }
+    const embeddings = new Map();
+    const view = new DataView(PATTERN_EMBEDDINGS_BINARY.buffer);
+    const embeddingSize = 384; // Standard size
+    CORE_PATTERNS.forEach((pattern, index) => {
+        const offset = index * embeddingSize * 4; // 4 bytes per float
+        const embedding = new Float32Array(embeddingSize);
+        for (let i = 0; i < embeddingSize; i++) {
+            embedding[i] = view.getFloat32(offset + i * 4, true);
+        }
+        embeddings.set(pattern.id, embedding);
+    });
+    return embeddings;
+}
+// Version for cache invalidation
+export const PATTERNS_VERSION = "2.0.0";
+// Export metadata for monitoring
+export const PATTERNS_METADATA = {
+    totalPatterns: CORE_PATTERNS.length,
+    categories: [...new Set(CORE_PATTERNS.map(p => p.category))],
+    embeddingDimensions: 384,
+    storageSize: {
+        patterns: "24KB",
+        embeddings: "98KB",
+        total: "122KB"
+    }
+};
+//# sourceMappingURL=patterns.js.map

package/dist/neural/staticPatternMatcher.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Static Pattern Matcher - NO runtime initialization, NO BrainyData needed
+ *
+ * All patterns and embeddings are pre-computed at build time
+ * This is pure pattern matching with zero dependencies
+ */
+import { EMBEDDED_PATTERNS } from './embeddedPatterns.js';
+import type { Vector } from '../coreTypes.js';
+import type { TripleQuery } from '../triple/TripleIntelligence.js';
+/**
+ * Match query against all patterns using embeddings
+ */
+export declare function findBestPatterns(queryEmbedding: Vector, k?: number): Array<{
+    pattern: typeof EMBEDDED_PATTERNS[0];
+    similarity: number;
+}>;
+/**
+ * Match query against patterns using regex
+ */
+export declare function matchPatternByRegex(query: string): {
+    pattern: typeof EMBEDDED_PATTERNS[0];
+    slots: Record<string, string>;
+    query: TripleQuery;
+} | null;
+/**
+ * Convert natural language to structured query using STATIC patterns
+ * NO initialization needed, NO BrainyData required
+ */
+export declare function patternMatchQuery(query: string, queryEmbedding?: Vector): TripleQuery;
+export declare const PATTERN_STATS: {
+    totalPatterns: number;
+    categories: string[];
+    domains: string[];
+    hasEmbeddings: boolean;
+};

package/dist/neural/staticPatternMatcher.js

@@ -0,0 +1,153 @@
+/**
+ * Static Pattern Matcher - NO runtime initialization, NO BrainyData needed
+ *
+ * All patterns and embeddings are pre-computed at build time
+ * This is pure pattern matching with zero dependencies
+ */
+import { EMBEDDED_PATTERNS, getPatternEmbeddings } from './embeddedPatterns.js';
+// Pre-load patterns and embeddings at module load time (happens once)
+const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p]));
+const patternEmbeddings = getPatternEmbeddings();
+/**
+ * Cosine similarity between two vectors
+ */
+function cosineSimilarity(a, b) {
+    if (!a || !b || a.length !== b.length)
+        return 0;
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    return denominator === 0 ? 0 : dotProduct / denominator;
+}
+/**
+ * Extract slots from matched pattern
+ */
+function extractSlots(query, pattern) {
+    try {
+        const regex = new RegExp(pattern, 'i');
+        const match = query.match(regex);
+        if (!match)
+            return null;
+        const slots = {};
+        for (let i = 1; i < match.length; i++) {
+            if (match[i]) {
+                slots[`$${i}`] = match[i];
+            }
+        }
+        return Object.keys(slots).length > 0 ? slots : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Apply template with extracted slots
+ */
+function applyTemplate(template, slots) {
+    if (!template || !slots)
+        return template;
+    const result = JSON.parse(JSON.stringify(template));
+    const applySlots = (obj) => {
+        if (typeof obj === 'string') {
+            return obj.replace(/\$\{(\d+)\}/g, (_, num) => slots[`$${num}`] || '');
+        }
+        if (Array.isArray(obj)) {
+            return obj.map(applySlots);
+        }
+        if (typeof obj === 'object' && obj !== null) {
+            const newObj = {};
+            for (const [key, value] of Object.entries(obj)) {
+                newObj[key] = applySlots(value);
+            }
+            return newObj;
+        }
+        return obj;
+    };
+    return applySlots(result);
+}
+/**
+ * Match query against all patterns using embeddings
+ */
+export function findBestPatterns(queryEmbedding, k = 3) {
+    const matches = [];
+    for (const pattern of EMBEDDED_PATTERNS) {
+        const patternEmbedding = patternEmbeddings.get(pattern.id);
+        if (!patternEmbedding)
+            continue;
+        // Pass Float32Array directly, no need for Array.from()!
+        const similarity = cosineSimilarity(queryEmbedding, patternEmbedding);
+        if (similarity > 0.5) { // Threshold for relevance
+            matches.push({ pattern, similarity });
+        }
+    }
+    // Sort by similarity and return top k
+    return matches
+        .sort((a, b) => b.similarity - a.similarity)
+        .slice(0, k);
+}
+/**
+ * Match query against patterns using regex
+ */
+export function matchPatternByRegex(query) {
+    // Try direct regex matching first (fastest)
+    for (const pattern of EMBEDDED_PATTERNS) {
+        const slots = extractSlots(query, pattern.pattern);
+        if (slots) {
+            const templatedQuery = applyTemplate(pattern.template, slots);
+            return {
+                pattern,
+                slots,
+                query: templatedQuery
+            };
+        }
+    }
+    return null;
+}
+/**
+ * Convert natural language to structured query using STATIC patterns
+ * NO initialization needed, NO BrainyData required
+ */
+export function patternMatchQuery(query, queryEmbedding) {
+    // ALWAYS use vector similarity when we have embeddings (which we always do!)
+    if (queryEmbedding && queryEmbedding.length === 384) {
+        const bestPatterns = findBestPatterns(queryEmbedding, 5); // Get top 5 matches
+        // Try to extract slots from best matching patterns
+        for (const { pattern, similarity } of bestPatterns) {
+            // Only try patterns with good similarity
+            if (similarity < 0.7)
+                break;
+            const slots = extractSlots(query, pattern.pattern);
+            if (slots) {
+                // Found a good match with extractable slots!
+                const result = applyTemplate(pattern.template, slots);
+                console.log('[NLP] Applied template with slots:', JSON.stringify(result));
+                return result;
+            }
+        }
+        // If no slots extracted but we have a good match, use the template as-is
+        if (bestPatterns.length > 0 && bestPatterns[0].similarity > 0.75) {
+            console.log('[NLP] Returning template as-is:', JSON.stringify(bestPatterns[0].pattern.template));
+            return bestPatterns[0].pattern.template;
+        }
+    }
+    // Fallback: simple vector search (should rarely happen)
+    console.log('[NLP] Fallback - returning simple query');
+    return {
+        like: query,
+        limit: 10
+    };
+}
+// Export pattern statistics for monitoring
+export const PATTERN_STATS = {
+    totalPatterns: EMBEDDED_PATTERNS.length,
+    categories: [...new Set(EMBEDDED_PATTERNS.map(p => p.category))],
+    domains: [...new Set(EMBEDDED_PATTERNS.filter(p => p.domain).map(p => p.domain))],
+    hasEmbeddings: patternEmbeddings.size > 0
+};
+//# sourceMappingURL=staticPatternMatcher.js.map

package/dist/scripts/precomputePatternEmbeddings.d.ts

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+/**
+ * 🧠 Pre-compute Pattern Embeddings Script
+ *
+ * This script pre-computes embeddings for all patterns and saves them to disk.
+ * Run this once after adding new patterns to avoid runtime embedding costs.
+ *
+ * How it works:
+ * 1. Load all patterns from library.json
+ * 2. Use Brainy's embedding model to encode each pattern's examples
+ * 3. Average the example embeddings to get a robust pattern representation
+ * 4. Save embeddings to patterns/embeddings.bin for instant loading
+ *
+ * Benefits:
+ * - Pattern matching becomes pure math (cosine similarity)
+ * - No embedding model calls during query processing
+ * - Patterns load instantly with pre-computed vectors
+ */
+export {};