@soulcraft/brainy 2.0.2 → 2.1.0

package/README.md

@@ -11,7 +11,7 @@
 
 **🧠 Brainy 2.0 - The Universal Knowledge Protocol™**
 
-**World's first Triple Intelligence™ database**—unifying vector similarity, graph relationships, and document filtering in one magical API. Model ANY data from ANY domain using 24 standardized noun types × 40 verb types.
+**World's first Triple Intelligence™ database**—unifying vector similarity, graph relationships, and document filtering in one magical API. Model ANY data from ANY domain using 31 standardized noun types × 40 verb types.
 
 **Why Brainy Leads**: We're the first to solve the impossible—combining three different database paradigms (vector, graph, document) into one unified query interface. This breakthrough enables us to be the Universal Knowledge Protocol where all tools, augmentations, and AI models speak the same language.
 
@@ -20,7 +20,7 @@
 ## 🎉 What's New in 2.0
 
 - **World's First Triple Intelligence™**: Unified vector + graph + document in ONE query
-- **Universal Knowledge Protocol**: 24 nouns × 40 verbs standardize all knowledge
+- **Universal Knowledge Protocol**: 31 nouns × 40 verbs standardize all knowledge
 - **Infinite Expressiveness**: Model ANY data with unlimited metadata
 - **API Consolidation**: 15+ methods → 2 clean APIs (`search()` and `find()`)
 - **Natural Language**: Ask questions in plain English

package/dist/augmentations/neuralImport.d.ts

@@ -59,6 +59,7 @@ export declare class NeuralImportAugmentation extends BaseAugmentation {
     readonly priority = 80;
     private config;
     private analysisCache;
+    private typeMatcher;
     constructor(config?: Partial<NeuralImportConfig>);
     protected onInitialize(): Promise<void>;
     protected onShutdown(): Promise<void>;
@@ -79,15 +80,23 @@ export declare class NeuralImportAugmentation extends BaseAugmentation {
      */
     private parseRawData;
     /**
-     * Parse CSV data
+     * Parse CSV data - handles quoted values, escaped quotes, and edge cases
      */
     private parseCSV;
+    /**
+     * Parse YAML data
+     */
+    private parseYAML;
+    /**
+     * Parse a YAML value (handle strings, numbers, booleans, null)
+     */
+    private parseYAMLValue;
     /**
      * Perform neural analysis on parsed data
      */
     private performNeuralAnalysis;
     /**
-     * Infer noun type from object structure
+     * Infer noun type from object structure using intelligent type matching
      */
     private inferNounType;
     /**
@@ -95,7 +104,7 @@ export declare class NeuralImportAugmentation extends BaseAugmentation {
      */
     private detectRelationships;
     /**
-     * Infer verb type from field name
+     * Infer verb type from field name using intelligent type matching
      */
     private inferVerbType;
     /**

package/dist/augmentations/neuralImport.js

@@ -8,6 +8,8 @@
  */
 import { BaseAugmentation } from './brainyAugmentation.js';
 import * as path from '../universal/path.js';
+import { getTypeMatcher } from './typeMatching/intelligentTypeMatcher.js';
+import { prodLog } from '../utils/logger.js';
 /**
  * Neural Import Augmentation - Unified Implementation
  * Processes data with AI before storage operations
@@ -20,6 +22,7 @@ export class NeuralImportAugmentation extends BaseAugmentation {
         this.operations = ['add', 'addNoun', 'addVerb', 'all']; // Use 'all' to catch batch operations
         this.priority = 80; // High priority for data processing
         this.analysisCache = new Map();
+        this.typeMatcher = null;
         this.config = {
             confidenceThreshold: 0.7,
             enableWeights: true,
@@ -29,7 +32,13 @@ export class NeuralImportAugmentation extends BaseAugmentation {
         };
     }
     async onInitialize() {
-        this.log('🧠 Neural Import augmentation initialized');
+        try {
+            this.typeMatcher = await getTypeMatcher();
+            this.log('🧠 Neural Import augmentation initialized with intelligent type matching');
+        }
+        catch (error) {
+            this.log('⚠️ Failed to initialize type matcher, falling back to heuristics', 'warn');
+        }
     }
     async onShutdown() {
         this.analysisCache.clear();
@@ -128,13 +137,7 @@ export class NeuralImportAugmentation extends BaseAugmentation {
                 return this.parseCSV(content);
             case 'yaml':
             case 'yml':
-                // For now, basic YAML support - in full implementation would use yaml parser
-                try {
-                    return JSON.parse(content); // Placeholder
-                }
-                catch {
-                    return [{ text: content }];
-                }
+                return this.parseYAML(content);
             case 'txt':
             case 'text':
                 // Split text into sentences/paragraphs for analysis
@@ -145,24 +148,174 @@ export class NeuralImportAugmentation extends BaseAugmentation {
         }
     }
     /**
-     * Parse CSV data
+     * Parse CSV data - handles quoted values, escaped quotes, and edge cases
      */
     parseCSV(content) {
-        const lines = content.split('\n').filter(line => line.trim());
+        const lines = content.split('\n');
         if (lines.length === 0)
             return [];
-        const headers = lines[0].split(',').map(h => h.trim());
+        // Parse a CSV line handling quotes
+        const parseLine = (line) => {
+            const result = [];
+            let current = '';
+            let inQuotes = false;
+            let i = 0;
+            while (i < line.length) {
+                const char = line[i];
+                const nextChar = line[i + 1];
+                if (char === '"') {
+                    if (inQuotes && nextChar === '"') {
+                        // Escaped quote
+                        current += '"';
+                        i += 2;
+                    }
+                    else {
+                        // Toggle quote mode
+                        inQuotes = !inQuotes;
+                        i++;
+                    }
+                }
+                else if (char === ',' && !inQuotes) {
+                    // Field separator
+                    result.push(current.trim());
+                    current = '';
+                    i++;
+                }
+                else {
+                    current += char;
+                    i++;
+                }
+            }
+            // Add last field
+            result.push(current.trim());
+            return result;
+        };
+        // Parse headers
+        const headers = parseLine(lines[0]);
         const data = [];
+        // Parse data rows
         for (let i = 1; i < lines.length; i++) {
-            const values = lines[i].split(',').map(v => v.trim());
+            const line = lines[i].trim();
+            if (!line)
+                continue; // Skip empty lines
+            const values = parseLine(line);
             const row = {};
             headers.forEach((header, index) => {
-                row[header] = values[index] || '';
+                const value = values[index] || '';
+                // Try to parse numbers
+                const num = Number(value);
+                row[header] = !isNaN(num) && value !== '' ? num : value;
             });
             data.push(row);
         }
         return data;
     }
+    /**
+     * Parse YAML data
+     */
+    parseYAML(content) {
+        try {
+            // Simple YAML parser for basic structures
+            // For full YAML support, we'd use js-yaml library
+            const lines = content.split('\n');
+            const result = [];
+            let currentObject = null;
+            let currentIndent = 0;
+            for (const line of lines) {
+                const trimmed = line.trim();
+                if (!trimmed || trimmed.startsWith('#'))
+                    continue; // Skip empty lines and comments
+                // Calculate indentation
+                const indent = line.length - line.trimStart().length;
+                // Check for array item
+                if (trimmed.startsWith('- ')) {
+                    const value = trimmed.substring(2).trim();
+                    if (indent === 0) {
+                        // Top-level array item
+                        if (value.includes(':')) {
+                            // Object in array
+                            currentObject = {};
+                            result.push(currentObject);
+                            const [key, val] = value.split(':').map(s => s.trim());
+                            currentObject[key] = this.parseYAMLValue(val);
+                        }
+                        else {
+                            result.push(this.parseYAMLValue(value));
+                        }
+                    }
+                    else if (currentObject) {
+                        // Nested array
+                        const lastKey = Object.keys(currentObject).pop();
+                        if (lastKey) {
+                            if (!Array.isArray(currentObject[lastKey])) {
+                                currentObject[lastKey] = [];
+                            }
+                            currentObject[lastKey].push(this.parseYAMLValue(value));
+                        }
+                    }
+                }
+                else if (trimmed.includes(':')) {
+                    // Key-value pair
+                    const colonIndex = trimmed.indexOf(':');
+                    const key = trimmed.substring(0, colonIndex).trim();
+                    const value = trimmed.substring(colonIndex + 1).trim();
+                    if (indent === 0) {
+                        // Top-level object
+                        if (!currentObject) {
+                            currentObject = {};
+                            result.push(currentObject);
+                        }
+                        currentObject[key] = this.parseYAMLValue(value);
+                        currentIndent = 0;
+                    }
+                    else if (currentObject) {
+                        // Nested object
+                        if (indent > currentIndent && !value) {
+                            // Start of nested object
+                            const lastKey = Object.keys(currentObject).pop();
+                            if (lastKey) {
+                                currentObject[lastKey] = { [key]: '' };
+                            }
+                        }
+                        else {
+                            currentObject[key] = this.parseYAMLValue(value);
+                        }
+                        currentIndent = indent;
+                    }
+                }
+            }
+            // If we built a single object and not an array, wrap it
+            if (result.length === 0 && currentObject) {
+                result.push(currentObject);
+            }
+            return result.length > 0 ? result : [{ text: content }];
+        }
+        catch (error) {
+            prodLog.warn('YAML parsing failed, treating as text:', error);
+            return [{ text: content }];
+        }
+    }
+    /**
+     * Parse a YAML value (handle strings, numbers, booleans, null)
+     */
+    parseYAMLValue(value) {
+        if (!value || value === '~' || value === 'null')
+            return null;
+        if (value === 'true')
+            return true;
+        if (value === 'false')
+            return false;
+        // Remove quotes if present
+        if ((value.startsWith('"') && value.endsWith('"')) ||
+            (value.startsWith("'") && value.endsWith("'"))) {
+            return value.slice(1, -1);
+        }
+        // Try to parse as number
+        const num = Number(value);
+        if (!isNaN(num) && value !== '')
+            return num;
+        return value;
+    }
     /**
      * Perform neural analysis on parsed data
      */
@@ -177,14 +330,14 @@ export class NeuralImportAugmentation extends BaseAugmentation {
                 const entityId = item.id || item.name || item.title || `entity_${Date.now()}_${Math.random()}`;
                 detectedEntities.push({
                     originalData: item,
-                    nounType: this.inferNounType(item),
+                    nounType: await this.inferNounType(item),
                     confidence: 0.85,
                     suggestedId: String(entityId),
                     reasoning: 'Detected from structured data',
                     alternativeTypes: []
                 });
                 // Detect relationships from references
-                this.detectRelationships(item, entityId, detectedRelationships);
+                await this.detectRelationships(item, entityId, detectedRelationships);
             }
         }
         // Generate insights
@@ -216,36 +369,31 @@ export class NeuralImportAugmentation extends BaseAugmentation {
         };
     }
     /**
-     * Infer noun type from object structure
+     * Infer noun type from object structure using intelligent type matching
      */
-    inferNounType(obj) {
-        // Simple heuristics for type detection
-        if (obj.email || obj.username)
-            return 'Person';
-        if (obj.title && obj.content)
-            return 'Document';
-        if (obj.price || obj.product)
-            return 'Product';
-        if (obj.date || obj.timestamp)
-            return 'Event';
-        if (obj.url || obj.link)
-            return 'Resource';
-        if (obj.lat || obj.longitude)
-            return 'Location';
-        // Default fallback
-        return 'Entity';
+    async inferNounType(obj) {
+        if (!this.typeMatcher) {
+            // Initialize type matcher if not available
+            this.typeMatcher = await getTypeMatcher();
+        }
+        const result = await this.typeMatcher.matchNounType(obj);
+        // Log if confidence is low for debugging
+        if (result.confidence < 0.5) {
+            this.log(`Low confidence (${result.confidence.toFixed(2)}) for noun type: ${result.type}`, 'warn');
+        }
+        return result.type;
     }
     /**
      * Detect relationships from object references
      */
-    detectRelationships(obj, sourceId, relationships) {
+    async detectRelationships(obj, sourceId, relationships) {
         // Look for reference patterns
         for (const [key, value] of Object.entries(obj)) {
             if (key.endsWith('Id') || key.endsWith('_id') || key === 'parentId' || key === 'userId') {
                 relationships.push({
                     sourceId,
                     targetId: String(value),
-                    verbType: this.inferVerbType(key),
+                    verbType: await this.inferVerbType(key, obj, { id: value }),
                     confidence: 0.75,
                     weight: 1,
                     reasoning: `Reference detected in field: ${key}`,
@@ -259,7 +407,7 @@ export class NeuralImportAugmentation extends BaseAugmentation {
                         relationships.push({
                             sourceId,
                             targetId: String(targetId),
-                            verbType: this.inferVerbType(key),
+                            verbType: await this.inferVerbType(key, obj, { id: targetId }),
                             confidence: 0.7,
                             weight: 1,
                             reasoning: `Array reference in field: ${key}`,
@@ -271,27 +419,19 @@ export class NeuralImportAugmentation extends BaseAugmentation {
         }
     }
     /**
-     * Infer verb type from field name
+     * Infer verb type from field name using intelligent type matching
      */
-    inferVerbType(fieldName) {
-        const normalized = fieldName.toLowerCase();
-        if (normalized.includes('parent'))
-            return 'childOf';
-        if (normalized.includes('user'))
-            return 'belongsTo';
-        if (normalized.includes('author'))
-            return 'authoredBy';
-        if (normalized.includes('owner'))
-            return 'ownedBy';
-        if (normalized.includes('creator'))
-            return 'createdBy';
-        if (normalized.includes('member'))
-            return 'memberOf';
-        if (normalized.includes('tag'))
-            return 'taggedWith';
-        if (normalized.includes('category'))
-            return 'categorizedAs';
-        return 'relatedTo';
+    async inferVerbType(fieldName, sourceObj, targetObj) {
+        if (!this.typeMatcher) {
+            // Initialize type matcher if not available
+            this.typeMatcher = await getTypeMatcher();
+        }
+        const result = await this.typeMatcher.matchVerbType(sourceObj, targetObj, fieldName);
+        // Log if confidence is low for debugging
+        if (result.confidence < 0.5) {
+            this.log(`Low confidence (${result.confidence.toFixed(2)}) for verb type: ${result.type}`, 'warn');
+        }
+        return result.type;
     }
     /**
      * Group entities by type

package/dist/augmentations/typeMatching/intelligentTypeMatcher.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Intelligent Type Matcher - Uses embeddings for semantic type detection
+ *
+ * This module uses our existing TransformerEmbedding and similarity functions
+ * to intelligently match data to our 31 noun types and 40 verb types.
+ *
+ * Features:
+ * - Semantic similarity matching using embeddings
+ * - Context-aware type detection
+ * - Confidence scoring
+ * - Caching for performance
+ */
+/**
+ * Result of type matching with confidence scores
+ */
+export interface TypeMatchResult {
+    type: string;
+    confidence: number;
+    reasoning: string;
+    alternatives: Array<{
+        type: string;
+        confidence: number;
+    }>;
+}
+/**
+ * Intelligent Type Matcher using semantic embeddings
+ */
+export declare class IntelligentTypeMatcher {
+    private embedder;
+    private nounEmbeddings;
+    private verbEmbeddings;
+    private initialized;
+    private cache;
+    constructor();
+    /**
+     * Initialize the type matcher by generating embeddings for all types
+     */
+    init(): Promise<void>;
+    /**
+     * Match an object to the most appropriate noun type
+     */
+    matchNounType(obj: any): Promise<TypeMatchResult>;
+    /**
+     * Match a relationship to the most appropriate verb type
+     */
+    matchVerbType(sourceObj: any, targetObj: any, relationshipHint?: string): Promise<TypeMatchResult>;
+    /**
+     * Create text representation of an object for embedding
+     */
+    private createTextRepresentation;
+    /**
+     * Create text representation of a relationship
+     */
+    private createRelationshipText;
+    /**
+     * Get a brief summary of an object
+     */
+    private getObjectSummary;
+    /**
+     * Apply heuristic rules for noun type detection
+     */
+    private applyNounHeuristics;
+    /**
+     * Apply heuristic rules for verb type detection
+     */
+    private applyVerbHeuristics;
+    /**
+     * Generate human-readable reasoning for the type selection
+     */
+    private generateReasoning;
+    /**
+     * Clear the cache
+     */
+    clearCache(): void;
+    /**
+     * Dispose of resources
+     */
+    dispose(): Promise<void>;
+}
+/**
+ * Get or create the global type matcher instance
+ */
+export declare function getTypeMatcher(): Promise<IntelligentTypeMatcher>;