@soulcraft/brainy 4.1.4 → 4.2.0

package/dist/importers/SmartYAMLImporter.d.ts

@@ -0,0 +1,121 @@
+/**
+ * Smart YAML Importer
+ *
+ * Extracts entities and relationships from YAML files using:
+ * - YAML parsing to JSON-like structure
+ * - Recursive traversal of nested structures
+ * - NeuralEntityExtractor for entity extraction from text values
+ * - NaturalLanguageProcessor for relationship inference
+ * - Hierarchical relationship creation (parent-child, contains, etc.)
+ *
+ * v4.2.0: New format handler
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+export interface SmartYAMLOptions {
+    /** Enable neural entity extraction from string values */
+    enableNeuralExtraction?: boolean;
+    /** Enable hierarchical relationship creation */
+    enableHierarchicalRelationships?: boolean;
+    /** Enable concept extraction for tagging */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities (0-1) */
+    confidenceThreshold?: number;
+    /** Maximum depth to traverse */
+    maxDepth?: number;
+    /** Minimum string length to process for entity extraction */
+    minStringLength?: number;
+    /** Keys that indicate entity names */
+    nameKeys?: string[];
+    /** Keys that indicate entity descriptions */
+    descriptionKeys?: string[];
+    /** Keys that indicate entity types */
+    typeKeys?: string[];
+    /** Progress callback */
+    onProgress?: (stats: {
+        processed: number;
+        entities: number;
+        relationships: number;
+    }) => void;
+}
+export interface ExtractedYAMLEntity {
+    /** Entity ID */
+    id: string;
+    /** Entity name */
+    name: string;
+    /** Entity type */
+    type: NounType;
+    /** Entity description/value */
+    description: string;
+    /** Confidence score */
+    confidence: number;
+    /** Weight/importance score */
+    weight?: number;
+    /** YAML path to this entity */
+    path: string;
+    /** Parent path in YAML hierarchy */
+    parentPath: string | null;
+    /** Metadata */
+    metadata: Record<string, any>;
+}
+export interface ExtractedYAMLRelationship {
+    from: string;
+    to: string;
+    type: VerbType;
+    confidence: number;
+    weight?: number;
+    evidence: string;
+}
+export interface SmartYAMLResult {
+    /** Total nodes processed */
+    nodesProcessed: number;
+    /** Entities extracted */
+    entitiesExtracted: number;
+    /** Relationships inferred */
+    relationshipsInferred: number;
+    /** All extracted entities */
+    entities: ExtractedYAMLEntity[];
+    /** All relationships */
+    relationships: ExtractedYAMLRelationship[];
+    /** Entity ID mapping (path -> ID) */
+    entityMap: Map<string, string>;
+    /** Processing time in ms */
+    processingTime: number;
+    /** Extraction statistics */
+    stats: {
+        byType: Record<string, number>;
+        byDepth: Record<number, number>;
+        byConfidence: {
+            high: number;
+            medium: number;
+            low: number;
+        };
+    };
+}
+/**
+ * SmartYAMLImporter - Extracts structured knowledge from YAML files
+ */
+export declare class SmartYAMLImporter {
+    private brain;
+    private extractor;
+    private nlp;
+    private relationshipExtractor;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the importer
+     */
+    init(): Promise<void>;
+    /**
+     * Extract entities and relationships from YAML string or buffer
+     */
+    extract(yamlContent: string | Buffer, options?: SmartYAMLOptions): Promise<SmartYAMLResult>;
+    /**
+     * Extract entities and relationships from parsed YAML data
+     */
+    private extractFromData;
+    /**
+     * Extract an entity from a YAML object node
+     */
+    private extractEntityFromObject;
+}

package/dist/importers/SmartYAMLImporter.js

@@ -0,0 +1,275 @@
+/**
+ * Smart YAML Importer
+ *
+ * Extracts entities and relationships from YAML files using:
+ * - YAML parsing to JSON-like structure
+ * - Recursive traversal of nested structures
+ * - NeuralEntityExtractor for entity extraction from text values
+ * - NaturalLanguageProcessor for relationship inference
+ * - Hierarchical relationship creation (parent-child, contains, etc.)
+ *
+ * v4.2.0: New format handler
+ * NO MOCKS - Production-ready implementation
+ */
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+import * as yaml from 'js-yaml';
+/**
+ * SmartYAMLImporter - Extracts structured knowledge from YAML files
+ */
+export class SmartYAMLImporter {
+    constructor(brain) {
+        this.brain = brain;
+        this.extractor = new NeuralEntityExtractor(brain);
+        this.nlp = new NaturalLanguageProcessor(brain);
+        this.relationshipExtractor = new SmartRelationshipExtractor(brain);
+    }
+    /**
+     * Initialize the importer
+     */
+    async init() {
+        await this.nlp.init();
+    }
+    /**
+     * Extract entities and relationships from YAML string or buffer
+     */
+    async extract(yamlContent, options = {}) {
+        const startTime = Date.now();
+        // Parse YAML to JavaScript object
+        const yamlString = typeof yamlContent === 'string'
+            ? yamlContent
+            : yamlContent.toString('utf-8');
+        let data;
+        try {
+            data = yaml.load(yamlString);
+        }
+        catch (error) {
+            throw new Error(`Failed to parse YAML: ${error.message}`);
+        }
+        // Process as JSON-like structure
+        const result = await this.extractFromData(data, options);
+        result.processingTime = Date.now() - startTime;
+        return result;
+    }
+    /**
+     * Extract entities and relationships from parsed YAML data
+     */
+    async extractFromData(data, options) {
+        const opts = {
+            enableNeuralExtraction: options.enableNeuralExtraction !== false,
+            enableHierarchicalRelationships: options.enableHierarchicalRelationships !== false,
+            enableConceptExtraction: options.enableConceptExtraction !== false,
+            confidenceThreshold: options.confidenceThreshold || 0.6,
+            maxDepth: options.maxDepth || 10,
+            minStringLength: options.minStringLength || 3,
+            nameKeys: options.nameKeys || ['name', 'title', 'label', 'id'],
+            descriptionKeys: options.descriptionKeys || ['description', 'desc', 'summary', 'value'],
+            typeKeys: options.typeKeys || ['type', 'kind', 'category'],
+            onProgress: options.onProgress
+        };
+        const entities = [];
+        const relationships = [];
+        const entityMap = new Map();
+        let nodesProcessed = 0;
+        const stats = {
+            byType: {},
+            byDepth: {},
+            byConfidence: { high: 0, medium: 0, low: 0 }
+        };
+        // Traverse YAML structure recursively
+        const traverse = async (obj, path = '$', depth = 0, parentPath = null) => {
+            if (depth > opts.maxDepth)
+                return;
+            nodesProcessed++;
+            stats.byDepth[depth] = (stats.byDepth[depth] || 0) + 1;
+            // Report progress
+            if (options.onProgress && nodesProcessed % 10 === 0) {
+                options.onProgress({
+                    processed: nodesProcessed,
+                    entities: entities.length,
+                    relationships: relationships.length
+                });
+            }
+            // Handle different value types
+            if (obj === null || obj === undefined) {
+                return;
+            }
+            // Handle arrays
+            if (Array.isArray(obj)) {
+                for (let i = 0; i < obj.length; i++) {
+                    await traverse(obj[i], `${path}[${i}]`, depth + 1, path);
+                }
+                return;
+            }
+            // Handle objects
+            if (typeof obj === 'object') {
+                // Extract entity from object
+                const entity = await this.extractEntityFromObject(obj, path, parentPath, depth, opts);
+                if (entity) {
+                    entities.push(entity);
+                    entityMap.set(path, entity.id);
+                    // Update stats
+                    stats.byType[entity.type] = (stats.byType[entity.type] || 0) + 1;
+                    if (entity.confidence > 0.8)
+                        stats.byConfidence.high++;
+                    else if (entity.confidence >= 0.6)
+                        stats.byConfidence.medium++;
+                    else
+                        stats.byConfidence.low++;
+                    // Create hierarchical relationship
+                    if (opts.enableHierarchicalRelationships && parentPath) {
+                        const parentId = entityMap.get(parentPath);
+                        if (parentId) {
+                            // Extract parent name from path for better context
+                            const parentName = parentPath.split('.').pop()?.replace(/\[(\d+)\]/, 'item $1') || 'parent';
+                            const childName = entity.name;
+                            // Infer relationship type using SmartRelationshipExtractor
+                            const context = `Hierarchical YAML structure: ${parentName} contains ${childName}. Parent path: ${parentPath}, Child path: ${entity.path}`;
+                            const inferredRelationship = await this.relationshipExtractor.infer(parentName, childName, context, {
+                                objectType: entity.type // Pass child entity type as hint
+                            });
+                            relationships.push({
+                                from: parentId,
+                                to: entity.id,
+                                type: inferredRelationship?.type || VerbType.Contains, // Fallback to Contains for hierarchical relationships
+                                confidence: inferredRelationship?.confidence || 0.9,
+                                weight: inferredRelationship?.weight || 1.0,
+                                evidence: inferredRelationship?.evidence || 'Hierarchical parent-child relationship in YAML structure'
+                            });
+                        }
+                    }
+                }
+                // Traverse nested objects
+                for (const [key, value] of Object.entries(obj)) {
+                    await traverse(value, `${path}.${key}`, depth + 1, path);
+                }
+                return;
+            }
+            // Handle primitive values (strings, numbers, booleans)
+            if (typeof obj === 'string' && obj.length >= opts.minStringLength) {
+                // Extract entities from string values
+                if (opts.enableNeuralExtraction) {
+                    const extractedEntities = await this.extractor.extract(obj, {
+                        confidence: opts.confidenceThreshold
+                    });
+                    for (const extracted of extractedEntities) {
+                        const entityId = `${path}:${extracted.text}`;
+                        const entity = {
+                            id: entityId,
+                            name: extracted.text,
+                            type: extracted.type,
+                            description: obj,
+                            confidence: extracted.confidence,
+                            weight: extracted.weight || 1.0,
+                            path,
+                            parentPath,
+                            metadata: {
+                                position: extracted.position,
+                                extractedFrom: 'string-value'
+                            }
+                        };
+                        entities.push(entity);
+                        entityMap.set(entityId, entityId);
+                        // Update stats
+                        stats.byType[entity.type] = (stats.byType[entity.type] || 0) + 1;
+                        if (entity.confidence > 0.8)
+                            stats.byConfidence.high++;
+                        else if (entity.confidence >= 0.6)
+                            stats.byConfidence.medium++;
+                        else
+                            stats.byConfidence.low++;
+                    }
+                }
+            }
+        };
+        // Start traversal
+        await traverse(data);
+        // Final progress report
+        if (options.onProgress) {
+            options.onProgress({
+                processed: nodesProcessed,
+                entities: entities.length,
+                relationships: relationships.length
+            });
+        }
+        return {
+            nodesProcessed,
+            entitiesExtracted: entities.length,
+            relationshipsInferred: relationships.length,
+            entities,
+            relationships,
+            entityMap,
+            processingTime: 0, // Will be set by caller
+            stats
+        };
+    }
+    /**
+     * Extract an entity from a YAML object node
+     */
+    async extractEntityFromObject(obj, path, parentPath, depth, opts) {
+        // Try to find name
+        let name = null;
+        for (const key of opts.nameKeys) {
+            if (obj[key] && typeof obj[key] === 'string') {
+                name = obj[key];
+                break;
+            }
+        }
+        // If no explicit name, use path segment
+        if (!name) {
+            const segments = path.split('.');
+            name = segments[segments.length - 1];
+            if (name === '$')
+                name = 'root';
+        }
+        // Try to find description
+        let description = name;
+        for (const key of opts.descriptionKeys) {
+            if (obj[key] && typeof obj[key] === 'string') {
+                description = obj[key];
+                break;
+            }
+        }
+        // Try to find explicit type
+        let explicitType = null;
+        for (const key of opts.typeKeys) {
+            if (obj[key] && typeof obj[key] === 'string') {
+                explicitType = obj[key];
+                break;
+            }
+        }
+        // Classify entity type using SmartExtractor
+        const classification = await this.extractor.extract(description, {
+            confidence: opts.confidenceThreshold
+        });
+        const entityType = classification.length > 0
+            ? classification[0].type
+            : NounType.Thing;
+        const confidence = classification.length > 0
+            ? classification[0].confidence
+            : 0.5;
+        const weight = classification.length > 0
+            ? classification[0].weight || 1.0
+            : 1.0;
+        // Create entity
+        const entity = {
+            id: path,
+            name,
+            type: entityType,
+            description,
+            confidence,
+            weight,
+            path,
+            parentPath,
+            metadata: {
+                depth,
+                explicitType,
+                yamlKeys: Object.keys(obj)
+            }
+        };
+        return entity;
+    }
+}
+//# sourceMappingURL=SmartYAMLImporter.js.map

package/dist/importers/VFSStructureGenerator.js

@@ -187,6 +187,14 @@ export class VFSStructureGenerator {
      */
     groupEntities(importResult, options) {
         const groups = new Map();
+        // Handle sheet-based grouping (v4.2.0)
+        if (options.groupBy === 'sheet' && importResult.sheets && importResult.sheets.length > 0) {
+            for (const sheet of importResult.sheets) {
+                groups.set(sheet.name, sheet.rows);
+            }
+            return groups;
+        }
+        // Handle other grouping strategies
         for (const extracted of importResult.rows) {
             let groupName;
             switch (options.groupBy) {
@@ -201,6 +209,10 @@ export class VFSStructureGenerator {
                         options.customGrouping(extracted.entity) :
                         'entities';
                     break;
+                case 'sheet':
+                    // Fallback if sheets data not available
+                    groupName = 'entities';
+                    break;
                 default:
                     groupName = 'entities';
             }

package/dist/neural/SmartExtractor.d.ts

@@ -0,0 +1,279 @@
+/**
+ * SmartExtractor - Unified entity type extraction using ensemble of neural signals
+ *
+ * PRODUCTION-READY: Single orchestration class for all entity type classification
+ *
+ * Design Philosophy:
+ * - Simplicity over complexity (KISS principle)
+ * - One class instead of multiple strategy layers
+ * - Clear execution path for debugging
+ * - Comprehensive format intelligence built-in
+ *
+ * Ensemble Architecture:
+ * - ExactMatchSignal (40%) - Explicit patterns and exact keywords
+ * - EmbeddingSignal (35%) - Neural similarity with type embeddings
+ * - PatternSignal (20%) - Regex patterns and naming conventions
+ * - ContextSignal (5%) - Relationship-based inference
+ *
+ * Format Intelligence:
+ * Supports 7 major formats with automatic hint extraction:
+ * - Excel (.xlsx): Column headers, sheet names, "Related Terms" detection
+ * - CSV (.csv): Header row patterns, naming conventions
+ * - PDF (.pdf): Form field names and labels
+ * - YAML (.yaml, .yml): Semantic key names
+ * - DOCX (.docx): Heading levels and structure
+ * - JSON (.json): Field name patterns
+ * - Markdown (.md): Heading hierarchy
+ *
+ * Performance:
+ * - Parallel signal execution (~15ms total)
+ * - LRU caching for hot entities
+ * - Confidence boosting when signals agree
+ * - Graceful degradation on errors
+ */
+import type { Brainy } from '../brainy.js';
+import type { NounType } from '../types/graphTypes.js';
+/**
+ * Extraction result with full traceability
+ */
+export interface ExtractionResult {
+    type: NounType;
+    confidence: number;
+    source: 'ensemble' | 'exact-match' | 'pattern' | 'embedding' | 'context';
+    evidence: string;
+    metadata?: {
+        signalResults?: Array<{
+            signal: string;
+            type: NounType;
+            confidence: number;
+            weight: number;
+        }>;
+        agreementBoost?: number;
+        formatHints?: string[];
+        formatContext?: FormatContext;
+    };
+}
+/**
+ * Format context for classification
+ */
+export interface FormatContext {
+    format?: 'excel' | 'csv' | 'pdf' | 'yaml' | 'docx' | 'json' | 'markdown';
+    columnHeader?: string;
+    fieldName?: string;
+    yamlKey?: string;
+    headingLevel?: number;
+    sheetName?: string;
+    metadata?: Record<string, any>;
+}
+/**
+ * Options for SmartExtractor
+ */
+export interface SmartExtractorOptions {
+    minConfidence?: number;
+    enableFormatHints?: boolean;
+    enableEnsemble?: boolean;
+    cacheSize?: number;
+    weights?: {
+        exactMatch?: number;
+        embedding?: number;
+        pattern?: number;
+        context?: number;
+    };
+}
+/**
+ * SmartExtractor - Unified entity type classification
+ *
+ * This is the single entry point for all entity type extraction.
+ * It orchestrates all 4 signals, applies format intelligence,
+ * and combines results using ensemble weighting.
+ *
+ * Production features:
+ * - Parallel signal execution for performance
+ * - Format-specific hint extraction
+ * - Ensemble voting with confidence boosting
+ * - Comprehensive statistics and observability
+ * - LRU caching for hot paths
+ * - Graceful error handling
+ */
+export declare class SmartExtractor {
+    private brain;
+    private options;
+    private exactMatchSignal;
+    private patternSignal;
+    private embeddingSignal;
+    private contextSignal;
+    private cache;
+    private cacheOrder;
+    private stats;
+    constructor(brain: Brainy, options?: SmartExtractorOptions);
+    /**
+     * Extract entity type using ensemble of signals
+     *
+     * Main entry point - orchestrates all signals and combines results
+     *
+     * @param candidate Entity text to classify
+     * @param context Classification context with format hints
+     * @returns ExtractionResult with type and confidence
+     */
+    extract(candidate: string, context?: {
+        definition?: string;
+        formatContext?: FormatContext;
+        allTerms?: string[];
+        metadata?: any;
+    }): Promise<ExtractionResult | null>;
+    /**
+     * Extract format-specific hints from context
+     *
+     * Returns array of hint strings that can help with classification
+     */
+    private extractFormatHints;
+    /**
+     * Extract Excel-specific hints
+     */
+    private extractExcelHints;
+    /**
+     * Extract CSV-specific hints
+     */
+    private extractCsvHints;
+    /**
+     * Extract PDF-specific hints
+     */
+    private extractPdfHints;
+    /**
+     * Extract YAML-specific hints
+     */
+    private extractYamlHints;
+    /**
+     * Extract DOCX-specific hints
+     */
+    private extractDocxHints;
+    /**
+     * Extract JSON-specific hints
+     */
+    private extractJsonHints;
+    /**
+     * Extract Markdown-specific hints
+     */
+    private extractMarkdownHints;
+    /**
+     * Combine signal results using ensemble voting
+     *
+     * Applies weighted voting with confidence boosting when signals agree
+     */
+    private combineEnsemble;
+    /**
+     * Select best single signal (when ensemble is disabled)
+     */
+    private selectBestSignal;
+    /**
+     * Update statistics based on result
+     */
+    private updateStatistics;
+    /**
+     * Get cache key from candidate and context
+     */
+    private getCacheKey;
+    /**
+     * Get from LRU cache
+     */
+    private getFromCache;
+    /**
+     * Add to LRU cache with eviction
+     */
+    private addToCache;
+    /**
+     * Get comprehensive statistics
+     */
+    getStats(): {
+        cacheSize: number;
+        cacheHitRate: number;
+        ensembleRate: number;
+        formatHintRate: number;
+        signalStats: {
+            exactMatch: {
+                indexSize: number;
+                cacheSize: number;
+                cacheHitRate: number;
+                termMatchRate: number;
+                metadataMatchRate: number;
+                formatMatchRate: number;
+                calls: number;
+                cacheHits: number;
+                termMatches: number;
+                metadataMatches: number;
+                formatMatches: number;
+            };
+            pattern: {
+                cacheSize: number;
+                patternCount: number;
+                cacheHitRate: number;
+                regexMatchRate: number;
+                namingMatchRate: number;
+                structuralMatchRate: number;
+                calls: number;
+                cacheHits: number;
+                regexMatches: number;
+                namingMatches: number;
+                structuralMatches: number;
+            };
+            embedding: {
+                cacheSize: number;
+                historySize: number;
+                cacheHitRate: number;
+                typeMatchRate: number;
+                graphMatchRate: number;
+                historyMatchRate: number;
+                calls: number;
+                cacheHits: number;
+                typeMatches: number;
+                graphMatches: number;
+                historyMatches: number;
+                combinedBoosts: number;
+            };
+            context: {
+                cacheSize: number;
+                cacheHitRate: number;
+                relationshipMatchRate: number;
+                attributeMatchRate: number;
+                calls: number;
+                cacheHits: number;
+                relationshipMatches: number;
+                attributeMatches: number;
+                combinedMatches: number;
+            };
+        };
+        calls: number;
+        cacheHits: number;
+        exactMatchWins: number;
+        patternWins: number;
+        embeddingWins: number;
+        contextWins: number;
+        ensembleWins: number;
+        agreementBoosts: number;
+        formatHintsUsed: number;
+        averageConfidence: number;
+        averageSignalsUsed: number;
+    };
+    /**
+     * Reset all statistics
+     */
+    resetStats(): void;
+    /**
+     * Clear all caches
+     */
+    clearCache(): void;
+    /**
+     * Add entity to historical data (for embedding signal temporal boosting)
+     */
+    addToHistory(text: string, type: NounType, vector: number[]): void;
+    /**
+     * Clear historical data
+     */
+    clearHistory(): void;
+}
+/**
+ * Create a new SmartExtractor instance
+ *
+ * Convenience factory function
+ */
+export declare function createSmartExtractor(brain: Brainy, options?: SmartExtractorOptions): SmartExtractor;