@soulcraft/brainy 3.27.1 → 3.28.0

package/dist/importers/SmartJSONImporter.js

@@ -0,0 +1,325 @@
+/**
+ * Smart JSON Importer
+ *
+ * Extracts entities and relationships from JSON files using:
+ * - Recursive traversal of nested structures
+ * - NeuralEntityExtractor for entity extraction from text values
+ * - NaturalLanguageProcessor for relationship inference
+ * - Hierarchical relationship creation (parent-child, contains, etc.)
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+/**
+ * SmartJSONImporter - Extracts structured knowledge from JSON files
+ */
+export class SmartJSONImporter {
+    constructor(brain) {
+        this.brain = brain;
+        this.extractor = new NeuralEntityExtractor(brain);
+        this.nlp = new NaturalLanguageProcessor(brain);
+    }
+    /**
+     * Initialize the importer
+     */
+    async init() {
+        await this.nlp.init();
+    }
+    /**
+     * Extract entities and relationships from JSON data
+     */
+    async extract(data, options = {}) {
+        const startTime = Date.now();
+        // Set defaults
+        const opts = {
+            enableNeuralExtraction: true,
+            enableHierarchicalRelationships: true,
+            enableConceptExtraction: true,
+            confidenceThreshold: 0.6,
+            maxDepth: 10,
+            minStringLength: 20,
+            nameKeys: ['name', 'title', 'label', 'id', 'key'],
+            descriptionKeys: ['description', 'desc', 'details', 'text', 'content', 'summary'],
+            typeKeys: ['type', 'kind', 'category', 'class'],
+            onProgress: () => { },
+            ...options
+        };
+        // Parse JSON if string
+        let jsonData;
+        if (typeof data === 'string') {
+            try {
+                jsonData = JSON.parse(data);
+            }
+            catch (error) {
+                throw new Error(`Invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        else {
+            jsonData = data;
+        }
+        // Traverse and extract
+        const entities = [];
+        const relationships = [];
+        const entityMap = new Map();
+        const stats = {
+            byType: {},
+            byDepth: {},
+            byConfidence: { high: 0, medium: 0, low: 0 }
+        };
+        let nodesProcessed = 0;
+        // Recursive traversal
+        await this.traverseJSON(jsonData, '', null, 0, opts, entities, relationships, entityMap, stats, () => {
+            nodesProcessed++;
+            if (nodesProcessed % 10 === 0) {
+                opts.onProgress({
+                    processed: nodesProcessed,
+                    entities: entities.length,
+                    relationships: relationships.length
+                });
+            }
+        });
+        return {
+            nodesProcessed,
+            entitiesExtracted: entities.length,
+            relationshipsInferred: relationships.length,
+            entities,
+            relationships,
+            entityMap,
+            processingTime: Date.now() - startTime,
+            stats
+        };
+    }
+    /**
+     * Recursively traverse JSON structure
+     */
+    async traverseJSON(node, path, parentPath, depth, options, entities, relationships, entityMap, stats, onNode) {
+        // Stop if max depth reached
+        if (depth > options.maxDepth)
+            return;
+        onNode();
+        stats.byDepth[depth] = (stats.byDepth[depth] || 0) + 1;
+        // Handle null/undefined
+        if (node === null || node === undefined)
+            return;
+        // Handle arrays
+        if (Array.isArray(node)) {
+            for (let i = 0; i < node.length; i++) {
+                await this.traverseJSON(node[i], `${path}[${i}]`, path, depth + 1, options, entities, relationships, entityMap, stats, onNode);
+            }
+            return;
+        }
+        // Handle objects
+        if (typeof node === 'object') {
+            // Extract entity from this object
+            const entity = await this.extractEntityFromObject(node, path, parentPath, depth, options, stats);
+            if (entity) {
+                entities.push(entity);
+                entityMap.set(path, entity.id);
+                // Create hierarchical relationship if parent exists
+                if (options.enableHierarchicalRelationships && parentPath && entityMap.has(parentPath)) {
+                    const parentId = entityMap.get(parentPath);
+                    relationships.push({
+                        from: parentId,
+                        to: entity.id,
+                        type: VerbType.Contains,
+                        confidence: 0.95,
+                        evidence: `Hierarchical relationship: ${parentPath} contains ${path}`
+                    });
+                }
+            }
+            // Traverse child properties
+            for (const [key, value] of Object.entries(node)) {
+                const childPath = path ? `${path}.${key}` : key;
+                await this.traverseJSON(value, childPath, path, depth + 1, options, entities, relationships, entityMap, stats, onNode);
+            }
+            return;
+        }
+        // Handle primitive values (strings)
+        if (typeof node === 'string' && node.length >= options.minStringLength) {
+            // Extract entities from text
+            if (options.enableNeuralExtraction) {
+                const extractedEntities = await this.extractor.extract(node, {
+                    confidence: options.confidenceThreshold,
+                    neuralMatching: true,
+                    cache: { enabled: true }
+                });
+                for (const extracted of extractedEntities) {
+                    const entity = {
+                        id: this.generateEntityId(extracted.text, path),
+                        name: extracted.text,
+                        type: extracted.type,
+                        description: node,
+                        confidence: extracted.confidence,
+                        path,
+                        parentPath,
+                        metadata: {
+                            source: 'json',
+                            depth,
+                            extractedAt: Date.now()
+                        }
+                    };
+                    entities.push(entity);
+                    this.updateStats(stats, entity.type, entity.confidence, depth);
+                    // Link to parent if exists
+                    if (options.enableHierarchicalRelationships && parentPath && entityMap.has(parentPath)) {
+                        const parentId = entityMap.get(parentPath);
+                        relationships.push({
+                            from: parentId,
+                            to: entity.id,
+                            type: VerbType.RelatedTo,
+                            confidence: extracted.confidence * 0.9,
+                            evidence: `Found in: ${path}`
+                        });
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Extract entity from JSON object
+     */
+    async extractEntityFromObject(obj, path, parentPath, depth, options, stats) {
+        // Find name
+        const name = this.findValue(obj, options.nameKeys);
+        if (!name)
+            return null;
+        // Find description
+        const description = this.findValue(obj, options.descriptionKeys) || name;
+        // Find type
+        const typeString = this.findValue(obj, options.typeKeys);
+        const type = typeString ? this.mapTypeString(typeString) : this.inferTypeFromStructure(obj);
+        // Extract concepts if enabled
+        let concepts = [];
+        if (options.enableConceptExtraction && description.length > 0) {
+            try {
+                concepts = await this.brain.extractConcepts(description, { limit: 10 });
+            }
+            catch (error) {
+                concepts = [];
+            }
+        }
+        const entity = {
+            id: this.generateEntityId(name, path),
+            name,
+            type,
+            description,
+            confidence: 0.9, // Objects with explicit structure have high confidence
+            path,
+            parentPath,
+            metadata: {
+                source: 'json',
+                depth,
+                originalObject: obj,
+                concepts,
+                extractedAt: Date.now()
+            }
+        };
+        this.updateStats(stats, entity.type, entity.confidence, depth);
+        return entity;
+    }
+    /**
+     * Find value in object by key patterns
+     */
+    findValue(obj, keys) {
+        for (const key of keys) {
+            if (obj[key] !== undefined && obj[key] !== null) {
+                const value = String(obj[key]).trim();
+                if (value.length > 0) {
+                    return value;
+                }
+            }
+        }
+        // Try case-insensitive match
+        for (const key of keys) {
+            const found = Object.keys(obj).find(k => k.toLowerCase() === key.toLowerCase());
+            if (found && obj[found] !== undefined && obj[found] !== null) {
+                const value = String(obj[found]).trim();
+                if (value.length > 0) {
+                    return value;
+                }
+            }
+        }
+        return null;
+    }
+    /**
+     * Infer type from JSON structure
+     */
+    inferTypeFromStructure(obj) {
+        const keys = Object.keys(obj).map(k => k.toLowerCase());
+        // Check for common patterns
+        if (keys.some(k => k.includes('person') || k.includes('user') || k.includes('author'))) {
+            return NounType.Person;
+        }
+        if (keys.some(k => k.includes('location') || k.includes('place') || k.includes('address'))) {
+            return NounType.Location;
+        }
+        if (keys.some(k => k.includes('organization') || k.includes('company') || k.includes('org'))) {
+            return NounType.Organization;
+        }
+        if (keys.some(k => k.includes('event') || k.includes('date') || k.includes('time'))) {
+            return NounType.Event;
+        }
+        if (keys.some(k => k.includes('project') || k.includes('task'))) {
+            return NounType.Project;
+        }
+        if (keys.some(k => k.includes('document') || k.includes('file') || k.includes('url'))) {
+            return NounType.Document;
+        }
+        return NounType.Thing;
+    }
+    /**
+     * Map type string to NounType
+     */
+    mapTypeString(typeString) {
+        const normalized = typeString.toLowerCase().trim();
+        const mapping = {
+            'person': NounType.Person,
+            'user': NounType.Person,
+            'character': NounType.Person,
+            'place': NounType.Location,
+            'location': NounType.Location,
+            'organization': NounType.Organization,
+            'company': NounType.Organization,
+            'org': NounType.Organization,
+            'concept': NounType.Concept,
+            'idea': NounType.Concept,
+            'event': NounType.Event,
+            'product': NounType.Product,
+            'item': NounType.Product,
+            'document': NounType.Document,
+            'file': NounType.File,
+            'project': NounType.Project,
+            'thing': NounType.Thing
+        };
+        return mapping[normalized] || NounType.Thing;
+    }
+    /**
+     * Generate consistent entity ID
+     */
+    generateEntityId(name, path) {
+        const normalized = name.toLowerCase().trim().replace(/\s+/g, '_');
+        const pathNorm = path.replace(/[^a-zA-Z0-9]/g, '_');
+        return `ent_${normalized}_${pathNorm}_${Date.now()}`;
+    }
+    /**
+     * Update statistics
+     */
+    updateStats(stats, type, confidence, depth) {
+        // Track by type
+        const typeName = String(type);
+        stats.byType[typeName] = (stats.byType[typeName] || 0) + 1;
+        // Track by confidence
+        if (confidence > 0.8) {
+            stats.byConfidence.high++;
+        }
+        else if (confidence >= 0.6) {
+            stats.byConfidence.medium++;
+        }
+        else {
+            stats.byConfidence.low++;
+        }
+    }
+}
+//# sourceMappingURL=SmartJSONImporter.js.map

package/dist/importers/SmartMarkdownImporter.d.ts

@@ -0,0 +1,159 @@
+/**
+ * Smart Markdown Importer
+ *
+ * Extracts entities and relationships from Markdown files using:
+ * - Heading structure for entity organization
+ * - Link relationships
+ * - NeuralEntityExtractor for entity extraction from text
+ * - Section-based grouping
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+export interface SmartMarkdownOptions {
+    /** Enable neural entity extraction from text */
+    enableNeuralExtraction?: boolean;
+    /** Enable relationship inference */
+    enableRelationshipInference?: boolean;
+    /** Enable concept extraction for tagging */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities (0-1) */
+    confidenceThreshold?: number;
+    /** Extract code blocks as entities */
+    extractCodeBlocks?: boolean;
+    /** Minimum section text length to process */
+    minSectionLength?: number;
+    /** Group by heading level */
+    groupByHeading?: boolean;
+    /** Progress callback */
+    onProgress?: (stats: {
+        processed: number;
+        total: number;
+        entities: number;
+        relationships: number;
+    }) => void;
+}
+export interface MarkdownSection {
+    /** Section ID */
+    id: string;
+    /** Heading text (if this section has a heading) */
+    heading: string | null;
+    /** Heading level (1-6) */
+    level: number;
+    /** Section content */
+    content: string;
+    /** Entities extracted from this section */
+    entities: Array<{
+        id: string;
+        name: string;
+        type: NounType;
+        description: string;
+        confidence: number;
+        metadata: Record<string, any>;
+    }>;
+    /** Links found in this section */
+    links: Array<{
+        text: string;
+        url: string;
+        type: 'internal' | 'external';
+    }>;
+    /** Code blocks in this section */
+    codeBlocks?: Array<{
+        language: string;
+        code: string;
+    }>;
+    /** Relationships */
+    relationships: Array<{
+        from: string;
+        to: string;
+        type: VerbType;
+        confidence: number;
+        evidence: string;
+    }>;
+    /** Concepts */
+    concepts?: string[];
+}
+export interface SmartMarkdownResult {
+    /** Total sections processed */
+    sectionsProcessed: number;
+    /** Entities extracted */
+    entitiesExtracted: number;
+    /** Relationships inferred */
+    relationshipsInferred: number;
+    /** All extracted sections */
+    sections: MarkdownSection[];
+    /** Entity ID mapping (name -> ID) */
+    entityMap: Map<string, string>;
+    /** Processing time in ms */
+    processingTime: number;
+    /** Extraction statistics */
+    stats: {
+        byType: Record<string, number>;
+        byHeadingLevel: Record<number, number>;
+        byConfidence: {
+            high: number;
+            medium: number;
+            low: number;
+        };
+        linksFound: number;
+        codeBlocksFound: number;
+    };
+}
+/**
+ * SmartMarkdownImporter - Extracts structured knowledge from Markdown files
+ */
+export declare class SmartMarkdownImporter {
+    private brain;
+    private extractor;
+    private nlp;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the importer
+     */
+    init(): Promise<void>;
+    /**
+     * Extract entities and relationships from Markdown content
+     */
+    extract(markdown: string, options?: SmartMarkdownOptions): Promise<SmartMarkdownResult>;
+    /**
+     * Parse markdown into sections
+     */
+    private parseMarkdown;
+    /**
+     * Process a single section
+     */
+    private processSection;
+    /**
+     * Extract markdown links
+     */
+    private extractLinks;
+    /**
+     * Extract code blocks
+     */
+    private extractCodeBlocks;
+    /**
+     * Remove code blocks from content
+     */
+    private removeCodeBlocks;
+    /**
+     * Infer type from heading
+     */
+    private inferTypeFromHeading;
+    /**
+     * Check if entities are related by proximity
+     */
+    private entitiesAreRelated;
+    /**
+     * Infer relationship type from context
+     */
+    private inferRelationship;
+    /**
+     * Generate consistent entity ID
+     */
+    private generateEntityId;
+    /**
+     * Update statistics
+     */
+    private updateStats;
+}