@soulcraft/brainy 3.27.1 → 3.29.0

package/dist/importers/SmartMarkdownImporter.js

@@ -0,0 +1,369 @@
+/**
+ * Smart Markdown Importer
+ *
+ * Extracts entities and relationships from Markdown files using:
+ * - Heading structure for entity organization
+ * - Link relationships
+ * - NeuralEntityExtractor for entity extraction from text
+ * - Section-based grouping
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { NeuralEntityExtractor } from '../neural/entityExtractor.js';
+import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+/**
+ * SmartMarkdownImporter - Extracts structured knowledge from Markdown files
+ */
+export class SmartMarkdownImporter {
+    constructor(brain) {
+        this.brain = brain;
+        this.extractor = new NeuralEntityExtractor(brain);
+        this.nlp = new NaturalLanguageProcessor(brain);
+    }
+    /**
+     * Initialize the importer
+     */
+    async init() {
+        await this.nlp.init();
+    }
+    /**
+     * Extract entities and relationships from Markdown content
+     */
+    async extract(markdown, options = {}) {
+        const startTime = Date.now();
+        // Set defaults
+        const opts = {
+            enableNeuralExtraction: true,
+            enableRelationshipInference: true,
+            enableConceptExtraction: true,
+            confidenceThreshold: 0.6,
+            extractCodeBlocks: true,
+            minSectionLength: 50,
+            groupByHeading: true,
+            onProgress: () => { },
+            ...options
+        };
+        // Parse markdown into sections
+        const parsedSections = this.parseMarkdown(markdown, opts);
+        // Process each section
+        const sections = [];
+        const entityMap = new Map();
+        const stats = {
+            byType: {},
+            byHeadingLevel: {},
+            byConfidence: { high: 0, medium: 0, low: 0 },
+            linksFound: 0,
+            codeBlocksFound: 0
+        };
+        for (let i = 0; i < parsedSections.length; i++) {
+            const parsed = parsedSections[i];
+            const section = await this.processSection(parsed, opts, stats, entityMap);
+            sections.push(section);
+            opts.onProgress({
+                processed: i + 1,
+                total: parsedSections.length,
+                entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
+                relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0)
+            });
+        }
+        return {
+            sectionsProcessed: sections.length,
+            entitiesExtracted: sections.reduce((sum, s) => sum + s.entities.length, 0),
+            relationshipsInferred: sections.reduce((sum, s) => sum + s.relationships.length, 0),
+            sections,
+            entityMap,
+            processingTime: Date.now() - startTime,
+            stats
+        };
+    }
+    /**
+     * Parse markdown into sections
+     */
+    parseMarkdown(markdown, options) {
+        const lines = markdown.split('\n');
+        const sections = [];
+        let currentSection = {
+            heading: null,
+            level: 0,
+            lines: []
+        };
+        let sectionCounter = 0;
+        for (const line of lines) {
+            // Check for heading
+            const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+            if (headingMatch) {
+                // Save current section if it has content
+                if (currentSection.lines.length > 0) {
+                    const content = currentSection.lines.join('\n').trim();
+                    if (content.length >= (options.minSectionLength || 50)) {
+                        sections.push({
+                            id: `section_${sectionCounter++}`,
+                            heading: currentSection.heading,
+                            level: currentSection.level,
+                            content
+                        });
+                    }
+                }
+                // Start new section
+                const level = headingMatch[1].length;
+                const heading = headingMatch[2].trim();
+                currentSection = {
+                    heading,
+                    level,
+                    lines: []
+                };
+            }
+            else {
+                currentSection.lines.push(line);
+            }
+        }
+        // Add last section
+        if (currentSection.lines.length > 0) {
+            const content = currentSection.lines.join('\n').trim();
+            if (content.length >= (options.minSectionLength || 50)) {
+                sections.push({
+                    id: `section_${sectionCounter}`,
+                    heading: currentSection.heading,
+                    level: currentSection.level,
+                    content
+                });
+            }
+        }
+        return sections;
+    }
+    /**
+     * Process a single section
+     */
+    async processSection(parsed, options, stats, entityMap) {
+        // Track heading level
+        stats.byHeadingLevel[parsed.level] = (stats.byHeadingLevel[parsed.level] || 0) + 1;
+        // Extract links
+        const links = this.extractLinks(parsed.content);
+        stats.linksFound += links.length;
+        // Extract code blocks
+        const codeBlocks = options.extractCodeBlocks ? this.extractCodeBlocks(parsed.content) : [];
+        stats.codeBlocksFound += codeBlocks.length;
+        // Remove code blocks from content for entity extraction
+        const contentWithoutCode = this.removeCodeBlocks(parsed.content);
+        // Extract entities
+        let extractedEntities = [];
+        if (options.enableNeuralExtraction && contentWithoutCode.length > 0) {
+            extractedEntities = await this.extractor.extract(contentWithoutCode, {
+                confidence: options.confidenceThreshold || 0.6,
+                neuralMatching: true,
+                cache: { enabled: true }
+            });
+        }
+        // If section has a heading, treat it as an entity
+        if (parsed.heading) {
+            const headingEntity = {
+                text: parsed.heading,
+                type: this.inferTypeFromHeading(parsed.heading, parsed.level),
+                confidence: 0.9,
+                position: { start: 0, end: parsed.heading.length }
+            };
+            extractedEntities.unshift(headingEntity);
+        }
+        // Extract concepts
+        let concepts = [];
+        if (options.enableConceptExtraction && contentWithoutCode.length > 0) {
+            try {
+                concepts = await this.brain.extractConcepts(contentWithoutCode, { limit: 10 });
+            }
+            catch (error) {
+                concepts = [];
+            }
+        }
+        // Create entity objects
+        const entities = extractedEntities.map(e => {
+            const entityId = this.generateEntityId(e.text, parsed.id);
+            entityMap.set(e.text.toLowerCase(), entityId);
+            // Update statistics
+            this.updateStats(stats, e.type, e.confidence);
+            return {
+                id: entityId,
+                name: e.text,
+                type: e.type,
+                description: contentWithoutCode.substring(0, 200),
+                confidence: e.confidence,
+                metadata: {
+                    source: 'markdown',
+                    section: parsed.id,
+                    heading: parsed.heading,
+                    level: parsed.level,
+                    extractedAt: Date.now()
+                }
+            };
+        });
+        // Infer relationships
+        const relationships = [];
+        // Link-based relationships
+        if (options.enableRelationshipInference) {
+            for (const link of links) {
+                // Find entity that might be the source
+                const sourceEntity = entities.find(e => contentWithoutCode.toLowerCase().includes(e.name.toLowerCase()));
+                if (sourceEntity) {
+                    // Create relationship to linked entity
+                    const targetId = this.generateEntityId(link.text, 'link');
+                    relationships.push({
+                        from: sourceEntity.id,
+                        to: link.text,
+                        type: VerbType.References,
+                        confidence: 0.85,
+                        evidence: `Markdown link: [${link.text}](${link.url})`
+                    });
+                }
+            }
+            // Entity proximity-based relationships
+            for (let i = 0; i < entities.length; i++) {
+                for (let j = i + 1; j < entities.length; j++) {
+                    const entity1 = entities[i];
+                    const entity2 = entities[j];
+                    if (this.entitiesAreRelated(contentWithoutCode, entity1.name, entity2.name)) {
+                        const verbType = await this.inferRelationship(entity1.name, entity2.name, contentWithoutCode);
+                        relationships.push({
+                            from: entity1.id,
+                            to: entity2.id,
+                            type: verbType,
+                            confidence: Math.min(entity1.confidence, entity2.confidence) * 0.8,
+                            evidence: `Co-occurrence in section: ${parsed.heading || parsed.id}`
+                        });
+                    }
+                }
+            }
+        }
+        return {
+            id: parsed.id,
+            heading: parsed.heading,
+            level: parsed.level,
+            content: parsed.content,
+            entities,
+            links,
+            codeBlocks,
+            relationships,
+            concepts
+        };
+    }
+    /**
+     * Extract markdown links
+     */
+    extractLinks(content) {
+        const links = [];
+        const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+        let match;
+        while ((match = linkRegex.exec(content)) !== null) {
+            const text = match[1];
+            const url = match[2];
+            const type = url.startsWith('http') ? 'external' : 'internal';
+            links.push({ text, url, type });
+        }
+        return links;
+    }
+    /**
+     * Extract code blocks
+     */
+    extractCodeBlocks(content) {
+        const codeBlocks = [];
+        const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+        let match;
+        while ((match = codeBlockRegex.exec(content)) !== null) {
+            const language = match[1] || 'text';
+            const code = match[2].trim();
+            codeBlocks.push({ language, code });
+        }
+        return codeBlocks;
+    }
+    /**
+     * Remove code blocks from content
+     */
+    removeCodeBlocks(content) {
+        return content.replace(/```[\s\S]*?```/g, '');
+    }
+    /**
+     * Infer type from heading
+     */
+    inferTypeFromHeading(heading, level) {
+        const lower = heading.toLowerCase();
+        if (lower.includes('person') || lower.includes('people') || lower.includes('author') || lower.includes('user')) {
+            return NounType.Person;
+        }
+        if (lower.includes('location') || lower.includes('place')) {
+            return NounType.Location;
+        }
+        if (lower.includes('organization') || lower.includes('company')) {
+            return NounType.Organization;
+        }
+        if (lower.includes('event')) {
+            return NounType.Event;
+        }
+        if (lower.includes('project')) {
+            return NounType.Project;
+        }
+        if (lower.includes('document') || lower.includes('file')) {
+            return NounType.Document;
+        }
+        // Top-level headings are often concepts/topics
+        if (level <= 2) {
+            return NounType.Concept;
+        }
+        return NounType.Thing;
+    }
+    /**
+     * Check if entities are related by proximity
+     */
+    entitiesAreRelated(text, entity1, entity2) {
+        const lowerText = text.toLowerCase();
+        const index1 = lowerText.indexOf(entity1.toLowerCase());
+        const index2 = lowerText.indexOf(entity2.toLowerCase());
+        if (index1 === -1 || index2 === -1)
+            return false;
+        return Math.abs(index1 - index2) < 300;
+    }
+    /**
+     * Infer relationship type from context
+     */
+    async inferRelationship(fromEntity, toEntity, context) {
+        const lowerContext = context.toLowerCase();
+        const patterns = [
+            [new RegExp(`${toEntity}.*of.*${fromEntity}`, 'i'), VerbType.PartOf],
+            [new RegExp(`${fromEntity}.*contains.*${toEntity}`, 'i'), VerbType.Contains],
+            [new RegExp(`${fromEntity}.*in.*${toEntity}`, 'i'), VerbType.LocatedAt],
+            [new RegExp(`${fromEntity}.*created.*${toEntity}`, 'i'), VerbType.Creates],
+            [new RegExp(`${fromEntity}.*and.*${toEntity}`, 'i'), VerbType.RelatedTo]
+        ];
+        for (const [pattern, verbType] of patterns) {
+            if (pattern.test(lowerContext)) {
+                return verbType;
+            }
+        }
+        return VerbType.RelatedTo;
+    }
+    /**
+     * Generate consistent entity ID
+     */
+    generateEntityId(name, section) {
+        const normalized = name.toLowerCase().trim().replace(/\s+/g, '_');
+        const sectionNorm = section.replace(/\s+/g, '_');
+        return `ent_${normalized}_${sectionNorm}_${Date.now()}`;
+    }
+    /**
+     * Update statistics
+     */
+    updateStats(stats, type, confidence) {
+        // Track by type
+        const typeName = String(type);
+        stats.byType[typeName] = (stats.byType[typeName] || 0) + 1;
+        // Track by confidence
+        if (confidence > 0.8) {
+            stats.byConfidence.high++;
+        }
+        else if (confidence >= 0.6) {
+            stats.byConfidence.medium++;
+        }
+        else {
+            stats.byConfidence.low++;
+        }
+    }
+}
+//# sourceMappingURL=SmartMarkdownImporter.js.map

package/dist/importers/SmartPDFImporter.d.ts

@@ -0,0 +1,154 @@
+/**
+ * Smart PDF Importer
+ *
+ * Extracts entities and relationships from PDF files using:
+ * - NeuralEntityExtractor for entity extraction
+ * - NaturalLanguageProcessor for relationship inference
+ * - brain.extractConcepts() for tagging
+ * - Section-based organization (by page or detected structure)
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js';
+export interface SmartPDFOptions extends FormatHandlerOptions {
+    /** Enable neural entity extraction */
+    enableNeuralExtraction?: boolean;
+    /** Enable relationship inference from text */
+    enableRelationshipInference?: boolean;
+    /** Enable concept extraction for tagging */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities (0-1) */
+    confidenceThreshold?: number;
+    /** Minimum paragraph length to process (characters) */
+    minParagraphLength?: number;
+    /** Extract from tables */
+    extractFromTables?: boolean;
+    /** Group by page or full document */
+    groupBy?: 'page' | 'document';
+    /** Progress callback */
+    onProgress?: (stats: {
+        processed: number;
+        total: number;
+        entities: number;
+        relationships: number;
+    }) => void;
+}
+export interface ExtractedSection {
+    /** Section identifier (page number or section name) */
+    sectionId: string;
+    /** Section type */
+    sectionType: 'page' | 'paragraph' | 'table';
+    /** Entities extracted from this section */
+    entities: Array<{
+        id: string;
+        name: string;
+        type: NounType;
+        description: string;
+        confidence: number;
+        metadata: Record<string, any>;
+    }>;
+    /** Relationships inferred in this section */
+    relationships: Array<{
+        from: string;
+        to: string;
+        type: VerbType;
+        confidence: number;
+        evidence: string;
+    }>;
+    /** Concepts extracted */
+    concepts?: string[];
+    /** Original text */
+    text: string;
+}
+export interface SmartPDFResult {
+    /** Total sections processed */
+    sectionsProcessed: number;
+    /** Total pages processed */
+    pagesProcessed: number;
+    /** Entities extracted */
+    entitiesExtracted: number;
+    /** Relationships inferred */
+    relationshipsInferred: number;
+    /** All extracted sections */
+    sections: ExtractedSection[];
+    /** Entity ID mapping (name -> ID) */
+    entityMap: Map<string, string>;
+    /** Processing time in ms */
+    processingTime: number;
+    /** Extraction statistics */
+    stats: {
+        byType: Record<string, number>;
+        byConfidence: {
+            high: number;
+            medium: number;
+            low: number;
+        };
+        bySource: {
+            paragraphs: number;
+            tables: number;
+        };
+    };
+    /** PDF metadata */
+    pdfMetadata: {
+        pageCount: number;
+        title?: string;
+        author?: string;
+        subject?: string;
+    };
+}
+/**
+ * SmartPDFImporter - Extracts structured knowledge from PDF files
+ */
+export declare class SmartPDFImporter {
+    private brain;
+    private extractor;
+    private nlp;
+    private pdfHandler;
+    constructor(brain: Brainy);
+    /**
+     * Initialize the importer
+     */
+    init(): Promise<void>;
+    /**
+     * Extract entities and relationships from PDF file
+     */
+    extract(buffer: Buffer, options?: SmartPDFOptions): Promise<SmartPDFResult>;
+    /**
+     * Group data by strategy
+     */
+    private groupData;
+    /**
+     * Process a single section
+     */
+    private processSection;
+    /**
+     * Extract context around an entity mention
+     */
+    private extractContextAroundEntity;
+    /**
+     * Check if two entities are related based on proximity in text
+     */
+    private entitiesAreRelated;
+    /**
+     * Extract context showing relationship between entities
+     */
+    private extractRelationshipContext;
+    /**
+     * Infer relationship type from context
+     */
+    private inferRelationship;
+    /**
+     * Generate consistent entity ID
+     */
+    private generateEntityId;
+    /**
+     * Update statistics
+     */
+    private updateStats;
+    /**
+     * Create empty result
+     */
+    private emptyResult;
+}