@soulcraft/brainy 3.27.0 → 3.28.0

package/dist/import/FormatDetector.js

@@ -0,0 +1,263 @@
+/**
+ * Format Detector
+ *
+ * Unified format detection for all import types using:
+ * - Magic byte signatures (PDF, Excel, images)
+ * - File extensions
+ * - Content analysis (JSON, Markdown, CSV)
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+/**
+ * FormatDetector - Detect file format from various inputs
+ */
+export class FormatDetector {
+    /**
+     * Detect format from buffer
+     */
+    detectFromBuffer(buffer) {
+        // Check magic bytes first (most reliable)
+        const magicResult = this.detectByMagicBytes(buffer);
+        if (magicResult)
+            return magicResult;
+        // Try content analysis
+        const contentResult = this.detectByContent(buffer);
+        if (contentResult)
+            return contentResult;
+        return null;
+    }
+    /**
+     * Detect format from file path
+     */
+    detectFromPath(path) {
+        const ext = this.getExtension(path).toLowerCase();
+        const extensionMap = {
+            '.xlsx': 'excel',
+            '.xls': 'excel',
+            '.pdf': 'pdf',
+            '.csv': 'csv',
+            '.json': 'json',
+            '.md': 'markdown',
+            '.markdown': 'markdown'
+        };
+        const format = extensionMap[ext];
+        if (format) {
+            return {
+                format,
+                confidence: 0.9,
+                evidence: [`File extension: ${ext}`]
+            };
+        }
+        return null;
+    }
+    /**
+     * Detect format from string content
+     */
+    detectFromString(content) {
+        const trimmed = content.trim();
+        // JSON detection
+        if (this.looksLikeJSON(trimmed)) {
+            return {
+                format: 'json',
+                confidence: 0.95,
+                evidence: ['Content starts with { or [', 'Valid JSON structure']
+            };
+        }
+        // Markdown detection
+        if (this.looksLikeMarkdown(trimmed)) {
+            return {
+                format: 'markdown',
+                confidence: 0.85,
+                evidence: ['Contains markdown heading markers (#)', 'Text-based content']
+            };
+        }
+        // CSV detection
+        if (this.looksLikeCSV(trimmed)) {
+            return {
+                format: 'csv',
+                confidence: 0.8,
+                evidence: ['Contains delimiter-separated values', 'Consistent column structure']
+            };
+        }
+        return null;
+    }
+    /**
+     * Detect format from object
+     */
+    detectFromObject(obj) {
+        if (typeof obj === 'object' && obj !== null) {
+            return {
+                format: 'json',
+                confidence: 1.0,
+                evidence: ['JavaScript object']
+            };
+        }
+        return null;
+    }
+    /**
+     * Detect by magic bytes
+     */
+    detectByMagicBytes(buffer) {
+        if (buffer.length < 4)
+            return null;
+        // PDF: %PDF (25 50 44 46)
+        if (buffer[0] === 0x25 && buffer[1] === 0x50 && buffer[2] === 0x44 && buffer[3] === 0x46) {
+            return {
+                format: 'pdf',
+                confidence: 1.0,
+                evidence: ['PDF magic bytes: %PDF']
+            };
+        }
+        // Excel (ZIP-based): PK (50 4B)
+        if (buffer[0] === 0x50 && buffer[1] === 0x4B) {
+            // Check for [Content_Types].xml which is specific to Office Open XML
+            const content = buffer.toString('utf8', 0, Math.min(1000, buffer.length));
+            if (content.includes('[Content_Types].xml') || content.includes('xl/')) {
+                return {
+                    format: 'excel',
+                    confidence: 1.0,
+                    evidence: ['ZIP magic bytes: PK', 'Contains Office Open XML structure']
+                };
+            }
+        }
+        return null;
+    }
+    /**
+     * Detect by content analysis
+     */
+    detectByContent(buffer) {
+        // Try to decode as UTF-8
+        let content;
+        try {
+            content = buffer.toString('utf8').trim();
+        }
+        catch {
+            return null;
+        }
+        // Check if it's text-based content
+        if (!this.isTextContent(content)) {
+            return null;
+        }
+        // JSON detection
+        if (this.looksLikeJSON(content)) {
+            return {
+                format: 'json',
+                confidence: 0.95,
+                evidence: ['Content starts with { or [', 'Valid JSON structure']
+            };
+        }
+        // Markdown detection
+        if (this.looksLikeMarkdown(content)) {
+            return {
+                format: 'markdown',
+                confidence: 0.85,
+                evidence: ['Contains markdown heading markers (#)', 'Text-based content']
+            };
+        }
+        // CSV detection
+        if (this.looksLikeCSV(content)) {
+            return {
+                format: 'csv',
+                confidence: 0.8,
+                evidence: ['Contains delimiter-separated values', 'Consistent column structure']
+            };
+        }
+        return null;
+    }
+    /**
+     * Check if content looks like JSON
+     */
+    looksLikeJSON(content) {
+        const trimmed = content.trim();
+        if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) {
+            return false;
+        }
+        try {
+            JSON.parse(trimmed);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Check if content looks like Markdown
+     */
+    looksLikeMarkdown(content) {
+        const lines = content.split('\n').slice(0, 50); // Check first 50 lines
+        // Count markdown indicators
+        let indicators = 0;
+        for (const line of lines) {
+            // Headings
+            if (/^#{1,6}\s+.+/.test(line))
+                indicators += 2;
+            // Lists
+            if (/^[\*\-\+]\s+.+/.test(line))
+                indicators++;
+            if (/^\d+\.\s+.+/.test(line))
+                indicators++;
+            // Links
+            if (/\[.+\]\(.+\)/.test(line))
+                indicators++;
+            // Code blocks
+            if (/^```/.test(line))
+                indicators += 2;
+            // Bold/Italic
+            if (/\*\*.+\*\*/.test(line) || /\*.+\*/.test(line))
+                indicators++;
+        }
+        // If we have at least 3 markdown indicators, it's likely markdown
+        return indicators >= 3;
+    }
+    /**
+     * Check if content looks like CSV
+     */
+    looksLikeCSV(content) {
+        const lines = content.split('\n').filter(l => l.trim()).slice(0, 20);
+        if (lines.length < 2)
+            return false;
+        // Try common delimiters
+        const delimiters = [',', ';', '\t', '|'];
+        for (const delimiter of delimiters) {
+            const columnCounts = lines.map(line => {
+                // Simple split (doesn't handle quoted delimiters, but good enough for detection)
+                return line.split(delimiter).length;
+            });
+            // Check if all rows have the same number of columns (within 1)
+            const firstCount = columnCounts[0];
+            const consistent = columnCounts.filter(c => Math.abs(c - firstCount) <= 1).length;
+            // If >80% of rows have consistent column counts, it's likely CSV
+            if (consistent / columnCounts.length > 0.8 && firstCount > 1) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Check if content is text-based (not binary)
+     */
+    isTextContent(content) {
+        // Check for null bytes (common in binary files)
+        if (content.includes('\0'))
+            return false;
+        // Check if mostly printable characters
+        const printable = content.split('').filter(c => {
+            const code = c.charCodeAt(0);
+            return (code >= 32 && code <= 126) || code === 9 || code === 10 || code === 13;
+        }).length;
+        const ratio = printable / content.length;
+        return ratio > 0.9;
+    }
+    /**
+     * Get file extension from path
+     */
+    getExtension(path) {
+        const lastDot = path.lastIndexOf('.');
+        const lastSlash = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\'));
+        if (lastDot > lastSlash && lastDot !== -1) {
+            return path.substring(lastDot);
+        }
+        return '';
+    }
+}
+//# sourceMappingURL=FormatDetector.js.map

package/dist/import/ImportCoordinator.d.ts

@@ -0,0 +1,160 @@
+/**
+ * Import Coordinator
+ *
+ * Unified import orchestrator that:
+ * - Auto-detects file formats
+ * - Routes to appropriate handlers
+ * - Coordinates dual storage (VFS + Graph)
+ * - Provides simple, unified API
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import { Brainy } from '../brainy.js';
+import { SupportedFormat } from './FormatDetector.js';
+import { ImportHistory } from './ImportHistory.js';
+import { NounType, VerbType } from '../types/graphTypes.js';
+export interface ImportSource {
+    /** Source type */
+    type: 'buffer' | 'path' | 'string' | 'object';
+    /** Source data */
+    data: Buffer | string | object;
+    /** Optional filename hint */
+    filename?: string;
+}
+export interface ImportOptions {
+    /** Force specific format (skip auto-detection) */
+    format?: SupportedFormat;
+    /** VFS root path for imported files */
+    vfsPath?: string;
+    /** Grouping strategy for VFS */
+    groupBy?: 'type' | 'sheet' | 'flat' | 'custom';
+    /** Custom grouping function */
+    customGrouping?: (entity: any) => string;
+    /** Create entities in knowledge graph */
+    createEntities?: boolean;
+    /** Create relationships in knowledge graph */
+    createRelationships?: boolean;
+    /** Preserve source file in VFS */
+    preserveSource?: boolean;
+    /** Enable neural entity extraction */
+    enableNeuralExtraction?: boolean;
+    /** Enable relationship inference */
+    enableRelationshipInference?: boolean;
+    /** Enable concept extraction */
+    enableConceptExtraction?: boolean;
+    /** Confidence threshold for entities */
+    confidenceThreshold?: number;
+    /** Enable entity deduplication across imports */
+    enableDeduplication?: boolean;
+    /** Similarity threshold for deduplication (0-1) */
+    deduplicationThreshold?: number;
+    /** Enable import history tracking */
+    enableHistory?: boolean;
+    /** Chunk size for streaming large imports (0 = no streaming) */
+    chunkSize?: number;
+    /** Progress callback */
+    onProgress?: (progress: ImportProgress) => void;
+}
+export interface ImportProgress {
+    stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete';
+    message: string;
+    processed?: number;
+    total?: number;
+    entities?: number;
+    relationships?: number;
+}
+export interface ImportResult {
+    /** Import ID for history tracking */
+    importId: string;
+    /** Detected format */
+    format: SupportedFormat;
+    /** Format detection confidence */
+    formatConfidence: number;
+    /** VFS paths created */
+    vfs: {
+        rootPath: string;
+        directories: string[];
+        files: Array<{
+            path: string;
+            entityId?: string;
+            type: 'entity' | 'metadata' | 'source' | 'relationships';
+        }>;
+    };
+    /** Knowledge graph entities created */
+    entities: Array<{
+        id: string;
+        name: string;
+        type: NounType;
+        vfsPath?: string;
+    }>;
+    /** Knowledge graph relationships created */
+    relationships: Array<{
+        id: string;
+        from: string;
+        to: string;
+        type: VerbType;
+    }>;
+    /** Import statistics */
+    stats: {
+        entitiesExtracted: number;
+        relationshipsInferred: number;
+        vfsFilesCreated: number;
+        graphNodesCreated: number;
+        graphEdgesCreated: number;
+        entitiesMerged: number;
+        entitiesNew: number;
+        processingTime: number;
+    };
+}
+/**
+ * ImportCoordinator - Main entry point for all imports
+ */
+export declare class ImportCoordinator {
+    private brain;
+    private detector;
+    private deduplicator;
+    private history;
+    private excelImporter;
+    private pdfImporter;
+    private csvImporter;
+    private jsonImporter;
+    private markdownImporter;
+    private vfsGenerator;
+    constructor(brain: Brainy);
+    /**
+     * Initialize all importers
+     */
+    init(): Promise<void>;
+    /**
+     * Get import history
+     */
+    getHistory(): ImportHistory;
+    /**
+     * Import from any source with auto-detection
+     */
+    import(source: Buffer | string | object, options?: ImportOptions): Promise<ImportResult>;
+    /**
+     * Normalize source to ImportSource
+     */
+    private normalizeSource;
+    /**
+     * Check if string is a file path
+     */
+    private isFilePath;
+    /**
+     * Detect format from source
+     */
+    private detectFormat;
+    /**
+     * Extract entities using format-specific importer
+     */
+    private extract;
+    /**
+     * Create entities and relationships in knowledge graph
+     */
+    private createGraphEntities;
+    /**
+     * Normalize extraction result to unified format (Excel-like structure)
+     */
+    private normalizeExtractionResult;
+}