@soulcraft/brainy 3.20.5 → 3.22.0

package/dist/augmentations/intelligentImport/handlers/pdfHandler.js

@@ -0,0 +1,247 @@
+/**
+ * PDF Format Handler
+ * Handles PDF files with:
+ * - Text extraction with layout preservation
+ * - Table detection and extraction
+ * - Metadata extraction (author, dates, etc.)
+ * - Page-by-page processing
+ */
+import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs';
+import { BaseFormatHandler } from './base.js';
+// Use built-in worker for Node.js environments
+// In production, this can be customized via options
+const initializeWorker = () => {
+    if (typeof pdfjsLib.GlobalWorkerOptions.workerSrc === 'undefined' ||
+        pdfjsLib.GlobalWorkerOptions.workerSrc === '') {
+        // Use a data URL to avoid file system dependencies
+        // This tells pdfjs to use the built-in fallback worker
+        try {
+            pdfjsLib.GlobalWorkerOptions.workerSrc = 'data:,';
+        }
+        catch {
+            // Ignore if already set or in incompatible environment
+        }
+    }
+};
+initializeWorker();
+export class PDFHandler extends BaseFormatHandler {
+    constructor() {
+        super(...arguments);
+        this.format = 'pdf';
+    }
+    canHandle(data) {
+        const ext = this.detectExtension(data);
+        if (ext === 'pdf')
+            return true;
+        // Check for PDF magic bytes
+        if (Buffer.isBuffer(data)) {
+            const header = data.slice(0, 5).toString('ascii');
+            return header === '%PDF-';
+        }
+        return false;
+    }
+    async process(data, options) {
+        const startTime = Date.now();
+        // Convert to buffer
+        const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary');
+        try {
+            // Load PDF document
+            const loadingTask = pdfjsLib.getDocument({
+                data: new Uint8Array(buffer),
+                useSystemFonts: true,
+                standardFontDataUrl: undefined
+            });
+            const pdfDoc = await loadingTask.promise;
+            // Extract metadata
+            const metadata = await pdfDoc.getMetadata();
+            const numPages = pdfDoc.numPages;
+            // Extract text and structure from all pages
+            const allData = [];
+            let totalTextLength = 0;
+            let detectedTables = 0;
+            for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+                const page = await pdfDoc.getPage(pageNum);
+                const textContent = await page.getTextContent();
+                // Extract text items with positions
+                const textItems = textContent.items.map((item) => ({
+                    text: item.str,
+                    x: item.transform[4],
+                    y: item.transform[5],
+                    width: item.width,
+                    height: item.height
+                }));
+                // Combine text items into lines (group by similar Y position)
+                const lines = this.groupIntoLines(textItems);
+                // Detect tables if requested
+                if (options.pdfExtractTables !== false) {
+                    const tables = this.detectTables(lines);
+                    if (tables.length > 0) {
+                        detectedTables += tables.length;
+                        for (const table of tables) {
+                            allData.push(...table.rows);
+                        }
+                    }
+                }
+                // Extract paragraphs from non-table lines
+                const paragraphs = this.extractParagraphs(lines);
+                for (let i = 0; i < paragraphs.length; i++) {
+                    const text = paragraphs[i].trim();
+                    if (text.length > 0) {
+                        totalTextLength += text.length;
+                        allData.push({
+                            _page: pageNum,
+                            _type: 'paragraph',
+                            _index: i,
+                            text
+                        });
+                    }
+                }
+            }
+            const processingTime = Date.now() - startTime;
+            // Get all unique fields (excluding metadata fields)
+            const fields = allData.length > 0
+                ? Object.keys(allData[0]).filter(k => !k.startsWith('_'))
+                : [];
+            return {
+                format: this.format,
+                data: allData,
+                metadata: this.createMetadata(allData.length, fields, processingTime, {
+                    pageCount: numPages,
+                    textLength: totalTextLength,
+                    tableCount: detectedTables,
+                    pdfMetadata: {
+                        title: metadata.info?.Title || null,
+                        author: metadata.info?.Author || null,
+                        subject: metadata.info?.Subject || null,
+                        creator: metadata.info?.Creator || null,
+                        producer: metadata.info?.Producer || null,
+                        creationDate: metadata.info?.CreationDate || null,
+                        modificationDate: metadata.info?.ModDate || null
+                    }
+                }),
+                filename: options.filename
+            };
+        }
+        catch (error) {
+            throw new Error(`PDF parsing failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Group text items into lines based on Y position
+     */
+    groupIntoLines(items) {
+        if (items.length === 0)
+            return [];
+        // Sort by Y position (descending, since PDF coordinates go bottom-up)
+        const sorted = [...items].sort((a, b) => b.y - a.y);
+        const lines = [];
+        let currentLine = [];
+        let currentY = sorted[0].y;
+        for (const item of sorted) {
+            // If Y position differs by more than half the height, it's a new line
+            if (Math.abs(item.y - currentY) > (item.height / 2)) {
+                if (currentLine.length > 0) {
+                    // Sort line items by X position
+                    currentLine.sort((a, b) => a.x - b.x);
+                    lines.push(currentLine);
+                }
+                currentLine = [];
+                currentY = item.y;
+            }
+            if (item.text.trim()) {
+                currentLine.push({ text: item.text, x: item.x });
+            }
+        }
+        // Add last line
+        if (currentLine.length > 0) {
+            currentLine.sort((a, b) => a.x - b.x);
+            lines.push(currentLine);
+        }
+        return lines;
+    }
+    /**
+     * Detect tables from lines
+     * Tables are detected when multiple consecutive lines have similar structure
+     */
+    detectTables(lines) {
+        const tables = [];
+        let potentialTable = [];
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            // A line with multiple items could be part of a table
+            if (line.length >= 2) {
+                potentialTable.push(line);
+            }
+            else {
+                // End of potential table
+                if (potentialTable.length >= 3) { // Need at least header + 2 rows
+                    const table = this.parseTable(potentialTable);
+                    if (table) {
+                        tables.push(table);
+                    }
+                }
+                potentialTable = [];
+            }
+        }
+        // Check last potential table
+        if (potentialTable.length >= 3) {
+            const table = this.parseTable(potentialTable);
+            if (table) {
+                tables.push(table);
+            }
+        }
+        return tables;
+    }
+    /**
+     * Parse a potential table into structured rows
+     */
+    parseTable(lines) {
+        if (lines.length < 2)
+            return null;
+        // First line is headers
+        const headerLine = lines[0];
+        const headers = headerLine.map(item => this.sanitizeFieldName(item.text));
+        // Remaining lines are data
+        const rows = [];
+        for (let i = 1; i < lines.length; i++) {
+            const line = lines[i];
+            const row = { _type: 'table_row' };
+            // Match each item to closest header by X position
+            for (let j = 0; j < line.length && j < headers.length; j++) {
+                const header = headers[j];
+                const value = line[j].text.trim();
+                row[header] = value || null;
+            }
+            if (Object.keys(row).length > 1) { // More than just _type
+                rows.push(row);
+            }
+        }
+        return rows.length > 0 ? { rows } : null;
+    }
+    /**
+     * Extract paragraphs from lines
+     */
+    extractParagraphs(lines) {
+        const paragraphs = [];
+        let currentParagraph = [];
+        for (const line of lines) {
+            const lineText = line.map(item => item.text).join(' ').trim();
+            if (lineText.length === 0) {
+                // Empty line - end paragraph
+                if (currentParagraph.length > 0) {
+                    paragraphs.push(currentParagraph.join(' '));
+                    currentParagraph = [];
+                }
+            }
+            else {
+                currentParagraph.push(lineText);
+            }
+        }
+        // Add last paragraph
+        if (currentParagraph.length > 0) {
+            paragraphs.push(currentParagraph.join(' '));
+        }
+        return paragraphs;
+    }
+}
+//# sourceMappingURL=pdfHandler.js.map

package/dist/augmentations/intelligentImport/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Intelligent Import Module
+ * Exports main augmentation and types
+ */
+export { IntelligentImportAugmentation } from './IntelligentImportAugmentation.js';
+export type { FormatHandler, FormatHandlerOptions, ProcessedData, IntelligentImportConfig } from './types.js';
+export { CSVHandler } from './handlers/csvHandler.js';
+export { ExcelHandler } from './handlers/excelHandler.js';
+export { PDFHandler } from './handlers/pdfHandler.js';

package/dist/augmentations/intelligentImport/index.js

@@ -0,0 +1,9 @@
+/**
+ * Intelligent Import Module
+ * Exports main augmentation and types
+ */
+export { IntelligentImportAugmentation } from './IntelligentImportAugmentation.js';
+export { CSVHandler } from './handlers/csvHandler.js';
+export { ExcelHandler } from './handlers/excelHandler.js';
+export { PDFHandler } from './handlers/pdfHandler.js';
+//# sourceMappingURL=index.js.map

package/dist/augmentations/intelligentImport/types.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Types for Intelligent Import Augmentation
+ * Handles Excel, PDF, and CSV import with intelligent extraction
+ */
+export interface FormatHandler {
+    /**
+     * Format name (e.g., 'csv', 'xlsx', 'pdf')
+     */
+    readonly format: string;
+    /**
+     * Process raw data into structured format
+     * @param data Raw file data (Buffer or string)
+     * @param options Format-specific options
+     * @returns Structured data ready for entity extraction
+     */
+    process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    /**
+     * Detect if this handler can process the given data
+     * @param data Raw data or filename
+     * @returns true if handler supports this format
+     */
+    canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+}
+export interface FormatHandlerOptions {
+    /** Source filename (for extension detection) */
+    filename?: string;
+    /** File extension (if known) */
+    ext?: string;
+    /** Encoding (auto-detected if not specified) */
+    encoding?: string;
+    /** CSV-specific: delimiter character */
+    csvDelimiter?: string;
+    /** CSV-specific: whether first row is headers */
+    csvHeaders?: boolean;
+    /** Excel-specific: sheet names to extract (or 'all') */
+    excelSheets?: string[] | 'all';
+    /** Excel-specific: whether to evaluate formulas */
+    excelEvaluateFormulas?: boolean;
+    /** PDF-specific: whether to extract tables */
+    pdfExtractTables?: boolean;
+    /** PDF-specific: whether to preserve layout */
+    pdfPreserveLayout?: boolean;
+    /** Maximum rows to process (for large files) */
+    maxRows?: number;
+    /** Whether to stream large files */
+    streaming?: boolean;
+}
+export interface ProcessedData {
+    /** Format that was processed */
+    format: string;
+    /** Structured data (array of objects) */
+    data: Array<Record<string, any>>;
+    /** Metadata about the processed data */
+    metadata: {
+        /** Number of rows/entities extracted */
+        rowCount: number;
+        /** Column/field names */
+        fields: string[];
+        /** Detected encoding (for text formats) */
+        encoding?: string;
+        /** Excel: sheet names */
+        sheets?: string[];
+        /** PDF: page count */
+        pageCount?: number;
+        /** PDF: extracted text length */
+        textLength?: number;
+        /** PDF: number of tables detected */
+        tableCount?: number;
+        /** Processing time in milliseconds */
+        processingTime: number;
+        /** Any warnings during processing */
+        warnings?: string[];
+        /** Format-specific metadata */
+        [key: string]: any;
+    };
+    /** Original filename (if available) */
+    filename?: string;
+}
+export interface HandlerRegistry {
+    /** Registered handlers by format extension */
+    handlers: Map<string, () => Promise<FormatHandler>>;
+    /** Loaded handler instances (lazy-loaded) */
+    loaded: Map<string, FormatHandler>;
+    /** Register a new handler */
+    register(extensions: string[], loader: () => Promise<FormatHandler>): void;
+    /** Get handler for a file/format */
+    getHandler(filenameOrExt: string): Promise<FormatHandler | null>;
+}
+export interface IntelligentImportConfig {
+    /** Enable CSV handler */
+    enableCSV: boolean;
+    /** Enable Excel handler */
+    enableExcel: boolean;
+    /** Enable PDF handler */
+    enablePDF: boolean;
+    /** Default options for CSV */
+    csvDefaults?: Partial<FormatHandlerOptions>;
+    /** Default options for Excel */
+    excelDefaults?: Partial<FormatHandlerOptions>;
+    /** Default options for PDF */
+    pdfDefaults?: Partial<FormatHandlerOptions>;
+    /** Maximum file size to process (bytes) */
+    maxFileSize?: number;
+    /** Enable caching of processed data */
+    enableCache?: boolean;
+    /** Cache TTL in milliseconds */
+    cacheTTL?: number;
+}

package/dist/augmentations/intelligentImport/types.js

@@ -0,0 +1,6 @@
+/**
+ * Types for Intelligent Import Augmentation
+ * Handles Excel, PDF, and CSV import with intelligent extraction
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/neural/entityExtractionCache.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Entity Extraction Cache
+ *
+ * Caches entity extraction results to avoid re-processing unchanged content.
+ * Uses file mtime or content hash for invalidation.
+ *
+ * PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
+ */
+import { ExtractedEntity } from './entityExtractor.js';
+/**
+ * Cache entry for extracted entities
+ */
+export interface EntityCacheEntry {
+    entities: ExtractedEntity[];
+    extractedAt: number;
+    expiresAt: number;
+    mtime?: number;
+    contentHash?: string;
+}
+/**
+ * Cache options
+ */
+export interface EntityCacheOptions {
+    enabled?: boolean;
+    ttl?: number;
+    invalidateOn?: 'mtime' | 'hash' | 'both';
+    maxEntries?: number;
+}
+/**
+ * Cache statistics
+ */
+export interface EntityCacheStats {
+    hits: number;
+    misses: number;
+    evictions: number;
+    totalEntries: number;
+    hitRate: number;
+    averageEntitiesPerEntry: number;
+    cacheSize: number;
+}
+/**
+ * Entity Extraction Cache with LRU eviction
+ */
+export declare class EntityExtractionCache {
+    private cache;
+    private accessOrder;
+    private stats;
+    private accessCounter;
+    private maxEntries;
+    private defaultTtl;
+    constructor(options?: EntityCacheOptions);
+    /**
+     * Get cached entities
+     */
+    get(key: string, options?: {
+        mtime?: number;
+        contentHash?: string;
+    }): ExtractedEntity[] | null;
+    /**
+     * Set cached entities
+     */
+    set(key: string, entities: ExtractedEntity[], options?: {
+        ttl?: number;
+        mtime?: number;
+        contentHash?: string;
+    }): void;
+    /**
+     * Invalidate cache entry
+     */
+    invalidate(key: string): boolean;
+    /**
+     * Invalidate all entries matching a prefix
+     */
+    invalidatePrefix(prefix: string): number;
+    /**
+     * Clear entire cache
+     */
+    clear(): void;
+    /**
+     * Evict least recently used entry
+     */
+    private evictLRU;
+    /**
+     * Cleanup expired entries
+     */
+    cleanup(): number;
+    /**
+     * Get cache statistics
+     */
+    getStats(): EntityCacheStats;
+    /**
+     * Get cache size (number of entries)
+     */
+    size(): number;
+    /**
+     * Check if cache has key
+     */
+    has(key: string): boolean;
+}
+/**
+ * Helper: Generate cache key from file path
+ */
+export declare function generateFileCacheKey(path: string): string;
+/**
+ * Helper: Generate cache key from content hash
+ */
+export declare function generateContentCacheKey(content: string): string;
+/**
+ * Helper: Compute content hash
+ */
+export declare function computeContentHash(content: string): string;

package/dist/neural/entityExtractionCache.js

@@ -0,0 +1,208 @@
+/**
+ * Entity Extraction Cache
+ *
+ * Caches entity extraction results to avoid re-processing unchanged content.
+ * Uses file mtime or content hash for invalidation.
+ *
+ * PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
+ */
+import { createHash } from 'crypto';
+/**
+ * Entity Extraction Cache with LRU eviction
+ */
+export class EntityExtractionCache {
+    constructor(options = {}) {
+        this.cache = new Map();
+        this.accessOrder = new Map(); // Track access time for LRU
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            evictions: 0
+        };
+        this.accessCounter = 0;
+        this.maxEntries = options.maxEntries || 1000;
+        this.defaultTtl = options.ttl || 7 * 24 * 60 * 60 * 1000; // 7 days default
+    }
+    /**
+     * Get cached entities
+     */
+    get(key, options) {
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.stats.misses++;
+            return null;
+        }
+        // Check expiration
+        if (Date.now() > entry.expiresAt) {
+            this.cache.delete(key);
+            this.accessOrder.delete(key);
+            this.stats.misses++;
+            return null;
+        }
+        // Check mtime invalidation
+        if (options?.mtime !== undefined && entry.mtime !== undefined) {
+            if (options.mtime !== entry.mtime) {
+                this.cache.delete(key);
+                this.accessOrder.delete(key);
+                this.stats.misses++;
+                return null;
+            }
+        }
+        // Check content hash invalidation
+        if (options?.contentHash !== undefined && entry.contentHash !== undefined) {
+            if (options.contentHash !== entry.contentHash) {
+                this.cache.delete(key);
+                this.accessOrder.delete(key);
+                this.stats.misses++;
+                return null;
+            }
+        }
+        // Cache hit - update access time
+        this.accessOrder.set(key, ++this.accessCounter);
+        this.stats.hits++;
+        return entry.entities;
+    }
+    /**
+     * Set cached entities
+     */
+    set(key, entities, options) {
+        // Check if we need to evict
+        if (this.cache.size >= this.maxEntries && !this.cache.has(key)) {
+            this.evictLRU();
+        }
+        const ttl = options?.ttl || this.defaultTtl;
+        const entry = {
+            entities,
+            extractedAt: Date.now(),
+            expiresAt: Date.now() + ttl,
+            mtime: options?.mtime,
+            contentHash: options?.contentHash
+        };
+        this.cache.set(key, entry);
+        this.accessOrder.set(key, ++this.accessCounter);
+    }
+    /**
+     * Invalidate cache entry
+     */
+    invalidate(key) {
+        const had = this.cache.has(key);
+        this.cache.delete(key);
+        this.accessOrder.delete(key);
+        return had;
+    }
+    /**
+     * Invalidate all entries matching a prefix
+     */
+    invalidatePrefix(prefix) {
+        let count = 0;
+        for (const key of this.cache.keys()) {
+            if (key.startsWith(prefix)) {
+                this.cache.delete(key);
+                this.accessOrder.delete(key);
+                count++;
+            }
+        }
+        return count;
+    }
+    /**
+     * Clear entire cache
+     */
+    clear() {
+        this.cache.clear();
+        this.accessOrder.clear();
+        this.stats.hits = 0;
+        this.stats.misses = 0;
+        this.stats.evictions = 0;
+        this.accessCounter = 0;
+    }
+    /**
+     * Evict least recently used entry
+     */
+    evictLRU() {
+        let lruKey = null;
+        let lruAccess = Infinity;
+        for (const [key, access] of this.accessOrder.entries()) {
+            if (access < lruAccess) {
+                lruAccess = access;
+                lruKey = key;
+            }
+        }
+        if (lruKey) {
+            this.cache.delete(lruKey);
+            this.accessOrder.delete(lruKey);
+            this.stats.evictions++;
+        }
+    }
+    /**
+     * Cleanup expired entries
+     */
+    cleanup() {
+        const now = Date.now();
+        let cleaned = 0;
+        for (const [key, entry] of this.cache.entries()) {
+            if (now > entry.expiresAt) {
+                this.cache.delete(key);
+                this.accessOrder.delete(key);
+                cleaned++;
+            }
+        }
+        return cleaned;
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        const total = this.stats.hits + this.stats.misses;
+        const hitRate = total > 0 ? this.stats.hits / total : 0;
+        let totalEntities = 0;
+        let totalSize = 0;
+        for (const entry of this.cache.values()) {
+            totalEntities += entry.entities.length;
+            // Rough estimate: each entity ~500 bytes
+            totalSize += entry.entities.length * 500;
+        }
+        return {
+            hits: this.stats.hits,
+            misses: this.stats.misses,
+            evictions: this.stats.evictions,
+            totalEntries: this.cache.size,
+            hitRate: Math.round(hitRate * 100) / 100,
+            averageEntitiesPerEntry: this.cache.size > 0
+                ? Math.round((totalEntities / this.cache.size) * 10) / 10
+                : 0,
+            cacheSize: totalSize
+        };
+    }
+    /**
+     * Get cache size (number of entries)
+     */
+    size() {
+        return this.cache.size;
+    }
+    /**
+     * Check if cache has key
+     */
+    has(key) {
+        return this.cache.has(key);
+    }
+}
+/**
+ * Helper: Generate cache key from file path
+ */
+export function generateFileCacheKey(path) {
+    return `file:${path}`;
+}
+/**
+ * Helper: Generate cache key from content hash
+ */
+export function generateContentCacheKey(content) {
+    const hash = createHash('sha256').update(content).digest('hex');
+    return `hash:${hash.substring(0, 16)}`; // Use first 16 chars for brevity
+}
+/**
+ * Helper: Compute content hash
+ */
+export function computeContentHash(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+//# sourceMappingURL=entityExtractionCache.js.map