@soulcraft/brainy 3.21.0 → 3.22.0

package/dist/augmentations/intelligentImport/handlers/csvHandler.js

@@ -0,0 +1,185 @@
+/**
+ * CSV Format Handler
+ * Handles CSV files with:
+ * - Automatic encoding detection
+ * - Automatic delimiter detection
+ * - Streaming for large files
+ * - Type inference
+ */
+import { parse } from 'csv-parse/sync';
+import { detect as detectEncoding } from 'chardet';
+import { BaseFormatHandler } from './base.js';
+export class CSVHandler extends BaseFormatHandler {
+    constructor() {
+        super(...arguments);
+        this.format = 'csv';
+    }
+    canHandle(data) {
+        const ext = this.detectExtension(data);
+        if (ext === 'csv' || ext === 'tsv' || ext === 'txt')
+            return true;
+        // Check content if it's a buffer
+        if (Buffer.isBuffer(data)) {
+            const sample = data.slice(0, 1024).toString('utf-8');
+            return this.looksLikeCSV(sample);
+        }
+        if (typeof data === 'string') {
+            return this.looksLikeCSV(data.slice(0, 1024));
+        }
+        return false;
+    }
+    async process(data, options) {
+        const startTime = Date.now();
+        // Convert to buffer if string
+        const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8');
+        // Detect encoding
+        const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer);
+        const text = buffer.toString(detectedEncoding);
+        // Detect delimiter if not specified
+        const delimiter = options.csvDelimiter || this.detectDelimiter(text);
+        // Parse CSV
+        const hasHeaders = options.csvHeaders !== false;
+        const maxRows = options.maxRows;
+        try {
+            const records = parse(text, {
+                columns: hasHeaders,
+                skip_empty_lines: true,
+                trim: true,
+                delimiter,
+                relax_column_count: true,
+                to: maxRows,
+                cast: false // We'll do type inference ourselves
+            });
+            // Convert to array of objects
+            const data = Array.isArray(records) ? records : [records];
+            // Infer types and convert values
+            const fields = data.length > 0 ? Object.keys(data[0]) : [];
+            const types = this.inferFieldTypes(data);
+            const convertedData = data.map(row => {
+                const converted = {};
+                for (const [key, value] of Object.entries(row)) {
+                    converted[key] = this.convertValue(value, types[key] || 'string');
+                }
+                return converted;
+            });
+            const processingTime = Date.now() - startTime;
+            return {
+                format: this.format,
+                data: convertedData,
+                metadata: this.createMetadata(convertedData.length, fields, processingTime, {
+                    encoding: detectedEncoding,
+                    delimiter,
+                    hasHeaders,
+                    types
+                }),
+                filename: options.filename
+            };
+        }
+        catch (error) {
+            throw new Error(`CSV parsing failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Check if text looks like CSV
+     */
+    looksLikeCSV(text) {
+        const lines = text.split('\n').filter(l => l.trim());
+        if (lines.length < 2)
+            return false;
+        // Check for common delimiters
+        const delimiters = [',', ';', '\t', '|'];
+        for (const delimiter of delimiters) {
+            const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+            if (firstCount === 0)
+                continue;
+            const secondCount = (lines[1].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+            if (firstCount === secondCount)
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Detect CSV delimiter
+     */
+    detectDelimiter(text) {
+        const sample = text.split('\n').slice(0, 10).join('\n');
+        const delimiters = [',', ';', '\t', '|'];
+        const counts = {};
+        for (const delimiter of delimiters) {
+            const lines = sample.split('\n').filter(l => l.trim());
+            if (lines.length < 2)
+                continue;
+            // Count delimiter in first line
+            const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+            if (firstCount === 0)
+                continue;
+            // Check if count is consistent across lines
+            let consistent = true;
+            for (let i = 1; i < Math.min(5, lines.length); i++) {
+                const count = (lines[i].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+                if (count !== firstCount) {
+                    consistent = false;
+                    break;
+                }
+            }
+            if (consistent) {
+                counts[delimiter] = firstCount;
+            }
+        }
+        // Return delimiter with highest count
+        const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0];
+        return best ? best[0] : ',';
+    }
+    /**
+     * Detect encoding safely (with fallback)
+     */
+    detectEncodingSafe(buffer) {
+        try {
+            const detected = detectEncoding(buffer);
+            if (!detected)
+                return 'utf-8';
+            // Normalize encoding to Node.js-supported names
+            return this.normalizeEncoding(detected);
+        }
+        catch {
+            return 'utf-8';
+        }
+    }
+    /**
+     * Normalize encoding names to Node.js-supported encodings
+     */
+    normalizeEncoding(encoding) {
+        const normalized = encoding.toLowerCase().replace(/[_-]/g, '');
+        // Map common encodings to Node.js names
+        const mappings = {
+            'iso88591': 'latin1',
+            'iso88592': 'latin1',
+            'iso88593': 'latin1',
+            'iso88594': 'latin1',
+            'iso88595': 'latin1',
+            'iso88596': 'latin1',
+            'iso88597': 'latin1',
+            'iso88598': 'latin1',
+            'iso88599': 'latin1',
+            'iso885910': 'latin1',
+            'iso885913': 'latin1',
+            'iso885914': 'latin1',
+            'iso885915': 'latin1',
+            'iso885916': 'latin1',
+            'usascii': 'ascii',
+            'utf8': 'utf8',
+            'utf16le': 'utf16le',
+            'utf16be': 'utf16le',
+            'windows1252': 'latin1',
+            'windows1251': 'utf8', // Cyrillic - best effort
+            'big5': 'utf8', // Chinese - best effort
+            'gbk': 'utf8', // Chinese - best effort
+            'gb2312': 'utf8', // Chinese - best effort
+            'shiftjis': 'utf8', // Japanese - best effort
+            'eucjp': 'utf8', // Japanese - best effort
+            'euckr': 'utf8' // Korean - best effort
+        };
+        return mappings[normalized] || 'utf8';
+    }
+}
+//# sourceMappingURL=csvHandler.js.map

package/dist/augmentations/intelligentImport/handlers/excelHandler.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Excel Format Handler
+ * Handles Excel files (.xlsx, .xls, .xlsb) with:
+ * - Multi-sheet extraction
+ * - Type inference
+ * - Formula evaluation
+ * - Metadata extraction
+ */
+import { BaseFormatHandler } from './base.js';
+import { FormatHandlerOptions, ProcessedData } from '../types.js';
+export declare class ExcelHandler extends BaseFormatHandler {
+    readonly format = "excel";
+    canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+    process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    /**
+     * Determine which sheets to process
+     */
+    private getSheetsToProcess;
+    /**
+     * Check if a number is likely an Excel date
+     * Excel stores dates as days since 1900-01-01
+     */
+    private isExcelDate;
+    /**
+     * Convert Excel date (days since 1900-01-01) to JS Date
+     */
+    private excelDateToJSDate;
+}

package/dist/augmentations/intelligentImport/handlers/excelHandler.js

@@ -0,0 +1,148 @@
+/**
+ * Excel Format Handler
+ * Handles Excel files (.xlsx, .xls, .xlsb) with:
+ * - Multi-sheet extraction
+ * - Type inference
+ * - Formula evaluation
+ * - Metadata extraction
+ */
+import * as XLSX from 'xlsx';
+import { BaseFormatHandler } from './base.js';
+export class ExcelHandler extends BaseFormatHandler {
+    constructor() {
+        super(...arguments);
+        this.format = 'excel';
+    }
+    canHandle(data) {
+        const ext = this.detectExtension(data);
+        return ['xlsx', 'xls', 'xlsb', 'xlsm', 'xlt', 'xltx', 'xltm'].includes(ext || '');
+    }
+    async process(data, options) {
+        const startTime = Date.now();
+        // Convert to buffer if string (though Excel should always be binary)
+        const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary');
+        try {
+            // Read workbook
+            const workbook = XLSX.read(buffer, {
+                type: 'buffer',
+                cellDates: true,
+                cellNF: true,
+                cellStyles: true
+            });
+            // Determine which sheets to process
+            const sheetsToProcess = this.getSheetsToProcess(workbook, options);
+            // Extract data from sheets
+            const allData = [];
+            const sheetMetadata = {};
+            for (const sheetName of sheetsToProcess) {
+                const sheet = workbook.Sheets[sheetName];
+                if (!sheet)
+                    continue;
+                // Convert sheet to JSON with headers
+                const sheetData = XLSX.utils.sheet_to_json(sheet, {
+                    header: 1, // Get as array of arrays first
+                    defval: null,
+                    blankrows: false,
+                    raw: false // Convert to formatted strings
+                });
+                if (sheetData.length === 0)
+                    continue;
+                // First row is headers
+                const headers = sheetData[0].map((h) => this.sanitizeFieldName(String(h || '')));
+                // Skip if no headers
+                if (headers.length === 0)
+                    continue;
+                // Convert rows to objects
+                for (let i = 1; i < sheetData.length; i++) {
+                    const row = sheetData[i];
+                    const rowObj = {};
+                    // Add sheet name to each row
+                    rowObj._sheet = sheetName;
+                    for (let j = 0; j < headers.length; j++) {
+                        const header = headers[j];
+                        let value = row[j];
+                        // Convert Excel dates
+                        if (value && typeof value === 'number' && this.isExcelDate(value)) {
+                            value = this.excelDateToJSDate(value);
+                        }
+                        rowObj[header] = value === undefined ? null : value;
+                    }
+                    allData.push(rowObj);
+                }
+                // Store sheet metadata
+                sheetMetadata[sheetName] = {
+                    rowCount: sheetData.length - 1, // Exclude header row
+                    columnCount: headers.length,
+                    headers
+                };
+            }
+            // Infer types (excluding _sheet field)
+            const fields = allData.length > 0 ? Object.keys(allData[0]).filter(k => k !== '_sheet') : [];
+            const types = this.inferFieldTypes(allData);
+            // Convert values to appropriate types
+            const convertedData = allData.map(row => {
+                const converted = {};
+                for (const [key, value] of Object.entries(row)) {
+                    if (key === '_sheet') {
+                        converted[key] = value;
+                    }
+                    else {
+                        converted[key] = this.convertValue(value, types[key] || 'string');
+                    }
+                }
+                return converted;
+            });
+            const processingTime = Date.now() - startTime;
+            return {
+                format: this.format,
+                data: convertedData,
+                metadata: this.createMetadata(convertedData.length, fields, processingTime, {
+                    sheets: sheetsToProcess,
+                    sheetCount: sheetsToProcess.length,
+                    sheetMetadata,
+                    types,
+                    workbookInfo: {
+                        sheetNames: workbook.SheetNames,
+                        properties: workbook.Props || {}
+                    }
+                }),
+                filename: options.filename
+            };
+        }
+        catch (error) {
+            throw new Error(`Excel parsing failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Determine which sheets to process
+     */
+    getSheetsToProcess(workbook, options) {
+        const allSheets = workbook.SheetNames;
+        // If specific sheets requested
+        if (options.excelSheets && options.excelSheets !== 'all') {
+            return options.excelSheets.filter(name => allSheets.includes(name));
+        }
+        // Otherwise process all sheets
+        return allSheets;
+    }
+    /**
+     * Check if a number is likely an Excel date
+     * Excel stores dates as days since 1900-01-01
+     */
+    isExcelDate(value) {
+        // Excel dates are typically between 1 and 60000 (1900 to 2064)
+        // This is a heuristic - not perfect but catches most cases
+        return value > 0 && value < 100000 && Number.isInteger(value);
+    }
+    /**
+     * Convert Excel date (days since 1900-01-01) to JS Date
+     */
+    excelDateToJSDate(excelDate) {
+        // Excel's epoch is 1900-01-01, but there's a bug where it thinks 1900 is a leap year
+        // So dates before March 1, 1900 are off by one day
+        const epoch = new Date(1899, 11, 30); // Dec 30, 1899
+        const msPerDay = 24 * 60 * 60 * 1000;
+        return new Date(epoch.getTime() + excelDate * msPerDay);
+    }
+}
+//# sourceMappingURL=excelHandler.js.map

package/dist/augmentations/intelligentImport/handlers/pdfHandler.d.ts

@@ -0,0 +1,35 @@
+/**
+ * PDF Format Handler
+ * Handles PDF files with:
+ * - Text extraction with layout preservation
+ * - Table detection and extraction
+ * - Metadata extraction (author, dates, etc.)
+ * - Page-by-page processing
+ */
+import { BaseFormatHandler } from './base.js';
+import { FormatHandlerOptions, ProcessedData } from '../types.js';
+export declare class PDFHandler extends BaseFormatHandler {
+    readonly format = "pdf";
+    canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+    process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    /**
+     * Group text items into lines based on Y position
+     */
+    private groupIntoLines;
+    /**
+     * Detect tables from lines
+     * Tables are detected when multiple consecutive lines have similar structure
+     */
+    private detectTables;
+    /**
+     * Parse a potential table into structured rows
+     */
+    private parseTable;
+    /**
+     * Extract paragraphs from lines
+     */
+    private extractParagraphs;
+}

package/dist/augmentations/intelligentImport/handlers/pdfHandler.js

@@ -0,0 +1,247 @@
+/**
+ * PDF Format Handler
+ * Handles PDF files with:
+ * - Text extraction with layout preservation
+ * - Table detection and extraction
+ * - Metadata extraction (author, dates, etc.)
+ * - Page-by-page processing
+ */
+import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs';
+import { BaseFormatHandler } from './base.js';
+// Use built-in worker for Node.js environments
+// In production, this can be customized via options
+const initializeWorker = () => {
+    if (typeof pdfjsLib.GlobalWorkerOptions.workerSrc === 'undefined' ||
+        pdfjsLib.GlobalWorkerOptions.workerSrc === '') {
+        // Use a data URL to avoid file system dependencies
+        // This tells pdfjs to use the built-in fallback worker
+        try {
+            pdfjsLib.GlobalWorkerOptions.workerSrc = 'data:,';
+        }
+        catch {
+            // Ignore if already set or in incompatible environment
+        }
+    }
+};
+initializeWorker();
+export class PDFHandler extends BaseFormatHandler {
+    constructor() {
+        super(...arguments);
+        this.format = 'pdf';
+    }
+    canHandle(data) {
+        const ext = this.detectExtension(data);
+        if (ext === 'pdf')
+            return true;
+        // Check for PDF magic bytes
+        if (Buffer.isBuffer(data)) {
+            const header = data.slice(0, 5).toString('ascii');
+            return header === '%PDF-';
+        }
+        return false;
+    }
+    async process(data, options) {
+        const startTime = Date.now();
+        // Convert to buffer
+        const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary');
+        try {
+            // Load PDF document
+            const loadingTask = pdfjsLib.getDocument({
+                data: new Uint8Array(buffer),
+                useSystemFonts: true,
+                standardFontDataUrl: undefined
+            });
+            const pdfDoc = await loadingTask.promise;
+            // Extract metadata
+            const metadata = await pdfDoc.getMetadata();
+            const numPages = pdfDoc.numPages;
+            // Extract text and structure from all pages
+            const allData = [];
+            let totalTextLength = 0;
+            let detectedTables = 0;
+            for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+                const page = await pdfDoc.getPage(pageNum);
+                const textContent = await page.getTextContent();
+                // Extract text items with positions
+                const textItems = textContent.items.map((item) => ({
+                    text: item.str,
+                    x: item.transform[4],
+                    y: item.transform[5],
+                    width: item.width,
+                    height: item.height
+                }));
+                // Combine text items into lines (group by similar Y position)
+                const lines = this.groupIntoLines(textItems);
+                // Detect tables if requested
+                if (options.pdfExtractTables !== false) {
+                    const tables = this.detectTables(lines);
+                    if (tables.length > 0) {
+                        detectedTables += tables.length;
+                        for (const table of tables) {
+                            allData.push(...table.rows);
+                        }
+                    }
+                }
+                // Extract paragraphs from non-table lines
+                const paragraphs = this.extractParagraphs(lines);
+                for (let i = 0; i < paragraphs.length; i++) {
+                    const text = paragraphs[i].trim();
+                    if (text.length > 0) {
+                        totalTextLength += text.length;
+                        allData.push({
+                            _page: pageNum,
+                            _type: 'paragraph',
+                            _index: i,
+                            text
+                        });
+                    }
+                }
+            }
+            const processingTime = Date.now() - startTime;
+            // Get all unique fields (excluding metadata fields)
+            const fields = allData.length > 0
+                ? Object.keys(allData[0]).filter(k => !k.startsWith('_'))
+                : [];
+            return {
+                format: this.format,
+                data: allData,
+                metadata: this.createMetadata(allData.length, fields, processingTime, {
+                    pageCount: numPages,
+                    textLength: totalTextLength,
+                    tableCount: detectedTables,
+                    pdfMetadata: {
+                        title: metadata.info?.Title || null,
+                        author: metadata.info?.Author || null,
+                        subject: metadata.info?.Subject || null,
+                        creator: metadata.info?.Creator || null,
+                        producer: metadata.info?.Producer || null,
+                        creationDate: metadata.info?.CreationDate || null,
+                        modificationDate: metadata.info?.ModDate || null
+                    }
+                }),
+                filename: options.filename
+            };
+        }
+        catch (error) {
+            throw new Error(`PDF parsing failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Group text items into lines based on Y position
+     */
+    groupIntoLines(items) {
+        if (items.length === 0)
+            return [];
+        // Sort by Y position (descending, since PDF coordinates go bottom-up)
+        const sorted = [...items].sort((a, b) => b.y - a.y);
+        const lines = [];
+        let currentLine = [];
+        let currentY = sorted[0].y;
+        for (const item of sorted) {
+            // If Y position differs by more than half the height, it's a new line
+            if (Math.abs(item.y - currentY) > (item.height / 2)) {
+                if (currentLine.length > 0) {
+                    // Sort line items by X position
+                    currentLine.sort((a, b) => a.x - b.x);
+                    lines.push(currentLine);
+                }
+                currentLine = [];
+                currentY = item.y;
+            }
+            if (item.text.trim()) {
+                currentLine.push({ text: item.text, x: item.x });
+            }
+        }
+        // Add last line
+        if (currentLine.length > 0) {
+            currentLine.sort((a, b) => a.x - b.x);
+            lines.push(currentLine);
+        }
+        return lines;
+    }
+    /**
+     * Detect tables from lines
+     * Tables are detected when multiple consecutive lines have similar structure
+     */
+    detectTables(lines) {
+        const tables = [];
+        let potentialTable = [];
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            // A line with multiple items could be part of a table
+            if (line.length >= 2) {
+                potentialTable.push(line);
+            }
+            else {
+                // End of potential table
+                if (potentialTable.length >= 3) { // Need at least header + 2 rows
+                    const table = this.parseTable(potentialTable);
+                    if (table) {
+                        tables.push(table);
+                    }
+                }
+                potentialTable = [];
+            }
+        }
+        // Check last potential table
+        if (potentialTable.length >= 3) {
+            const table = this.parseTable(potentialTable);
+            if (table) {
+                tables.push(table);
+            }
+        }
+        return tables;
+    }
+    /**
+     * Parse a potential table into structured rows
+     */
+    parseTable(lines) {
+        if (lines.length < 2)
+            return null;
+        // First line is headers
+        const headerLine = lines[0];
+        const headers = headerLine.map(item => this.sanitizeFieldName(item.text));
+        // Remaining lines are data
+        const rows = [];
+        for (let i = 1; i < lines.length; i++) {
+            const line = lines[i];
+            const row = { _type: 'table_row' };
+            // Match each item to closest header by X position
+            for (let j = 0; j < line.length && j < headers.length; j++) {
+                const header = headers[j];
+                const value = line[j].text.trim();
+                row[header] = value || null;
+            }
+            if (Object.keys(row).length > 1) { // More than just _type
+                rows.push(row);
+            }
+        }
+        return rows.length > 0 ? { rows } : null;
+    }
+    /**
+     * Extract paragraphs from lines
+     */
+    extractParagraphs(lines) {
+        const paragraphs = [];
+        let currentParagraph = [];
+        for (const line of lines) {
+            const lineText = line.map(item => item.text).join(' ').trim();
+            if (lineText.length === 0) {
+                // Empty line - end paragraph
+                if (currentParagraph.length > 0) {
+                    paragraphs.push(currentParagraph.join(' '));
+                    currentParagraph = [];
+                }
+            }
+            else {
+                currentParagraph.push(lineText);
+            }
+        }
+        // Add last paragraph
+        if (currentParagraph.length > 0) {
+            paragraphs.push(currentParagraph.join(' '));
+        }
+        return paragraphs;
+    }
+}
+//# sourceMappingURL=pdfHandler.js.map

package/dist/augmentations/intelligentImport/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Intelligent Import Module
+ * Exports main augmentation and types
+ */
+export { IntelligentImportAugmentation } from './IntelligentImportAugmentation.js';
+export type { FormatHandler, FormatHandlerOptions, ProcessedData, IntelligentImportConfig } from './types.js';
+export { CSVHandler } from './handlers/csvHandler.js';
+export { ExcelHandler } from './handlers/excelHandler.js';
+export { PDFHandler } from './handlers/pdfHandler.js';

package/dist/augmentations/intelligentImport/index.js

@@ -0,0 +1,9 @@
+/**
+ * Intelligent Import Module
+ * Exports main augmentation and types
+ */
+export { IntelligentImportAugmentation } from './IntelligentImportAugmentation.js';
+export { CSVHandler } from './handlers/csvHandler.js';
+export { ExcelHandler } from './handlers/excelHandler.js';
+export { PDFHandler } from './handlers/pdfHandler.js';
+//# sourceMappingURL=index.js.map