@soulcraft/brainy 3.20.5 → 3.22.0

package/dist/augmentations/intelligentImport/handlers/base.js

@@ -0,0 +1,149 @@
+/**
+ * Base Format Handler
+ * Abstract class providing common functionality for all format handlers
+ */
+export class BaseFormatHandler {
+    /**
+     * Detect file extension from various inputs
+     */
+    detectExtension(data) {
+        if (typeof data === 'object' && 'filename' in data && data.filename) {
+            return this.getExtension(data.filename);
+        }
+        if (typeof data === 'object' && 'ext' in data && data.ext) {
+            return data.ext.toLowerCase().replace(/^\./, '');
+        }
+        return null;
+    }
+    /**
+     * Extract extension from filename
+     */
+    getExtension(filename) {
+        const match = filename.match(/\.([^.]+)$/);
+        return match ? match[1].toLowerCase() : '';
+    }
+    /**
+     * Infer field types from data
+     * Analyzes multiple rows to determine the most appropriate type
+     */
+    inferFieldTypes(data) {
+        if (data.length === 0)
+            return {};
+        const types = {};
+        const firstRow = data[0];
+        const sampleSize = Math.min(10, data.length);
+        for (const key of Object.keys(firstRow)) {
+            // Check first few rows to get more accurate type
+            const sampleTypes = new Set();
+            for (let i = 0; i < sampleSize; i++) {
+                const value = data[i][key];
+                const type = this.inferType(value);
+                sampleTypes.add(type);
+            }
+            // If we see both integer and float, use float
+            if (sampleTypes.has('float') || (sampleTypes.has('integer') && sampleTypes.has('float'))) {
+                types[key] = 'float';
+            }
+            else if (sampleTypes.has('integer')) {
+                types[key] = 'integer';
+            }
+            else if (sampleTypes.has('date')) {
+                types[key] = 'date';
+            }
+            else if (sampleTypes.has('boolean')) {
+                types[key] = 'boolean';
+            }
+            else {
+                types[key] = 'string';
+            }
+        }
+        return types;
+    }
+    /**
+     * Infer type of a single value
+     */
+    inferType(value) {
+        if (value === null || value === undefined || value === '')
+            return 'string';
+        if (typeof value === 'number')
+            return 'number';
+        if (typeof value === 'boolean')
+            return 'boolean';
+        if (typeof value === 'string') {
+            // Check if it's a number
+            if (/^-?\d+$/.test(value))
+                return 'integer';
+            if (/^-?\d+\.\d+$/.test(value))
+                return 'float';
+            // Check if it's a date
+            if (this.isDateString(value))
+                return 'date';
+            // Check if it's a boolean
+            if (/^(true|false|yes|no|y|n)$/i.test(value))
+                return 'boolean';
+        }
+        return 'string';
+    }
+    /**
+     * Check if string looks like a date
+     */
+    isDateString(value) {
+        // ISO 8601
+        if (/^\d{4}-\d{2}-\d{2}/.test(value))
+            return true;
+        // Common date formats
+        if (/^\d{1,2}\/\d{1,2}\/\d{2,4}$/.test(value))
+            return true;
+        if (/^\d{1,2}-\d{1,2}-\d{2,4}$/.test(value))
+            return true;
+        return false;
+    }
+    /**
+     * Sanitize field names for use as object keys
+     */
+    sanitizeFieldName(name) {
+        return name
+            .trim()
+            .replace(/[^a-zA-Z0-9_\s-]/g, '')
+            .replace(/\s+/g, '_')
+            .replace(/-+/g, '_')
+            .replace(/_+/g, '_')
+            .replace(/^_|_$/g, '')
+            || 'field';
+    }
+    /**
+     * Convert value to appropriate type
+     */
+    convertValue(value, type) {
+        if (value === null || value === undefined || value === '')
+            return null;
+        switch (type) {
+            case 'integer':
+                return parseInt(String(value), 10);
+            case 'float':
+            case 'number':
+                return parseFloat(String(value));
+            case 'boolean':
+                if (typeof value === 'boolean')
+                    return value;
+                const str = String(value).toLowerCase();
+                return ['true', 'yes', 'y', '1'].includes(str);
+            case 'date':
+                return new Date(value);
+            default:
+                return value;
+        }
+    }
+    /**
+     * Create metadata object with common fields
+     */
+    createMetadata(rowCount, fields, processingTime, extra = {}) {
+        return {
+            rowCount,
+            fields,
+            processingTime,
+            ...extra
+        };
+    }
+}
+//# sourceMappingURL=base.js.map

package/dist/augmentations/intelligentImport/handlers/csvHandler.d.ts

@@ -0,0 +1,34 @@
+/**
+ * CSV Format Handler
+ * Handles CSV files with:
+ * - Automatic encoding detection
+ * - Automatic delimiter detection
+ * - Streaming for large files
+ * - Type inference
+ */
+import { BaseFormatHandler } from './base.js';
+import { FormatHandlerOptions, ProcessedData } from '../types.js';
+export declare class CSVHandler extends BaseFormatHandler {
+    readonly format = "csv";
+    canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+    process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    /**
+     * Check if text looks like CSV
+     */
+    private looksLikeCSV;
+    /**
+     * Detect CSV delimiter
+     */
+    private detectDelimiter;
+    /**
+     * Detect encoding safely (with fallback)
+     */
+    private detectEncodingSafe;
+    /**
+     * Normalize encoding names to Node.js-supported encodings
+     */
+    private normalizeEncoding;
+}

package/dist/augmentations/intelligentImport/handlers/csvHandler.js

@@ -0,0 +1,185 @@
+/**
+ * CSV Format Handler
+ * Handles CSV files with:
+ * - Automatic encoding detection
+ * - Automatic delimiter detection
+ * - Streaming for large files
+ * - Type inference
+ */
+import { parse } from 'csv-parse/sync';
+import { detect as detectEncoding } from 'chardet';
+import { BaseFormatHandler } from './base.js';
+export class CSVHandler extends BaseFormatHandler {
+    constructor() {
+        super(...arguments);
+        this.format = 'csv';
+    }
+    canHandle(data) {
+        const ext = this.detectExtension(data);
+        if (ext === 'csv' || ext === 'tsv' || ext === 'txt')
+            return true;
+        // Check content if it's a buffer
+        if (Buffer.isBuffer(data)) {
+            const sample = data.slice(0, 1024).toString('utf-8');
+            return this.looksLikeCSV(sample);
+        }
+        if (typeof data === 'string') {
+            return this.looksLikeCSV(data.slice(0, 1024));
+        }
+        return false;
+    }
+    async process(data, options) {
+        const startTime = Date.now();
+        // Convert to buffer if string
+        const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8');
+        // Detect encoding
+        const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer);
+        const text = buffer.toString(detectedEncoding);
+        // Detect delimiter if not specified
+        const delimiter = options.csvDelimiter || this.detectDelimiter(text);
+        // Parse CSV
+        const hasHeaders = options.csvHeaders !== false;
+        const maxRows = options.maxRows;
+        try {
+            const records = parse(text, {
+                columns: hasHeaders,
+                skip_empty_lines: true,
+                trim: true,
+                delimiter,
+                relax_column_count: true,
+                to: maxRows,
+                cast: false // We'll do type inference ourselves
+            });
+            // Convert to array of objects
+            const data = Array.isArray(records) ? records : [records];
+            // Infer types and convert values
+            const fields = data.length > 0 ? Object.keys(data[0]) : [];
+            const types = this.inferFieldTypes(data);
+            const convertedData = data.map(row => {
+                const converted = {};
+                for (const [key, value] of Object.entries(row)) {
+                    converted[key] = this.convertValue(value, types[key] || 'string');
+                }
+                return converted;
+            });
+            const processingTime = Date.now() - startTime;
+            return {
+                format: this.format,
+                data: convertedData,
+                metadata: this.createMetadata(convertedData.length, fields, processingTime, {
+                    encoding: detectedEncoding,
+                    delimiter,
+                    hasHeaders,
+                    types
+                }),
+                filename: options.filename
+            };
+        }
+        catch (error) {
+            throw new Error(`CSV parsing failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Check if text looks like CSV
+     */
+    looksLikeCSV(text) {
+        const lines = text.split('\n').filter(l => l.trim());
+        if (lines.length < 2)
+            return false;
+        // Check for common delimiters
+        const delimiters = [',', ';', '\t', '|'];
+        for (const delimiter of delimiters) {
+            const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+            if (firstCount === 0)
+                continue;
+            const secondCount = (lines[1].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+            if (firstCount === secondCount)
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Detect CSV delimiter
+     */
+    detectDelimiter(text) {
+        const sample = text.split('\n').slice(0, 10).join('\n');
+        const delimiters = [',', ';', '\t', '|'];
+        const counts = {};
+        for (const delimiter of delimiters) {
+            const lines = sample.split('\n').filter(l => l.trim());
+            if (lines.length < 2)
+                continue;
+            // Count delimiter in first line
+            const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+            if (firstCount === 0)
+                continue;
+            // Check if count is consistent across lines
+            let consistent = true;
+            for (let i = 1; i < Math.min(5, lines.length); i++) {
+                const count = (lines[i].match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+                if (count !== firstCount) {
+                    consistent = false;
+                    break;
+                }
+            }
+            if (consistent) {
+                counts[delimiter] = firstCount;
+            }
+        }
+        // Return delimiter with highest count
+        const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0];
+        return best ? best[0] : ',';
+    }
+    /**
+     * Detect encoding safely (with fallback)
+     */
+    detectEncodingSafe(buffer) {
+        try {
+            const detected = detectEncoding(buffer);
+            if (!detected)
+                return 'utf-8';
+            // Normalize encoding to Node.js-supported names
+            return this.normalizeEncoding(detected);
+        }
+        catch {
+            return 'utf-8';
+        }
+    }
+    /**
+     * Normalize encoding names to Node.js-supported encodings
+     */
+    normalizeEncoding(encoding) {
+        const normalized = encoding.toLowerCase().replace(/[_-]/g, '');
+        // Map common encodings to Node.js names
+        const mappings = {
+            'iso88591': 'latin1',
+            'iso88592': 'latin1',
+            'iso88593': 'latin1',
+            'iso88594': 'latin1',
+            'iso88595': 'latin1',
+            'iso88596': 'latin1',
+            'iso88597': 'latin1',
+            'iso88598': 'latin1',
+            'iso88599': 'latin1',
+            'iso885910': 'latin1',
+            'iso885913': 'latin1',
+            'iso885914': 'latin1',
+            'iso885915': 'latin1',
+            'iso885916': 'latin1',
+            'usascii': 'ascii',
+            'utf8': 'utf8',
+            'utf16le': 'utf16le',
+            'utf16be': 'utf16le',
+            'windows1252': 'latin1',
+            'windows1251': 'utf8', // Cyrillic - best effort
+            'big5': 'utf8', // Chinese - best effort
+            'gbk': 'utf8', // Chinese - best effort
+            'gb2312': 'utf8', // Chinese - best effort
+            'shiftjis': 'utf8', // Japanese - best effort
+            'eucjp': 'utf8', // Japanese - best effort
+            'euckr': 'utf8' // Korean - best effort
+        };
+        return mappings[normalized] || 'utf8';
+    }
+}
+//# sourceMappingURL=csvHandler.js.map

package/dist/augmentations/intelligentImport/handlers/excelHandler.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Excel Format Handler
+ * Handles Excel files (.xlsx, .xls, .xlsb) with:
+ * - Multi-sheet extraction
+ * - Type inference
+ * - Formula evaluation
+ * - Metadata extraction
+ */
+import { BaseFormatHandler } from './base.js';
+import { FormatHandlerOptions, ProcessedData } from '../types.js';
+export declare class ExcelHandler extends BaseFormatHandler {
+    readonly format = "excel";
+    canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+    process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    /**
+     * Determine which sheets to process
+     */
+    private getSheetsToProcess;
+    /**
+     * Check if a number is likely an Excel date
+     * Excel stores dates as days since 1900-01-01
+     */
+    private isExcelDate;
+    /**
+     * Convert Excel date (days since 1900-01-01) to JS Date
+     */
+    private excelDateToJSDate;
+}

package/dist/augmentations/intelligentImport/handlers/excelHandler.js

@@ -0,0 +1,148 @@
+/**
+ * Excel Format Handler
+ * Handles Excel files (.xlsx, .xls, .xlsb) with:
+ * - Multi-sheet extraction
+ * - Type inference
+ * - Formula evaluation
+ * - Metadata extraction
+ */
+import * as XLSX from 'xlsx';
+import { BaseFormatHandler } from './base.js';
+export class ExcelHandler extends BaseFormatHandler {
+    constructor() {
+        super(...arguments);
+        this.format = 'excel';
+    }
+    canHandle(data) {
+        const ext = this.detectExtension(data);
+        return ['xlsx', 'xls', 'xlsb', 'xlsm', 'xlt', 'xltx', 'xltm'].includes(ext || '');
+    }
+    async process(data, options) {
+        const startTime = Date.now();
+        // Convert to buffer if string (though Excel should always be binary)
+        const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary');
+        try {
+            // Read workbook
+            const workbook = XLSX.read(buffer, {
+                type: 'buffer',
+                cellDates: true,
+                cellNF: true,
+                cellStyles: true
+            });
+            // Determine which sheets to process
+            const sheetsToProcess = this.getSheetsToProcess(workbook, options);
+            // Extract data from sheets
+            const allData = [];
+            const sheetMetadata = {};
+            for (const sheetName of sheetsToProcess) {
+                const sheet = workbook.Sheets[sheetName];
+                if (!sheet)
+                    continue;
+                // Convert sheet to JSON with headers
+                const sheetData = XLSX.utils.sheet_to_json(sheet, {
+                    header: 1, // Get as array of arrays first
+                    defval: null,
+                    blankrows: false,
+                    raw: false // Convert to formatted strings
+                });
+                if (sheetData.length === 0)
+                    continue;
+                // First row is headers
+                const headers = sheetData[0].map((h) => this.sanitizeFieldName(String(h || '')));
+                // Skip if no headers
+                if (headers.length === 0)
+                    continue;
+                // Convert rows to objects
+                for (let i = 1; i < sheetData.length; i++) {
+                    const row = sheetData[i];
+                    const rowObj = {};
+                    // Add sheet name to each row
+                    rowObj._sheet = sheetName;
+                    for (let j = 0; j < headers.length; j++) {
+                        const header = headers[j];
+                        let value = row[j];
+                        // Convert Excel dates
+                        if (value && typeof value === 'number' && this.isExcelDate(value)) {
+                            value = this.excelDateToJSDate(value);
+                        }
+                        rowObj[header] = value === undefined ? null : value;
+                    }
+                    allData.push(rowObj);
+                }
+                // Store sheet metadata
+                sheetMetadata[sheetName] = {
+                    rowCount: sheetData.length - 1, // Exclude header row
+                    columnCount: headers.length,
+                    headers
+                };
+            }
+            // Infer types (excluding _sheet field)
+            const fields = allData.length > 0 ? Object.keys(allData[0]).filter(k => k !== '_sheet') : [];
+            const types = this.inferFieldTypes(allData);
+            // Convert values to appropriate types
+            const convertedData = allData.map(row => {
+                const converted = {};
+                for (const [key, value] of Object.entries(row)) {
+                    if (key === '_sheet') {
+                        converted[key] = value;
+                    }
+                    else {
+                        converted[key] = this.convertValue(value, types[key] || 'string');
+                    }
+                }
+                return converted;
+            });
+            const processingTime = Date.now() - startTime;
+            return {
+                format: this.format,
+                data: convertedData,
+                metadata: this.createMetadata(convertedData.length, fields, processingTime, {
+                    sheets: sheetsToProcess,
+                    sheetCount: sheetsToProcess.length,
+                    sheetMetadata,
+                    types,
+                    workbookInfo: {
+                        sheetNames: workbook.SheetNames,
+                        properties: workbook.Props || {}
+                    }
+                }),
+                filename: options.filename
+            };
+        }
+        catch (error) {
+            throw new Error(`Excel parsing failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Determine which sheets to process
+     */
+    getSheetsToProcess(workbook, options) {
+        const allSheets = workbook.SheetNames;
+        // If specific sheets requested
+        if (options.excelSheets && options.excelSheets !== 'all') {
+            return options.excelSheets.filter(name => allSheets.includes(name));
+        }
+        // Otherwise process all sheets
+        return allSheets;
+    }
+    /**
+     * Check if a number is likely an Excel date
+     * Excel stores dates as days since 1900-01-01
+     */
+    isExcelDate(value) {
+        // Excel dates are typically between 1 and 60000 (1900 to 2064)
+        // This is a heuristic - not perfect but catches most cases
+        return value > 0 && value < 100000 && Number.isInteger(value);
+    }
+    /**
+     * Convert Excel date (days since 1900-01-01) to JS Date
+     */
+    excelDateToJSDate(excelDate) {
+        // Excel's epoch is 1900-01-01, but there's a bug where it thinks 1900 is a leap year
+        // So dates before March 1, 1900 are off by one day
+        const epoch = new Date(1899, 11, 30); // Dec 30, 1899
+        const msPerDay = 24 * 60 * 60 * 1000;
+        return new Date(epoch.getTime() + excelDate * msPerDay);
+    }
+}
+//# sourceMappingURL=excelHandler.js.map

package/dist/augmentations/intelligentImport/handlers/pdfHandler.d.ts

@@ -0,0 +1,35 @@
+/**
+ * PDF Format Handler
+ * Handles PDF files with:
+ * - Text extraction with layout preservation
+ * - Table detection and extraction
+ * - Metadata extraction (author, dates, etc.)
+ * - Page-by-page processing
+ */
+import { BaseFormatHandler } from './base.js';
+import { FormatHandlerOptions, ProcessedData } from '../types.js';
+export declare class PDFHandler extends BaseFormatHandler {
+    readonly format = "pdf";
+    canHandle(data: Buffer | string | {
+        filename?: string;
+        ext?: string;
+    }): boolean;
+    process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>;
+    /**
+     * Group text items into lines based on Y position
+     */
+    private groupIntoLines;
+    /**
+     * Detect tables from lines
+     * Tables are detected when multiple consecutive lines have similar structure
+     */
+    private detectTables;
+    /**
+     * Parse a potential table into structured rows
+     */
+    private parseTable;
+    /**
+     * Extract paragraphs from lines
+     */
+    private extractParagraphs;
+}