juxscript 1.1.159 → 1.1.161

package/lib/storage/TabularDriver.ts

@@ -0,0 +1,491 @@
+import { DataFrame } from './DataFrame.js';
+
+export interface StoredTable {
+    id: string;
+    name: string;
+    columns: string[];
+    rows: any[][];
+    rowCount: number;
+    delimiter: string;
+    timestamp: number;
+    metadata?: Record<string, any>;
+}
+
+export interface ParseOptions {
+    delimiter?: string;
+    hasHeader?: boolean;
+    chunkSize?: number;
+    onProgress?: (parsed: number, total: number | null) => void;
+    maxRows?: number;
+    skipRows?: number;
+    columns?: string[];
+}
+
+export class TabularDriver {
+    private _dbName: string;
+    private _storeName: string;
+    private _db: IDBDatabase | null = null;
+
+    constructor(dbName: string = 'jux-tabular', storeName: string = 'tables') {
+        this._dbName = dbName;
+        this._storeName = storeName;
+    }
+
+    async open(): Promise<IDBDatabase> {
+        if (this._db) return this._db;
+
+        return new Promise((resolve, reject) => {
+            const request = indexedDB.open(this._dbName, 1);
+
+            request.onupgradeneeded = (e) => {
+                const db = (e.target as IDBOpenDBRequest).result;
+                if (!db.objectStoreNames.contains(this._storeName)) {
+                    const store = db.createObjectStore(this._storeName, { keyPath: 'id' });
+                    store.createIndex('name', 'name', { unique: false });
+                    store.createIndex('timestamp', 'timestamp', { unique: false });
+                }
+            };
+
+            request.onsuccess = (e) => {
+                this._db = (e.target as IDBOpenDBRequest).result;
+                resolve(this._db);
+            };
+
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * CSV / TSV PARSING
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Parse a CSV/TSV string into a DataFrame
+     */
+    parseCSV(text: string, options: ParseOptions = {}): DataFrame {
+        const {
+            delimiter = ',',
+            hasHeader = true,
+            maxRows,
+            skipRows = 0,
+            columns: selectCols
+        } = options;
+
+        const lines = this._splitLines(text);
+        let startIdx = skipRows;
+        let headers: string[];
+
+        if (hasHeader && lines.length > startIdx) {
+            headers = this._parseLine(lines[startIdx], delimiter);
+            startIdx++;
+        } else {
+            const firstLine = this._parseLine(lines[startIdx] || '', delimiter);
+            headers = firstLine.map((_, i) => `col_${i}`);
+        }
+
+        const rows: Record<string, any>[] = [];
+        const end = maxRows ? Math.min(startIdx + maxRows, lines.length) : lines.length;
+
+        for (let i = startIdx; i < end; i++) {
+            if (!lines[i].trim()) continue;
+            const values = this._parseLine(lines[i], delimiter);
+            const row: Record<string, any> = {};
+
+            headers.forEach((h, j) => {
+                row[h] = this._autoType(values[j]);
+            });
+
+            rows.push(row);
+        }
+
+        let df = new DataFrame(rows);
+
+        if (selectCols) {
+            df = df.select(...selectCols);
+        }
+
+        return df;
+    }
+
+    /**
+     * Stream-parse a File into a DataFrame, with progress callback
+     */
+    async streamFile(file: File, options: ParseOptions = {}): Promise<DataFrame> {
+        const {
+            delimiter = file.name.endsWith('.tsv') ? '\t' : ',',
+            hasHeader = true,
+            chunkSize = 64 * 1024,
+            onProgress,
+            maxRows,
+            skipRows = 0,
+            columns: selectCols
+        } = options;
+
+        const totalSize = file.size;
+        let bytesRead = 0;
+        let headers: string[] | null = null;
+        let lineBuffer = '';
+        let rowCount = 0;
+        let skipped = 0;
+        const rows: Record<string, any>[] = [];
+
+        const reader = file.stream().getReader();
+        const decoder = new TextDecoder();
+
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+
+            bytesRead += value.byteLength;
+            lineBuffer += decoder.decode(value, { stream: !done });
+
+            const lines = lineBuffer.split('\n');
+            lineBuffer = lines.pop() || '';
+
+            for (const line of lines) {
+                if (!line.trim()) continue;
+
+                if (!headers && hasHeader) {
+                    headers = this._parseLine(line, delimiter);
+                    continue;
+                }
+
+                if (!headers) {
+                    const vals = this._parseLine(line, delimiter);
+                    headers = vals.map((_, i) => `col_${i}`);
+                }
+
+                if (skipped < skipRows) {
+                    skipped++;
+                    continue;
+                }
+
+                if (maxRows && rowCount >= maxRows) break;
+
+                const values = this._parseLine(line, delimiter);
+                const row: Record<string, any> = {};
+                headers.forEach((h, j) => {
+                    row[h] = this._autoType(values[j]);
+                });
+                rows.push(row);
+                rowCount++;
+            }
+
+            if (onProgress) {
+                onProgress(bytesRead, totalSize);
+            }
+
+            if (maxRows && rowCount >= maxRows) break;
+        }
+
+        // Handle remaining buffer
+        if (lineBuffer.trim() && (!maxRows || rowCount < maxRows)) {
+            if (!headers) {
+                headers = this._parseLine(lineBuffer, delimiter).map((_, i) => `col_${i}`);
+            } else {
+                const values = this._parseLine(lineBuffer, delimiter);
+                const row: Record<string, any> = {};
+                headers.forEach((h, j) => {
+                    row[h] = this._autoType(values[j]);
+                });
+                rows.push(row);
+            }
+        }
+
+        if (onProgress) onProgress(totalSize, totalSize);
+
+        let df = new DataFrame(rows);
+        if (selectCols) df = df.select(...selectCols);
+
+        return df;
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * INDEXEDDB PERSISTENCE
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Store a DataFrame to IndexedDB
+     */
+    async store(name: string, df: DataFrame, metadata?: Record<string, any>): Promise<string> {
+        const db = await this.open();
+
+        const table: StoredTable = {
+            id: `${name}-${Date.now()}`,
+            name,
+            columns: df.columns,
+            rows: df.toRows().map(row => df.columns.map(c => row[c])),
+            rowCount: df.height,
+            delimiter: ',',
+            timestamp: Date.now(),
+            metadata
+        };
+
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction(this._storeName, 'readwrite');
+            const store = tx.objectStore(this._storeName);
+            const request = store.put(table);
+            request.onsuccess = () => resolve(table.id);
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    /**
+     * Load a DataFrame from IndexedDB by ID
+     */
+    async load(id: string): Promise<DataFrame | null> {
+        const db = await this.open();
+
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction(this._storeName, 'readonly');
+            const store = tx.objectStore(this._storeName);
+            const request = store.get(id);
+
+            request.onsuccess = () => {
+                const table: StoredTable = request.result;
+                if (!table) return resolve(null);
+
+                const rows = table.rows.map(row => {
+                    const obj: Record<string, any> = {};
+                    table.columns.forEach((col, i) => { obj[col] = row[i]; });
+                    return obj;
+                });
+
+                resolve(new DataFrame(rows));
+            };
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    /**
+     * Load the most recent table by name
+     */
+    async loadByName(name: string): Promise<DataFrame | null> {
+        const db = await this.open();
+
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction(this._storeName, 'readonly');
+            const store = tx.objectStore(this._storeName);
+            const index = store.index('name');
+            const request = index.getAll(name);
+
+            request.onsuccess = () => {
+                const tables: StoredTable[] = request.result || [];
+                if (tables.length === 0) return resolve(null);
+
+                tables.sort((a, b) => b.timestamp - a.timestamp);
+                const table = tables[0];
+
+                const rows = table.rows.map(row => {
+                    const obj: Record<string, any> = {};
+                    table.columns.forEach((col, i) => { obj[col] = row[i]; });
+                    return obj;
+                });
+
+                resolve(new DataFrame(rows));
+            };
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    /**
+     * List all stored tables (metadata only)
+     */
+    async list(): Promise<Array<{ id: string; name: string; columns: string[]; rowCount: number; timestamp: number }>> {
+        const db = await this.open();
+
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction(this._storeName, 'readonly');
+            const store = tx.objectStore(this._storeName);
+            const request = store.getAll();
+
+            request.onsuccess = () => {
+                resolve((request.result || []).map((t: StoredTable) => ({
+                    id: t.id,
+                    name: t.name,
+                    columns: t.columns,
+                    rowCount: t.rowCount,
+                    timestamp: t.timestamp
+                })));
+            };
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    /**
+     * Delete a stored table
+     */
+    async delete(id: string): Promise<void> {
+        const db = await this.open();
+
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction(this._storeName, 'readwrite');
+            const store = tx.objectStore(this._storeName);
+            const request = store.delete(id);
+            request.onsuccess = () => resolve();
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    async clear(): Promise<void> {
+        const db = await this.open();
+
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction(this._storeName, 'readwrite');
+            const store = tx.objectStore(this._storeName);
+            const request = store.clear();
+            request.onsuccess = () => resolve();
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * STREAM FROM URL
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Fetch and stream-parse a remote CSV/TSV file
+     */
+    async fetch(url: string, options: ParseOptions = {}): Promise<DataFrame> {
+        const response = await globalThis.fetch(url);
+        if (!response.ok) throw new Error(`Failed to fetch: ${response.status}`);
+        if (!response.body) throw new Error('No response body');
+
+        const contentLength = Number(response.headers.get('content-length')) || null;
+        const ext = url.split('.').pop()?.toLowerCase();
+        const delimiter = options.delimiter ?? (ext === 'tsv' ? '\t' : ',');
+
+        const { hasHeader = true, chunkSize, onProgress, maxRows, skipRows = 0, columns: selectCols } = options;
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+
+        let headers: string[] | null = null;
+        let lineBuffer = '';
+        let rowCount = 0;
+        let skipped = 0;
+        let bytesRead = 0;
+        const rows: Record<string, any>[] = [];
+
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+
+            bytesRead += value.byteLength;
+            lineBuffer += decoder.decode(value, { stream: true });
+
+            const lines = lineBuffer.split('\n');
+            lineBuffer = lines.pop() || '';
+
+            for (const line of lines) {
+                if (!line.trim()) continue;
+
+                if (!headers && hasHeader) {
+                    headers = this._parseLine(line, delimiter);
+                    continue;
+                }
+
+                if (!headers) {
+                    const vals = this._parseLine(line, delimiter);
+                    headers = vals.map((_, i) => `col_${i}`);
+                }
+
+                if (skipped < skipRows) { skipped++; continue; }
+                if (maxRows && rowCount >= maxRows) break;
+
+                const values = this._parseLine(line, delimiter);
+                const row: Record<string, any> = {};
+                headers.forEach((h, j) => { row[h] = this._autoType(values[j]); });
+                rows.push(row);
+                rowCount++;
+            }
+
+            if (onProgress) onProgress(bytesRead, contentLength);
+            if (maxRows && rowCount >= maxRows) break;
+        }
+
+        if (lineBuffer.trim() && (!maxRows || rowCount < maxRows)) {
+            if (headers) {
+                const values = this._parseLine(lineBuffer, delimiter);
+                const row: Record<string, any> = {};
+                headers.forEach((h, j) => { row[h] = this._autoType(values[j]); });
+                rows.push(row);
+            }
+        }
+
+        if (onProgress) onProgress(bytesRead, bytesRead);
+
+        let df = new DataFrame(rows);
+        if (selectCols) df = df.select(...selectCols);
+        return df;
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * INTERNAL PARSING HELPERS
+     * ═══════════════════════════════════════════════════ */
+
+    private _splitLines(text: string): string[] {
+        return text.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n');
+    }
+
+    private _parseLine(line: string, delimiter: string): string[] {
+        const result: string[] = [];
+        let current = '';
+        let inQuotes = false;
+
+        for (let i = 0; i < line.length; i++) {
+            const ch = line[i];
+
+            if (inQuotes) {
+                if (ch === '"') {
+                    if (i + 1 < line.length && line[i + 1] === '"') {
+                        current += '"';
+                        i++;
+                    } else {
+                        inQuotes = false;
+                    }
+                } else {
+                    current += ch;
+                }
+            } else {
+                if (ch === '"') {
+                    inQuotes = true;
+                } else if (ch === delimiter) {
+                    result.push(current.trim());
+                    current = '';
+                } else {
+                    current += ch;
+                }
+            }
+        }
+
+        result.push(current.trim());
+        return result;
+    }
+
+    private _autoType(value: string | undefined): any {
+        if (value === undefined || value === '' || value === null) return null;
+
+        if (value === 'true') return true;
+        if (value === 'false') return false;
+
+        const num = Number(value);
+        if (!isNaN(num) && value.trim() !== '') return num;
+
+        return value;
+    }
+
+    close(): void {
+        if (this._db) {
+            this._db.close();
+            this._db = null;
+        }
+    }
+}
+
+/**
+ * Factory
+ */
+export function tabularDriver(dbName?: string, storeName?: string): TabularDriver {
+    return new TabularDriver(dbName, storeName);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "juxscript",
-  "version": "1.1.159",
+  "version": "1.1.161",
   "type": "module",
   "description": "A JavaScript UX authorship platform",
   "main": "index.js",