juxscript 1.1.231 → 1.1.232

package/lib/storage/DataFrameSource.ts

@@ -0,0 +1,556 @@
+import { DataFrame } from './DataFrame.js';
+import { TabularDriver } from './TabularDriver.js';
+
+export interface SourceOptions {
+    // Storage
+    dbName?: string;
+    storeName?: string;
+    persistToIndexedDB?: boolean;
+
+    // Parsing
+    inferTypes?: boolean;
+    maxFileSize?: number;  // MB
+    maxSheetSize?: number;
+    headerRow?: number;
+    delimiter?: string;
+}
+
+export interface APIOptions {
+    method?: 'GET' | 'POST' | 'PUT' | 'DELETE';
+    headers?: Record<string, string>;
+    body?: any;
+    credentials?: 'omit' | 'same-origin' | 'include';
+    dataPath?: string;  // JSONPath-like: "data.results" or "response.items[0].records"
+}
+
+export interface StreamOptions {
+    mimeType?: string;
+    chunkSize?: number;
+    cacheToIndexedDB?: boolean;
+}
+
+export interface MultiSheetResult {
+    sheets: Map<string, DataFrame>;
+    sheetNames: string[];
+    activeSheet: string;
+}
+
+/**
+ * DataFrameSource - Data acquisition layer for DataFrames
+ * 
+ * Responsibilities:
+ * - Load from files (CSV, TSV, Excel)
+ * - Load from URLs (fetch, stream)
+ * - Load from APIs (with auth, JSON traversal)
+ * - Load from IndexedDB
+ * - Load from inline data
+ * - Multi-sheet support
+ * - Progress callbacks
+ * 
+ * This is a PURE data layer - no UI concerns
+ */
+export class DataFrameSource {
+    private _driver: TabularDriver;
+    private _options: SourceOptions;
+    private _sheets: Map<string, DataFrame> = new Map();
+    private _activeSheet: string = '';
+    private _sourceName: string = '';
+    private _rawFile: File | null = null;
+    private _rawText: string | null = null;
+
+    constructor(options: SourceOptions = {}) {
+        this._options = {
+            dbName: options.dbName ?? 'jux-dataframes',
+            storeName: options.storeName ?? 'frames',
+            persistToIndexedDB: options.persistToIndexedDB ?? false,
+            inferTypes: options.inferTypes ?? true,
+            maxFileSize: options.maxFileSize ?? 50,
+            maxSheetSize: options.maxSheetSize ?? 100000,
+            headerRow: options.headerRow ?? 0,
+            delimiter: options.delimiter
+        };
+
+        this._driver = new TabularDriver(
+            this._options.dbName!,
+            this._options.storeName!
+        );
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * FROM FILE
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Load from a File object (CSV, TSV, Excel)
+     */
+    async fromFile(
+        file: File,
+        onProgress?: (loaded: number, total: number | null) => void
+    ): Promise<DataFrame | MultiSheetResult> {
+        const fileSizeMB = file.size / (1024 * 1024);
+
+        if (fileSizeMB > this._options.maxFileSize!) {
+            throw new Error(`File too large (${fileSizeMB.toFixed(1)}MB). Max: ${this._options.maxFileSize}MB`);
+        }
+
+        this._rawFile = file;
+        this._sourceName = file.name;
+
+        const isExcel = /\.(xlsx?|xls)$/i.test(file.name);
+
+        if (isExcel) {
+            const sheets = await this._driver.streamFileMultiSheet(file, {
+                maxSheetSize: this._options.maxSheetSize,
+                headerRow: this._options.headerRow,
+                onProgress
+            });
+
+            this._sheets.clear();
+            const sheetNames = Object.keys(sheets);
+
+            sheetNames.forEach(name => {
+                this._sheets.set(name, sheets[name]);
+            });
+
+            this._activeSheet = sheetNames[0] || '';
+
+            if (this._options.persistToIndexedDB) {
+                await this._driver.store(file.name, sheets[sheetNames[0]], { source: file.name });
+            }
+
+            if (sheetNames.length > 1) {
+                return {
+                    sheets: this._sheets,
+                    sheetNames,
+                    activeSheet: this._activeSheet
+                };
+            } else {
+                return this._sheets.get(this._activeSheet)!;
+            }
+        } else {
+            const text = await file.text();
+            this._rawText = text;
+
+            const df = this._driver.parseCSV(text, {
+                autoDetectDelimiter: !this._options.delimiter,
+                delimiter: this._options.delimiter,
+                headerRow: this._options.headerRow,
+                hasHeader: true
+            });
+
+            this._sheets.set('Sheet1', df);
+            this._activeSheet = 'Sheet1';
+
+            if (this._options.persistToIndexedDB) {
+                await this._driver.store(file.name, df, { source: file.name });
+            }
+
+            return df;
+        }
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * FROM URL (Simple Fetch)
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Fetch CSV/TSV from URL
+     */
+    async fromUrl(
+        url: string,
+        onProgress?: (loaded: number, total: number | null) => void
+    ): Promise<DataFrame> {
+        this._sourceName = url.split('/').pop() || url;
+
+        const df = await this._driver.fetch(url, {
+            autoDetectDelimiter: true,
+            hasHeader: true,
+            headerRow: this._options.headerRow,
+            onProgress
+        });
+
+        this._sheets.set('Sheet1', df);
+        this._activeSheet = 'Sheet1';
+
+        if (this._options.persistToIndexedDB) {
+            await this._driver.store(this._sourceName, df, { source: url });
+        }
+
+        return df;
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * FROM API (JSON with traversal)
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Fetch data from API endpoint
+     * Supports JSON response with path traversal
+     */
+    async fromAPI(url: string, options: APIOptions = {}): Promise<DataFrame> {
+        const {
+            method = 'GET',
+            headers = {},
+            body,
+            credentials = 'same-origin',
+            dataPath
+        } = options;
+
+        const fetchOptions: RequestInit = {
+            method,
+            headers: {
+                'Accept': 'application/json',
+                ...headers
+            },
+            credentials
+        };
+
+        if (body && method !== 'GET') {
+            fetchOptions.body = typeof body === 'string' ? body : JSON.stringify(body);
+            if (!headers['Content-Type']) {
+                (fetchOptions.headers as Record<string, string>)['Content-Type'] = 'application/json';
+            }
+        }
+
+        const response = await fetch(url, fetchOptions);
+
+        if (!response.ok) {
+            throw new Error(`API request failed: ${response.status} ${response.statusText}`);
+        }
+
+        let data = await response.json();
+
+        // Traverse to data path if specified
+        if (dataPath) {
+            data = this._traversePath(data, dataPath);
+        }
+
+        // Convert to DataFrame
+        if (!Array.isArray(data)) {
+            throw new Error('API response must be an array of objects (or use dataPath to navigate to array)');
+        }
+
+        const df = new DataFrame(data, { inferTypes: this._options.inferTypes });
+
+        this._sourceName = url;
+        this._sheets.set('Sheet1', df);
+        this._activeSheet = 'Sheet1';
+
+        if (this._options.persistToIndexedDB) {
+            await this._driver.store(this._sourceName, df, { source: url });
+        }
+
+        return df;
+    }
+
+    /**
+     * Traverse JSON path like "data.results" or "response.items[0].records"
+     */
+    private _traversePath(obj: any, path: string): any {
+        const parts = path.split(/\.|\[|\]/).filter(Boolean);
+        let current = obj;
+
+        for (const part of parts) {
+            if (current === null || current === undefined) {
+                throw new Error(`Cannot traverse path '${path}': null at '${part}'`);
+            }
+
+            const index = Number(part);
+            if (!isNaN(index)) {
+                current = current[index];
+            } else {
+                current = current[part];
+            }
+        }
+
+        return current;
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * FROM STREAM (Binary data with MIME type)
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Stream binary data from URL, auto-detect format from MIME type
+     */
+    async fromStream(
+        url: string,
+        options: StreamOptions = {},
+        onProgress?: (loaded: number, total: number | null) => void
+    ): Promise<DataFrame | MultiSheetResult> {
+        const {
+            mimeType,
+            cacheToIndexedDB = false
+        } = options;
+
+        const response = await fetch(url);
+
+        if (!response.ok) {
+            throw new Error(`Stream request failed: ${response.status}`);
+        }
+
+        const contentType = mimeType || response.headers.get('content-type') || '';
+        const fileName = url.split('/').pop() || 'stream-data';
+
+        // Get the blob
+        const blob = await response.blob();
+        const file = new File([blob], fileName, { type: contentType });
+
+        // Cache to IndexedDB if requested
+        if (cacheToIndexedDB) {
+            // Store raw blob for potential re-parsing
+            await this._cacheBlob(fileName, blob);
+        }
+
+        // Parse based on content type
+        return this.fromFile(file, onProgress);
+    }
+
+    private async _cacheBlob(name: string, blob: Blob): Promise<void> {
+        // Simple blob caching in IndexedDB
+        const db = await this._driver.open();
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction('blobs', 'readwrite');
+            const store = tx.objectStore('blobs');
+            store.put({ name, blob, timestamp: Date.now() });
+            tx.oncomplete = () => resolve();
+            tx.onerror = () => reject(tx.error);
+        });
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * FROM INDEXEDDB
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Load from IndexedDB by key/name
+     */
+    async fromStorage(key: string): Promise<DataFrame | null> {
+        const df = await this._driver.loadByName(key);
+
+        if (df) {
+            this._sourceName = key;
+            this._sheets.set('Sheet1', df);
+            this._activeSheet = 'Sheet1';
+        }
+
+        return df;
+    }
+
+    /**
+     * List all stored DataFrames
+     */
+    async listStored(): Promise<Array<{ id: string; name: string; rowCount: number; timestamp: number }>> {
+        return this._driver.list();
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * FROM INLINE DATA
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Create from inline data
+     */
+    fromData(
+        data: Record<string, any>[] | Record<string, any[]>,
+        name: string = 'inline'
+    ): DataFrame {
+        const df = new DataFrame(data, { inferTypes: this._options.inferTypes });
+
+        this._sourceName = name;
+        this._sheets.set('Sheet1', df);
+        this._activeSheet = 'Sheet1';
+
+        return df;
+    }
+
+    /**
+     * Create empty DataFrame with specified columns
+     */
+    empty(columns: string[]): DataFrame {
+        const data: Record<string, any[]> = {};
+        columns.forEach(c => { data[c] = []; });
+        return this.fromData(data, 'empty');
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * RE-IMPORT / RE-PARSE
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Re-parse with different options
+     */
+    async reimport(options: {
+        headerRow?: number;
+        delimiter?: string;
+        sheetName?: string;
+    }): Promise<DataFrame | MultiSheetResult> {
+        if (this._rawFile) {
+            // Update options
+            if (options.headerRow !== undefined) {
+                this._options.headerRow = options.headerRow;
+            }
+            if (options.delimiter !== undefined) {
+                this._options.delimiter = options.delimiter;
+            }
+
+            return this.fromFile(this._rawFile);
+        } else if (this._rawText) {
+            const df = this._driver.parseCSV(this._rawText, {
+                delimiter: options.delimiter || this._options.delimiter,
+                headerRow: options.headerRow ?? this._options.headerRow,
+                hasHeader: true
+            });
+
+            this._sheets.set(this._activeSheet, df);
+            return df;
+        }
+
+        throw new Error('No raw data available for reimport');
+    }
+
+    /**
+     * Get raw preview rows for header selection UI
+     */
+    async getPreviewRows(maxRows: number = 15): Promise<{ row: number; values: any[] }[]> {
+        if (this._rawFile && /\.(xlsx?|xls)$/i.test(this._rawFile.name)) {
+            const rows = await this._driver.readRawExcelRows(this._rawFile, maxRows);
+            return rows.map(r => ({ row: r.sheetRow, values: r.values }));
+        } else if (this._rawText) {
+            const lines = this._rawText.split('\n').slice(0, maxRows);
+            const delimiter = (this._driver as any)._detectDelimiter(this._rawText);
+            return lines.map((line, i) => ({
+                row: i,
+                values: (this._driver as any)._parseLine(line, delimiter)
+            }));
+        }
+
+        return [];
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * MULTI-SHEET ACCESS
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Get all sheet names
+     */
+    get sheetNames(): string[] {
+        return Array.from(this._sheets.keys());
+    }
+
+    /**
+     * Get active sheet
+     */
+    get activeSheet(): string {
+        return this._activeSheet;
+    }
+
+    /**
+     * Set active sheet
+     */
+    setActiveSheet(name: string): DataFrame {
+        if (!this._sheets.has(name)) {
+            throw new Error(`Sheet '${name}' not found`);
+        }
+        this._activeSheet = name;
+        return this._sheets.get(name)!;
+    }
+
+    /**
+     * Get specific sheet by name
+     */
+    getSheet(name: string): DataFrame | undefined {
+        return this._sheets.get(name);
+    }
+
+    /**
+     * Get current DataFrame (active sheet)
+     */
+    get df(): DataFrame | null {
+        return this._sheets.get(this._activeSheet) || null;
+    }
+
+    /**
+     * Get all sheets
+     */
+    get sheets(): Map<string, DataFrame> {
+        return new Map(this._sheets);
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * PERSISTENCE
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Save current DataFrame to IndexedDB
+     */
+    async save(key?: string): Promise<string | null> {
+        const df = this.df;
+        if (!df) return null;
+
+        const name = key || this._sourceName;
+        return this._driver.store(name, df);
+    }
+
+    /**
+     * Save all sheets
+     */
+    async saveAll(prefix?: string): Promise<string[]> {
+        const ids: string[] = [];
+
+        for (const [sheetName, df] of this._sheets) {
+            const key = prefix ? `${prefix}_${sheetName}` : `${this._sourceName}_${sheetName}`;
+            const id = await this._driver.store(key, df);
+            ids.push(id);
+        }
+
+        return ids;
+    }
+
+    /**
+     * Delete from storage
+     */
+    async deleteStored(key: string): Promise<void> {
+        const tables = await this._driver.list();
+        const matching = tables.filter(t => t.name === key);
+
+        for (const table of matching) {
+            await this._driver.delete(table.id);
+        }
+    }
+
+    /**
+     * Clear all data
+     */
+    clear(): void {
+        this._sheets.clear();
+        this._activeSheet = '';
+        this._sourceName = '';
+        this._rawFile = null;
+        this._rawText = null;
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * SOURCE INFO
+     * ═══════════════════════════════════════════════════ */
+
+    get sourceName(): string {
+        return this._sourceName;
+    }
+
+    get hasRawData(): boolean {
+        return this._rawFile !== null || this._rawText !== null;
+    }
+
+    get isExcel(): boolean {
+        return this._rawFile !== null && /\.(xlsx?|xls)$/i.test(this._rawFile.name);
+    }
+}
+
+/**
+ * Factory function
+ */
+export function dataFrameSource(options: SourceOptions = {}): DataFrameSource {
+    return new DataFrameSource(options);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "juxscript",
-  "version": "1.1.231",
+  "version": "1.1.232",
   "type": "module",
   "description": "A JavaScript UX authorship platform",
   "main": "index.js",