juxscript 1.1.240 → 1.1.243

package/lib/data/DataPipeline.js

@@ -1,359 +0,0 @@
-import { DataFrame } from '../storage/DataFrame.js';
-import { TabularDriver } from '../storage/TabularDriver.js';
-/**
- * DataPipeline — Headless, promise-based data loading and transformation.
- *
- * Usage:
- *   const df = await jux.gather('raw/myfile.xlsx').result();
- *   const df = await jux.gather('https://api.com/data', { bearer: 'xxx' }).result();
- *   const df = await jux.gather(myUpload).select('name').result();
- *   const df = await jux.gather([{a:1}, {b:2}]).result();
- *   await jux.gather(key).filter(r => r.active).intoTable(myTable);
- */
-export class DataPipeline {
-    constructor(options = {}) {
-        this._pending = null;
-        this._transforms = [];
-        this._driver = new TabularDriver(options.dbName ?? 'jux-dataframes', options.storeName ?? 'frames');
-        this._gatherOptions = {};
-    }
-    /* ═══════════════════════════════════════════════════
-     * SMART SOURCE DETECTION
-     * ═══════════════════════════════════════════════════ */
-    /**
-     * Auto-detect source type and load accordingly.
-     *
-     * - string starting with http/https → URL fetch
-     * - string → IndexedDB storage key
-     * - File → parse file
-     * - Array → row objects
-     * - DataFrame → clone
-     * - object with .storageKey → FileUpload cached key
-     * - object with string keys + array values → column-oriented data
-     */
-    from(source, options = {}) {
-        this._transforms = [];
-        this._gatherOptions = options;
-        // Update driver if custom db/store
-        if (options.dbName || options.storeName) {
-            this._driver = new TabularDriver(options.dbName ?? 'jux-dataframes', options.storeName ?? 'frames');
-        }
-        if (source instanceof DataFrame) {
-            this._pending = Promise.resolve(source.clone());
-        }
-        else if (Array.isArray(source)) {
-            this._pending = Promise.resolve(new DataFrame(source));
-        }
-        else if (typeof source === 'string') {
-            if (/^https?:\/\//i.test(source)) {
-                this._pending = this._loadFromUrl(source, options);
-            }
-            else {
-                this._pending = this._loadFromStorage(source);
-            }
-        }
-        else if (source instanceof File) {
-            this._pending = this._loadFromFile(source, options);
-        }
-        else if (source && typeof source === 'object' && 'storageKey' in source) {
-            // FileUpload component with .cache() enabled
-            const key = source.storageKey;
-            if (!key) {
-                this._pending = Promise.reject(new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.'));
-            }
-            else {
-                this._pending = this._loadFromStorage(key);
-            }
-        }
-        else if (source && typeof source === 'object' && !Array.isArray(source)) {
-            // Column-oriented: { col1: [...], col2: [...] }
-            this._pending = Promise.resolve(new DataFrame(source));
-        }
-        else {
-            this._pending = Promise.reject(new Error('Unrecognized source type passed to gather()'));
-        }
-        return this;
-    }
-    /* ═══════════════════════════════════════════════════
-     * EXPLICIT SOURCES (still available)
-     * ═══════════════════════════════════════════════════ */
-    fromStorage(key) {
-        this._transforms = [];
-        this._pending = this._loadFromStorage(key);
-        return this;
-    }
-    fromUrl(url, options = {}) {
-        this._transforms = [];
-        this._gatherOptions = options;
-        this._pending = this._loadFromUrl(url, options);
-        return this;
-    }
-    fromFile(file, options = {}) {
-        this._transforms = [];
-        this._pending = this._loadFromFile(file, options);
-        return this;
-    }
-    fromRows(rows, columns) {
-        this._transforms = [];
-        this._pending = Promise.resolve(new DataFrame(rows, { columns }));
-        return this;
-    }
-    fromColumns(columns) {
-        this._transforms = [];
-        this._pending = Promise.resolve(new DataFrame(columns));
-        return this;
-    }
-    fromJson(json) {
-        this._transforms = [];
-        const parsed = typeof json === 'string' ? JSON.parse(json) : json;
-        this._pending = Promise.resolve(new DataFrame(parsed));
-        return this;
-    }
-    fromUpload(upload) {
-        this._transforms = [];
-        const key = upload.storageKey;
-        if (!key) {
-            this._pending = Promise.reject(new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.'));
-        }
-        else {
-            this._pending = this._loadFromStorage(key);
-        }
-        return this;
-    }
-    /* ═══════════════════════════════════════════════════
-     * TRANSFORMS (chained, lazy — applied on terminal)
-     * ═══════════════════════════════════════════════════ */
-    transform(fn) {
-        this._transforms.push(fn);
-        return this;
-    }
-    select(...cols) {
-        return this.transform(df => df.select(...cols));
-    }
-    filter(predicate) {
-        return this.transform(df => df.filter(predicate));
-    }
-    sort(column, descending) {
-        return this.transform(df => df.sort(column, descending));
-    }
-    head(n = 5) {
-        return this.transform(df => df.head(n));
-    }
-    tail(n = 5) {
-        return this.transform(df => df.tail(n));
-    }
-    rename(mapping) {
-        return this.transform(df => df.rename(mapping));
-    }
-    withColumn(name, fn) {
-        return this.transform(df => df.withColumn(name, fn));
-    }
-    dropna(columns) {
-        return this.transform(df => df.dropna(columns));
-    }
-    dropDuplicates(columns) {
-        return this.transform(df => df.dropDuplicates(columns));
-    }
-    /* ═══════════════════════════════════════════════════
-     * TERMINAL OPERATIONS
-     * ═══════════════════════════════════════════════════ */
-    async result() {
-        if (!this._pending)
-            throw new Error('No data source specified. Call gather(source) or from*(source) first.');
-        let df = await this._pending;
-        for (const fn of this._transforms) {
-            df = fn(df);
-        }
-        return df;
-    }
-    async toRows() {
-        return (await this.result()).toRows();
-    }
-    async toArray(column) {
-        return (await this.result()).col(column);
-    }
-    async toDistinct(column) {
-        return (await this.result()).distinct(column);
-    }
-    async toOptions(labelKey, valueKey) {
-        const df = await this.result();
-        const vKey = valueKey ?? labelKey;
-        return df.toRows().map(row => ({
-            label: String(row[labelKey] ?? ''),
-            value: row[vKey]
-        }));
-    }
-    async toList(column) {
-        const df = await this.result();
-        return df.col(column).map(v => String(v ?? ''));
-    }
-    async intoTable(table, columnLabels) {
-        const df = await this.result();
-        const columnDefs = df.columns.map(col => ({
-            key: col,
-            label: columnLabels?.[col] ?? col
-        }));
-        table.columns(columnDefs).rows(df.toRows());
-        return df;
-    }
-    async intoDropdown(dropdown, labelKey, valueKey) {
-        const df = await this.result();
-        const vKey = valueKey ?? labelKey;
-        const opts = df.toRows().map(row => ({
-            label: String(row[labelKey] ?? ''),
-            value: row[vKey]
-        }));
-        dropdown.options(opts);
-        return df;
-    }
-    async intoList(list, column) {
-        const df = await this.result();
-        list.items(df.col(column).map((v) => String(v ?? '')));
-        return df;
-    }
-    async into(target, options = {}) {
-        const df = await this.result();
-        const rows = df.toRows();
-        if (typeof target.columns === 'function' && typeof target.rows === 'function') {
-            const cols = options.columns ?? df.columns;
-            const columnDefs = cols.map((col) => ({
-                key: col,
-                label: options.columnLabels?.[col] ?? col
-            }));
-            target.columns(columnDefs).rows(rows);
-            return df;
-        }
-        if (typeof target.fromData === 'function') {
-            target.fromData(rows);
-            return df;
-        }
-        if (typeof target.options === 'function') {
-            if (!options.labelKey) {
-                throw new Error('Cannot push into dropdown/select without labelKey. ' +
-                    'Use .intoDropdown(target, labelKey, valueKey) or pass { labelKey, valueKey } to .into()');
-            }
-            const opts = rows.map(row => ({
-                label: String(row[options.labelKey] ?? ''),
-                value: row[options.valueKey ?? options.labelKey]
-            }));
-            target.options(opts);
-            return df;
-        }
-        if (typeof target.items === 'function') {
-            if (!options.labelKey) {
-                throw new Error('Cannot push into list without labelKey. ' +
-                    'Use .intoList(target, column) or pass { labelKey } to .into()');
-            }
-            target.items(rows.map(row => String(row[options.labelKey] ?? '')));
-            return df;
-        }
-        throw new Error('Target does not have a recognized API (.columns/.rows, .fromData, .options, or .items)');
-    }
-    async store(key, metadata) {
-        const df = await this.result();
-        await this._driver.store(key, df, metadata);
-        return df;
-    }
-    /* ═══════════════════════════════════════════════════
-     * PRIVATE
-     * ═══════════════════════════════════════════════════ */
-    async _loadFromStorage(key) {
-        // Try by name first, then by id
-        let df = await this._driver.loadByName(key);
-        if (df)
-            return df;
-        df = await this._driver.load(key);
-        if (df)
-            return df;
-        throw new Error(`No data found in storage with key: "${key}"`);
-    }
-    async _loadFromUrl(url, options) {
-        const mimeType = options.mimeType ?? this._detectMimeType(url);
-        // Build headers from auth options
-        const headers = { ...options.headers };
-        if (options.bearer) {
-            headers['Authorization'] = `Bearer ${options.bearer}`;
-        }
-        if (options.apiKey) {
-            headers['x-api-key'] = options.apiKey;
-        }
-        if (mimeType === 'json') {
-            const response = await fetch(url, { headers });
-            if (!response.ok)
-                throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
-            const json = await response.json();
-            return new DataFrame(json);
-        }
-        // CSV/TSV — fetch with headers, then parse
-        if (Object.keys(headers).length > 0) {
-            const response = await fetch(url, { headers });
-            if (!response.ok)
-                throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
-            const text = await response.text();
-            return this._driver.parseCSV(text, {
-                autoDetectDelimiter: !options.delimiter,
-                delimiter: options.delimiter,
-                hasHeader: options.hasHeader ?? true,
-                headerRow: options.headerRow
-            });
-        }
-        return this._driver.fetch(url, {
-            autoDetectDelimiter: !options.delimiter,
-            delimiter: options.delimiter,
-            hasHeader: options.hasHeader ?? true,
-            headerRow: options.headerRow
-        });
-    }
-    async _loadFromFile(file, options) {
-        const isExcel = /\.(xlsx?|xls)$/i.test(file.name);
-        if (isExcel) {
-            const sheets = await this._driver.streamFileMultiSheet(file, {
-                headerRow: options.headerRow
-            });
-            const sheetNames = Object.keys(sheets);
-            if (sheetNames.length === 0)
-                throw new Error('No data found in file');
-            return sheets[sheetNames[0]];
-        }
-        const text = await file.text();
-        return this._driver.parseCSV(text, {
-            autoDetectDelimiter: !options.delimiter,
-            delimiter: options.delimiter,
-            hasHeader: options.hasHeader ?? true,
-            headerRow: options.headerRow
-        });
-    }
-    _detectMimeType(url) {
-        const lower = url.toLowerCase().split('?')[0];
-        if (lower.endsWith('.json'))
-            return 'json';
-        if (lower.endsWith('.tsv'))
-            return 'tsv';
-        if (lower.endsWith('.csv'))
-            return 'csv';
-        return 'auto';
-    }
-}
-/**
- * Factory function — primary entry point via jux.gather()
- *
- * Smart single-arg:
- *   jux.gather('raw/myfile.xlsx')                              // storage key
- *   jux.gather('https://api.com/data', { bearer: 'xxx' })     // URL with auth
- *   jux.gather(myUpload)                                       // FileUpload
- *   jux.gather([{a:1}, {b:2}])                                 // inline rows
- *   jux.gather(existingDf)                                     // clone DataFrame
- *
- * Explicit source:
- *   jux.gather().fromStorage(key)
- *   jux.gather().fromUrl(url, opts)
- */
-export function gather(source, options) {
-    const pipe = new DataPipeline(options?.dbName || options?.storeName
-        ? { dbName: options.dbName, storeName: options.storeName }
-        : {});
-    if (source !== undefined) {
-        pipe.from(source, options ?? {});
-    }
-    return pipe;
-}