juxscript 1.1.232 → 1.1.234

package/lib/data/DataPipeline.ts

@@ -0,0 +1,388 @@
+import { DataFrame } from '../storage/DataFrame.js';
+import { TabularDriver } from '../storage/TabularDriver.js';
+
+export interface PipelineOptions {
+    dbName?: string;
+    storeName?: string;
+}
+
+export interface FetchOptions {
+    delimiter?: string;
+    hasHeader?: boolean;
+    headerRow?: number;
+    mimeType?: 'csv' | 'tsv' | 'json' | 'auto';
+}
+
+/**
+ * DataPipeline — Headless, promise-based data loading and transformation.
+ *
+ * Usage:
+ *   const df = await data.fromUrl('https://example.com/report.csv').result();
+ *   const df = await data.fromStorage('patients').result();
+ *   const df = await data.fromJson(apiResponse).result();
+ *
+ *   // Transform and distribute:
+ *   await data.fromUrl(url)
+ *       .transform(df => df.select('name', 'age').filter(r => r.age > 30))
+ *       .into(myTable);
+ *
+ *   await data.fromStorage('encounters')
+ *       .transform(df => df.distinct('provider'))
+ *       .into(myDropdown, { labelKey: 'provider', valueKey: 'provider' });
+ */
+export class DataPipeline {
+    private _driver: TabularDriver;
+    private _pending: Promise<DataFrame> | null = null;
+    private _transforms: ((df: DataFrame) => DataFrame)[] = [];
+
+    constructor(options: PipelineOptions = {}) {
+        this._driver = new TabularDriver(
+            options.dbName ?? 'jux-dataframes',
+            options.storeName ?? 'frames'
+        );
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * SOURCES
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Load from IndexedDB by key
+     */
+    fromStorage(key: string): this {
+        this._transforms = [];
+        this._pending = this._driver.loadByName(key).then(df => {
+            if (!df) throw new Error(`No data found in storage with key: "${key}"`);
+            return df;
+        });
+        return this;
+    }
+
+    /**
+     * Load from a URL (CSV, TSV, or JSON)
+     */
+    fromUrl(url: string, options: FetchOptions = {}): this {
+        this._transforms = [];
+        this._pending = this._loadFromUrl(url, options);
+        return this;
+    }
+
+    /**
+     * Load from a File object directly
+     */
+    fromFile(file: File, options: FetchOptions = {}): this {
+        this._transforms = [];
+        this._pending = this._loadFromFile(file, options);
+        return this;
+    }
+
+    /**
+     * Load from an array of row objects
+     */
+    fromRows(rows: Record<string, any>[], columns?: string[]): this {
+        this._transforms = [];
+        this._pending = Promise.resolve(new DataFrame(rows, { columns }));
+        return this;
+    }
+
+    /**
+     * Load from column-oriented data
+     */
+    fromColumns(columns: Record<string, any[]>): this {
+        this._transforms = [];
+        this._pending = Promise.resolve(new DataFrame(columns));
+        return this;
+    }
+
+    /**
+     * Load from a JSON response (array of objects or column-oriented)
+     */
+    fromJson(json: Record<string, any>[] | Record<string, any[]> | string): this {
+        this._transforms = [];
+        const parsed = typeof json === 'string' ? JSON.parse(json) : json;
+        this._pending = Promise.resolve(new DataFrame(parsed));
+        return this;
+    }
+
+    /**
+     * Load from a FileUpload component's cached storage key.
+     * Requires .cache() enabled on the FileUpload and a file already uploaded.
+     *
+     * Usage:
+     *   await data().fromUpload(myUpload).select('name').into(myTable);
+     */
+    fromUpload(upload: any): this {
+        this._transforms = [];
+        const key = upload.storageKey;
+        if (!key) {
+            this._pending = Promise.reject(
+                new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.')
+            );
+        } else {
+            this._pending = this._driver.loadByName(key).then((df: any) => {
+                if (!df) throw new Error(`No cached data found for key: "${key}"`);
+                return df;
+            });
+        }
+        return this;
+    }
+
+    /**
+     * Start from an existing DataFrame
+     */
+    from(df: DataFrame): this {
+        this._transforms = [];
+        this._pending = Promise.resolve(df.clone());
+        return this;
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * TRANSFORMS (chained, lazy — applied on .result())
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Apply an arbitrary transform
+     */
+    transform(fn: (df: DataFrame) => DataFrame): this {
+        this._transforms.push(fn);
+        return this;
+    }
+
+    /**
+     * Select columns
+     */
+    select(...cols: string[]): this {
+        return this.transform(df => df.select(...cols));
+    }
+
+    /**
+     * Filter rows
+     */
+    filter(predicate: (row: Record<string, any>, index: number) => boolean): this {
+        return this.transform(df => df.filter(predicate));
+    }
+
+    /**
+     * Sort by column
+     */
+    sort(column: string, descending?: boolean): this {
+        return this.transform(df => df.sort(column, descending));
+    }
+
+    /**
+     * Take first N rows
+     */
+    head(n: number = 5): this {
+        return this.transform(df => df.head(n));
+    }
+
+    /**
+     * Take last N rows
+     */
+    tail(n: number = 5): this {
+        return this.transform(df => df.tail(n));
+    }
+
+    /**
+     * Rename columns
+     */
+    rename(mapping: Record<string, string>): this {
+        return this.transform(df => df.rename(mapping));
+    }
+
+    /**
+     * Add a computed column
+     */
+    withColumn(name: string, fn: (row: Record<string, any>, index: number) => any): this {
+        return this.transform(df => df.withColumn(name, fn));
+    }
+
+    /**
+     * Drop nulls
+     */
+    dropna(columns?: string[]): this {
+        return this.transform(df => df.dropna(columns));
+    }
+
+    /**
+     * Drop duplicates
+     */
+    dropDuplicates(columns?: string[]): this {
+        return this.transform(df => df.dropDuplicates(columns));
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * TERMINAL OPERATIONS
+     * ═══════════════════════════════════════════════════ */
+
+    /**
+     * Execute the pipeline and return the DataFrame
+     */
+    async result(): Promise<DataFrame> {
+        if (!this._pending) throw new Error('No data source specified. Call fromStorage(), fromUrl(), etc. first.');
+
+        let df = await this._pending;
+        for (const fn of this._transforms) {
+            df = fn(df);
+        }
+        return df;
+    }
+
+    /**
+     * Execute and return rows as plain objects
+     */
+    async toRows(): Promise<Record<string, any>[]> {
+        return (await this.result()).toRows();
+    }
+
+    /**
+     * Execute and return a single column as an array
+     */
+    async toArray(column: string): Promise<any[]> {
+        return (await this.result()).col(column);
+    }
+
+    /**
+     * Execute and return distinct values from a column
+     */
+    async toDistinct(column: string): Promise<any[]> {
+        return (await this.result()).distinct(column);
+    }
+
+    /**
+     * Execute and push results into a jux component.
+     *
+     * Supported targets:
+     * - Table: sets columns + rows
+     * - Dropdown/Select: sets options from labelKey/valueKey
+     * - List: sets items
+     * - DataFrameComponent: loads data
+     * - Any object with a .rows() or .options() or .items() method
+     */
+    async into(target: any, options: IntoOptions = {}): Promise<DataFrame> {
+        const df = await this.result();
+        const rows = df.toRows();
+
+        // Table-like: has .columns() and .rows()
+        if (typeof target.columns === 'function' && typeof target.rows === 'function') {
+            const columnDefs = (options.columns ?? df.columns).map((col: string) => {
+                const label = options.columnLabels?.[col] ?? col;
+                return { key: col, label };
+            });
+            target.columns(columnDefs).rows(rows);
+            return df;
+        }
+
+        // DataFrameComponent: has .fromData()
+        if (typeof target.fromData === 'function') {
+            target.fromData(rows);
+            return df;
+        }
+
+        // Dropdown/Select-like: has .options()
+        if (typeof target.options === 'function') {
+            const labelKey = options.labelKey ?? df.columns[0];
+            const valueKey = options.valueKey ?? (df.columns[1] ?? df.columns[0]);
+            const opts = rows.map(row => ({
+                label: String(row[labelKey] ?? ''),
+                value: row[valueKey]
+            }));
+            target.options(opts);
+            return df;
+        }
+
+        // List-like: has .items()
+        if (typeof target.items === 'function') {
+            const key = options.labelKey ?? df.columns[0];
+            target.items(rows.map(row => String(row[key] ?? '')));
+            return df;
+        }
+
+        throw new Error(`Target does not have a recognized API (.columns/.rows, .fromData, .options, or .items)`);
+    }
+
+    /**
+     * Execute and persist to IndexedDB
+     */
+    async store(key: string, metadata?: Record<string, any>): Promise<DataFrame> {
+        const df = await this.result();
+        await this._driver.store(key, df, metadata);
+        return df;
+    }
+
+    /* ═══════════════════════════════════════════════════
+     * PRIVATE
+     * ═══════════════════════════════════════════════════ */
+
+    private async _loadFromUrl(url: string, options: FetchOptions): Promise<DataFrame> {
+        const mimeType = options.mimeType ?? this._detectMimeType(url);
+
+        if (mimeType === 'json') {
+            const response = await fetch(url);
+            if (!response.ok) throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
+            const json = await response.json();
+            return new DataFrame(json);
+        }
+
+        // CSV/TSV — delegate to driver
+        return this._driver.fetch(url, {
+            autoDetectDelimiter: !options.delimiter,
+            delimiter: options.delimiter,
+            hasHeader: options.hasHeader ?? true,
+            headerRow: options.headerRow
+        });
+    }
+
+    private async _loadFromFile(file: File, options: FetchOptions): Promise<DataFrame> {
+        const isExcel = /\.(xlsx?|xls)$/i.test(file.name);
+
+        if (isExcel) {
+            const sheets = await this._driver.streamFileMultiSheet(file, {
+                headerRow: options.headerRow
+            });
+            const sheetNames = Object.keys(sheets);
+            if (sheetNames.length === 0) throw new Error('No data found in file');
+            return sheets[sheetNames[0]];
+        }
+
+        const text = await file.text();
+        return this._driver.parseCSV(text, {
+            autoDetectDelimiter: !options.delimiter,
+            delimiter: options.delimiter,
+            hasHeader: options.hasHeader ?? true,
+            headerRow: options.headerRow
+        });
+    }
+
+    private _detectMimeType(url: string): 'csv' | 'tsv' | 'json' | 'auto' {
+        const lower = url.toLowerCase().split('?')[0];
+        if (lower.endsWith('.json')) return 'json';
+        if (lower.endsWith('.tsv')) return 'tsv';
+        if (lower.endsWith('.csv')) return 'csv';
+        return 'auto';
+    }
+}
+
+export interface IntoOptions {
+    /** Override which columns to use (for Table targets) */
+    columns?: string[];
+    /** Column name → display label mapping */
+    columnLabels?: Record<string, string>;
+    /** Key for option labels (Dropdown/Select/List targets) */
+    labelKey?: string;
+    /** Key for option values (Dropdown/Select targets) */
+    valueKey?: string;
+}
+
+/**
+ * Factory function — the primary entry point
+ *
+ * Usage:
+ *   import { data } from 'juxscript';
+ *
+ *   const df = await data().fromStorage('patients').select('name', 'dob').result();
+ *   await data().fromUrl(presignedUrl).filter(r => r.status === 'active').into(myTable);
+ */
+export function data(options: PipelineOptions = {}): DataPipeline {
+    return new DataPipeline(options);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "juxscript",
-  "version": "1.1.232",
+  "version": "1.1.234",
   "type": "module",
   "description": "A JavaScript UX authorship platform",
   "main": "index.js",