juxscript 1.1.237 → 1.1.239

package/lib/data/DataPipeline.ts

@@ -13,97 +13,148 @@ export interface FetchOptions {
     mimeType?: 'csv' | 'tsv' | 'json' | 'auto';
 }
 
+export interface GatherOptions extends FetchOptions {
+    /** Authorization bearer token */
+    bearer?: string;
+    /** API key (sent as x-api-key header) */
+    apiKey?: string;
+    /** Custom headers for URL fetches */
+    headers?: Record<string, string>;
+    /** IndexedDB database name override */
+    dbName?: string;
+    /** IndexedDB store name override */
+    storeName?: string;
+}
+
+export interface IntoOptions {
+    /** Override which columns to use (for Table targets) */
+    columns?: string[];
+    /** Column name → display label mapping */
+    columnLabels?: Record<string, string>;
+    /** Key for option labels (Dropdown/Select/List targets) */
+    labelKey?: string;
+    /** Key for option values (Dropdown/Select targets) */
+    valueKey?: string;
+}
+
 /**
  * DataPipeline — Headless, promise-based data loading and transformation.
  *
  * Usage:
- *   const df = await data.fromUrl('https://example.com/report.csv').result();
- *   const df = await data.fromStorage('patients').result();
- *   const df = await data.fromJson(apiResponse).result();
- *
- *   // Transform and distribute:
- *   await data.fromUrl(url)
- *       .transform(df => df.select('name', 'age').filter(r => r.age > 30))
- *       .into(myTable);
- *
- *   await data.fromStorage('encounters')
- *       .transform(df => df.distinct('provider'))
- *       .into(myDropdown, { labelKey: 'provider', valueKey: 'provider' });
+ *   const df = await jux.gather('raw/myfile.xlsx').result();
+ *   const df = await jux.gather('https://api.com/data', { bearer: 'xxx' }).result();
+ *   const df = await jux.gather(myUpload).select('name').result();
+ *   const df = await jux.gather([{a:1}, {b:2}]).result();
+ *   await jux.gather(key).filter(r => r.active).intoTable(myTable);
  */
 export class DataPipeline {
     private _driver: TabularDriver;
     private _pending: Promise<DataFrame> | null = null;
     private _transforms: ((df: DataFrame) => DataFrame)[] = [];
+    private _gatherOptions: GatherOptions;
 
     constructor(options: PipelineOptions = {}) {
         this._driver = new TabularDriver(
             options.dbName ?? 'jux-dataframes',
             options.storeName ?? 'frames'
         );
+        this._gatherOptions = {};
     }
 
     /* ═══════════════════════════════════════════════════
-     * SOURCES
+     * SMART SOURCE DETECTION
      * ═══════════════════════════════════════════════════ */
 
     /**
-     * Load from IndexedDB by key (searches by name first, then by id)
+     * Auto-detect source type and load accordingly.
+     *
+     * - string starting with http/https → URL fetch
+     * - string → IndexedDB storage key
+     * - File → parse file
+     * - Array → row objects
+     * - DataFrame → clone
+     * - object with .storageKey → FileUpload cached key
+     * - object with string keys + array values → column-oriented data
      */
-    fromStorage(key: string): this {
+    from(source: any, options: GatherOptions = {}): this {
         this._transforms = [];
-        this._pending = (async () => {
-            // Try by name index first
-            let df = await this._driver.loadByName(key);
-            if (df) return df;
+        this._gatherOptions = options;
+
+        // Update driver if custom db/store
+        if (options.dbName || options.storeName) {
+            this._driver = new TabularDriver(
+                options.dbName ?? 'jux-dataframes',
+                options.storeName ?? 'frames'
+            );
+        }
 
-            // Fall back to loading by primary key (id)
-            df = await this._driver.load(key);
-            if (df) return df;
+        if (source instanceof DataFrame) {
+            this._pending = Promise.resolve(source.clone());
+        } else if (Array.isArray(source)) {
+            this._pending = Promise.resolve(new DataFrame(source));
+        } else if (typeof source === 'string') {
+            if (/^https?:\/\//i.test(source)) {
+                this._pending = this._loadFromUrl(source, options);
+            } else {
+                this._pending = this._loadFromStorage(source);
+            }
+        } else if (source instanceof File) {
+            this._pending = this._loadFromFile(source, options);
+        } else if (source && typeof source === 'object' && 'storageKey' in source) {
+            // FileUpload component with .cache() enabled
+            const key = source.storageKey;
+            if (!key) {
+                this._pending = Promise.reject(
+                    new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.')
+                );
+            } else {
+                this._pending = this._loadFromStorage(key);
+            }
+        } else if (source && typeof source === 'object' && !Array.isArray(source)) {
+            // Column-oriented: { col1: [...], col2: [...] }
+            this._pending = Promise.resolve(new DataFrame(source));
+        } else {
+            this._pending = Promise.reject(new Error('Unrecognized source type passed to gather()'));
+        }
 
-            throw new Error(`No data found in storage with key: "${key}"`);
-        })();
         return this;
     }
 
-    /**
-     * Load from a URL (CSV, TSV, or JSON)
-     */
-    fromUrl(url: string, options: FetchOptions = {}): this {
+    /* ═══════════════════════════════════════════════════
+     * EXPLICIT SOURCES (still available)
+     * ═══════════════════════════════════════════════════ */
+
+    fromStorage(key: string): this {
+        this._transforms = [];
+        this._pending = this._loadFromStorage(key);
+        return this;
+    }
+
+    fromUrl(url: string, options: GatherOptions = {}): this {
         this._transforms = [];
+        this._gatherOptions = options;
         this._pending = this._loadFromUrl(url, options);
         return this;
     }
 
-    /**
-     * Load from a File object directly
-     */
     fromFile(file: File, options: FetchOptions = {}): this {
         this._transforms = [];
         this._pending = this._loadFromFile(file, options);
         return this;
     }
 
-    /**
-     * Load from an array of row objects
-     */
     fromRows(rows: Record<string, any>[], columns?: string[]): this {
         this._transforms = [];
         this._pending = Promise.resolve(new DataFrame(rows, { columns }));
         return this;
     }
 
-    /**
-     * Load from column-oriented data
-     */
     fromColumns(columns: Record<string, any[]>): this {
         this._transforms = [];
         this._pending = Promise.resolve(new DataFrame(columns));
         return this;
     }
 
-    /**
-     * Load from a JSON response (array of objects or column-oriented)
-     */
     fromJson(json: Record<string, any>[] | Record<string, any[]> | string): this {
         this._transforms = [];
         const parsed = typeof json === 'string' ? JSON.parse(json) : json;
@@ -111,13 +162,6 @@ export class DataPipeline {
         return this;
     }
 
-    /**
-     * Load from a FileUpload component's cached storage key.
-     * Requires .cache() enabled on the FileUpload and a file already uploaded.
-     *
-     * Usage:
-     *   await data().fromUpload(myUpload).select('name').into(myTable);
-     */
     fromUpload(upload: any): this {
         this._transforms = [];
         const key = upload.storageKey;
@@ -126,94 +170,52 @@ export class DataPipeline {
                 new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.')
             );
         } else {
-            this._pending = this._driver.loadByName(key).then((df: any) => {
-                if (!df) throw new Error(`No cached data found for key: "${key}"`);
-                return df;
-            });
+            this._pending = this._loadFromStorage(key);
         }
         return this;
     }
 
-    /**
-     * Start from an existing DataFrame
-     */
-    from(df: DataFrame): this {
-        this._transforms = [];
-        this._pending = Promise.resolve(df.clone());
-        return this;
-    }
-
     /* ═══════════════════════════════════════════════════
-     * TRANSFORMS (chained, lazy — applied on .result())
+     * TRANSFORMS (chained, lazy — applied on terminal)
      * ═══════════════════════════════════════════════════ */
 
-    /**
-     * Apply an arbitrary transform
-     */
     transform(fn: (df: DataFrame) => DataFrame): this {
         this._transforms.push(fn);
         return this;
     }
 
-    /**
-     * Select columns
-     */
     select(...cols: string[]): this {
         return this.transform(df => df.select(...cols));
     }
 
-    /**
-     * Filter rows
-     */
     filter(predicate: (row: Record<string, any>, index: number) => boolean): this {
         return this.transform(df => df.filter(predicate));
     }
 
-    /**
-     * Sort by column
-     */
     sort(column: string, descending?: boolean): this {
         return this.transform(df => df.sort(column, descending));
     }
 
-    /**
-     * Take first N rows
-     */
     head(n: number = 5): this {
         return this.transform(df => df.head(n));
     }
 
-    /**
-     * Take last N rows
-     */
     tail(n: number = 5): this {
         return this.transform(df => df.tail(n));
     }
 
-    /**
-     * Rename columns
-     */
     rename(mapping: Record<string, string>): this {
         return this.transform(df => df.rename(mapping));
     }
 
-    /**
-     * Add a computed column
-     */
     withColumn(name: string, fn: (row: Record<string, any>, index: number) => any): this {
         return this.transform(df => df.withColumn(name, fn));
     }
 
-    /**
-     * Drop nulls
-     */
     dropna(columns?: string[]): this {
         return this.transform(df => df.dropna(columns));
     }
 
-    /**
-     * Drop duplicates
-     */
     dropDuplicates(columns?: string[]): this {
         return this.transform(df => df.dropDuplicates(columns));
     }
@@ -222,12 +224,8 @@ export class DataPipeline {
      * TERMINAL OPERATIONS
      * ═══════════════════════════════════════════════════ */
 
-    /**
-     * Execute the pipeline and return the DataFrame
-     */
     async result(): Promise<DataFrame> {
-        if (!this._pending) throw new Error('No data source specified. Call fromStorage(), fromUrl(), etc. first.');
-
+        if (!this._pending) throw new Error('No data source specified. Call gather(source) or from*(source) first.');
         let df = await this._pending;
         for (const fn of this._transforms) {
             df = fn(df);
@@ -235,82 +233,107 @@ export class DataPipeline {
         return df;
     }
 
-    /**
-     * Execute and return rows as plain objects
-     */
     async toRows(): Promise<Record<string, any>[]> {
         return (await this.result()).toRows();
     }
 
-    /**
-     * Execute and return a single column as an array
-     */
     async toArray(column: string): Promise<any[]> {
         return (await this.result()).col(column);
     }
 
-    /**
-     * Execute and return distinct values from a column
-     */
     async toDistinct(column: string): Promise<any[]> {
         return (await this.result()).distinct(column);
     }
 
-    /**
-     * Execute and push results into a jux component.
-     *
-     * Supported targets:
-     * - Table: sets columns + rows
-     * - Dropdown/Select: sets options from labelKey/valueKey
-     * - List: sets items
-     * - DataFrameComponent: loads data
-     * - Any object with a .rows() or .options() or .items() method
-     */
+    async toOptions(labelKey: string, valueKey?: string): Promise<{ label: string; value: any }[]> {
+        const df = await this.result();
+        const vKey = valueKey ?? labelKey;
+        return df.toRows().map(row => ({
+            label: String(row[labelKey] ?? ''),
+            value: row[vKey]
+        }));
+    }
+
+    async toList(column: string): Promise<string[]> {
+        const df = await this.result();
+        return df.col(column).map(v => String(v ?? ''));
+    }
+
+    async intoTable(table: any, columnLabels?: Record<string, string>): Promise<DataFrame> {
+        const df = await this.result();
+        const columnDefs = df.columns.map(col => ({
+            key: col,
+            label: columnLabels?.[col] ?? col
+        }));
+        table.columns(columnDefs).rows(df.toRows());
+        return df;
+    }
+
+    async intoDropdown(dropdown: any, labelKey: string, valueKey?: string): Promise<DataFrame> {
+        const df = await this.result();
+        const vKey = valueKey ?? labelKey;
+        const opts = df.toRows().map(row => ({
+            label: String(row[labelKey] ?? ''),
+            value: row[vKey]
+        }));
+        dropdown.options(opts);
+        return df;
+    }
+
+    async intoList(list: any, column: string): Promise<DataFrame> {
+        const df = await this.result();
+        list.items(df.col(column).map((v: any) => String(v ?? '')));
+        return df;
+    }
+
     async into(target: any, options: IntoOptions = {}): Promise<DataFrame> {
         const df = await this.result();
         const rows = df.toRows();
 
-        // Table-like: has .columns() and .rows()
         if (typeof target.columns === 'function' && typeof target.rows === 'function') {
-            const columnDefs = (options.columns ?? df.columns).map((col: string) => {
-                const label = options.columnLabels?.[col] ?? col;
-                return { key: col, label };
-            });
+            const cols = options.columns ?? df.columns;
+            const columnDefs = cols.map((col: string) => ({
+                key: col,
+                label: options.columnLabels?.[col] ?? col
+            }));
             target.columns(columnDefs).rows(rows);
             return df;
         }
 
-        // DataFrameComponent: has .fromData()
         if (typeof target.fromData === 'function') {
             target.fromData(rows);
             return df;
         }
 
-        // Dropdown/Select-like: has .options()
         if (typeof target.options === 'function') {
-            const labelKey = options.labelKey ?? df.columns[0];
-            const valueKey = options.valueKey ?? (df.columns[1] ?? df.columns[0]);
+            if (!options.labelKey) {
+                throw new Error(
+                    'Cannot push into dropdown/select without labelKey. ' +
+                    'Use .intoDropdown(target, labelKey, valueKey) or pass { labelKey, valueKey } to .into()'
+                );
+            }
             const opts = rows.map(row => ({
-                label: String(row[labelKey] ?? ''),
-                value: row[valueKey]
+                label: String(row[options.labelKey!] ?? ''),
+                value: row[options.valueKey ?? options.labelKey!]
             }));
             target.options(opts);
             return df;
         }
 
-        // List-like: has .items()
         if (typeof target.items === 'function') {
-            const key = options.labelKey ?? df.columns[0];
-            target.items(rows.map(row => String(row[key] ?? '')));
+            if (!options.labelKey) {
+                throw new Error(
+                    'Cannot push into list without labelKey. ' +
+                    'Use .intoList(target, column) or pass { labelKey } to .into()'
+                );
+            }
+            target.items(rows.map(row => String(row[options.labelKey!] ?? '')));
             return df;
         }
 
-        throw new Error(`Target does not have a recognized API (.columns/.rows, .fromData, .options, or .items)`);
+        throw new Error('Target does not have a recognized API (.columns/.rows, .fromData, .options, or .items)');
     }
 
-    /**
-     * Execute and persist to IndexedDB
-     */
     async store(key: string, metadata?: Record<string, any>): Promise<DataFrame> {
         const df = await this.result();
         await this._driver.store(key, df, metadata);
@@ -321,17 +344,49 @@ export class DataPipeline {
      * PRIVATE
      * ═══════════════════════════════════════════════════ */
 
-    private async _loadFromUrl(url: string, options: FetchOptions): Promise<DataFrame> {
+    private async _loadFromStorage(key: string): Promise<DataFrame> {
+        // Try by name first, then by id
+        let df = await this._driver.loadByName(key);
+        if (df) return df;
+
+        df = await this._driver.load(key);
+        if (df) return df;
+
+        throw new Error(`No data found in storage with key: "${key}"`);
+    }
+
+    private async _loadFromUrl(url: string, options: GatherOptions): Promise<DataFrame> {
         const mimeType = options.mimeType ?? this._detectMimeType(url);
 
+        // Build headers from auth options
+        const headers: Record<string, string> = { ...options.headers };
+        if (options.bearer) {
+            headers['Authorization'] = `Bearer ${options.bearer}`;
+        }
+        if (options.apiKey) {
+            headers['x-api-key'] = options.apiKey;
+        }
+
         if (mimeType === 'json') {
-            const response = await fetch(url);
+            const response = await fetch(url, { headers });
             if (!response.ok) throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
             const json = await response.json();
             return new DataFrame(json);
         }
 
-        // CSV/TSV — delegate to driver
+        // CSV/TSV — fetch with headers, then parse
+        if (Object.keys(headers).length > 0) {
+            const response = await fetch(url, { headers });
+            if (!response.ok) throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
+            const text = await response.text();
+            return this._driver.parseCSV(text, {
+                autoDetectDelimiter: !options.delimiter,
+                delimiter: options.delimiter,
+                hasHeader: options.hasHeader ?? true,
+                headerRow: options.headerRow
+            });
+        }
+
         return this._driver.fetch(url, {
             autoDetectDelimiter: !options.delimiter,
             delimiter: options.delimiter,
@@ -370,26 +425,28 @@ export class DataPipeline {
     }
 }
 
-export interface IntoOptions {
-    /** Override which columns to use (for Table targets) */
-    columns?: string[];
-    /** Column name → display label mapping */
-    columnLabels?: Record<string, string>;
-    /** Key for option labels (Dropdown/Select/List targets) */
-    labelKey?: string;
-    /** Key for option values (Dropdown/Select targets) */
-    valueKey?: string;
-}
-
 /**
- * Factory function — the primary entry point
+ * Factory function — primary entry point via jux.gather()
  *
- * Usage:
- *   import { data } from 'juxscript';
+ * Smart single-arg:
+ *   jux.gather('raw/myfile.xlsx')                              // storage key
+ *   jux.gather('https://api.com/data', { bearer: 'xxx' })     // URL with auth
+ *   jux.gather(myUpload)                                       // FileUpload
+ *   jux.gather([{a:1}, {b:2}])                                 // inline rows
+ *   jux.gather(existingDf)                                     // clone DataFrame
  *
- *   const df = await data().fromStorage('patients').select('name', 'dob').result();
- *   await data().fromUrl(presignedUrl).filter(r => r.status === 'active').into(myTable);
+ * Explicit source:
+ *   jux.gather().fromStorage(key)
+ *   jux.gather().fromUrl(url, opts)
  */
-export function data(options: PipelineOptions = {}): DataPipeline {
-    return new DataPipeline(options);
+export function gather(source?: any, options?: GatherOptions): DataPipeline {
+    const pipe = new DataPipeline(
+        options?.dbName || options?.storeName
+            ? { dbName: options.dbName, storeName: options.storeName }
+            : {}
+    );
+    if (source !== undefined) {
+        pipe.from(source, options ?? {});
+    }
+    return pipe;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "juxscript",
-  "version": "1.1.237",
+  "version": "1.1.239",
   "type": "module",
   "description": "A JavaScript UX authorship platform",
   "main": "index.js",