juxscript 1.1.236 → 1.1.238
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +4 -4
- package/index.d.ts.map +1 -1
- package/index.js +3 -3
- package/lib/data/DataPipeline.d.ts +60 -112
- package/lib/data/DataPipeline.d.ts.map +1 -1
- package/lib/data/DataPipeline.js +170 -135
- package/lib/data/DataPipeline.ts +212 -148
- package/package.json +1 -1
package/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { checkbox } from './lib/components/checkbox.js';
|
|
|
8
8
|
import { code } from './lib/components/code.js';
|
|
9
9
|
import { container } from './lib/components/container.js';
|
|
10
10
|
import { data } from './lib/components/data.js';
|
|
11
|
-
import {
|
|
11
|
+
import { gather } from './lib/data/DataPipeline.js';
|
|
12
12
|
import { datepicker } from './lib/components/datepicker.js';
|
|
13
13
|
import { dialog } from './lib/components/dialog.js';
|
|
14
14
|
import { divider } from './lib/components/divider.js';
|
|
@@ -58,8 +58,8 @@ export { DataFrameSource, dataFrameSource } from './lib/storage/DataFrameSource.
|
|
|
58
58
|
export type { SourceOptions, APIOptions, StreamOptions, MultiSheetResult } from './lib/storage/DataFrameSource.js';
|
|
59
59
|
export { indexedDBDriver } from './lib/storage/IndexedDBDriver.js';
|
|
60
60
|
export { fileStorage } from './lib/storage/FileStorage.js';
|
|
61
|
-
export {
|
|
62
|
-
export type { PipelineOptions, FetchOptions, IntoOptions } from './lib/data/DataPipeline.js';
|
|
61
|
+
export { gather, DataPipeline } from './lib/data/DataPipeline.js';
|
|
62
|
+
export type { PipelineOptions, FetchOptions, GatherOptions, IntoOptions } from './lib/data/DataPipeline.js';
|
|
63
63
|
export { dataframe } from './lib/components/dataframe.js';
|
|
64
64
|
import { VStack, vstack } from './lib/components/stack/VStack.js';
|
|
65
65
|
import { HStack, hstack } from './lib/components/stack/HStack.js';
|
|
@@ -89,6 +89,7 @@ export declare const jux: {
|
|
|
89
89
|
dropdownMenu: typeof dropdownMenu;
|
|
90
90
|
element: typeof element;
|
|
91
91
|
fileupload: typeof fileupload;
|
|
92
|
+
gather: typeof gather;
|
|
92
93
|
grid: typeof grid;
|
|
93
94
|
heading: typeof heading;
|
|
94
95
|
hero: typeof hero;
|
|
@@ -106,7 +107,6 @@ export declare const jux: {
|
|
|
106
107
|
nav: typeof nav;
|
|
107
108
|
paragraph: typeof paragraph;
|
|
108
109
|
pen: typeof pen;
|
|
109
|
-
pipeline: typeof pipeline;
|
|
110
110
|
progress: typeof progress;
|
|
111
111
|
radio: typeof radio;
|
|
112
112
|
req: import("./lib/components/req.js").Req;
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,GAAG,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,GAAG,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,MAAM,EAAgB,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,GAAG,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAC1D,OAAO,EAAE,GAAG,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAElD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,0CAA0C,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAEtD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;AAGzC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACzE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC3F,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACpF,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACnH,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAG3D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAClE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAG5G,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAG1D,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAGlE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7E,YAAY,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAG5D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AAG3D;;GAEG;AACH,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Df,CAAC;AAGF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC"}
|
package/index.js
CHANGED
|
@@ -8,7 +8,7 @@ import { checkbox } from './lib/components/checkbox.js';
|
|
|
8
8
|
import { code } from './lib/components/code.js';
|
|
9
9
|
import { container } from './lib/components/container.js';
|
|
10
10
|
import { data } from './lib/components/data.js';
|
|
11
|
-
import {
|
|
11
|
+
import { gather } from './lib/data/DataPipeline.js';
|
|
12
12
|
import { datepicker } from './lib/components/datepicker.js';
|
|
13
13
|
import { dialog } from './lib/components/dialog.js';
|
|
14
14
|
import { divider } from './lib/components/divider.js';
|
|
@@ -60,7 +60,7 @@ export { DataFrameSource, dataFrameSource } from './lib/storage/DataFrameSource.
|
|
|
60
60
|
export { indexedDBDriver } from './lib/storage/IndexedDBDriver.js';
|
|
61
61
|
export { fileStorage } from './lib/storage/FileStorage.js';
|
|
62
62
|
// Data Pipeline
|
|
63
|
-
export {
|
|
63
|
+
export { gather, DataPipeline } from './lib/data/DataPipeline.js';
|
|
64
64
|
// DataFrame UI Component (separate from data layer)
|
|
65
65
|
export { dataframe } from './lib/components/dataframe.js';
|
|
66
66
|
// Import Stack components (already added earlier)
|
|
@@ -94,6 +94,7 @@ export const jux = {
|
|
|
94
94
|
dropdownMenu,
|
|
95
95
|
element,
|
|
96
96
|
fileupload,
|
|
97
|
+
gather,
|
|
97
98
|
grid,
|
|
98
99
|
heading,
|
|
99
100
|
hero,
|
|
@@ -111,7 +112,6 @@ export const jux = {
|
|
|
111
112
|
nav,
|
|
112
113
|
paragraph,
|
|
113
114
|
pen,
|
|
114
|
-
pipeline,
|
|
115
115
|
progress,
|
|
116
116
|
radio,
|
|
117
117
|
req,
|
|
@@ -9,157 +9,105 @@ export interface FetchOptions {
|
|
|
9
9
|
headerRow?: number;
|
|
10
10
|
mimeType?: 'csv' | 'tsv' | 'json' | 'auto';
|
|
11
11
|
}
|
|
12
|
+
export interface GatherOptions extends FetchOptions {
|
|
13
|
+
/** Authorization bearer token */
|
|
14
|
+
bearer?: string;
|
|
15
|
+
/** API key (sent as x-api-key header) */
|
|
16
|
+
apiKey?: string;
|
|
17
|
+
/** Custom headers for URL fetches */
|
|
18
|
+
headers?: Record<string, string>;
|
|
19
|
+
/** IndexedDB database name override */
|
|
20
|
+
dbName?: string;
|
|
21
|
+
/** IndexedDB store name override */
|
|
22
|
+
storeName?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface IntoOptions {
|
|
25
|
+
/** Override which columns to use (for Table targets) */
|
|
26
|
+
columns?: string[];
|
|
27
|
+
/** Column name → display label mapping */
|
|
28
|
+
columnLabels?: Record<string, string>;
|
|
29
|
+
/** Key for option labels (Dropdown/Select/List targets) */
|
|
30
|
+
labelKey?: string;
|
|
31
|
+
/** Key for option values (Dropdown/Select targets) */
|
|
32
|
+
valueKey?: string;
|
|
33
|
+
}
|
|
12
34
|
/**
|
|
13
35
|
* DataPipeline — Headless, promise-based data loading and transformation.
|
|
14
36
|
*
|
|
15
37
|
* Usage:
|
|
16
|
-
* const df = await
|
|
17
|
-
* const df = await
|
|
18
|
-
* const df = await
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* await data.fromUrl(url)
|
|
22
|
-
* .transform(df => df.select('name', 'age').filter(r => r.age > 30))
|
|
23
|
-
* .into(myTable);
|
|
24
|
-
*
|
|
25
|
-
* await data.fromStorage('encounters')
|
|
26
|
-
* .transform(df => df.distinct('provider'))
|
|
27
|
-
* .into(myDropdown, { labelKey: 'provider', valueKey: 'provider' });
|
|
38
|
+
* const df = await jux.gather('raw/myfile.xlsx').result();
|
|
39
|
+
* const df = await jux.gather('https://api.com/data', { bearer: 'xxx' }).result();
|
|
40
|
+
* const df = await jux.gather(myUpload).select('name').result();
|
|
41
|
+
* const df = await jux.gather([{a:1}, {b:2}]).result();
|
|
42
|
+
* await jux.gather(key).filter(r => r.active).intoTable(myTable);
|
|
28
43
|
*/
|
|
29
44
|
export declare class DataPipeline {
|
|
30
45
|
private _driver;
|
|
31
46
|
private _pending;
|
|
32
47
|
private _transforms;
|
|
48
|
+
private _gatherOptions;
|
|
33
49
|
constructor(options?: PipelineOptions);
|
|
34
50
|
/**
|
|
35
|
-
*
|
|
36
|
-
|
|
51
|
+
* Auto-detect source type and load accordingly.
|
|
52
|
+
*
|
|
53
|
+
* - string starting with http/https → URL fetch
|
|
54
|
+
* - string → IndexedDB storage key
|
|
55
|
+
* - File → parse file
|
|
56
|
+
* - Array → row objects
|
|
57
|
+
* - DataFrame → clone
|
|
58
|
+
* - object with .storageKey → FileUpload cached key
|
|
59
|
+
* - object with string keys + array values → column-oriented data
|
|
60
|
+
*/
|
|
61
|
+
from(source: any, options?: GatherOptions): this;
|
|
37
62
|
fromStorage(key: string): this;
|
|
38
|
-
|
|
39
|
-
* Load from a URL (CSV, TSV, or JSON)
|
|
40
|
-
*/
|
|
41
|
-
fromUrl(url: string, options?: FetchOptions): this;
|
|
42
|
-
/**
|
|
43
|
-
* Load from a File object directly
|
|
44
|
-
*/
|
|
63
|
+
fromUrl(url: string, options?: GatherOptions): this;
|
|
45
64
|
fromFile(file: File, options?: FetchOptions): this;
|
|
46
|
-
/**
|
|
47
|
-
* Load from an array of row objects
|
|
48
|
-
*/
|
|
49
65
|
fromRows(rows: Record<string, any>[], columns?: string[]): this;
|
|
50
|
-
/**
|
|
51
|
-
* Load from column-oriented data
|
|
52
|
-
*/
|
|
53
66
|
fromColumns(columns: Record<string, any[]>): this;
|
|
54
|
-
/**
|
|
55
|
-
* Load from a JSON response (array of objects or column-oriented)
|
|
56
|
-
*/
|
|
57
67
|
fromJson(json: Record<string, any>[] | Record<string, any[]> | string): this;
|
|
58
|
-
/**
|
|
59
|
-
* Load from a FileUpload component's cached storage key.
|
|
60
|
-
* Requires .cache() enabled on the FileUpload and a file already uploaded.
|
|
61
|
-
*
|
|
62
|
-
* Usage:
|
|
63
|
-
* await data().fromUpload(myUpload).select('name').into(myTable);
|
|
64
|
-
*/
|
|
65
68
|
fromUpload(upload: any): this;
|
|
66
|
-
/**
|
|
67
|
-
* Start from an existing DataFrame
|
|
68
|
-
*/
|
|
69
|
-
from(df: DataFrame): this;
|
|
70
|
-
/**
|
|
71
|
-
* Apply an arbitrary transform
|
|
72
|
-
*/
|
|
73
69
|
transform(fn: (df: DataFrame) => DataFrame): this;
|
|
74
|
-
/**
|
|
75
|
-
* Select columns
|
|
76
|
-
*/
|
|
77
70
|
select(...cols: string[]): this;
|
|
78
|
-
/**
|
|
79
|
-
* Filter rows
|
|
80
|
-
*/
|
|
81
71
|
filter(predicate: (row: Record<string, any>, index: number) => boolean): this;
|
|
82
|
-
/**
|
|
83
|
-
* Sort by column
|
|
84
|
-
*/
|
|
85
72
|
sort(column: string, descending?: boolean): this;
|
|
86
|
-
/**
|
|
87
|
-
* Take first N rows
|
|
88
|
-
*/
|
|
89
73
|
head(n?: number): this;
|
|
90
|
-
/**
|
|
91
|
-
* Take last N rows
|
|
92
|
-
*/
|
|
93
74
|
tail(n?: number): this;
|
|
94
|
-
/**
|
|
95
|
-
* Rename columns
|
|
96
|
-
*/
|
|
97
75
|
rename(mapping: Record<string, string>): this;
|
|
98
|
-
/**
|
|
99
|
-
* Add a computed column
|
|
100
|
-
*/
|
|
101
76
|
withColumn(name: string, fn: (row: Record<string, any>, index: number) => any): this;
|
|
102
|
-
/**
|
|
103
|
-
* Drop nulls
|
|
104
|
-
*/
|
|
105
77
|
dropna(columns?: string[]): this;
|
|
106
|
-
/**
|
|
107
|
-
* Drop duplicates
|
|
108
|
-
*/
|
|
109
78
|
dropDuplicates(columns?: string[]): this;
|
|
110
|
-
/**
|
|
111
|
-
* Execute the pipeline and return the DataFrame
|
|
112
|
-
*/
|
|
113
79
|
result(): Promise<DataFrame>;
|
|
114
|
-
/**
|
|
115
|
-
* Execute and return rows as plain objects
|
|
116
|
-
*/
|
|
117
80
|
toRows(): Promise<Record<string, any>[]>;
|
|
118
|
-
/**
|
|
119
|
-
* Execute and return a single column as an array
|
|
120
|
-
*/
|
|
121
81
|
toArray(column: string): Promise<any[]>;
|
|
122
|
-
/**
|
|
123
|
-
* Execute and return distinct values from a column
|
|
124
|
-
*/
|
|
125
82
|
toDistinct(column: string): Promise<any[]>;
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
* - Any object with a .rows() or .options() or .items() method
|
|
135
|
-
*/
|
|
83
|
+
toOptions(labelKey: string, valueKey?: string): Promise<{
|
|
84
|
+
label: string;
|
|
85
|
+
value: any;
|
|
86
|
+
}[]>;
|
|
87
|
+
toList(column: string): Promise<string[]>;
|
|
88
|
+
intoTable(table: any, columnLabels?: Record<string, string>): Promise<DataFrame>;
|
|
89
|
+
intoDropdown(dropdown: any, labelKey: string, valueKey?: string): Promise<DataFrame>;
|
|
90
|
+
intoList(list: any, column: string): Promise<DataFrame>;
|
|
136
91
|
into(target: any, options?: IntoOptions): Promise<DataFrame>;
|
|
137
|
-
/**
|
|
138
|
-
* Execute and persist to IndexedDB
|
|
139
|
-
*/
|
|
140
92
|
store(key: string, metadata?: Record<string, any>): Promise<DataFrame>;
|
|
93
|
+
private _loadFromStorage;
|
|
141
94
|
private _loadFromUrl;
|
|
142
95
|
private _loadFromFile;
|
|
143
96
|
private _detectMimeType;
|
|
144
97
|
}
|
|
145
|
-
export interface IntoOptions {
|
|
146
|
-
/** Override which columns to use (for Table targets) */
|
|
147
|
-
columns?: string[];
|
|
148
|
-
/** Column name → display label mapping */
|
|
149
|
-
columnLabels?: Record<string, string>;
|
|
150
|
-
/** Key for option labels (Dropdown/Select/List targets) */
|
|
151
|
-
labelKey?: string;
|
|
152
|
-
/** Key for option values (Dropdown/Select targets) */
|
|
153
|
-
valueKey?: string;
|
|
154
|
-
}
|
|
155
98
|
/**
|
|
156
|
-
* Factory function —
|
|
99
|
+
* Factory function — primary entry point via jux.gather()
|
|
157
100
|
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
101
|
+
* Smart single-arg:
|
|
102
|
+
* jux.gather('raw/myfile.xlsx') // storage key
|
|
103
|
+
* jux.gather('https://api.com/data', { bearer: 'xxx' }) // URL with auth
|
|
104
|
+
* jux.gather(myUpload) // FileUpload
|
|
105
|
+
* jux.gather([{a:1}, {b:2}]) // inline rows
|
|
106
|
+
* jux.gather(existingDf) // clone DataFrame
|
|
160
107
|
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
108
|
+
* Explicit source:
|
|
109
|
+
* jux.gather().fromStorage(key)
|
|
110
|
+
* jux.gather().fromUrl(url, opts)
|
|
163
111
|
*/
|
|
164
|
-
export declare function
|
|
112
|
+
export declare function gather(source?: any, options?: GatherOptions): DataPipeline;
|
|
165
113
|
//# sourceMappingURL=DataPipeline.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataPipeline.d.ts","sourceRoot":"","sources":["DataPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAGpD,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;CAC9C;AAED
|
|
1
|
+
{"version":3,"file":"DataPipeline.d.ts","sourceRoot":"","sources":["DataPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAGpD,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;CAC9C;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IAC/C,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IACxB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,0CAA0C;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;GASG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,WAAW,CAAwC;IAC3D,OAAO,CAAC,cAAc,CAAgB;gBAE1B,OAAO,GAAE,eAAoB;IAYzC;;;;;;;;;;OAUG;IACH,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,GAAE,aAAkB,GAAG,IAAI;IAgDpD,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAM9B,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,IAAI;IAOvD,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,YAAiB,GAAG,IAAI;IAMtD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI;IAM/D,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI;IAMjD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,MAAM,GAAG,IAAI;IAO5E,UAAU,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI;IAiB7B,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,IAAI;IAKjD,MAAM,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI;IAI/B,MAAM,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,IAAI;IAI7E,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,GAAG,IAAI;IAIhD,IAAI,CAAC,CAAC,GAAE,MAAU,GAAG,IAAI;IAIzB,IAAI,CAAC,CAAC,GAAE,MAAU,GAAG,IAAI;IAIzB,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAI7C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,GAAG,GAAG,IAAI;IAIpF,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI;IAIhC,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI;IAQlC,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC;IAS5B,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;IAIxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAIvC,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAI1C,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAA;KAAE,EAAE,CAAC;IASxF,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKzC,SAAS,CAAC,KAAK,EAAE,GAAG,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC;IAUhF,YAAY,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAWpF,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAMvD,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,SAAS,CAAC;IAgDhE,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC;YAU9D,gBAAgB;YAWhB,YAAY;YAwCZ,aAAa;IAqB3B,OAAO,CAAC,eAAe;CAO1B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,YAAY,CAU1E"}
|
package/lib/data/DataPipeline.js
CHANGED
|
@@ -4,88 +4,111 @@ import { TabularDriver } from '../storage/TabularDriver.js';
|
|
|
4
4
|
* DataPipeline — Headless, promise-based data loading and transformation.
|
|
5
5
|
*
|
|
6
6
|
* Usage:
|
|
7
|
-
* const df = await
|
|
8
|
-
* const df = await
|
|
9
|
-
* const df = await
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* await data.fromUrl(url)
|
|
13
|
-
* .transform(df => df.select('name', 'age').filter(r => r.age > 30))
|
|
14
|
-
* .into(myTable);
|
|
15
|
-
*
|
|
16
|
-
* await data.fromStorage('encounters')
|
|
17
|
-
* .transform(df => df.distinct('provider'))
|
|
18
|
-
* .into(myDropdown, { labelKey: 'provider', valueKey: 'provider' });
|
|
7
|
+
* const df = await jux.gather('raw/myfile.xlsx').result();
|
|
8
|
+
* const df = await jux.gather('https://api.com/data', { bearer: 'xxx' }).result();
|
|
9
|
+
* const df = await jux.gather(myUpload).select('name').result();
|
|
10
|
+
* const df = await jux.gather([{a:1}, {b:2}]).result();
|
|
11
|
+
* await jux.gather(key).filter(r => r.active).intoTable(myTable);
|
|
19
12
|
*/
|
|
20
13
|
export class DataPipeline {
|
|
21
14
|
constructor(options = {}) {
|
|
22
15
|
this._pending = null;
|
|
23
16
|
this._transforms = [];
|
|
24
17
|
this._driver = new TabularDriver(options.dbName ?? 'jux-dataframes', options.storeName ?? 'frames');
|
|
18
|
+
this._gatherOptions = {};
|
|
25
19
|
}
|
|
26
20
|
/* ═══════════════════════════════════════════════════
|
|
27
|
-
*
|
|
21
|
+
* SMART SOURCE DETECTION
|
|
28
22
|
* ═══════════════════════════════════════════════════ */
|
|
29
23
|
/**
|
|
30
|
-
*
|
|
24
|
+
* Auto-detect source type and load accordingly.
|
|
25
|
+
*
|
|
26
|
+
* - string starting with http/https → URL fetch
|
|
27
|
+
* - string → IndexedDB storage key
|
|
28
|
+
* - File → parse file
|
|
29
|
+
* - Array → row objects
|
|
30
|
+
* - DataFrame → clone
|
|
31
|
+
* - object with .storageKey → FileUpload cached key
|
|
32
|
+
* - object with string keys + array values → column-oriented data
|
|
31
33
|
*/
|
|
34
|
+
from(source, options = {}) {
|
|
35
|
+
this._transforms = [];
|
|
36
|
+
this._gatherOptions = options;
|
|
37
|
+
// Update driver if custom db/store
|
|
38
|
+
if (options.dbName || options.storeName) {
|
|
39
|
+
this._driver = new TabularDriver(options.dbName ?? 'jux-dataframes', options.storeName ?? 'frames');
|
|
40
|
+
}
|
|
41
|
+
if (source instanceof DataFrame) {
|
|
42
|
+
this._pending = Promise.resolve(source.clone());
|
|
43
|
+
}
|
|
44
|
+
else if (Array.isArray(source)) {
|
|
45
|
+
this._pending = Promise.resolve(new DataFrame(source));
|
|
46
|
+
}
|
|
47
|
+
else if (typeof source === 'string') {
|
|
48
|
+
if (/^https?:\/\//i.test(source)) {
|
|
49
|
+
this._pending = this._loadFromUrl(source, options);
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
this._pending = this._loadFromStorage(source);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
else if (source instanceof File) {
|
|
56
|
+
this._pending = this._loadFromFile(source, options);
|
|
57
|
+
}
|
|
58
|
+
else if (source && typeof source === 'object' && 'storageKey' in source) {
|
|
59
|
+
// FileUpload component with .cache() enabled
|
|
60
|
+
const key = source.storageKey;
|
|
61
|
+
if (!key) {
|
|
62
|
+
this._pending = Promise.reject(new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.'));
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
this._pending = this._loadFromStorage(key);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
else if (source && typeof source === 'object' && !Array.isArray(source)) {
|
|
69
|
+
// Column-oriented: { col1: [...], col2: [...] }
|
|
70
|
+
this._pending = Promise.resolve(new DataFrame(source));
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
this._pending = Promise.reject(new Error('Unrecognized source type passed to gather()'));
|
|
74
|
+
}
|
|
75
|
+
return this;
|
|
76
|
+
}
|
|
77
|
+
/* ═══════════════════════════════════════════════════
|
|
78
|
+
* EXPLICIT SOURCES (still available)
|
|
79
|
+
* ═══════════════════════════════════════════════════ */
|
|
32
80
|
fromStorage(key) {
|
|
33
81
|
this._transforms = [];
|
|
34
|
-
this._pending = this.
|
|
35
|
-
if (!df)
|
|
36
|
-
throw new Error(`No data found in storage with key: "${key}"`);
|
|
37
|
-
return df;
|
|
38
|
-
});
|
|
82
|
+
this._pending = this._loadFromStorage(key);
|
|
39
83
|
return this;
|
|
40
84
|
}
|
|
41
|
-
/**
|
|
42
|
-
* Load from a URL (CSV, TSV, or JSON)
|
|
43
|
-
*/
|
|
44
85
|
fromUrl(url, options = {}) {
|
|
45
86
|
this._transforms = [];
|
|
87
|
+
this._gatherOptions = options;
|
|
46
88
|
this._pending = this._loadFromUrl(url, options);
|
|
47
89
|
return this;
|
|
48
90
|
}
|
|
49
|
-
/**
|
|
50
|
-
* Load from a File object directly
|
|
51
|
-
*/
|
|
52
91
|
fromFile(file, options = {}) {
|
|
53
92
|
this._transforms = [];
|
|
54
93
|
this._pending = this._loadFromFile(file, options);
|
|
55
94
|
return this;
|
|
56
95
|
}
|
|
57
|
-
/**
|
|
58
|
-
* Load from an array of row objects
|
|
59
|
-
*/
|
|
60
96
|
fromRows(rows, columns) {
|
|
61
97
|
this._transforms = [];
|
|
62
98
|
this._pending = Promise.resolve(new DataFrame(rows, { columns }));
|
|
63
99
|
return this;
|
|
64
100
|
}
|
|
65
|
-
/**
|
|
66
|
-
* Load from column-oriented data
|
|
67
|
-
*/
|
|
68
101
|
fromColumns(columns) {
|
|
69
102
|
this._transforms = [];
|
|
70
103
|
this._pending = Promise.resolve(new DataFrame(columns));
|
|
71
104
|
return this;
|
|
72
105
|
}
|
|
73
|
-
/**
|
|
74
|
-
* Load from a JSON response (array of objects or column-oriented)
|
|
75
|
-
*/
|
|
76
106
|
fromJson(json) {
|
|
77
107
|
this._transforms = [];
|
|
78
108
|
const parsed = typeof json === 'string' ? JSON.parse(json) : json;
|
|
79
109
|
this._pending = Promise.resolve(new DataFrame(parsed));
|
|
80
110
|
return this;
|
|
81
111
|
}
|
|
82
|
-
/**
|
|
83
|
-
* Load from a FileUpload component's cached storage key.
|
|
84
|
-
* Requires .cache() enabled on the FileUpload and a file already uploaded.
|
|
85
|
-
*
|
|
86
|
-
* Usage:
|
|
87
|
-
* await data().fromUpload(myUpload).select('name').into(myTable);
|
|
88
|
-
*/
|
|
89
112
|
fromUpload(upload) {
|
|
90
113
|
this._transforms = [];
|
|
91
114
|
const key = upload.storageKey;
|
|
@@ -93,168 +116,139 @@ export class DataPipeline {
|
|
|
93
116
|
this._pending = Promise.reject(new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.'));
|
|
94
117
|
}
|
|
95
118
|
else {
|
|
96
|
-
this._pending = this.
|
|
97
|
-
if (!df)
|
|
98
|
-
throw new Error(`No cached data found for key: "${key}"`);
|
|
99
|
-
return df;
|
|
100
|
-
});
|
|
119
|
+
this._pending = this._loadFromStorage(key);
|
|
101
120
|
}
|
|
102
121
|
return this;
|
|
103
122
|
}
|
|
104
|
-
/**
|
|
105
|
-
* Start from an existing DataFrame
|
|
106
|
-
*/
|
|
107
|
-
from(df) {
|
|
108
|
-
this._transforms = [];
|
|
109
|
-
this._pending = Promise.resolve(df.clone());
|
|
110
|
-
return this;
|
|
111
|
-
}
|
|
112
123
|
/* ═══════════════════════════════════════════════════
|
|
113
|
-
* TRANSFORMS (chained, lazy — applied on
|
|
124
|
+
* TRANSFORMS (chained, lazy — applied on terminal)
|
|
114
125
|
* ═══════════════════════════════════════════════════ */
|
|
115
|
-
/**
|
|
116
|
-
* Apply an arbitrary transform
|
|
117
|
-
*/
|
|
118
126
|
transform(fn) {
|
|
119
127
|
this._transforms.push(fn);
|
|
120
128
|
return this;
|
|
121
129
|
}
|
|
122
|
-
/**
|
|
123
|
-
* Select columns
|
|
124
|
-
*/
|
|
125
130
|
select(...cols) {
|
|
126
131
|
return this.transform(df => df.select(...cols));
|
|
127
132
|
}
|
|
128
|
-
/**
|
|
129
|
-
* Filter rows
|
|
130
|
-
*/
|
|
131
133
|
filter(predicate) {
|
|
132
134
|
return this.transform(df => df.filter(predicate));
|
|
133
135
|
}
|
|
134
|
-
/**
|
|
135
|
-
* Sort by column
|
|
136
|
-
*/
|
|
137
136
|
sort(column, descending) {
|
|
138
137
|
return this.transform(df => df.sort(column, descending));
|
|
139
138
|
}
|
|
140
|
-
/**
|
|
141
|
-
* Take first N rows
|
|
142
|
-
*/
|
|
143
139
|
head(n = 5) {
|
|
144
140
|
return this.transform(df => df.head(n));
|
|
145
141
|
}
|
|
146
|
-
/**
|
|
147
|
-
* Take last N rows
|
|
148
|
-
*/
|
|
149
142
|
tail(n = 5) {
|
|
150
143
|
return this.transform(df => df.tail(n));
|
|
151
144
|
}
|
|
152
|
-
/**
|
|
153
|
-
* Rename columns
|
|
154
|
-
*/
|
|
155
145
|
rename(mapping) {
|
|
156
146
|
return this.transform(df => df.rename(mapping));
|
|
157
147
|
}
|
|
158
|
-
/**
|
|
159
|
-
* Add a computed column
|
|
160
|
-
*/
|
|
161
148
|
withColumn(name, fn) {
|
|
162
149
|
return this.transform(df => df.withColumn(name, fn));
|
|
163
150
|
}
|
|
164
|
-
/**
|
|
165
|
-
* Drop nulls
|
|
166
|
-
*/
|
|
167
151
|
dropna(columns) {
|
|
168
152
|
return this.transform(df => df.dropna(columns));
|
|
169
153
|
}
|
|
170
|
-
/**
|
|
171
|
-
* Drop duplicates
|
|
172
|
-
*/
|
|
173
154
|
dropDuplicates(columns) {
|
|
174
155
|
return this.transform(df => df.dropDuplicates(columns));
|
|
175
156
|
}
|
|
176
157
|
/* ═══════════════════════════════════════════════════
|
|
177
158
|
* TERMINAL OPERATIONS
|
|
178
159
|
* ═══════════════════════════════════════════════════ */
|
|
179
|
-
/**
|
|
180
|
-
* Execute the pipeline and return the DataFrame
|
|
181
|
-
*/
|
|
182
160
|
async result() {
|
|
183
161
|
if (!this._pending)
|
|
184
|
-
throw new Error('No data source specified. Call
|
|
162
|
+
throw new Error('No data source specified. Call gather(source) or from*(source) first.');
|
|
185
163
|
let df = await this._pending;
|
|
186
164
|
for (const fn of this._transforms) {
|
|
187
165
|
df = fn(df);
|
|
188
166
|
}
|
|
189
167
|
return df;
|
|
190
168
|
}
|
|
191
|
-
/**
|
|
192
|
-
* Execute and return rows as plain objects
|
|
193
|
-
*/
|
|
194
169
|
async toRows() {
|
|
195
170
|
return (await this.result()).toRows();
|
|
196
171
|
}
|
|
197
|
-
/**
|
|
198
|
-
* Execute and return a single column as an array
|
|
199
|
-
*/
|
|
200
172
|
async toArray(column) {
|
|
201
173
|
return (await this.result()).col(column);
|
|
202
174
|
}
|
|
203
|
-
/**
|
|
204
|
-
* Execute and return distinct values from a column
|
|
205
|
-
*/
|
|
206
175
|
async toDistinct(column) {
|
|
207
176
|
return (await this.result()).distinct(column);
|
|
208
177
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
178
|
+
async toOptions(labelKey, valueKey) {
|
|
179
|
+
const df = await this.result();
|
|
180
|
+
const vKey = valueKey ?? labelKey;
|
|
181
|
+
return df.toRows().map(row => ({
|
|
182
|
+
label: String(row[labelKey] ?? ''),
|
|
183
|
+
value: row[vKey]
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
async toList(column) {
|
|
187
|
+
const df = await this.result();
|
|
188
|
+
return df.col(column).map(v => String(v ?? ''));
|
|
189
|
+
}
|
|
190
|
+
async intoTable(table, columnLabels) {
|
|
191
|
+
const df = await this.result();
|
|
192
|
+
const columnDefs = df.columns.map(col => ({
|
|
193
|
+
key: col,
|
|
194
|
+
label: columnLabels?.[col] ?? col
|
|
195
|
+
}));
|
|
196
|
+
table.columns(columnDefs).rows(df.toRows());
|
|
197
|
+
return df;
|
|
198
|
+
}
|
|
199
|
+
async intoDropdown(dropdown, labelKey, valueKey) {
|
|
200
|
+
const df = await this.result();
|
|
201
|
+
const vKey = valueKey ?? labelKey;
|
|
202
|
+
const opts = df.toRows().map(row => ({
|
|
203
|
+
label: String(row[labelKey] ?? ''),
|
|
204
|
+
value: row[vKey]
|
|
205
|
+
}));
|
|
206
|
+
dropdown.options(opts);
|
|
207
|
+
return df;
|
|
208
|
+
}
|
|
209
|
+
async intoList(list, column) {
|
|
210
|
+
const df = await this.result();
|
|
211
|
+
list.items(df.col(column).map((v) => String(v ?? '')));
|
|
212
|
+
return df;
|
|
213
|
+
}
|
|
219
214
|
async into(target, options = {}) {
|
|
220
215
|
const df = await this.result();
|
|
221
216
|
const rows = df.toRows();
|
|
222
|
-
// Table-like: has .columns() and .rows()
|
|
223
217
|
if (typeof target.columns === 'function' && typeof target.rows === 'function') {
|
|
224
|
-
const
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
218
|
+
const cols = options.columns ?? df.columns;
|
|
219
|
+
const columnDefs = cols.map((col) => ({
|
|
220
|
+
key: col,
|
|
221
|
+
label: options.columnLabels?.[col] ?? col
|
|
222
|
+
}));
|
|
228
223
|
target.columns(columnDefs).rows(rows);
|
|
229
224
|
return df;
|
|
230
225
|
}
|
|
231
|
-
// DataFrameComponent: has .fromData()
|
|
232
226
|
if (typeof target.fromData === 'function') {
|
|
233
227
|
target.fromData(rows);
|
|
234
228
|
return df;
|
|
235
229
|
}
|
|
236
|
-
// Dropdown/Select-like: has .options()
|
|
237
230
|
if (typeof target.options === 'function') {
|
|
238
|
-
|
|
239
|
-
|
|
231
|
+
if (!options.labelKey) {
|
|
232
|
+
throw new Error('Cannot push into dropdown/select without labelKey. ' +
|
|
233
|
+
'Use .intoDropdown(target, labelKey, valueKey) or pass { labelKey, valueKey } to .into()');
|
|
234
|
+
}
|
|
240
235
|
const opts = rows.map(row => ({
|
|
241
|
-
label: String(row[labelKey] ?? ''),
|
|
242
|
-
value: row[valueKey]
|
|
236
|
+
label: String(row[options.labelKey] ?? ''),
|
|
237
|
+
value: row[options.valueKey ?? options.labelKey]
|
|
243
238
|
}));
|
|
244
239
|
target.options(opts);
|
|
245
240
|
return df;
|
|
246
241
|
}
|
|
247
|
-
// List-like: has .items()
|
|
248
242
|
if (typeof target.items === 'function') {
|
|
249
|
-
|
|
250
|
-
|
|
243
|
+
if (!options.labelKey) {
|
|
244
|
+
throw new Error('Cannot push into list without labelKey. ' +
|
|
245
|
+
'Use .intoList(target, column) or pass { labelKey } to .into()');
|
|
246
|
+
}
|
|
247
|
+
target.items(rows.map(row => String(row[options.labelKey] ?? '')));
|
|
251
248
|
return df;
|
|
252
249
|
}
|
|
253
|
-
throw new Error(
|
|
250
|
+
throw new Error('Target does not have a recognized API (.columns/.rows, .fromData, .options, or .items)');
|
|
254
251
|
}
|
|
255
|
-
/**
|
|
256
|
-
* Execute and persist to IndexedDB
|
|
257
|
-
*/
|
|
258
252
|
async store(key, metadata) {
|
|
259
253
|
const df = await this.result();
|
|
260
254
|
await this._driver.store(key, df, metadata);
|
|
@@ -263,16 +257,46 @@ export class DataPipeline {
|
|
|
263
257
|
/* ═══════════════════════════════════════════════════
|
|
264
258
|
* PRIVATE
|
|
265
259
|
* ═══════════════════════════════════════════════════ */
|
|
260
|
+
async _loadFromStorage(key) {
|
|
261
|
+
// Try by name first, then by id
|
|
262
|
+
let df = await this._driver.loadByName(key);
|
|
263
|
+
if (df)
|
|
264
|
+
return df;
|
|
265
|
+
df = await this._driver.load(key);
|
|
266
|
+
if (df)
|
|
267
|
+
return df;
|
|
268
|
+
throw new Error(`No data found in storage with key: "${key}"`);
|
|
269
|
+
}
|
|
266
270
|
async _loadFromUrl(url, options) {
|
|
267
271
|
const mimeType = options.mimeType ?? this._detectMimeType(url);
|
|
272
|
+
// Build headers from auth options
|
|
273
|
+
const headers = { ...options.headers };
|
|
274
|
+
if (options.bearer) {
|
|
275
|
+
headers['Authorization'] = `Bearer ${options.bearer}`;
|
|
276
|
+
}
|
|
277
|
+
if (options.apiKey) {
|
|
278
|
+
headers['x-api-key'] = options.apiKey;
|
|
279
|
+
}
|
|
268
280
|
if (mimeType === 'json') {
|
|
269
|
-
const response = await fetch(url);
|
|
281
|
+
const response = await fetch(url, { headers });
|
|
270
282
|
if (!response.ok)
|
|
271
283
|
throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
|
|
272
284
|
const json = await response.json();
|
|
273
285
|
return new DataFrame(json);
|
|
274
286
|
}
|
|
275
|
-
// CSV/TSV —
|
|
287
|
+
// CSV/TSV — fetch with headers, then parse
|
|
288
|
+
if (Object.keys(headers).length > 0) {
|
|
289
|
+
const response = await fetch(url, { headers });
|
|
290
|
+
if (!response.ok)
|
|
291
|
+
throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
|
|
292
|
+
const text = await response.text();
|
|
293
|
+
return this._driver.parseCSV(text, {
|
|
294
|
+
autoDetectDelimiter: !options.delimiter,
|
|
295
|
+
delimiter: options.delimiter,
|
|
296
|
+
hasHeader: options.hasHeader ?? true,
|
|
297
|
+
headerRow: options.headerRow
|
|
298
|
+
});
|
|
299
|
+
}
|
|
276
300
|
return this._driver.fetch(url, {
|
|
277
301
|
autoDetectDelimiter: !options.delimiter,
|
|
278
302
|
delimiter: options.delimiter,
|
|
@@ -311,14 +335,25 @@ export class DataPipeline {
|
|
|
311
335
|
}
|
|
312
336
|
}
|
|
313
337
|
/**
|
|
314
|
-
* Factory function —
|
|
338
|
+
* Factory function — primary entry point via jux.gather()
|
|
315
339
|
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
340
|
+
* Smart single-arg:
|
|
341
|
+
* jux.gather('raw/myfile.xlsx') // storage key
|
|
342
|
+
* jux.gather('https://api.com/data', { bearer: 'xxx' }) // URL with auth
|
|
343
|
+
* jux.gather(myUpload) // FileUpload
|
|
344
|
+
* jux.gather([{a:1}, {b:2}]) // inline rows
|
|
345
|
+
* jux.gather(existingDf) // clone DataFrame
|
|
318
346
|
*
|
|
319
|
-
*
|
|
320
|
-
*
|
|
347
|
+
* Explicit source:
|
|
348
|
+
* jux.gather().fromStorage(key)
|
|
349
|
+
* jux.gather().fromUrl(url, opts)
|
|
321
350
|
*/
|
|
322
|
-
export function
|
|
323
|
-
|
|
351
|
+
export function gather(source, options) {
|
|
352
|
+
const pipe = new DataPipeline(options?.dbName || options?.storeName
|
|
353
|
+
? { dbName: options.dbName, storeName: options.storeName }
|
|
354
|
+
: {});
|
|
355
|
+
if (source !== undefined) {
|
|
356
|
+
pipe.from(source, options ?? {});
|
|
357
|
+
}
|
|
358
|
+
return pipe;
|
|
324
359
|
}
|
package/lib/data/DataPipeline.ts
CHANGED
|
@@ -13,90 +13,148 @@ export interface FetchOptions {
|
|
|
13
13
|
mimeType?: 'csv' | 'tsv' | 'json' | 'auto';
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
export interface GatherOptions extends FetchOptions {
|
|
17
|
+
/** Authorization bearer token */
|
|
18
|
+
bearer?: string;
|
|
19
|
+
/** API key (sent as x-api-key header) */
|
|
20
|
+
apiKey?: string;
|
|
21
|
+
/** Custom headers for URL fetches */
|
|
22
|
+
headers?: Record<string, string>;
|
|
23
|
+
/** IndexedDB database name override */
|
|
24
|
+
dbName?: string;
|
|
25
|
+
/** IndexedDB store name override */
|
|
26
|
+
storeName?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface IntoOptions {
|
|
30
|
+
/** Override which columns to use (for Table targets) */
|
|
31
|
+
columns?: string[];
|
|
32
|
+
/** Column name → display label mapping */
|
|
33
|
+
columnLabels?: Record<string, string>;
|
|
34
|
+
/** Key for option labels (Dropdown/Select/List targets) */
|
|
35
|
+
labelKey?: string;
|
|
36
|
+
/** Key for option values (Dropdown/Select targets) */
|
|
37
|
+
valueKey?: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
16
40
|
/**
|
|
17
41
|
* DataPipeline — Headless, promise-based data loading and transformation.
|
|
18
42
|
*
|
|
19
43
|
* Usage:
|
|
20
|
-
* const df = await
|
|
21
|
-
* const df = await
|
|
22
|
-
* const df = await
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* await data.fromUrl(url)
|
|
26
|
-
* .transform(df => df.select('name', 'age').filter(r => r.age > 30))
|
|
27
|
-
* .into(myTable);
|
|
28
|
-
*
|
|
29
|
-
* await data.fromStorage('encounters')
|
|
30
|
-
* .transform(df => df.distinct('provider'))
|
|
31
|
-
* .into(myDropdown, { labelKey: 'provider', valueKey: 'provider' });
|
|
44
|
+
* const df = await jux.gather('raw/myfile.xlsx').result();
|
|
45
|
+
* const df = await jux.gather('https://api.com/data', { bearer: 'xxx' }).result();
|
|
46
|
+
* const df = await jux.gather(myUpload).select('name').result();
|
|
47
|
+
* const df = await jux.gather([{a:1}, {b:2}]).result();
|
|
48
|
+
* await jux.gather(key).filter(r => r.active).intoTable(myTable);
|
|
32
49
|
*/
|
|
33
50
|
export class DataPipeline {
|
|
34
51
|
private _driver: TabularDriver;
|
|
35
52
|
private _pending: Promise<DataFrame> | null = null;
|
|
36
53
|
private _transforms: ((df: DataFrame) => DataFrame)[] = [];
|
|
54
|
+
private _gatherOptions: GatherOptions;
|
|
37
55
|
|
|
38
56
|
constructor(options: PipelineOptions = {}) {
|
|
39
57
|
this._driver = new TabularDriver(
|
|
40
58
|
options.dbName ?? 'jux-dataframes',
|
|
41
59
|
options.storeName ?? 'frames'
|
|
42
60
|
);
|
|
61
|
+
this._gatherOptions = {};
|
|
43
62
|
}
|
|
44
63
|
|
|
45
64
|
/* ═══════════════════════════════════════════════════
|
|
46
|
-
*
|
|
65
|
+
* SMART SOURCE DETECTION
|
|
47
66
|
* ═══════════════════════════════════════════════════ */
|
|
48
67
|
|
|
49
68
|
/**
|
|
50
|
-
*
|
|
69
|
+
* Auto-detect source type and load accordingly.
|
|
70
|
+
*
|
|
71
|
+
* - string starting with http/https → URL fetch
|
|
72
|
+
* - string → IndexedDB storage key
|
|
73
|
+
* - File → parse file
|
|
74
|
+
* - Array → row objects
|
|
75
|
+
* - DataFrame → clone
|
|
76
|
+
* - object with .storageKey → FileUpload cached key
|
|
77
|
+
* - object with string keys + array values → column-oriented data
|
|
51
78
|
*/
|
|
79
|
+
from(source: any, options: GatherOptions = {}): this {
|
|
80
|
+
this._transforms = [];
|
|
81
|
+
this._gatherOptions = options;
|
|
82
|
+
|
|
83
|
+
// Update driver if custom db/store
|
|
84
|
+
if (options.dbName || options.storeName) {
|
|
85
|
+
this._driver = new TabularDriver(
|
|
86
|
+
options.dbName ?? 'jux-dataframes',
|
|
87
|
+
options.storeName ?? 'frames'
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (source instanceof DataFrame) {
|
|
92
|
+
this._pending = Promise.resolve(source.clone());
|
|
93
|
+
} else if (Array.isArray(source)) {
|
|
94
|
+
this._pending = Promise.resolve(new DataFrame(source));
|
|
95
|
+
} else if (typeof source === 'string') {
|
|
96
|
+
if (/^https?:\/\//i.test(source)) {
|
|
97
|
+
this._pending = this._loadFromUrl(source, options);
|
|
98
|
+
} else {
|
|
99
|
+
this._pending = this._loadFromStorage(source);
|
|
100
|
+
}
|
|
101
|
+
} else if (source instanceof File) {
|
|
102
|
+
this._pending = this._loadFromFile(source, options);
|
|
103
|
+
} else if (source && typeof source === 'object' && 'storageKey' in source) {
|
|
104
|
+
// FileUpload component with .cache() enabled
|
|
105
|
+
const key = source.storageKey;
|
|
106
|
+
if (!key) {
|
|
107
|
+
this._pending = Promise.reject(
|
|
108
|
+
new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.')
|
|
109
|
+
);
|
|
110
|
+
} else {
|
|
111
|
+
this._pending = this._loadFromStorage(key);
|
|
112
|
+
}
|
|
113
|
+
} else if (source && typeof source === 'object' && !Array.isArray(source)) {
|
|
114
|
+
// Column-oriented: { col1: [...], col2: [...] }
|
|
115
|
+
this._pending = Promise.resolve(new DataFrame(source));
|
|
116
|
+
} else {
|
|
117
|
+
this._pending = Promise.reject(new Error('Unrecognized source type passed to gather()'));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return this;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/* ═══════════════════════════════════════════════════
|
|
124
|
+
* EXPLICIT SOURCES (still available)
|
|
125
|
+
* ═══════════════════════════════════════════════════ */
|
|
126
|
+
|
|
52
127
|
fromStorage(key: string): this {
|
|
53
128
|
this._transforms = [];
|
|
54
|
-
this._pending = this.
|
|
55
|
-
if (!df) throw new Error(`No data found in storage with key: "${key}"`);
|
|
56
|
-
return df;
|
|
57
|
-
});
|
|
129
|
+
this._pending = this._loadFromStorage(key);
|
|
58
130
|
return this;
|
|
59
131
|
}
|
|
60
132
|
|
|
61
|
-
|
|
62
|
-
* Load from a URL (CSV, TSV, or JSON)
|
|
63
|
-
*/
|
|
64
|
-
fromUrl(url: string, options: FetchOptions = {}): this {
|
|
133
|
+
fromUrl(url: string, options: GatherOptions = {}): this {
|
|
65
134
|
this._transforms = [];
|
|
135
|
+
this._gatherOptions = options;
|
|
66
136
|
this._pending = this._loadFromUrl(url, options);
|
|
67
137
|
return this;
|
|
68
138
|
}
|
|
69
139
|
|
|
70
|
-
/**
|
|
71
|
-
* Load from a File object directly
|
|
72
|
-
*/
|
|
73
140
|
fromFile(file: File, options: FetchOptions = {}): this {
|
|
74
141
|
this._transforms = [];
|
|
75
142
|
this._pending = this._loadFromFile(file, options);
|
|
76
143
|
return this;
|
|
77
144
|
}
|
|
78
145
|
|
|
79
|
-
/**
|
|
80
|
-
* Load from an array of row objects
|
|
81
|
-
*/
|
|
82
146
|
fromRows(rows: Record<string, any>[], columns?: string[]): this {
|
|
83
147
|
this._transforms = [];
|
|
84
148
|
this._pending = Promise.resolve(new DataFrame(rows, { columns }));
|
|
85
149
|
return this;
|
|
86
150
|
}
|
|
87
151
|
|
|
88
|
-
/**
|
|
89
|
-
* Load from column-oriented data
|
|
90
|
-
*/
|
|
91
152
|
fromColumns(columns: Record<string, any[]>): this {
|
|
92
153
|
this._transforms = [];
|
|
93
154
|
this._pending = Promise.resolve(new DataFrame(columns));
|
|
94
155
|
return this;
|
|
95
156
|
}
|
|
96
157
|
|
|
97
|
-
/**
|
|
98
|
-
* Load from a JSON response (array of objects or column-oriented)
|
|
99
|
-
*/
|
|
100
158
|
fromJson(json: Record<string, any>[] | Record<string, any[]> | string): this {
|
|
101
159
|
this._transforms = [];
|
|
102
160
|
const parsed = typeof json === 'string' ? JSON.parse(json) : json;
|
|
@@ -104,13 +162,6 @@ export class DataPipeline {
|
|
|
104
162
|
return this;
|
|
105
163
|
}
|
|
106
164
|
|
|
107
|
-
/**
|
|
108
|
-
* Load from a FileUpload component's cached storage key.
|
|
109
|
-
* Requires .cache() enabled on the FileUpload and a file already uploaded.
|
|
110
|
-
*
|
|
111
|
-
* Usage:
|
|
112
|
-
* await data().fromUpload(myUpload).select('name').into(myTable);
|
|
113
|
-
*/
|
|
114
165
|
fromUpload(upload: any): this {
|
|
115
166
|
this._transforms = [];
|
|
116
167
|
const key = upload.storageKey;
|
|
@@ -119,94 +170,52 @@ export class DataPipeline {
|
|
|
119
170
|
new Error('FileUpload has no cached storage key. Ensure .cache() is enabled and a file has been uploaded.')
|
|
120
171
|
);
|
|
121
172
|
} else {
|
|
122
|
-
this._pending = this.
|
|
123
|
-
if (!df) throw new Error(`No cached data found for key: "${key}"`);
|
|
124
|
-
return df;
|
|
125
|
-
});
|
|
173
|
+
this._pending = this._loadFromStorage(key);
|
|
126
174
|
}
|
|
127
175
|
return this;
|
|
128
176
|
}
|
|
129
177
|
|
|
130
|
-
/**
|
|
131
|
-
* Start from an existing DataFrame
|
|
132
|
-
*/
|
|
133
|
-
from(df: DataFrame): this {
|
|
134
|
-
this._transforms = [];
|
|
135
|
-
this._pending = Promise.resolve(df.clone());
|
|
136
|
-
return this;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
178
|
/* ═══════════════════════════════════════════════════
|
|
140
|
-
* TRANSFORMS (chained, lazy — applied on
|
|
179
|
+
* TRANSFORMS (chained, lazy — applied on terminal)
|
|
141
180
|
* ═══════════════════════════════════════════════════ */
|
|
142
181
|
|
|
143
|
-
/**
|
|
144
|
-
* Apply an arbitrary transform
|
|
145
|
-
*/
|
|
146
182
|
transform(fn: (df: DataFrame) => DataFrame): this {
|
|
147
183
|
this._transforms.push(fn);
|
|
148
184
|
return this;
|
|
149
185
|
}
|
|
150
186
|
|
|
151
|
-
/**
|
|
152
|
-
* Select columns
|
|
153
|
-
*/
|
|
154
187
|
select(...cols: string[]): this {
|
|
155
188
|
return this.transform(df => df.select(...cols));
|
|
156
189
|
}
|
|
157
190
|
|
|
158
|
-
/**
|
|
159
|
-
* Filter rows
|
|
160
|
-
*/
|
|
161
191
|
filter(predicate: (row: Record<string, any>, index: number) => boolean): this {
|
|
162
192
|
return this.transform(df => df.filter(predicate));
|
|
163
193
|
}
|
|
164
194
|
|
|
165
|
-
/**
|
|
166
|
-
* Sort by column
|
|
167
|
-
*/
|
|
168
195
|
sort(column: string, descending?: boolean): this {
|
|
169
196
|
return this.transform(df => df.sort(column, descending));
|
|
170
197
|
}
|
|
171
198
|
|
|
172
|
-
/**
|
|
173
|
-
* Take first N rows
|
|
174
|
-
*/
|
|
175
199
|
head(n: number = 5): this {
|
|
176
200
|
return this.transform(df => df.head(n));
|
|
177
201
|
}
|
|
178
202
|
|
|
179
|
-
/**
|
|
180
|
-
* Take last N rows
|
|
181
|
-
*/
|
|
182
203
|
tail(n: number = 5): this {
|
|
183
204
|
return this.transform(df => df.tail(n));
|
|
184
205
|
}
|
|
185
206
|
|
|
186
|
-
/**
|
|
187
|
-
* Rename columns
|
|
188
|
-
*/
|
|
189
207
|
rename(mapping: Record<string, string>): this {
|
|
190
208
|
return this.transform(df => df.rename(mapping));
|
|
191
209
|
}
|
|
192
210
|
|
|
193
|
-
/**
|
|
194
|
-
* Add a computed column
|
|
195
|
-
*/
|
|
196
211
|
withColumn(name: string, fn: (row: Record<string, any>, index: number) => any): this {
|
|
197
212
|
return this.transform(df => df.withColumn(name, fn));
|
|
198
213
|
}
|
|
199
214
|
|
|
200
|
-
/**
|
|
201
|
-
* Drop nulls
|
|
202
|
-
*/
|
|
203
215
|
dropna(columns?: string[]): this {
|
|
204
216
|
return this.transform(df => df.dropna(columns));
|
|
205
217
|
}
|
|
206
218
|
|
|
207
|
-
/**
|
|
208
|
-
* Drop duplicates
|
|
209
|
-
*/
|
|
210
219
|
dropDuplicates(columns?: string[]): this {
|
|
211
220
|
return this.transform(df => df.dropDuplicates(columns));
|
|
212
221
|
}
|
|
@@ -215,12 +224,8 @@ export class DataPipeline {
|
|
|
215
224
|
* TERMINAL OPERATIONS
|
|
216
225
|
* ═══════════════════════════════════════════════════ */
|
|
217
226
|
|
|
218
|
-
/**
|
|
219
|
-
* Execute the pipeline and return the DataFrame
|
|
220
|
-
*/
|
|
221
227
|
async result(): Promise<DataFrame> {
|
|
222
|
-
if (!this._pending) throw new Error('No data source specified. Call
|
|
223
|
-
|
|
228
|
+
if (!this._pending) throw new Error('No data source specified. Call gather(source) or from*(source) first.');
|
|
224
229
|
let df = await this._pending;
|
|
225
230
|
for (const fn of this._transforms) {
|
|
226
231
|
df = fn(df);
|
|
@@ -228,82 +233,107 @@ export class DataPipeline {
|
|
|
228
233
|
return df;
|
|
229
234
|
}
|
|
230
235
|
|
|
231
|
-
/**
|
|
232
|
-
* Execute and return rows as plain objects
|
|
233
|
-
*/
|
|
234
236
|
async toRows(): Promise<Record<string, any>[]> {
|
|
235
237
|
return (await this.result()).toRows();
|
|
236
238
|
}
|
|
237
239
|
|
|
238
|
-
/**
|
|
239
|
-
* Execute and return a single column as an array
|
|
240
|
-
*/
|
|
241
240
|
async toArray(column: string): Promise<any[]> {
|
|
242
241
|
return (await this.result()).col(column);
|
|
243
242
|
}
|
|
244
243
|
|
|
245
|
-
/**
|
|
246
|
-
* Execute and return distinct values from a column
|
|
247
|
-
*/
|
|
248
244
|
async toDistinct(column: string): Promise<any[]> {
|
|
249
245
|
return (await this.result()).distinct(column);
|
|
250
246
|
}
|
|
251
247
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
248
|
+
async toOptions(labelKey: string, valueKey?: string): Promise<{ label: string; value: any }[]> {
|
|
249
|
+
const df = await this.result();
|
|
250
|
+
const vKey = valueKey ?? labelKey;
|
|
251
|
+
return df.toRows().map(row => ({
|
|
252
|
+
label: String(row[labelKey] ?? ''),
|
|
253
|
+
value: row[vKey]
|
|
254
|
+
}));
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
async toList(column: string): Promise<string[]> {
|
|
258
|
+
const df = await this.result();
|
|
259
|
+
return df.col(column).map(v => String(v ?? ''));
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async intoTable(table: any, columnLabels?: Record<string, string>): Promise<DataFrame> {
|
|
263
|
+
const df = await this.result();
|
|
264
|
+
const columnDefs = df.columns.map(col => ({
|
|
265
|
+
key: col,
|
|
266
|
+
label: columnLabels?.[col] ?? col
|
|
267
|
+
}));
|
|
268
|
+
table.columns(columnDefs).rows(df.toRows());
|
|
269
|
+
return df;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
async intoDropdown(dropdown: any, labelKey: string, valueKey?: string): Promise<DataFrame> {
|
|
273
|
+
const df = await this.result();
|
|
274
|
+
const vKey = valueKey ?? labelKey;
|
|
275
|
+
const opts = df.toRows().map(row => ({
|
|
276
|
+
label: String(row[labelKey] ?? ''),
|
|
277
|
+
value: row[vKey]
|
|
278
|
+
}));
|
|
279
|
+
dropdown.options(opts);
|
|
280
|
+
return df;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
async intoList(list: any, column: string): Promise<DataFrame> {
|
|
284
|
+
const df = await this.result();
|
|
285
|
+
list.items(df.col(column).map((v: any) => String(v ?? '')));
|
|
286
|
+
return df;
|
|
287
|
+
}
|
|
288
|
+
|
|
262
289
|
async into(target: any, options: IntoOptions = {}): Promise<DataFrame> {
|
|
263
290
|
const df = await this.result();
|
|
264
291
|
const rows = df.toRows();
|
|
265
292
|
|
|
266
|
-
// Table-like: has .columns() and .rows()
|
|
267
293
|
if (typeof target.columns === 'function' && typeof target.rows === 'function') {
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
294
|
+
const cols = options.columns ?? df.columns;
|
|
295
|
+
const columnDefs = cols.map((col: string) => ({
|
|
296
|
+
key: col,
|
|
297
|
+
label: options.columnLabels?.[col] ?? col
|
|
298
|
+
}));
|
|
272
299
|
target.columns(columnDefs).rows(rows);
|
|
273
300
|
return df;
|
|
274
301
|
}
|
|
275
302
|
|
|
276
|
-
// DataFrameComponent: has .fromData()
|
|
277
303
|
if (typeof target.fromData === 'function') {
|
|
278
304
|
target.fromData(rows);
|
|
279
305
|
return df;
|
|
280
306
|
}
|
|
281
307
|
|
|
282
|
-
// Dropdown/Select-like: has .options()
|
|
283
308
|
if (typeof target.options === 'function') {
|
|
284
|
-
|
|
285
|
-
|
|
309
|
+
if (!options.labelKey) {
|
|
310
|
+
throw new Error(
|
|
311
|
+
'Cannot push into dropdown/select without labelKey. ' +
|
|
312
|
+
'Use .intoDropdown(target, labelKey, valueKey) or pass { labelKey, valueKey } to .into()'
|
|
313
|
+
);
|
|
314
|
+
}
|
|
286
315
|
const opts = rows.map(row => ({
|
|
287
|
-
label: String(row[labelKey] ?? ''),
|
|
288
|
-
value: row[valueKey]
|
|
316
|
+
label: String(row[options.labelKey!] ?? ''),
|
|
317
|
+
value: row[options.valueKey ?? options.labelKey!]
|
|
289
318
|
}));
|
|
290
319
|
target.options(opts);
|
|
291
320
|
return df;
|
|
292
321
|
}
|
|
293
322
|
|
|
294
|
-
// List-like: has .items()
|
|
295
323
|
if (typeof target.items === 'function') {
|
|
296
|
-
|
|
297
|
-
|
|
324
|
+
if (!options.labelKey) {
|
|
325
|
+
throw new Error(
|
|
326
|
+
'Cannot push into list without labelKey. ' +
|
|
327
|
+
'Use .intoList(target, column) or pass { labelKey } to .into()'
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
target.items(rows.map(row => String(row[options.labelKey!] ?? '')));
|
|
298
331
|
return df;
|
|
299
332
|
}
|
|
300
333
|
|
|
301
|
-
throw new Error(
|
|
334
|
+
throw new Error('Target does not have a recognized API (.columns/.rows, .fromData, .options, or .items)');
|
|
302
335
|
}
|
|
303
336
|
|
|
304
|
-
/**
|
|
305
|
-
* Execute and persist to IndexedDB
|
|
306
|
-
*/
|
|
307
337
|
async store(key: string, metadata?: Record<string, any>): Promise<DataFrame> {
|
|
308
338
|
const df = await this.result();
|
|
309
339
|
await this._driver.store(key, df, metadata);
|
|
@@ -314,17 +344,49 @@ export class DataPipeline {
|
|
|
314
344
|
* PRIVATE
|
|
315
345
|
* ═══════════════════════════════════════════════════ */
|
|
316
346
|
|
|
317
|
-
private async
|
|
347
|
+
private async _loadFromStorage(key: string): Promise<DataFrame> {
|
|
348
|
+
// Try by name first, then by id
|
|
349
|
+
let df = await this._driver.loadByName(key);
|
|
350
|
+
if (df) return df;
|
|
351
|
+
|
|
352
|
+
df = await this._driver.load(key);
|
|
353
|
+
if (df) return df;
|
|
354
|
+
|
|
355
|
+
throw new Error(`No data found in storage with key: "${key}"`);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
private async _loadFromUrl(url: string, options: GatherOptions): Promise<DataFrame> {
|
|
318
359
|
const mimeType = options.mimeType ?? this._detectMimeType(url);
|
|
319
360
|
|
|
361
|
+
// Build headers from auth options
|
|
362
|
+
const headers: Record<string, string> = { ...options.headers };
|
|
363
|
+
if (options.bearer) {
|
|
364
|
+
headers['Authorization'] = `Bearer ${options.bearer}`;
|
|
365
|
+
}
|
|
366
|
+
if (options.apiKey) {
|
|
367
|
+
headers['x-api-key'] = options.apiKey;
|
|
368
|
+
}
|
|
369
|
+
|
|
320
370
|
if (mimeType === 'json') {
|
|
321
|
-
const response = await fetch(url);
|
|
371
|
+
const response = await fetch(url, { headers });
|
|
322
372
|
if (!response.ok) throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
|
|
323
373
|
const json = await response.json();
|
|
324
374
|
return new DataFrame(json);
|
|
325
375
|
}
|
|
326
376
|
|
|
327
|
-
// CSV/TSV —
|
|
377
|
+
// CSV/TSV — fetch with headers, then parse
|
|
378
|
+
if (Object.keys(headers).length > 0) {
|
|
379
|
+
const response = await fetch(url, { headers });
|
|
380
|
+
if (!response.ok) throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);
|
|
381
|
+
const text = await response.text();
|
|
382
|
+
return this._driver.parseCSV(text, {
|
|
383
|
+
autoDetectDelimiter: !options.delimiter,
|
|
384
|
+
delimiter: options.delimiter,
|
|
385
|
+
hasHeader: options.hasHeader ?? true,
|
|
386
|
+
headerRow: options.headerRow
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
|
|
328
390
|
return this._driver.fetch(url, {
|
|
329
391
|
autoDetectDelimiter: !options.delimiter,
|
|
330
392
|
delimiter: options.delimiter,
|
|
@@ -363,26 +425,28 @@ export class DataPipeline {
|
|
|
363
425
|
}
|
|
364
426
|
}
|
|
365
427
|
|
|
366
|
-
export interface IntoOptions {
|
|
367
|
-
/** Override which columns to use (for Table targets) */
|
|
368
|
-
columns?: string[];
|
|
369
|
-
/** Column name → display label mapping */
|
|
370
|
-
columnLabels?: Record<string, string>;
|
|
371
|
-
/** Key for option labels (Dropdown/Select/List targets) */
|
|
372
|
-
labelKey?: string;
|
|
373
|
-
/** Key for option values (Dropdown/Select targets) */
|
|
374
|
-
valueKey?: string;
|
|
375
|
-
}
|
|
376
|
-
|
|
377
428
|
/**
|
|
378
|
-
* Factory function —
|
|
429
|
+
* Factory function — primary entry point via jux.gather()
|
|
379
430
|
*
|
|
380
|
-
*
|
|
381
|
-
*
|
|
431
|
+
* Smart single-arg:
|
|
432
|
+
* jux.gather('raw/myfile.xlsx') // storage key
|
|
433
|
+
* jux.gather('https://api.com/data', { bearer: 'xxx' }) // URL with auth
|
|
434
|
+
* jux.gather(myUpload) // FileUpload
|
|
435
|
+
* jux.gather([{a:1}, {b:2}]) // inline rows
|
|
436
|
+
* jux.gather(existingDf) // clone DataFrame
|
|
382
437
|
*
|
|
383
|
-
*
|
|
384
|
-
*
|
|
438
|
+
* Explicit source:
|
|
439
|
+
* jux.gather().fromStorage(key)
|
|
440
|
+
* jux.gather().fromUrl(url, opts)
|
|
385
441
|
*/
|
|
386
|
-
export function
|
|
387
|
-
|
|
442
|
+
export function gather(source?: any, options?: GatherOptions): DataPipeline {
|
|
443
|
+
const pipe = new DataPipeline(
|
|
444
|
+
options?.dbName || options?.storeName
|
|
445
|
+
? { dbName: options.dbName, storeName: options.storeName }
|
|
446
|
+
: {}
|
|
447
|
+
);
|
|
448
|
+
if (source !== undefined) {
|
|
449
|
+
pipe.from(source, options ?? {});
|
|
450
|
+
}
|
|
451
|
+
return pipe;
|
|
388
452
|
}
|