juxscript 1.1.180 → 1.1.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/components/dataframe.d.ts +12 -0
- package/lib/components/dataframe.d.ts.map +1 -1
- package/lib/components/dataframe.js +36 -12
- package/lib/components/dataframe.ts +42 -12
- package/lib/storage/TabularDriver.d.ts +5 -4
- package/lib/storage/TabularDriver.d.ts.map +1 -1
- package/lib/storage/TabularDriver.js +73 -11
- package/lib/storage/TabularDriver.ts +88 -12
- package/package.json +1 -1
|
@@ -14,6 +14,8 @@ export interface DataFrameOptions {
|
|
|
14
14
|
rowsPerPage?: number;
|
|
15
15
|
showStatus?: boolean;
|
|
16
16
|
icon?: string;
|
|
17
|
+
maxSheetSize?: number;
|
|
18
|
+
sheetChunkSize?: number;
|
|
17
19
|
style?: string;
|
|
18
20
|
class?: string;
|
|
19
21
|
}
|
|
@@ -36,6 +38,8 @@ export declare class DataFrameComponent extends BaseComponent<DataFrameState> {
|
|
|
36
38
|
private _inlineUpload;
|
|
37
39
|
private _showStatus;
|
|
38
40
|
private _icon;
|
|
41
|
+
private _maxSheetSize;
|
|
42
|
+
private _sheetChunkSize;
|
|
39
43
|
constructor(id: string, options?: DataFrameOptions);
|
|
40
44
|
protected getTriggerEvents(): readonly string[];
|
|
41
45
|
protected getCallbackEvents(): readonly string[];
|
|
@@ -68,6 +72,14 @@ export declare class DataFrameComponent extends BaseComponent<DataFrameState> {
|
|
|
68
72
|
filterable(v: boolean): this;
|
|
69
73
|
paginated(v: boolean): this;
|
|
70
74
|
rowsPerPage(v: number): this;
|
|
75
|
+
/**
|
|
76
|
+
* ✅ NEW: Set max rows per sheet (prevents memory issues with huge Excel files)
|
|
77
|
+
*/
|
|
78
|
+
maxSheetSize(v: number): this;
|
|
79
|
+
/**
|
|
80
|
+
* ✅ NEW: Set chunk size for processing large sheets
|
|
81
|
+
*/
|
|
82
|
+
sheetChunkSize(v: number): this;
|
|
71
83
|
/**
|
|
72
84
|
* ✅ FIXED: Render multiple Excel sheets as tabs
|
|
73
85
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataframe.d.ts","sourceRoot":"","sources":["dataframe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAOnC,MAAM,WAAW,gBAAgB;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,KAAK,cAAc,GAAG,SAAS,GAAG;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,aAAa,CAAC,cAAc,CAAC;IACjE,OAAO,CAAC,GAAG,CAA0B;IACrC,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,aAAa,CAOnB;IACF,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,cAAc,CAAsC;IAC5D,OAAO,CAAC,aAAa,CAAgE;IACrF,OAAO,CAAC,WAAW,CAAiB;IACpC,OAAO,CAAC,KAAK,CAAc;
|
|
1
|
+
{"version":3,"file":"dataframe.d.ts","sourceRoot":"","sources":["dataframe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAOnC,MAAM,WAAW,gBAAgB;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,KAAK,cAAc,GAAG,SAAS,GAAG;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,aAAa,CAAC,cAAc,CAAC;IACjE,OAAO,CAAC,GAAG,CAA0B;IACrC,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,aAAa,CAOnB;IACF,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,cAAc,CAAsC;IAC5D,OAAO,CAAC,aAAa,CAAgE;IACrF,OAAO,CAAC,WAAW,CAAiB;IACpC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,eAAe,CAAiB;gBAE5B,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB;IAiCtD,SAAS,CAAC,gBAAgB,IAAI,SAAS,MAAM,EAAE;IAC/C,SAAS,CAAC,iBAAiB,IAAI,SAAS,MAAM,EAAE;IAMhD,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAwB9B,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAiDpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI;IAiBnE,UAAU,CAAC,KAAK,GAAE,MAAsB,EAAE,MAAM,GAAE,MAAoC,EAAE,IAAI,GAAE,MAAiB,GAAG,IAAI;IAStH,UAAU,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IAC5B,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAM3B,KAAK,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,IAAI;IAQ7C,MAAM,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,IAAI;IAI7E,MAAM,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI;IAI/B,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,GAAG,IAAI;IAI7C,IAAI,CAAC,CAAC,GAAE,MAAU,GAAG,IAAI;IAIzB,IAAI,CAAC,CAAC,GAAE,MAAU,GAAG,IAAI;IAIzB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,GAAG,GAAG,IAAI;IAIpF,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,UAAU,GAAG,YAAY,GAAG,UAAU,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI;IAQxH,IAAI,EAAE,IAAI,SAAS,GAAG,IAAI,CAAqB;IAC/C,IAAI,MAAM,IAAI,aAAa,CAAyB;IACpD,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,CAAwB;IACjD,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IACtC,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IACjC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;IAC/B,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAsC;IACnE,IAAI,OAAO,IAAI,MAAM,EAAE,CAAoC;IAErD,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAUhD,OAAO,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IACzB,SAAS,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IAC3B,QAAQ,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IAC1B,UAAU,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IAC5B,SAAS,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IAC3B,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAE5B;;OAEG;IACH,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK7B;;OAEG;IACH,cAAc,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAS/B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAkIzB,OAAO,CAAC,aAAa;IAuBrB,OAAO,CAAC,aAAa;IA0BrB,OAAO,CAAC,YAAY;IAwCpB,OAAO,CAAC,gBAAgB;IA+CxB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,IAAI;IAMtC,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,IAAI;CAyGrE;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,kBAAkB,CAExF"}
|
|
@@ -30,6 +30,8 @@ export class DataFrameComponent extends BaseComponent {
|
|
|
30
30
|
this._inlineUpload = null;
|
|
31
31
|
this._showStatus = true;
|
|
32
32
|
this._icon = '';
|
|
33
|
+
this._maxSheetSize = 100000; // ✅ Default 100k rows
|
|
34
|
+
this._sheetChunkSize = 10000; // ✅ Default 10k chunk
|
|
33
35
|
this._driver = new TabularDriver(options.dbName ?? 'jux-dataframes', options.storeName ?? 'frames');
|
|
34
36
|
this._showStatus = options.showStatus ?? true;
|
|
35
37
|
this._icon = options.icon ?? '';
|
|
@@ -41,6 +43,8 @@ export class DataFrameComponent extends BaseComponent {
|
|
|
41
43
|
paginated: options.paginated ?? true,
|
|
42
44
|
rowsPerPage: options.rowsPerPage ?? 25
|
|
43
45
|
};
|
|
46
|
+
this._maxSheetSize = options.maxSheetSize ?? 100000;
|
|
47
|
+
this._sheetChunkSize = options.sheetChunkSize ?? 10000;
|
|
44
48
|
}
|
|
45
49
|
getTriggerEvents() { return TRIGGER_EVENTS; }
|
|
46
50
|
getCallbackEvents() { return CALLBACK_EVENTS; }
|
|
@@ -86,25 +90,29 @@ export class DataFrameComponent extends BaseComponent {
|
|
|
86
90
|
this.state.loading = true;
|
|
87
91
|
this._updateStatus('⏳ Parsing ' + file.name + '...', 'loading');
|
|
88
92
|
try {
|
|
89
|
-
// ✅ Check if multi-sheet Excel
|
|
90
93
|
const isExcel = file.name.toLowerCase().endsWith('.xlsx') ||
|
|
91
94
|
file.name.toLowerCase().endsWith('.xls');
|
|
92
95
|
if (isExcel) {
|
|
93
|
-
|
|
96
|
+
// ✅ Pass chunking options for large files
|
|
97
|
+
const sheets = await this._driver.streamFileMultiSheet(file, {
|
|
98
|
+
maxSheetSize: this._maxSheetSize,
|
|
99
|
+
sheetChunkSize: this._sheetChunkSize,
|
|
100
|
+
onProgress: (loaded, total) => {
|
|
101
|
+
const pct = total ? Math.round((loaded / total) * 100) : 0;
|
|
102
|
+
this._updateStatus(`⏳ Parsing ${file.name}... ${pct}%`, 'loading');
|
|
103
|
+
}
|
|
104
|
+
});
|
|
94
105
|
const sheetNames = Object.keys(sheets);
|
|
95
|
-
// Store first sheet to IndexedDB
|
|
96
106
|
await this._driver.store(file.name, sheets[sheetNames[0]], { source: file.name });
|
|
97
107
|
if (sheetNames.length > 1) {
|
|
98
|
-
// ✅ Multi-sheet: render tabs
|
|
99
108
|
this._renderMultiSheet(sheets, file.name);
|
|
100
109
|
}
|
|
101
110
|
else {
|
|
102
|
-
// Single sheet: render normally
|
|
103
111
|
this._setDataFrame(sheets[sheetNames[0]], file.name);
|
|
104
112
|
}
|
|
105
113
|
}
|
|
106
114
|
else {
|
|
107
|
-
// CSV/TSV:
|
|
115
|
+
// CSV/TSV: use existing streaming
|
|
108
116
|
const df = await this._driver.streamFile(file);
|
|
109
117
|
await this._driver.store(file.name, df, { source: file.name });
|
|
110
118
|
this._setDataFrame(df, file.name);
|
|
@@ -208,6 +216,20 @@ export class DataFrameComponent extends BaseComponent {
|
|
|
208
216
|
filterable(v) { this._tableOptions.filterable = v; return this; }
|
|
209
217
|
paginated(v) { this._tableOptions.paginated = v; return this; }
|
|
210
218
|
rowsPerPage(v) { this._tableOptions.rowsPerPage = v; return this; }
|
|
219
|
+
/**
|
|
220
|
+
* ✅ NEW: Set max rows per sheet (prevents memory issues with huge Excel files)
|
|
221
|
+
*/
|
|
222
|
+
maxSheetSize(v) {
|
|
223
|
+
this._maxSheetSize = v;
|
|
224
|
+
return this;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* ✅ NEW: Set chunk size for processing large sheets
|
|
228
|
+
*/
|
|
229
|
+
sheetChunkSize(v) {
|
|
230
|
+
this._sheetChunkSize = v;
|
|
231
|
+
return this;
|
|
232
|
+
}
|
|
211
233
|
/* ═══════════════════════════════════════════════════
|
|
212
234
|
* MULTI-SHEET RENDERING
|
|
213
235
|
* ═══════════════════════════════════════════════════ */
|
|
@@ -460,25 +482,27 @@ export class DataFrameComponent extends BaseComponent {
|
|
|
460
482
|
this.state.loading = true;
|
|
461
483
|
this._updateStatus('⏳ Parsing ' + file.name + '...', 'loading');
|
|
462
484
|
try {
|
|
463
|
-
// ✅ Check if multi-sheet Excel
|
|
464
485
|
const isExcel = file.name.toLowerCase().endsWith('.xlsx') ||
|
|
465
486
|
file.name.toLowerCase().endsWith('.xls');
|
|
466
487
|
if (isExcel) {
|
|
467
|
-
const sheets = await this._driver.streamFileMultiSheet(file
|
|
488
|
+
const sheets = await this._driver.streamFileMultiSheet(file, {
|
|
489
|
+
maxSheetSize: this._maxSheetSize,
|
|
490
|
+
sheetChunkSize: this._sheetChunkSize,
|
|
491
|
+
onProgress: (loaded, total) => {
|
|
492
|
+
const pct = total ? Math.round((loaded / total) * 100) : 0;
|
|
493
|
+
this._updateStatus(`⏳ Parsing ${file.name}... ${pct}%`, 'loading');
|
|
494
|
+
}
|
|
495
|
+
});
|
|
468
496
|
const sheetNames = Object.keys(sheets);
|
|
469
|
-
// Store first sheet to IndexedDB
|
|
470
497
|
await this._driver.store(file.name, sheets[sheetNames[0]], { source: file.name });
|
|
471
498
|
if (sheetNames.length > 1) {
|
|
472
|
-
// ✅ Multi-sheet: render tabs
|
|
473
499
|
this._renderMultiSheet(sheets, file.name);
|
|
474
500
|
}
|
|
475
501
|
else {
|
|
476
|
-
// Single sheet: render normally
|
|
477
502
|
this._setDataFrame(sheets[sheetNames[0]], file.name);
|
|
478
503
|
}
|
|
479
504
|
}
|
|
480
505
|
else {
|
|
481
|
-
// CSV/TSV: single sheet
|
|
482
506
|
const df = await this._driver.streamFile(file);
|
|
483
507
|
await this._driver.store(file.name, df, { source: file.name });
|
|
484
508
|
this._setDataFrame(df, file.name);
|
|
@@ -20,6 +20,8 @@ export interface DataFrameOptions {
|
|
|
20
20
|
rowsPerPage?: number;
|
|
21
21
|
showStatus?: boolean;
|
|
22
22
|
icon?: string;
|
|
23
|
+
maxSheetSize?: number; // ✅ NEW: Max rows per sheet (default: 100k)
|
|
24
|
+
sheetChunkSize?: number; // ✅ NEW: Chunk size for large sheets (default: 10k)
|
|
23
25
|
style?: string;
|
|
24
26
|
class?: string;
|
|
25
27
|
}
|
|
@@ -51,6 +53,8 @@ export class DataFrameComponent extends BaseComponent<DataFrameState> {
|
|
|
51
53
|
private _inlineUpload: { label: string; accept: string; icon: string } | null = null;
|
|
52
54
|
private _showStatus: boolean = true;
|
|
53
55
|
private _icon: string = '';
|
|
56
|
+
private _maxSheetSize: number = 100000; // ✅ Default 100k rows
|
|
57
|
+
private _sheetChunkSize: number = 10000; // ✅ Default 10k chunk
|
|
54
58
|
|
|
55
59
|
constructor(id: string, options: DataFrameOptions = {}) {
|
|
56
60
|
super(id, {
|
|
@@ -81,6 +85,8 @@ export class DataFrameComponent extends BaseComponent<DataFrameState> {
|
|
|
81
85
|
paginated: options.paginated ?? true,
|
|
82
86
|
rowsPerPage: options.rowsPerPage ?? 25
|
|
83
87
|
};
|
|
88
|
+
this._maxSheetSize = options.maxSheetSize ?? 100000;
|
|
89
|
+
this._sheetChunkSize = options.sheetChunkSize ?? 10000;
|
|
84
90
|
}
|
|
85
91
|
|
|
86
92
|
protected getTriggerEvents(): readonly string[] { return TRIGGER_EVENTS; }
|
|
@@ -124,26 +130,31 @@ export class DataFrameComponent extends BaseComponent<DataFrameState> {
|
|
|
124
130
|
this._updateStatus('⏳ Parsing ' + file.name + '...', 'loading');
|
|
125
131
|
|
|
126
132
|
try {
|
|
127
|
-
// ✅ Check if multi-sheet Excel
|
|
128
133
|
const isExcel = file.name.toLowerCase().endsWith('.xlsx') ||
|
|
129
134
|
file.name.toLowerCase().endsWith('.xls');
|
|
130
135
|
|
|
131
136
|
if (isExcel) {
|
|
132
|
-
|
|
137
|
+
// ✅ Pass chunking options for large files
|
|
138
|
+
const sheets = await this._driver.streamFileMultiSheet(file, {
|
|
139
|
+
maxSheetSize: this._maxSheetSize,
|
|
140
|
+
sheetChunkSize: this._sheetChunkSize,
|
|
141
|
+
onProgress: (loaded, total) => {
|
|
142
|
+
const pct = total ? Math.round((loaded / total) * 100) : 0;
|
|
143
|
+
this._updateStatus(`⏳ Parsing ${file.name}... ${pct}%`, 'loading');
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
|
|
133
147
|
const sheetNames = Object.keys(sheets);
|
|
134
148
|
|
|
135
|
-
// Store first sheet to IndexedDB
|
|
136
149
|
await this._driver.store(file.name, sheets[sheetNames[0]], { source: file.name });
|
|
137
150
|
|
|
138
151
|
if (sheetNames.length > 1) {
|
|
139
|
-
// ✅ Multi-sheet: render tabs
|
|
140
152
|
this._renderMultiSheet(sheets, file.name);
|
|
141
153
|
} else {
|
|
142
|
-
// Single sheet: render normally
|
|
143
154
|
this._setDataFrame(sheets[sheetNames[0]], file.name);
|
|
144
155
|
}
|
|
145
156
|
} else {
|
|
146
|
-
// CSV/TSV:
|
|
157
|
+
// CSV/TSV: use existing streaming
|
|
147
158
|
const df = await this._driver.streamFile(file);
|
|
148
159
|
await this._driver.store(file.name, df, { source: file.name });
|
|
149
160
|
this._setDataFrame(df, file.name);
|
|
@@ -257,6 +268,22 @@ export class DataFrameComponent extends BaseComponent<DataFrameState> {
|
|
|
257
268
|
paginated(v: boolean): this { this._tableOptions.paginated = v; return this; }
|
|
258
269
|
rowsPerPage(v: number): this { this._tableOptions.rowsPerPage = v; return this; }
|
|
259
270
|
|
|
271
|
+
/**
|
|
272
|
+
* ✅ NEW: Set max rows per sheet (prevents memory issues with huge Excel files)
|
|
273
|
+
*/
|
|
274
|
+
maxSheetSize(v: number): this {
|
|
275
|
+
this._maxSheetSize = v;
|
|
276
|
+
return this;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* ✅ NEW: Set chunk size for processing large sheets
|
|
281
|
+
*/
|
|
282
|
+
sheetChunkSize(v: number): this {
|
|
283
|
+
this._sheetChunkSize = v;
|
|
284
|
+
return this;
|
|
285
|
+
}
|
|
286
|
+
|
|
260
287
|
/* ═══════════════════════════════════════════════════
|
|
261
288
|
* MULTI-SHEET RENDERING
|
|
262
289
|
* ═══════════════════════════════════════════════════ */
|
|
@@ -567,26 +594,29 @@ export class DataFrameComponent extends BaseComponent<DataFrameState> {
|
|
|
567
594
|
this._updateStatus('⏳ Parsing ' + file.name + '...', 'loading');
|
|
568
595
|
|
|
569
596
|
try {
|
|
570
|
-
// ✅ Check if multi-sheet Excel
|
|
571
597
|
const isExcel = file.name.toLowerCase().endsWith('.xlsx') ||
|
|
572
598
|
file.name.toLowerCase().endsWith('.xls');
|
|
573
599
|
|
|
574
600
|
if (isExcel) {
|
|
575
|
-
const sheets = await this._driver.streamFileMultiSheet(file
|
|
601
|
+
const sheets = await this._driver.streamFileMultiSheet(file, {
|
|
602
|
+
maxSheetSize: this._maxSheetSize,
|
|
603
|
+
sheetChunkSize: this._sheetChunkSize,
|
|
604
|
+
onProgress: (loaded, total) => {
|
|
605
|
+
const pct = total ? Math.round((loaded / total) * 100) : 0;
|
|
606
|
+
this._updateStatus(`⏳ Parsing ${file.name}... ${pct}%`, 'loading');
|
|
607
|
+
}
|
|
608
|
+
});
|
|
609
|
+
|
|
576
610
|
const sheetNames = Object.keys(sheets);
|
|
577
611
|
|
|
578
|
-
// Store first sheet to IndexedDB
|
|
579
612
|
await this._driver.store(file.name, sheets[sheetNames[0]], { source: file.name });
|
|
580
613
|
|
|
581
614
|
if (sheetNames.length > 1) {
|
|
582
|
-
// ✅ Multi-sheet: render tabs
|
|
583
615
|
this._renderMultiSheet(sheets, file.name);
|
|
584
616
|
} else {
|
|
585
|
-
// Single sheet: render normally
|
|
586
617
|
this._setDataFrame(sheets[sheetNames[0]], file.name);
|
|
587
618
|
}
|
|
588
619
|
} else {
|
|
589
|
-
// CSV/TSV: single sheet
|
|
590
620
|
const df = await this._driver.streamFile(file);
|
|
591
621
|
await this._driver.store(file.name, df, { source: file.name });
|
|
592
622
|
this._setDataFrame(df, file.name);
|
|
@@ -18,6 +18,8 @@ export interface ParseOptions {
|
|
|
18
18
|
skipRows?: number;
|
|
19
19
|
columns?: string[];
|
|
20
20
|
sheet?: string | number;
|
|
21
|
+
maxSheetSize?: number;
|
|
22
|
+
sheetChunkSize?: number;
|
|
21
23
|
}
|
|
22
24
|
export declare class TabularDriver {
|
|
23
25
|
private _dbName;
|
|
@@ -74,11 +76,10 @@ export declare class TabularDriver {
|
|
|
74
76
|
*/
|
|
75
77
|
fetch(url: string, options?: ParseOptions): Promise<DataFrame>;
|
|
76
78
|
/**
|
|
77
|
-
* ✅
|
|
78
|
-
*
|
|
79
|
-
* @returns Record<sheetName, DataFrame>
|
|
79
|
+
* ✅ OPTIMIZED: Stream Excel file with chunked row processing
|
|
80
|
+
* Handles large files by processing rows in batches
|
|
80
81
|
*/
|
|
81
|
-
streamFileMultiSheet(file: File): Promise<Record<string, DataFrame>>;
|
|
82
|
+
streamFileMultiSheet(file: File, options?: ParseOptions): Promise<Record<string, DataFrame>>;
|
|
82
83
|
private _splitLines;
|
|
83
84
|
private _parseLine;
|
|
84
85
|
private _autoType;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TabularDriver.d.ts","sourceRoot":"","sources":["TabularDriver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,KAAK,IAAI,CAAC;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC3B;AAED,qBAAa,aAAa;IACtB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,GAAG,CAA4B;gBAE3B,MAAM,GAAE,MAAsB,EAAE,SAAS,GAAE,MAAiB;IAKlE,IAAI,IAAI,OAAO,CAAC,WAAW,CAAC;IA4BlC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,SAAS;IA6C7D;;;OAGG;IACG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;IAoG5E;;OAEG;YACW,UAAU;IAuExB;;OAEG;IACG,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiBlD;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBzF;;OAEG;IACG,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAwBjD;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IA4BzD;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAqBlH;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB5B;;OAEG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;IA0ExE
|
|
1
|
+
{"version":3,"file":"TabularDriver.d.ts","sourceRoot":"","sources":["TabularDriver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,KAAK,IAAI,CAAC;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,aAAa;IACtB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,GAAG,CAA4B;gBAE3B,MAAM,GAAE,MAAsB,EAAE,SAAS,GAAE,MAAiB;IAKlE,IAAI,IAAI,OAAO,CAAC,WAAW,CAAC;IA4BlC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,SAAS;IA6C7D;;;OAGG;IACG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;IAoG5E;;OAEG;YACW,UAAU;IAuExB;;OAEG;IACG,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiBlD;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBzF;;OAEG;IACG,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAwBjD;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IA4BzD;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAqBlH;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB5B;;OAEG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;IA0ExE;;;OAGG;IACG,oBAAoB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAyGtG,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,UAAU;IAmClB,OAAO,CAAC,SAAS;IAYjB,KAAK,IAAI,IAAI;CAMhB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,aAAa,CAEhF"}
|
|
@@ -419,12 +419,11 @@ export class TabularDriver {
|
|
|
419
419
|
return df;
|
|
420
420
|
}
|
|
421
421
|
/**
|
|
422
|
-
* ✅
|
|
423
|
-
*
|
|
424
|
-
* @returns Record<sheetName, DataFrame>
|
|
422
|
+
* ✅ OPTIMIZED: Stream Excel file with chunked row processing
|
|
423
|
+
* Handles large files by processing rows in batches
|
|
425
424
|
*/
|
|
426
|
-
async streamFileMultiSheet(file) {
|
|
427
|
-
|
|
425
|
+
async streamFileMultiSheet(file, options = {}) {
|
|
426
|
+
const { maxSheetSize = 100000, sheetChunkSize = 10000, onProgress } = options;
|
|
428
427
|
let XLSX;
|
|
429
428
|
try {
|
|
430
429
|
XLSX = await import('xlsx');
|
|
@@ -432,16 +431,79 @@ export class TabularDriver {
|
|
|
432
431
|
catch {
|
|
433
432
|
throw new Error('XLSX support requires the "xlsx" package. Install it with: npm install xlsx');
|
|
434
433
|
}
|
|
434
|
+
if (onProgress)
|
|
435
|
+
onProgress(0, file.size);
|
|
435
436
|
const buffer = await file.arrayBuffer();
|
|
436
|
-
|
|
437
|
+
if (onProgress)
|
|
438
|
+
onProgress(file.size * 0.3, file.size);
|
|
439
|
+
const workbook = XLSX.read(buffer, {
|
|
440
|
+
type: 'array',
|
|
441
|
+
sheetRows: maxSheetSize, // ✅ Limit rows read per sheet
|
|
442
|
+
dense: false // ✅ Use sparse format for memory efficiency
|
|
443
|
+
});
|
|
437
444
|
const sheets = {};
|
|
438
|
-
workbook.SheetNames.
|
|
445
|
+
const totalSheets = workbook.SheetNames.length;
|
|
446
|
+
let processedSheets = 0;
|
|
447
|
+
for (const sheetName of workbook.SheetNames) {
|
|
439
448
|
const worksheet = workbook.Sheets[sheetName];
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
449
|
+
// ✅ Get sheet range to determine size
|
|
450
|
+
const range = XLSX.utils.decode_range(worksheet['!ref'] || 'A1');
|
|
451
|
+
const totalRows = range.e.r - range.s.r + 1;
|
|
452
|
+
if (totalRows === 0) {
|
|
453
|
+
processedSheets++;
|
|
454
|
+
continue;
|
|
443
455
|
}
|
|
444
|
-
|
|
456
|
+
// ✅ Process in chunks if sheet is large
|
|
457
|
+
if (totalRows > sheetChunkSize) {
|
|
458
|
+
const rows = [];
|
|
459
|
+
let headers = [];
|
|
460
|
+
for (let startRow = 0; startRow < totalRows; startRow += sheetChunkSize) {
|
|
461
|
+
const endRow = Math.min(startRow + sheetChunkSize, totalRows);
|
|
462
|
+
// ✅ Read chunk with limited row range
|
|
463
|
+
const chunkData = XLSX.utils.sheet_to_json(worksheet, {
|
|
464
|
+
range: startRow,
|
|
465
|
+
header: startRow === 0 ? undefined : headers,
|
|
466
|
+
defval: null,
|
|
467
|
+
raw: false, // ✅ Convert dates/numbers to strings to reduce memory
|
|
468
|
+
blankrows: false
|
|
469
|
+
});
|
|
470
|
+
if (startRow === 0 && chunkData.length > 0) {
|
|
471
|
+
headers = Object.keys(chunkData[0]);
|
|
472
|
+
}
|
|
473
|
+
rows.push(...chunkData);
|
|
474
|
+
if (onProgress) {
|
|
475
|
+
const progress = 0.3 + (0.6 * (processedSheets + (startRow / totalRows)) / totalSheets);
|
|
476
|
+
onProgress(file.size * progress, file.size);
|
|
477
|
+
}
|
|
478
|
+
// ✅ Stop if we hit max rows
|
|
479
|
+
if (rows.length >= maxSheetSize) {
|
|
480
|
+
console.warn(`⚠️ Sheet "${sheetName}" truncated to ${maxSheetSize} rows`);
|
|
481
|
+
break;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
if (rows.length > 0) {
|
|
485
|
+
sheets[sheetName] = new DataFrame(rows);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
else {
|
|
489
|
+
// ✅ Small sheet: process normally
|
|
490
|
+
const jsonData = XLSX.utils.sheet_to_json(worksheet, {
|
|
491
|
+
defval: null,
|
|
492
|
+
raw: false,
|
|
493
|
+
blankrows: false
|
|
494
|
+
});
|
|
495
|
+
if (jsonData.length > 0) {
|
|
496
|
+
sheets[sheetName] = new DataFrame(jsonData);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
processedSheets++;
|
|
500
|
+
if (onProgress) {
|
|
501
|
+
const progress = 0.3 + (0.6 * (processedSheets / totalSheets));
|
|
502
|
+
onProgress(file.size * progress, file.size);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
if (onProgress)
|
|
506
|
+
onProgress(file.size, file.size);
|
|
445
507
|
return sheets;
|
|
446
508
|
}
|
|
447
509
|
/* ═══════════════════════════════════════════════════
|
|
@@ -19,7 +19,9 @@ export interface ParseOptions {
|
|
|
19
19
|
maxRows?: number;
|
|
20
20
|
skipRows?: number;
|
|
21
21
|
columns?: string[];
|
|
22
|
-
sheet?: string | number;
|
|
22
|
+
sheet?: string | number;
|
|
23
|
+
maxSheetSize?: number; // ✅ NEW: Max rows per sheet to prevent memory issues
|
|
24
|
+
sheetChunkSize?: number; // ✅ NEW: Process Excel in chunks
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
export class TabularDriver {
|
|
@@ -523,12 +525,12 @@ export class TabularDriver {
|
|
|
523
525
|
}
|
|
524
526
|
|
|
525
527
|
/**
|
|
526
|
-
* ✅
|
|
527
|
-
*
|
|
528
|
-
* @returns Record<sheetName, DataFrame>
|
|
528
|
+
* ✅ OPTIMIZED: Stream Excel file with chunked row processing
|
|
529
|
+
* Handles large files by processing rows in batches
|
|
529
530
|
*/
|
|
530
|
-
async streamFileMultiSheet(file: File): Promise<Record<string, DataFrame>> {
|
|
531
|
-
|
|
531
|
+
async streamFileMultiSheet(file: File, options: ParseOptions = {}): Promise<Record<string, DataFrame>> {
|
|
532
|
+
const { maxSheetSize = 100000, sheetChunkSize = 10000, onProgress } = options;
|
|
533
|
+
|
|
532
534
|
let XLSX: any;
|
|
533
535
|
try {
|
|
534
536
|
XLSX = await import('xlsx');
|
|
@@ -536,19 +538,93 @@ export class TabularDriver {
|
|
|
536
538
|
throw new Error('XLSX support requires the "xlsx" package. Install it with: npm install xlsx');
|
|
537
539
|
}
|
|
538
540
|
|
|
541
|
+
if (onProgress) onProgress(0, file.size);
|
|
542
|
+
|
|
539
543
|
const buffer = await file.arrayBuffer();
|
|
540
|
-
|
|
544
|
+
|
|
545
|
+
if (onProgress) onProgress(file.size * 0.3, file.size);
|
|
546
|
+
|
|
547
|
+
const workbook = XLSX.read(buffer, {
|
|
548
|
+
type: 'array',
|
|
549
|
+
sheetRows: maxSheetSize, // ✅ Limit rows read per sheet
|
|
550
|
+
dense: false // ✅ Use sparse format for memory efficiency
|
|
551
|
+
});
|
|
541
552
|
|
|
542
553
|
const sheets: Record<string, DataFrame> = {};
|
|
554
|
+
const totalSheets = workbook.SheetNames.length;
|
|
555
|
+
let processedSheets = 0;
|
|
543
556
|
|
|
544
|
-
workbook.SheetNames
|
|
557
|
+
for (const sheetName of workbook.SheetNames) {
|
|
545
558
|
const worksheet = workbook.Sheets[sheetName];
|
|
546
|
-
const jsonData: Record<string, any>[] = XLSX.utils.sheet_to_json(worksheet, { defval: null });
|
|
547
559
|
|
|
548
|
-
|
|
549
|
-
|
|
560
|
+
// ✅ Get sheet range to determine size
|
|
561
|
+
const range = XLSX.utils.decode_range(worksheet['!ref'] || 'A1');
|
|
562
|
+
const totalRows = range.e.r - range.s.r + 1;
|
|
563
|
+
|
|
564
|
+
if (totalRows === 0) {
|
|
565
|
+
processedSheets++;
|
|
566
|
+
continue;
|
|
550
567
|
}
|
|
551
|
-
|
|
568
|
+
|
|
569
|
+
// ✅ Process in chunks if sheet is large
|
|
570
|
+
if (totalRows > sheetChunkSize) {
|
|
571
|
+
const rows: Record<string, any>[] = [];
|
|
572
|
+
let headers: string[] = [];
|
|
573
|
+
|
|
574
|
+
for (let startRow = 0; startRow < totalRows; startRow += sheetChunkSize) {
|
|
575
|
+
const endRow = Math.min(startRow + sheetChunkSize, totalRows);
|
|
576
|
+
|
|
577
|
+
// ✅ Read chunk with limited row range
|
|
578
|
+
const chunkData: Record<string, any>[] = XLSX.utils.sheet_to_json(worksheet, {
|
|
579
|
+
range: startRow,
|
|
580
|
+
header: startRow === 0 ? undefined : headers,
|
|
581
|
+
defval: null,
|
|
582
|
+
raw: false, // ✅ Convert dates/numbers to strings to reduce memory
|
|
583
|
+
blankrows: false
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
if (startRow === 0 && chunkData.length > 0) {
|
|
587
|
+
headers = Object.keys(chunkData[0]);
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
rows.push(...chunkData);
|
|
591
|
+
|
|
592
|
+
if (onProgress) {
|
|
593
|
+
const progress = 0.3 + (0.6 * (processedSheets + (startRow / totalRows)) / totalSheets);
|
|
594
|
+
onProgress(file.size * progress, file.size);
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
// ✅ Stop if we hit max rows
|
|
598
|
+
if (rows.length >= maxSheetSize) {
|
|
599
|
+
console.warn(`⚠️ Sheet "${sheetName}" truncated to ${maxSheetSize} rows`);
|
|
600
|
+
break;
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
if (rows.length > 0) {
|
|
605
|
+
sheets[sheetName] = new DataFrame(rows);
|
|
606
|
+
}
|
|
607
|
+
} else {
|
|
608
|
+
// ✅ Small sheet: process normally
|
|
609
|
+
const jsonData: Record<string, any>[] = XLSX.utils.sheet_to_json(worksheet, {
|
|
610
|
+
defval: null,
|
|
611
|
+
raw: false,
|
|
612
|
+
blankrows: false
|
|
613
|
+
});
|
|
614
|
+
|
|
615
|
+
if (jsonData.length > 0) {
|
|
616
|
+
sheets[sheetName] = new DataFrame(jsonData);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
processedSheets++;
|
|
621
|
+
if (onProgress) {
|
|
622
|
+
const progress = 0.3 + (0.6 * (processedSheets / totalSheets));
|
|
623
|
+
onProgress(file.size * progress, file.size);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
if (onProgress) onProgress(file.size, file.size);
|
|
552
628
|
|
|
553
629
|
return sheets;
|
|
554
630
|
}
|