@loaders.gl/csv 4.0.0-beta.1 → 4.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/csv-loader.d.ts.map +1 -0
- package/dist/{esm/csv-loader.js → csv-loader.js} +5 -5
- package/dist/csv-loader.js.map +1 -0
- package/dist/csv-writer.d.ts.map +1 -0
- package/dist/{esm/csv-writer.js → csv-writer.js} +1 -1
- package/dist/csv-writer.js.map +1 -0
- package/dist/{dist.min.js → dist.dev.js} +604 -602
- package/dist/index.cjs +1133 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/encoders/encode-csv.d.ts.map +1 -0
- package/dist/lib/encoders/encode-csv.js.map +1 -0
- package/dist/papaparse/async-iterator-streamer.d.ts.map +1 -0
- package/dist/{esm/papaparse → papaparse}/async-iterator-streamer.js +1 -1
- package/dist/papaparse/async-iterator-streamer.js.map +1 -0
- package/dist/papaparse/papaparse.d.ts.map +1 -0
- package/dist/papaparse/papaparse.js.map +1 -0
- package/package.json +16 -8
- package/dist/es5/bundle.js +0 -6
- package/dist/es5/bundle.js.map +0 -1
- package/dist/es5/csv-loader.js +0 -252
- package/dist/es5/csv-loader.js.map +0 -1
- package/dist/es5/csv-writer.js +0 -47
- package/dist/es5/csv-writer.js.map +0 -1
- package/dist/es5/index.js +0 -20
- package/dist/es5/index.js.map +0 -1
- package/dist/es5/lib/encoders/encode-csv.js +0 -60
- package/dist/es5/lib/encoders/encode-csv.js.map +0 -1
- package/dist/es5/papaparse/async-iterator-streamer.js +0 -100
- package/dist/es5/papaparse/async-iterator-streamer.js.map +0 -1
- package/dist/es5/papaparse/papaparse.js +0 -703
- package/dist/es5/papaparse/papaparse.js.map +0 -1
- package/dist/esm/bundle.js +0 -4
- package/dist/esm/bundle.js.map +0 -1
- package/dist/esm/csv-loader.js.map +0 -1
- package/dist/esm/csv-writer.js.map +0 -1
- package/dist/esm/index.js +0 -3
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/lib/encoders/encode-csv.js.map +0 -1
- package/dist/esm/papaparse/async-iterator-streamer.js.map +0 -1
- package/dist/esm/papaparse/papaparse.js.map +0 -1
- package/dist/src/bundle.d.ts +0 -2
- package/dist/src/bundle.d.ts.map +0 -1
- package/dist/src/csv-loader.d.ts.map +0 -1
- package/dist/src/csv-writer.d.ts.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/lib/encoders/encode-csv.d.ts.map +0 -1
- package/dist/src/papaparse/async-iterator-streamer.d.ts.map +0 -1
- package/dist/src/papaparse/papaparse.d.ts.map +0 -1
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/src/bundle.ts +0 -4
- /package/dist/{src/csv-loader.d.ts → csv-loader.d.ts} +0 -0
- /package/dist/{src/csv-writer.d.ts → csv-writer.d.ts} +0 -0
- /package/dist/{src/index.d.ts → index.d.ts} +0 -0
- /package/dist/{src/lib → lib}/encoders/encode-csv.d.ts +0 -0
- /package/dist/{esm/lib → lib}/encoders/encode-csv.js +0 -0
- /package/dist/{src/papaparse → papaparse}/async-iterator-streamer.d.ts +0 -0
- /package/dist/{src/papaparse → papaparse}/papaparse.d.ts +0 -0
- /package/dist/{esm/papaparse → papaparse}/papaparse.js +0 -0
|
@@ -1,77 +1,90 @@
|
|
|
1
|
-
(()
|
|
1
|
+
(function webpackUniversalModuleDefinition(root, factory) {
|
|
2
|
+
if (typeof exports === 'object' && typeof module === 'object')
|
|
3
|
+
module.exports = factory();
|
|
4
|
+
else if (typeof define === 'function' && define.amd) define([], factory);
|
|
5
|
+
else if (typeof exports === 'object') exports['loader'] = factory();
|
|
6
|
+
else root['loader'] = factory();})(globalThis, function () {
|
|
7
|
+
"use strict";
|
|
8
|
+
var __exports__ = (() => {
|
|
2
9
|
var __defProp = Object.defineProperty;
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
|
|
6
|
-
};
|
|
7
|
-
var __commonJS = (cb, mod) => function __require() {
|
|
8
|
-
return mod || (0, cb[Object.keys(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
9
|
-
};
|
|
10
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
11
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
12
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
10
13
|
var __export = (target, all) => {
|
|
11
|
-
__markAsModule(target);
|
|
12
14
|
for (var name in all)
|
|
13
15
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
14
16
|
};
|
|
17
|
+
var __copyProps = (to, from, except, desc) => {
|
|
18
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
19
|
+
for (let key of __getOwnPropNames(from))
|
|
20
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
21
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
22
|
+
}
|
|
23
|
+
return to;
|
|
24
|
+
};
|
|
25
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
26
|
+
|
|
27
|
+
// src/index.ts
|
|
28
|
+
var src_exports = {};
|
|
29
|
+
__export(src_exports, {
|
|
30
|
+
CSVLoader: () => CSVLoader,
|
|
31
|
+
CSVWriter: () => CSVWriter
|
|
32
|
+
});
|
|
15
33
|
|
|
16
34
|
// ../schema/src/lib/table/batches/base-table-batch-aggregator.ts
|
|
17
|
-
var DEFAULT_ROW_COUNT
|
|
18
|
-
var
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
this._headers = [];
|
|
31
|
-
for (const key in schema) {
|
|
32
|
-
this._headers[schema[key].index] = schema[key].name;
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
rowCount() {
|
|
37
|
-
return this.length;
|
|
38
|
-
}
|
|
39
|
-
addArrayRow(row, cursor) {
|
|
40
|
-
if (Number.isFinite(cursor)) {
|
|
41
|
-
this.cursor = cursor;
|
|
42
|
-
}
|
|
43
|
-
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
44
|
-
this.rows[this.length] = row;
|
|
45
|
-
this.length++;
|
|
46
|
-
}
|
|
47
|
-
addObjectRow(row, cursor) {
|
|
48
|
-
if (Number.isFinite(cursor)) {
|
|
49
|
-
this.cursor = cursor;
|
|
50
|
-
}
|
|
51
|
-
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
52
|
-
this.rows[this.length] = row;
|
|
53
|
-
this.length++;
|
|
54
|
-
}
|
|
55
|
-
getBatch() {
|
|
56
|
-
let rows = this.rows;
|
|
57
|
-
if (!rows) {
|
|
58
|
-
return null;
|
|
59
|
-
}
|
|
60
|
-
rows = rows.slice(0, this.length);
|
|
61
|
-
this.rows = null;
|
|
62
|
-
const batch = {
|
|
63
|
-
shape: this.options.shape,
|
|
64
|
-
batchType: "data",
|
|
65
|
-
data: rows,
|
|
66
|
-
length: this.length,
|
|
67
|
-
schema: this.schema,
|
|
68
|
-
cursor: this.cursor
|
|
69
|
-
};
|
|
70
|
-
return batch;
|
|
35
|
+
var DEFAULT_ROW_COUNT = 100;
|
|
36
|
+
var BaseTableBatchAggregator = class {
|
|
37
|
+
length = 0;
|
|
38
|
+
rows = null;
|
|
39
|
+
cursor = 0;
|
|
40
|
+
_headers = [];
|
|
41
|
+
constructor(schema, options) {
|
|
42
|
+
this.options = options;
|
|
43
|
+
this.schema = schema;
|
|
44
|
+
if (!Array.isArray(schema)) {
|
|
45
|
+
this._headers = [];
|
|
46
|
+
for (const key in schema) {
|
|
47
|
+
this._headers[schema[key].index] = schema[key].name;
|
|
71
48
|
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
rowCount() {
|
|
52
|
+
return this.length;
|
|
53
|
+
}
|
|
54
|
+
addArrayRow(row, cursor) {
|
|
55
|
+
if (Number.isFinite(cursor)) {
|
|
56
|
+
this.cursor = cursor;
|
|
57
|
+
}
|
|
58
|
+
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
59
|
+
this.rows[this.length] = row;
|
|
60
|
+
this.length++;
|
|
61
|
+
}
|
|
62
|
+
addObjectRow(row, cursor) {
|
|
63
|
+
if (Number.isFinite(cursor)) {
|
|
64
|
+
this.cursor = cursor;
|
|
65
|
+
}
|
|
66
|
+
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
67
|
+
this.rows[this.length] = row;
|
|
68
|
+
this.length++;
|
|
69
|
+
}
|
|
70
|
+
getBatch() {
|
|
71
|
+
let rows = this.rows;
|
|
72
|
+
if (!rows) {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
rows = rows.slice(0, this.length);
|
|
76
|
+
this.rows = null;
|
|
77
|
+
const batch = {
|
|
78
|
+
shape: this.options.shape,
|
|
79
|
+
batchType: "data",
|
|
80
|
+
data: rows,
|
|
81
|
+
length: this.length,
|
|
82
|
+
schema: this.schema,
|
|
83
|
+
cursor: this.cursor
|
|
72
84
|
};
|
|
85
|
+
return batch;
|
|
73
86
|
}
|
|
74
|
-
}
|
|
87
|
+
};
|
|
75
88
|
|
|
76
89
|
// ../schema/src/lib/table/simple-table/row-utils.ts
|
|
77
90
|
function convertToObjectRow(arrayRow, headers) {
|
|
@@ -100,306 +113,295 @@
|
|
|
100
113
|
}
|
|
101
114
|
return arrayRow;
|
|
102
115
|
}
|
|
103
|
-
var init_row_utils = __esm({
|
|
104
|
-
"../schema/src/lib/table/simple-table/row-utils.ts"() {
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
116
|
|
|
108
117
|
// ../schema/src/lib/table/batches/row-table-batch-aggregator.ts
|
|
109
|
-
var DEFAULT_ROW_COUNT2
|
|
110
|
-
var
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
this.schema = schema;
|
|
123
|
-
if (!Array.isArray(schema)) {
|
|
124
|
-
this._headers = [];
|
|
125
|
-
for (const key in schema) {
|
|
126
|
-
this._headers[schema[key].index] = schema[key].name;
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
rowCount() {
|
|
131
|
-
return this.length;
|
|
132
|
-
}
|
|
133
|
-
addArrayRow(row, cursor) {
|
|
134
|
-
if (Number.isFinite(cursor)) {
|
|
135
|
-
this.cursor = cursor;
|
|
136
|
-
}
|
|
137
|
-
switch (this.options.shape) {
|
|
138
|
-
case "object-row-table":
|
|
139
|
-
const rowObject = convertToObjectRow(row, this._headers);
|
|
140
|
-
this.addObjectRow(rowObject, cursor);
|
|
141
|
-
break;
|
|
142
|
-
case "array-row-table":
|
|
143
|
-
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT2);
|
|
144
|
-
this.arrayRows[this.length] = row;
|
|
145
|
-
this.length++;
|
|
146
|
-
break;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
addObjectRow(row, cursor) {
|
|
150
|
-
if (Number.isFinite(cursor)) {
|
|
151
|
-
this.cursor = cursor;
|
|
152
|
-
}
|
|
153
|
-
switch (this.options.shape) {
|
|
154
|
-
case "array-row-table":
|
|
155
|
-
const rowArray = convertToArrayRow(row, this._headers);
|
|
156
|
-
this.addArrayRow(rowArray, cursor);
|
|
157
|
-
break;
|
|
158
|
-
case "object-row-table":
|
|
159
|
-
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT2);
|
|
160
|
-
this.objectRows[this.length] = row;
|
|
161
|
-
this.length++;
|
|
162
|
-
break;
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
getBatch() {
|
|
166
|
-
let rows = this.arrayRows || this.objectRows;
|
|
167
|
-
if (!rows) {
|
|
168
|
-
return null;
|
|
169
|
-
}
|
|
170
|
-
rows = rows.slice(0, this.length);
|
|
171
|
-
this.arrayRows = null;
|
|
172
|
-
this.objectRows = null;
|
|
173
|
-
return {
|
|
174
|
-
shape: this.options.shape,
|
|
175
|
-
batchType: "data",
|
|
176
|
-
data: rows,
|
|
177
|
-
length: this.length,
|
|
178
|
-
schema: this.schema,
|
|
179
|
-
cursor: this.cursor
|
|
180
|
-
};
|
|
118
|
+
var DEFAULT_ROW_COUNT2 = 100;
|
|
119
|
+
var RowTableBatchAggregator = class {
|
|
120
|
+
length = 0;
|
|
121
|
+
objectRows = null;
|
|
122
|
+
arrayRows = null;
|
|
123
|
+
cursor = 0;
|
|
124
|
+
_headers = [];
|
|
125
|
+
constructor(schema, options) {
|
|
126
|
+
this.options = options;
|
|
127
|
+
this.schema = schema;
|
|
128
|
+
if (!Array.isArray(schema)) {
|
|
129
|
+
this._headers = [];
|
|
130
|
+
for (const key in schema) {
|
|
131
|
+
this._headers[schema[key].index] = schema[key].name;
|
|
181
132
|
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
rowCount() {
|
|
136
|
+
return this.length;
|
|
137
|
+
}
|
|
138
|
+
addArrayRow(row, cursor) {
|
|
139
|
+
if (Number.isFinite(cursor)) {
|
|
140
|
+
this.cursor = cursor;
|
|
141
|
+
}
|
|
142
|
+
switch (this.options.shape) {
|
|
143
|
+
case "object-row-table":
|
|
144
|
+
const rowObject = convertToObjectRow(row, this._headers);
|
|
145
|
+
this.addObjectRow(rowObject, cursor);
|
|
146
|
+
break;
|
|
147
|
+
case "array-row-table":
|
|
148
|
+
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT2);
|
|
149
|
+
this.arrayRows[this.length] = row;
|
|
150
|
+
this.length++;
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
addObjectRow(row, cursor) {
|
|
155
|
+
if (Number.isFinite(cursor)) {
|
|
156
|
+
this.cursor = cursor;
|
|
157
|
+
}
|
|
158
|
+
switch (this.options.shape) {
|
|
159
|
+
case "array-row-table":
|
|
160
|
+
const rowArray = convertToArrayRow(row, this._headers);
|
|
161
|
+
this.addArrayRow(rowArray, cursor);
|
|
162
|
+
break;
|
|
163
|
+
case "object-row-table":
|
|
164
|
+
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT2);
|
|
165
|
+
this.objectRows[this.length] = row;
|
|
166
|
+
this.length++;
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
getBatch() {
|
|
171
|
+
let rows = this.arrayRows || this.objectRows;
|
|
172
|
+
if (!rows) {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
rows = rows.slice(0, this.length);
|
|
176
|
+
this.arrayRows = null;
|
|
177
|
+
this.objectRows = null;
|
|
178
|
+
return {
|
|
179
|
+
shape: this.options.shape,
|
|
180
|
+
batchType: "data",
|
|
181
|
+
data: rows,
|
|
182
|
+
length: this.length,
|
|
183
|
+
schema: this.schema,
|
|
184
|
+
cursor: this.cursor
|
|
182
185
|
};
|
|
183
186
|
}
|
|
184
|
-
}
|
|
187
|
+
};
|
|
185
188
|
|
|
186
189
|
// ../schema/src/lib/table/batches/columnar-table-batch-aggregator.ts
|
|
187
|
-
var DEFAULT_ROW_COUNT3
|
|
188
|
-
var
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
for (const fieldName in this.schema) {
|
|
222
|
-
const field = this.schema[fieldName];
|
|
223
|
-
columns[field.name] = this.columns[field.index];
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
this.columns = {};
|
|
227
|
-
const batch = {
|
|
228
|
-
shape: "columnar-table",
|
|
229
|
-
batchType: "data",
|
|
230
|
-
data: columns,
|
|
231
|
-
schema: this.schema,
|
|
232
|
-
length: this.length
|
|
233
|
-
};
|
|
234
|
-
return batch;
|
|
235
|
-
}
|
|
236
|
-
_reallocateColumns() {
|
|
237
|
-
if (this.length < this.allocated) {
|
|
238
|
-
return;
|
|
239
|
-
}
|
|
240
|
-
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT3;
|
|
241
|
-
this.columns = {};
|
|
242
|
-
for (const fieldName in this.schema) {
|
|
243
|
-
const field = this.schema[fieldName];
|
|
244
|
-
const ArrayType = field.type || Float32Array;
|
|
245
|
-
const oldColumn = this.columns[field.index];
|
|
246
|
-
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
247
|
-
const typedArray = new ArrayType(this.allocated);
|
|
248
|
-
typedArray.set(oldColumn);
|
|
249
|
-
this.columns[field.index] = typedArray;
|
|
250
|
-
} else if (oldColumn) {
|
|
251
|
-
oldColumn.length = this.allocated;
|
|
252
|
-
this.columns[field.index] = oldColumn;
|
|
253
|
-
} else {
|
|
254
|
-
this.columns[field.index] = new ArrayType(this.allocated);
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
_pruneColumns() {
|
|
259
|
-
for (const [columnName, column] of Object.entries(this.columns)) {
|
|
260
|
-
this.columns[columnName] = column.slice(0, this.length);
|
|
261
|
-
}
|
|
190
|
+
var DEFAULT_ROW_COUNT3 = 100;
|
|
191
|
+
var ColumnarTableBatchAggregator = class {
|
|
192
|
+
length = 0;
|
|
193
|
+
allocated = 0;
|
|
194
|
+
columns = {};
|
|
195
|
+
constructor(schema, options) {
|
|
196
|
+
this.schema = schema;
|
|
197
|
+
this._reallocateColumns();
|
|
198
|
+
}
|
|
199
|
+
rowCount() {
|
|
200
|
+
return this.length;
|
|
201
|
+
}
|
|
202
|
+
addArrayRow(row) {
|
|
203
|
+
this._reallocateColumns();
|
|
204
|
+
let i = 0;
|
|
205
|
+
for (const fieldName in this.columns) {
|
|
206
|
+
this.columns[fieldName][this.length] = row[i++];
|
|
207
|
+
}
|
|
208
|
+
this.length++;
|
|
209
|
+
}
|
|
210
|
+
addObjectRow(row) {
|
|
211
|
+
this._reallocateColumns();
|
|
212
|
+
for (const fieldName in row) {
|
|
213
|
+
this.columns[fieldName][this.length] = row[fieldName];
|
|
214
|
+
}
|
|
215
|
+
this.length++;
|
|
216
|
+
}
|
|
217
|
+
getBatch() {
|
|
218
|
+
this._pruneColumns();
|
|
219
|
+
const columns = Array.isArray(this.schema) ? this.columns : {};
|
|
220
|
+
if (!Array.isArray(this.schema)) {
|
|
221
|
+
for (const fieldName in this.schema) {
|
|
222
|
+
const field = this.schema[fieldName];
|
|
223
|
+
columns[field.name] = this.columns[field.index];
|
|
262
224
|
}
|
|
225
|
+
}
|
|
226
|
+
this.columns = {};
|
|
227
|
+
const batch = {
|
|
228
|
+
shape: "columnar-table",
|
|
229
|
+
batchType: "data",
|
|
230
|
+
data: columns,
|
|
231
|
+
schema: this.schema,
|
|
232
|
+
length: this.length
|
|
263
233
|
};
|
|
234
|
+
return batch;
|
|
264
235
|
}
|
|
265
|
-
|
|
236
|
+
// HELPERS
|
|
237
|
+
_reallocateColumns() {
|
|
238
|
+
if (this.length < this.allocated) {
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT3;
|
|
242
|
+
this.columns = {};
|
|
243
|
+
for (const fieldName in this.schema) {
|
|
244
|
+
const field = this.schema[fieldName];
|
|
245
|
+
const ArrayType = field.type || Float32Array;
|
|
246
|
+
const oldColumn = this.columns[field.index];
|
|
247
|
+
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
248
|
+
const typedArray = new ArrayType(this.allocated);
|
|
249
|
+
typedArray.set(oldColumn);
|
|
250
|
+
this.columns[field.index] = typedArray;
|
|
251
|
+
} else if (oldColumn) {
|
|
252
|
+
oldColumn.length = this.allocated;
|
|
253
|
+
this.columns[field.index] = oldColumn;
|
|
254
|
+
} else {
|
|
255
|
+
this.columns[field.index] = new ArrayType(this.allocated);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
_pruneColumns() {
|
|
260
|
+
for (const [columnName, column] of Object.entries(this.columns)) {
|
|
261
|
+
this.columns[columnName] = column.slice(0, this.length);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
};
|
|
266
265
|
|
|
267
266
|
// ../schema/src/lib/table/batches/table-batch-builder.ts
|
|
268
|
-
var DEFAULT_OPTIONS
|
|
269
|
-
|
|
270
|
-
"
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
267
|
+
var DEFAULT_OPTIONS = {
|
|
268
|
+
shape: "array-row-table",
|
|
269
|
+
batchSize: "auto",
|
|
270
|
+
batchDebounceMs: 0,
|
|
271
|
+
limit: 0,
|
|
272
|
+
_limitMB: 0
|
|
273
|
+
};
|
|
274
|
+
var ERR_MESSAGE = "TableBatchBuilder";
|
|
275
|
+
var TableBatchBuilder = class {
|
|
276
|
+
aggregator = null;
|
|
277
|
+
batchCount = 0;
|
|
278
|
+
bytesUsed = 0;
|
|
279
|
+
isChunkComplete = false;
|
|
280
|
+
lastBatchEmittedMs = Date.now();
|
|
281
|
+
totalLength = 0;
|
|
282
|
+
totalBytes = 0;
|
|
283
|
+
rowBytes = 0;
|
|
284
|
+
constructor(schema, options) {
|
|
285
|
+
this.schema = schema;
|
|
286
|
+
this.options = {
|
|
287
|
+
...DEFAULT_OPTIONS,
|
|
288
|
+
...options
|
|
280
289
|
};
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
290
|
+
}
|
|
291
|
+
limitReached() {
|
|
292
|
+
if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
|
|
293
|
+
return true;
|
|
294
|
+
}
|
|
295
|
+
if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
|
|
296
|
+
return true;
|
|
297
|
+
}
|
|
298
|
+
return false;
|
|
299
|
+
}
|
|
300
|
+
/** @deprecated Use addArrayRow or addObjectRow */
|
|
301
|
+
addRow(row) {
|
|
302
|
+
if (this.limitReached()) {
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
this.totalLength++;
|
|
306
|
+
this.rowBytes = this.rowBytes || this._estimateRowMB(row);
|
|
307
|
+
this.totalBytes += this.rowBytes;
|
|
308
|
+
if (Array.isArray(row)) {
|
|
309
|
+
this.addArrayRow(row);
|
|
310
|
+
} else {
|
|
311
|
+
this.addObjectRow(row);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
/** Add one row to the batch */
|
|
315
|
+
addArrayRow(row) {
|
|
316
|
+
if (!this.aggregator) {
|
|
317
|
+
const TableBatchType = this._getTableBatchType();
|
|
318
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
319
|
+
}
|
|
320
|
+
this.aggregator.addArrayRow(row);
|
|
321
|
+
}
|
|
322
|
+
/** Add one row to the batch */
|
|
323
|
+
addObjectRow(row) {
|
|
324
|
+
if (!this.aggregator) {
|
|
325
|
+
const TableBatchType = this._getTableBatchType();
|
|
326
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
327
|
+
}
|
|
328
|
+
this.aggregator.addObjectRow(row);
|
|
329
|
+
}
|
|
330
|
+
/** Mark an incoming raw memory chunk has completed */
|
|
331
|
+
chunkComplete(chunk) {
|
|
332
|
+
if (chunk instanceof ArrayBuffer) {
|
|
333
|
+
this.bytesUsed += chunk.byteLength;
|
|
334
|
+
}
|
|
335
|
+
if (typeof chunk === "string") {
|
|
336
|
+
this.bytesUsed += chunk.length;
|
|
337
|
+
}
|
|
338
|
+
this.isChunkComplete = true;
|
|
339
|
+
}
|
|
340
|
+
getFullBatch(options) {
|
|
341
|
+
return this._isFull() ? this._getBatch(options) : null;
|
|
342
|
+
}
|
|
343
|
+
getFinalBatch(options) {
|
|
344
|
+
return this._getBatch(options);
|
|
345
|
+
}
|
|
346
|
+
// INTERNAL
|
|
347
|
+
_estimateRowMB(row) {
|
|
348
|
+
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
349
|
+
}
|
|
350
|
+
_isFull() {
|
|
351
|
+
if (!this.aggregator || this.aggregator.rowCount() === 0) {
|
|
352
|
+
return false;
|
|
353
|
+
}
|
|
354
|
+
if (this.options.batchSize === "auto") {
|
|
355
|
+
if (!this.isChunkComplete) {
|
|
302
356
|
return false;
|
|
303
357
|
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
return this._getBatch(options);
|
|
345
|
-
}
|
|
346
|
-
_estimateRowMB(row) {
|
|
347
|
-
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
348
|
-
}
|
|
349
|
-
_isFull() {
|
|
350
|
-
if (!this.aggregator || this.aggregator.rowCount() === 0) {
|
|
351
|
-
return false;
|
|
352
|
-
}
|
|
353
|
-
if (this.options.batchSize === "auto") {
|
|
354
|
-
if (!this.isChunkComplete) {
|
|
355
|
-
return false;
|
|
356
|
-
}
|
|
357
|
-
} else if (this.options.batchSize > this.aggregator.rowCount()) {
|
|
358
|
-
return false;
|
|
359
|
-
}
|
|
360
|
-
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
|
|
361
|
-
return false;
|
|
362
|
-
}
|
|
363
|
-
this.isChunkComplete = false;
|
|
364
|
-
this.lastBatchEmittedMs = Date.now();
|
|
365
|
-
return true;
|
|
366
|
-
}
|
|
367
|
-
_getBatch(options) {
|
|
368
|
-
if (!this.aggregator) {
|
|
369
|
-
return null;
|
|
370
|
-
}
|
|
371
|
-
if (options?.bytesUsed) {
|
|
372
|
-
this.bytesUsed = options.bytesUsed;
|
|
373
|
-
}
|
|
374
|
-
const normalizedBatch = this.aggregator.getBatch();
|
|
375
|
-
normalizedBatch.count = this.batchCount;
|
|
376
|
-
normalizedBatch.bytesUsed = this.bytesUsed;
|
|
377
|
-
Object.assign(normalizedBatch, options);
|
|
378
|
-
this.batchCount++;
|
|
379
|
-
this.aggregator = null;
|
|
380
|
-
return normalizedBatch;
|
|
381
|
-
}
|
|
382
|
-
_getTableBatchType() {
|
|
383
|
-
switch (this.options.shape) {
|
|
384
|
-
case "row-table":
|
|
385
|
-
return BaseTableBatchAggregator;
|
|
386
|
-
case "array-row-table":
|
|
387
|
-
case "object-row-table":
|
|
388
|
-
return RowTableBatchAggregator;
|
|
389
|
-
case "columnar-table":
|
|
390
|
-
return ColumnarTableBatchAggregator;
|
|
391
|
-
case "arrow-table":
|
|
392
|
-
if (!TableBatchBuilder.ArrowBatch) {
|
|
393
|
-
throw new Error(ERR_MESSAGE);
|
|
394
|
-
}
|
|
395
|
-
return TableBatchBuilder.ArrowBatch;
|
|
396
|
-
default:
|
|
397
|
-
throw new Error(ERR_MESSAGE);
|
|
358
|
+
} else if (this.options.batchSize > this.aggregator.rowCount()) {
|
|
359
|
+
return false;
|
|
360
|
+
}
|
|
361
|
+
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
|
|
362
|
+
return false;
|
|
363
|
+
}
|
|
364
|
+
this.isChunkComplete = false;
|
|
365
|
+
this.lastBatchEmittedMs = Date.now();
|
|
366
|
+
return true;
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* bytesUsed can be set via chunkComplete or via getBatch*
|
|
370
|
+
*/
|
|
371
|
+
_getBatch(options) {
|
|
372
|
+
if (!this.aggregator) {
|
|
373
|
+
return null;
|
|
374
|
+
}
|
|
375
|
+
if (options?.bytesUsed) {
|
|
376
|
+
this.bytesUsed = options.bytesUsed;
|
|
377
|
+
}
|
|
378
|
+
const normalizedBatch = this.aggregator.getBatch();
|
|
379
|
+
normalizedBatch.count = this.batchCount;
|
|
380
|
+
normalizedBatch.bytesUsed = this.bytesUsed;
|
|
381
|
+
Object.assign(normalizedBatch, options);
|
|
382
|
+
this.batchCount++;
|
|
383
|
+
this.aggregator = null;
|
|
384
|
+
return normalizedBatch;
|
|
385
|
+
}
|
|
386
|
+
_getTableBatchType() {
|
|
387
|
+
switch (this.options.shape) {
|
|
388
|
+
case "row-table":
|
|
389
|
+
return BaseTableBatchAggregator;
|
|
390
|
+
case "array-row-table":
|
|
391
|
+
case "object-row-table":
|
|
392
|
+
return RowTableBatchAggregator;
|
|
393
|
+
case "columnar-table":
|
|
394
|
+
return ColumnarTableBatchAggregator;
|
|
395
|
+
case "arrow-table":
|
|
396
|
+
if (!TableBatchBuilder.ArrowBatch) {
|
|
397
|
+
throw new Error(ERR_MESSAGE);
|
|
398
398
|
}
|
|
399
|
-
|
|
400
|
-
|
|
399
|
+
return TableBatchBuilder.ArrowBatch;
|
|
400
|
+
default:
|
|
401
|
+
throw new Error(ERR_MESSAGE);
|
|
402
|
+
}
|
|
401
403
|
}
|
|
402
|
-
}
|
|
404
|
+
};
|
|
403
405
|
|
|
404
406
|
// ../schema/src/lib/table/simple-table/table-accessors.ts
|
|
405
407
|
function getTableLength(table) {
|
|
@@ -501,89 +503,110 @@
|
|
|
501
503
|
yield getTableRowAsArray(table, rowIndex, target);
|
|
502
504
|
}
|
|
503
505
|
}
|
|
504
|
-
var init_table_accessors = __esm({
|
|
505
|
-
"../schema/src/lib/table/simple-table/table-accessors.ts"() {
|
|
506
|
-
}
|
|
507
|
-
});
|
|
508
506
|
|
|
509
507
|
// ../schema/src/lib/utils/async-queue.ts
|
|
510
|
-
var ArrayQueue
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
throw new Error("Illegal internal state");
|
|
543
|
-
}
|
|
544
|
-
const settler = this._settlers.dequeue();
|
|
545
|
-
if (value instanceof Error) {
|
|
546
|
-
settler.reject(value);
|
|
547
|
-
} else {
|
|
548
|
-
settler.resolve({ value });
|
|
549
|
-
}
|
|
550
|
-
} else {
|
|
551
|
-
this._values.enqueue(value);
|
|
552
|
-
}
|
|
508
|
+
var ArrayQueue = class extends Array {
|
|
509
|
+
enqueue(value) {
|
|
510
|
+
return this.push(value);
|
|
511
|
+
}
|
|
512
|
+
dequeue() {
|
|
513
|
+
return this.shift();
|
|
514
|
+
}
|
|
515
|
+
};
|
|
516
|
+
var AsyncQueue = class {
|
|
517
|
+
constructor() {
|
|
518
|
+
this._values = new ArrayQueue();
|
|
519
|
+
this._settlers = new ArrayQueue();
|
|
520
|
+
this._closed = false;
|
|
521
|
+
}
|
|
522
|
+
close() {
|
|
523
|
+
while (this._settlers.length > 0) {
|
|
524
|
+
this._settlers.dequeue().resolve({
|
|
525
|
+
done: true
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
this._closed = true;
|
|
529
|
+
}
|
|
530
|
+
[Symbol.asyncIterator]() {
|
|
531
|
+
return this;
|
|
532
|
+
}
|
|
533
|
+
enqueue(value) {
|
|
534
|
+
if (this._closed) {
|
|
535
|
+
throw new Error("Closed");
|
|
536
|
+
}
|
|
537
|
+
if (this._settlers.length > 0) {
|
|
538
|
+
if (this._values.length > 0) {
|
|
539
|
+
throw new Error("Illegal internal state");
|
|
553
540
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
return Promise.resolve({ value });
|
|
561
|
-
}
|
|
562
|
-
if (this._closed) {
|
|
563
|
-
if (this._settlers.length > 0) {
|
|
564
|
-
throw new Error("Illegal internal state");
|
|
565
|
-
}
|
|
566
|
-
return Promise.resolve({ done: true });
|
|
567
|
-
}
|
|
568
|
-
return new Promise((resolve, reject) => {
|
|
569
|
-
this._settlers.enqueue({ resolve, reject });
|
|
541
|
+
const settler = this._settlers.dequeue();
|
|
542
|
+
if (value instanceof Error) {
|
|
543
|
+
settler.reject(value);
|
|
544
|
+
} else {
|
|
545
|
+
settler.resolve({
|
|
546
|
+
value
|
|
570
547
|
});
|
|
571
548
|
}
|
|
572
|
-
}
|
|
549
|
+
} else {
|
|
550
|
+
this._values.enqueue(value);
|
|
551
|
+
}
|
|
573
552
|
}
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
553
|
+
/**
|
|
554
|
+
* @returns a Promise for an IteratorResult
|
|
555
|
+
*/
|
|
556
|
+
next() {
|
|
557
|
+
if (this._values.length > 0) {
|
|
558
|
+
const value = this._values.dequeue();
|
|
559
|
+
if (value instanceof Error) {
|
|
560
|
+
return Promise.reject(value);
|
|
561
|
+
}
|
|
562
|
+
return Promise.resolve({
|
|
563
|
+
value
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
if (this._closed) {
|
|
567
|
+
if (this._settlers.length > 0) {
|
|
568
|
+
throw new Error("Illegal internal state");
|
|
569
|
+
}
|
|
570
|
+
return Promise.resolve({
|
|
571
|
+
done: true
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
return new Promise((resolve, reject) => {
|
|
575
|
+
this._settlers.enqueue({
|
|
576
|
+
resolve,
|
|
577
|
+
reject
|
|
578
|
+
});
|
|
579
|
+
});
|
|
583
580
|
}
|
|
584
|
-
}
|
|
581
|
+
};
|
|
585
582
|
|
|
586
583
|
// src/papaparse/papaparse.ts
|
|
584
|
+
var BYTE_ORDER_MARK = "\uFEFF";
|
|
585
|
+
var Papa = {
|
|
586
|
+
parse: CsvToJson,
|
|
587
|
+
unparse: JsonToCsv,
|
|
588
|
+
RECORD_SEP: String.fromCharCode(30),
|
|
589
|
+
UNIT_SEP: String.fromCharCode(31),
|
|
590
|
+
BYTE_ORDER_MARK,
|
|
591
|
+
BAD_DELIMITERS: ["\r", "\n", '"', BYTE_ORDER_MARK],
|
|
592
|
+
WORKERS_SUPPORTED: false,
|
|
593
|
+
// !IS_WORKER && !!globalThis.Worker
|
|
594
|
+
NODE_STREAM_INPUT: 1,
|
|
595
|
+
// Configurable chunk sizes for local and remote files, respectively
|
|
596
|
+
LocalChunkSize: 1024 * 1024 * 10,
|
|
597
|
+
// 10 M,
|
|
598
|
+
RemoteChunkSize: 1024 * 1024 * 5,
|
|
599
|
+
// 5 M,
|
|
600
|
+
DefaultDelimiter: ",",
|
|
601
|
+
// Used if not specified and detection fail,
|
|
602
|
+
// Exposed for testing and development only
|
|
603
|
+
Parser,
|
|
604
|
+
ParserHandle,
|
|
605
|
+
// BEGIN FORK
|
|
606
|
+
ChunkStreamer,
|
|
607
|
+
StringStreamer
|
|
608
|
+
};
|
|
609
|
+
var papaparse_default = Papa;
|
|
587
610
|
function CsvToJson(_input, _config, UserDefinedStreamer) {
|
|
588
611
|
_config = _config || {};
|
|
589
612
|
var dynamicTyping = _config.dynamicTyping || false;
|
|
@@ -838,6 +861,8 @@
|
|
|
838
861
|
return this.parseChunk(chunk);
|
|
839
862
|
};
|
|
840
863
|
}
|
|
864
|
+
StringStreamer.prototype = Object.create(StringStreamer.prototype);
|
|
865
|
+
StringStreamer.prototype.constructor = StringStreamer;
|
|
841
866
|
function ParserHandle(_config) {
|
|
842
867
|
var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
|
|
843
868
|
var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
|
|
@@ -851,6 +876,7 @@
|
|
|
851
876
|
var _delimiterError;
|
|
852
877
|
var _fields = [];
|
|
853
878
|
var _results = {
|
|
879
|
+
// The last results returned from the parser
|
|
854
880
|
data: [],
|
|
855
881
|
errors: [],
|
|
856
882
|
meta: {}
|
|
@@ -898,7 +924,15 @@
|
|
|
898
924
|
_parser = new Parser(parserConfig);
|
|
899
925
|
_results = _parser.parse(_input, baseIndex, ignoreLastRow);
|
|
900
926
|
processResults();
|
|
901
|
-
return _paused ? {
|
|
927
|
+
return _paused ? {
|
|
928
|
+
meta: {
|
|
929
|
+
paused: true
|
|
930
|
+
}
|
|
931
|
+
} : _results || {
|
|
932
|
+
meta: {
|
|
933
|
+
paused: false
|
|
934
|
+
}
|
|
935
|
+
};
|
|
902
936
|
};
|
|
903
937
|
this.paused = function() {
|
|
904
938
|
return _paused;
|
|
@@ -1169,6 +1203,7 @@
|
|
|
1169
1203
|
code: "MissingQuotes",
|
|
1170
1204
|
message: "Quoted field unterminated",
|
|
1171
1205
|
row: data.length,
|
|
1206
|
+
// row has yet to be inserted
|
|
1172
1207
|
index: cursor
|
|
1173
1208
|
});
|
|
1174
1209
|
}
|
|
@@ -1220,6 +1255,7 @@
|
|
|
1220
1255
|
code: "InvalidQuotes",
|
|
1221
1256
|
message: "Trailing quote on quoted field is malformed",
|
|
1222
1257
|
row: data.length,
|
|
1258
|
+
// row has yet to be inserted
|
|
1223
1259
|
index: cursor
|
|
1224
1260
|
});
|
|
1225
1261
|
quoteSearch++;
|
|
@@ -1333,34 +1369,11 @@
|
|
|
1333
1369
|
function isFunction(func) {
|
|
1334
1370
|
return typeof func === "function";
|
|
1335
1371
|
}
|
|
1336
|
-
var BYTE_ORDER_MARK, Papa, papaparse_default;
|
|
1337
|
-
var init_papaparse = __esm({
|
|
1338
|
-
"src/papaparse/papaparse.ts"() {
|
|
1339
|
-
BYTE_ORDER_MARK = "\uFEFF";
|
|
1340
|
-
Papa = {
|
|
1341
|
-
parse: CsvToJson,
|
|
1342
|
-
unparse: JsonToCsv,
|
|
1343
|
-
RECORD_SEP: String.fromCharCode(30),
|
|
1344
|
-
UNIT_SEP: String.fromCharCode(31),
|
|
1345
|
-
BYTE_ORDER_MARK,
|
|
1346
|
-
BAD_DELIMITERS: ["\r", "\n", '"', BYTE_ORDER_MARK],
|
|
1347
|
-
WORKERS_SUPPORTED: false,
|
|
1348
|
-
NODE_STREAM_INPUT: 1,
|
|
1349
|
-
LocalChunkSize: 1024 * 1024 * 10,
|
|
1350
|
-
RemoteChunkSize: 1024 * 1024 * 5,
|
|
1351
|
-
DefaultDelimiter: ",",
|
|
1352
|
-
Parser,
|
|
1353
|
-
ParserHandle,
|
|
1354
|
-
ChunkStreamer,
|
|
1355
|
-
StringStreamer
|
|
1356
|
-
};
|
|
1357
|
-
papaparse_default = Papa;
|
|
1358
|
-
StringStreamer.prototype = Object.create(StringStreamer.prototype);
|
|
1359
|
-
StringStreamer.prototype.constructor = StringStreamer;
|
|
1360
|
-
}
|
|
1361
|
-
});
|
|
1362
1372
|
|
|
1363
1373
|
// src/papaparse/async-iterator-streamer.ts
|
|
1374
|
+
var {
|
|
1375
|
+
ChunkStreamer: ChunkStreamer2
|
|
1376
|
+
} = papaparse_default;
|
|
1364
1377
|
function AsyncIteratorStreamer(config) {
|
|
1365
1378
|
config = config || {};
|
|
1366
1379
|
ChunkStreamer2.call(this, config);
|
|
@@ -1380,29 +1393,65 @@
|
|
|
1380
1393
|
this._nextChunk = function nextChunk() {
|
|
1381
1394
|
};
|
|
1382
1395
|
this.getStringChunk = function(chunk) {
|
|
1383
|
-
return typeof chunk === "string" ? chunk : this.textDecoder.decode(chunk, {
|
|
1396
|
+
return typeof chunk === "string" ? chunk : this.textDecoder.decode(chunk, {
|
|
1397
|
+
stream: true
|
|
1398
|
+
});
|
|
1384
1399
|
};
|
|
1385
1400
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
"src/papaparse/async-iterator-streamer.ts"() {
|
|
1389
|
-
init_papaparse();
|
|
1390
|
-
({ ChunkStreamer: ChunkStreamer2 } = papaparse_default);
|
|
1391
|
-
AsyncIteratorStreamer.prototype = Object.create(ChunkStreamer2.prototype);
|
|
1392
|
-
AsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;
|
|
1393
|
-
}
|
|
1394
|
-
});
|
|
1401
|
+
AsyncIteratorStreamer.prototype = Object.create(ChunkStreamer2.prototype);
|
|
1402
|
+
AsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;
|
|
1395
1403
|
|
|
1396
1404
|
// src/csv-loader.ts
|
|
1405
|
+
var VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
|
|
1406
|
+
var DEFAULT_CSV_SHAPE = "object-row-table";
|
|
1407
|
+
var CSVLoader = {
|
|
1408
|
+
id: "csv",
|
|
1409
|
+
module: "csv",
|
|
1410
|
+
name: "CSV",
|
|
1411
|
+
version: VERSION,
|
|
1412
|
+
extensions: ["csv", "tsv", "dsv"],
|
|
1413
|
+
mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"],
|
|
1414
|
+
category: "table",
|
|
1415
|
+
parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
|
|
1416
|
+
parseText: (text, options) => parseCSV(text, options),
|
|
1417
|
+
parseInBatches: parseCSVInBatches,
|
|
1418
|
+
// @ts-ignore
|
|
1419
|
+
// testText: null,
|
|
1420
|
+
options: {
|
|
1421
|
+
csv: {
|
|
1422
|
+
shape: DEFAULT_CSV_SHAPE,
|
|
1423
|
+
// 'object-row-table'
|
|
1424
|
+
optimizeMemoryUsage: false,
|
|
1425
|
+
// CSV options
|
|
1426
|
+
header: "auto",
|
|
1427
|
+
columnPrefix: "column",
|
|
1428
|
+
// delimiter: auto
|
|
1429
|
+
// newline: auto
|
|
1430
|
+
quoteChar: '"',
|
|
1431
|
+
escapeChar: '"',
|
|
1432
|
+
dynamicTyping: true,
|
|
1433
|
+
comments: false,
|
|
1434
|
+
skipEmptyLines: true,
|
|
1435
|
+
// transform: null?
|
|
1436
|
+
delimitersToGuess: [",", " ", "|", ";"]
|
|
1437
|
+
// fastMode: auto
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
};
|
|
1397
1441
|
async function parseCSV(csvText, options) {
|
|
1398
|
-
const csvOptions = {
|
|
1442
|
+
const csvOptions = {
|
|
1443
|
+
...CSVLoader.options.csv,
|
|
1444
|
+
...options?.csv
|
|
1445
|
+
};
|
|
1399
1446
|
const firstRow = readFirstRow(csvText);
|
|
1400
1447
|
const header = csvOptions.header === "auto" ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
|
|
1401
1448
|
const parseWithHeader = header;
|
|
1402
1449
|
const papaparseConfig = {
|
|
1450
|
+
// dynamicTyping: true,
|
|
1403
1451
|
...csvOptions,
|
|
1404
1452
|
header: parseWithHeader,
|
|
1405
1453
|
download: false,
|
|
1454
|
+
// We handle loading, no need for papaparse to do it for us
|
|
1406
1455
|
transformHeader: parseWithHeader ? duplicateColumnTransformer() : void 0,
|
|
1407
1456
|
error: (e) => {
|
|
1408
1457
|
throw new Error(e);
|
|
@@ -1428,22 +1477,40 @@
|
|
|
1428
1477
|
}
|
|
1429
1478
|
}
|
|
1430
1479
|
function parseCSVInBatches(asyncIterator, options) {
|
|
1431
|
-
options = {
|
|
1480
|
+
options = {
|
|
1481
|
+
...options
|
|
1482
|
+
};
|
|
1432
1483
|
if (options.batchSize === "auto") {
|
|
1433
1484
|
options.batchSize = 4e3;
|
|
1434
1485
|
}
|
|
1435
|
-
const csvOptions = {
|
|
1486
|
+
const csvOptions = {
|
|
1487
|
+
...CSVLoader.options.csv,
|
|
1488
|
+
...options?.csv
|
|
1489
|
+
};
|
|
1436
1490
|
const asyncQueue = new AsyncQueue();
|
|
1437
1491
|
let isFirstRow = true;
|
|
1438
1492
|
let headerRow = null;
|
|
1439
1493
|
let tableBatchBuilder = null;
|
|
1440
1494
|
let schema = null;
|
|
1441
1495
|
const config = {
|
|
1496
|
+
// dynamicTyping: true, // Convert numbers and boolean values in rows from strings,
|
|
1442
1497
|
...csvOptions,
|
|
1443
1498
|
header: false,
|
|
1499
|
+
// Unfortunately, header detection is not automatic and does not infer shapes
|
|
1444
1500
|
download: false,
|
|
1501
|
+
// We handle loading, no need for papaparse to do it for us
|
|
1502
|
+
// chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the
|
|
1503
|
+
// streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.
|
|
1504
|
+
// See https://github.com/mholt/PapaParse/issues/465
|
|
1445
1505
|
chunkSize: 1024 * 1024 * 5,
|
|
1506
|
+
// skipEmptyLines is set to a boolean value if supplied. Greedy is set to true
|
|
1507
|
+
// skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if
|
|
1508
|
+
// both of the skipEmptyLines and step callback options are provided:
|
|
1509
|
+
// - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465
|
|
1510
|
+
// - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825
|
|
1446
1511
|
skipEmptyLines: false,
|
|
1512
|
+
// step is called on every row
|
|
1513
|
+
// eslint-disable-next-line complexity, max-statements
|
|
1447
1514
|
step(results) {
|
|
1448
1515
|
let row = results.data;
|
|
1449
1516
|
if (csvOptions.skipEmptyLines) {
|
|
@@ -1471,13 +1538,19 @@
|
|
|
1471
1538
|
row = JSON.parse(JSON.stringify(row));
|
|
1472
1539
|
}
|
|
1473
1540
|
const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
|
|
1474
|
-
tableBatchBuilder = tableBatchBuilder || new TableBatchBuilder(
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1541
|
+
tableBatchBuilder = tableBatchBuilder || new TableBatchBuilder(
|
|
1542
|
+
// @ts-expect-error TODO this is not a proper schema
|
|
1543
|
+
schema,
|
|
1544
|
+
{
|
|
1545
|
+
shape,
|
|
1546
|
+
...options
|
|
1547
|
+
}
|
|
1548
|
+
);
|
|
1478
1549
|
try {
|
|
1479
1550
|
tableBatchBuilder.addRow(row);
|
|
1480
|
-
const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({
|
|
1551
|
+
const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({
|
|
1552
|
+
bytesUsed
|
|
1553
|
+
});
|
|
1481
1554
|
if (batch) {
|
|
1482
1555
|
asyncQueue.enqueue(batch);
|
|
1483
1556
|
}
|
|
@@ -1485,10 +1558,13 @@
|
|
|
1485
1558
|
asyncQueue.enqueue(error);
|
|
1486
1559
|
}
|
|
1487
1560
|
},
|
|
1561
|
+
// complete is called when all rows have been read
|
|
1488
1562
|
complete(results) {
|
|
1489
1563
|
try {
|
|
1490
1564
|
const bytesUsed = results.meta.cursor;
|
|
1491
|
-
const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({
|
|
1565
|
+
const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({
|
|
1566
|
+
bytesUsed
|
|
1567
|
+
});
|
|
1492
1568
|
if (batch) {
|
|
1493
1569
|
asyncQueue.enqueue(batch);
|
|
1494
1570
|
}
|
|
@@ -1513,7 +1589,7 @@
|
|
|
1513
1589
|
return result.data[0];
|
|
1514
1590
|
}
|
|
1515
1591
|
function duplicateColumnTransformer() {
|
|
1516
|
-
const observedColumns = new Set();
|
|
1592
|
+
const observedColumns = /* @__PURE__ */ new Set();
|
|
1517
1593
|
return (col) => {
|
|
1518
1594
|
let colName = col;
|
|
1519
1595
|
let counter = 1;
|
|
@@ -1540,53 +1616,30 @@
|
|
|
1540
1616
|
switch (typeof value) {
|
|
1541
1617
|
case "number":
|
|
1542
1618
|
case "boolean":
|
|
1543
|
-
schema[columnName] = {
|
|
1619
|
+
schema[columnName] = {
|
|
1620
|
+
name: String(columnName),
|
|
1621
|
+
index: i,
|
|
1622
|
+
type: Float32Array
|
|
1623
|
+
};
|
|
1544
1624
|
break;
|
|
1545
1625
|
case "string":
|
|
1546
1626
|
default:
|
|
1547
|
-
schema[columnName] = {
|
|
1627
|
+
schema[columnName] = {
|
|
1628
|
+
name: String(columnName),
|
|
1629
|
+
index: i,
|
|
1630
|
+
type: Array
|
|
1631
|
+
};
|
|
1548
1632
|
}
|
|
1549
1633
|
}
|
|
1550
1634
|
return schema;
|
|
1551
1635
|
}
|
|
1552
|
-
var VERSION, DEFAULT_CSV_SHAPE, CSVLoader;
|
|
1553
|
-
var init_csv_loader = __esm({
|
|
1554
|
-
"src/csv-loader.ts"() {
|
|
1555
|
-
init_src();
|
|
1556
|
-
init_papaparse();
|
|
1557
|
-
init_async_iterator_streamer();
|
|
1558
|
-
VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
|
|
1559
|
-
DEFAULT_CSV_SHAPE = "object-row-table";
|
|
1560
|
-
CSVLoader = {
|
|
1561
|
-
id: "csv",
|
|
1562
|
-
module: "csv",
|
|
1563
|
-
name: "CSV",
|
|
1564
|
-
version: VERSION,
|
|
1565
|
-
extensions: ["csv", "tsv", "dsv"],
|
|
1566
|
-
mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"],
|
|
1567
|
-
category: "table",
|
|
1568
|
-
parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
|
|
1569
|
-
parseText: (text, options) => parseCSV(text, options),
|
|
1570
|
-
parseInBatches: parseCSVInBatches,
|
|
1571
|
-
options: {
|
|
1572
|
-
csv: {
|
|
1573
|
-
shape: DEFAULT_CSV_SHAPE,
|
|
1574
|
-
optimizeMemoryUsage: false,
|
|
1575
|
-
header: "auto",
|
|
1576
|
-
columnPrefix: "column",
|
|
1577
|
-
quoteChar: '"',
|
|
1578
|
-
escapeChar: '"',
|
|
1579
|
-
dynamicTyping: true,
|
|
1580
|
-
comments: false,
|
|
1581
|
-
skipEmptyLines: true,
|
|
1582
|
-
delimitersToGuess: [",", " ", "|", ";"]
|
|
1583
|
-
}
|
|
1584
|
-
}
|
|
1585
|
-
};
|
|
1586
|
-
}
|
|
1587
|
-
});
|
|
1588
1636
|
|
|
1589
1637
|
// ../../node_modules/d3-dsv/src/dsv.js
|
|
1638
|
+
var EOL = {};
|
|
1639
|
+
var EOF = {};
|
|
1640
|
+
var QUOTE = 34;
|
|
1641
|
+
var NEWLINE = 10;
|
|
1642
|
+
var RETURN = 13;
|
|
1590
1643
|
function objectConverter(columns) {
|
|
1591
1644
|
return new Function("d", "return {" + columns.map(function(name, i) {
|
|
1592
1645
|
return JSON.stringify(name) + ": d[" + i + '] || ""';
|
|
@@ -1599,7 +1652,7 @@
|
|
|
1599
1652
|
};
|
|
1600
1653
|
}
|
|
1601
1654
|
function inferColumns(rows) {
|
|
1602
|
-
var columnSet = Object.create(null), columns = [];
|
|
1655
|
+
var columnSet = /* @__PURE__ */ Object.create(null), columns = [];
|
|
1603
1656
|
rows.forEach(function(row) {
|
|
1604
1657
|
for (var column in row) {
|
|
1605
1658
|
if (!(column in columnSet)) {
|
|
@@ -1716,42 +1769,23 @@
|
|
|
1716
1769
|
formatValue
|
|
1717
1770
|
};
|
|
1718
1771
|
}
|
|
1719
|
-
var EOL, EOF, QUOTE, NEWLINE, RETURN;
|
|
1720
|
-
var init_dsv = __esm({
|
|
1721
|
-
"../../node_modules/d3-dsv/src/dsv.js"() {
|
|
1722
|
-
EOL = {};
|
|
1723
|
-
EOF = {};
|
|
1724
|
-
QUOTE = 34;
|
|
1725
|
-
NEWLINE = 10;
|
|
1726
|
-
RETURN = 13;
|
|
1727
|
-
}
|
|
1728
|
-
});
|
|
1729
1772
|
|
|
1730
1773
|
// ../../node_modules/d3-dsv/src/csv.js
|
|
1731
|
-
var csv
|
|
1732
|
-
var
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
csvFormatBody = csv.formatBody;
|
|
1740
|
-
csvFormatRows = csv.formatRows;
|
|
1741
|
-
csvFormatRow = csv.formatRow;
|
|
1742
|
-
csvFormatValue = csv.formatValue;
|
|
1743
|
-
}
|
|
1744
|
-
});
|
|
1745
|
-
|
|
1746
|
-
// ../../node_modules/d3-dsv/src/index.js
|
|
1747
|
-
var init_src2 = __esm({
|
|
1748
|
-
"../../node_modules/d3-dsv/src/index.js"() {
|
|
1749
|
-
init_csv();
|
|
1750
|
-
}
|
|
1751
|
-
});
|
|
1774
|
+
var csv = dsv_default(",");
|
|
1775
|
+
var csvParse = csv.parse;
|
|
1776
|
+
var csvParseRows = csv.parseRows;
|
|
1777
|
+
var csvFormat = csv.format;
|
|
1778
|
+
var csvFormatBody = csv.formatBody;
|
|
1779
|
+
var csvFormatRows = csv.formatRows;
|
|
1780
|
+
var csvFormatRow = csv.formatRow;
|
|
1781
|
+
var csvFormatValue = csv.formatValue;
|
|
1752
1782
|
|
|
1753
1783
|
// src/lib/encoders/encode-csv.ts
|
|
1754
|
-
function encodeTableAsCSV(table, options = {
|
|
1784
|
+
function encodeTableAsCSV(table, options = {
|
|
1785
|
+
csv: {
|
|
1786
|
+
useDisplayNames: true
|
|
1787
|
+
}
|
|
1788
|
+
}) {
|
|
1755
1789
|
const useDisplayNames = options.useDisplayNames || options.csv?.useDisplayNames;
|
|
1756
1790
|
const fields = table.schema?.fields || [];
|
|
1757
1791
|
const columnNames = fields.map((f) => {
|
|
@@ -1769,73 +1803,39 @@
|
|
|
1769
1803
|
}
|
|
1770
1804
|
return csvFormatRows(formattedData);
|
|
1771
1805
|
}
|
|
1772
|
-
var preformatFieldValue
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
init_src();
|
|
1776
|
-
init_src2();
|
|
1777
|
-
preformatFieldValue = (value) => {
|
|
1778
|
-
if (value === null || value === void 0) {
|
|
1779
|
-
return null;
|
|
1780
|
-
}
|
|
1781
|
-
if (value instanceof Date) {
|
|
1782
|
-
return value.toISOString();
|
|
1783
|
-
}
|
|
1784
|
-
if (typeof value === "object") {
|
|
1785
|
-
return JSON.stringify(value);
|
|
1786
|
-
}
|
|
1787
|
-
return String(value);
|
|
1788
|
-
};
|
|
1806
|
+
var preformatFieldValue = (value) => {
|
|
1807
|
+
if (value === null || value === void 0) {
|
|
1808
|
+
return null;
|
|
1789
1809
|
}
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
// src/csv-writer.ts
|
|
1793
|
-
var CSVWriter;
|
|
1794
|
-
var init_csv_writer = __esm({
|
|
1795
|
-
"src/csv-writer.ts"() {
|
|
1796
|
-
init_encode_csv();
|
|
1797
|
-
CSVWriter = {
|
|
1798
|
-
id: "csv",
|
|
1799
|
-
version: "latest",
|
|
1800
|
-
module: "csv",
|
|
1801
|
-
name: "CSV",
|
|
1802
|
-
extensions: ["csv"],
|
|
1803
|
-
mimeTypes: ["text/csv"],
|
|
1804
|
-
options: {
|
|
1805
|
-
csv: {
|
|
1806
|
-
useDisplayNames: false
|
|
1807
|
-
},
|
|
1808
|
-
useDisplayNames: false
|
|
1809
|
-
},
|
|
1810
|
-
text: true,
|
|
1811
|
-
encode: async (table, options) => new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
|
|
1812
|
-
encodeTextSync: (table, options) => encodeTableAsCSV(table, options)
|
|
1813
|
-
};
|
|
1810
|
+
if (value instanceof Date) {
|
|
1811
|
+
return value.toISOString();
|
|
1814
1812
|
}
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
// src/index.ts
|
|
1818
|
-
var src_exports = {};
|
|
1819
|
-
__export(src_exports, {
|
|
1820
|
-
CSVLoader: () => CSVLoader,
|
|
1821
|
-
CSVWriter: () => CSVWriter
|
|
1822
|
-
});
|
|
1823
|
-
var init_src3 = __esm({
|
|
1824
|
-
"src/index.ts"() {
|
|
1825
|
-
init_csv_loader();
|
|
1826
|
-
init_csv_writer();
|
|
1813
|
+
if (typeof value === "object") {
|
|
1814
|
+
return JSON.stringify(value);
|
|
1827
1815
|
}
|
|
1828
|
-
|
|
1816
|
+
return String(value);
|
|
1817
|
+
};
|
|
1829
1818
|
|
|
1830
|
-
// src/
|
|
1831
|
-
var
|
|
1832
|
-
"
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1819
|
+
// src/csv-writer.ts
|
|
1820
|
+
var CSVWriter = {
|
|
1821
|
+
id: "csv",
|
|
1822
|
+
version: "latest",
|
|
1823
|
+
module: "csv",
|
|
1824
|
+
name: "CSV",
|
|
1825
|
+
extensions: ["csv"],
|
|
1826
|
+
mimeTypes: ["text/csv"],
|
|
1827
|
+
options: {
|
|
1828
|
+
csv: {
|
|
1829
|
+
useDisplayNames: false
|
|
1830
|
+
},
|
|
1831
|
+
/** @deprecated use csv.displayNames */
|
|
1832
|
+
useDisplayNames: false
|
|
1833
|
+
},
|
|
1834
|
+
text: true,
|
|
1835
|
+
encode: async (table, options) => new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
|
|
1836
|
+
encodeTextSync: (table, options) => encodeTableAsCSV(table, options)
|
|
1837
|
+
};
|
|
1838
|
+
return __toCommonJS(src_exports);
|
|
1839
1839
|
})();
|
|
1840
1840
|
/* @license
|
|
1841
1841
|
Papa Parse
|
|
@@ -1843,3 +1843,5 @@ v5.0.0-beta.0
|
|
|
1843
1843
|
https://github.com/mholt/PapaParse
|
|
1844
1844
|
License: MIT
|
|
1845
1845
|
*/
|
|
1846
|
+
return __exports__;
|
|
1847
|
+
});
|