@loaders.gl/csv 4.0.0-beta.2 → 4.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/csv-loader.d.ts.map +1 -0
- package/dist/{esm/csv-loader.js → csv-loader.js} +5 -5
- package/dist/csv-loader.js.map +1 -0
- package/dist/csv-writer.d.ts.map +1 -0
- package/dist/{esm/csv-writer.js → csv-writer.js} +1 -1
- package/dist/csv-writer.js.map +1 -0
- package/dist/{dist.min.js → dist.dev.js} +549 -598
- package/dist/index.cjs +1133 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/encoders/encode-csv.d.ts.map +1 -0
- package/dist/lib/encoders/encode-csv.js.map +1 -0
- package/dist/papaparse/async-iterator-streamer.d.ts.map +1 -0
- package/dist/{esm/papaparse → papaparse}/async-iterator-streamer.js +1 -1
- package/dist/papaparse/async-iterator-streamer.js.map +1 -0
- package/dist/papaparse/papaparse.d.ts.map +1 -0
- package/dist/papaparse/papaparse.js.map +1 -0
- package/package.json +16 -8
- package/dist/es5/bundle.js +0 -6
- package/dist/es5/bundle.js.map +0 -1
- package/dist/es5/csv-loader.js +0 -252
- package/dist/es5/csv-loader.js.map +0 -1
- package/dist/es5/csv-writer.js +0 -47
- package/dist/es5/csv-writer.js.map +0 -1
- package/dist/es5/index.js +0 -20
- package/dist/es5/index.js.map +0 -1
- package/dist/es5/lib/encoders/encode-csv.js +0 -60
- package/dist/es5/lib/encoders/encode-csv.js.map +0 -1
- package/dist/es5/papaparse/async-iterator-streamer.js +0 -100
- package/dist/es5/papaparse/async-iterator-streamer.js.map +0 -1
- package/dist/es5/papaparse/papaparse.js +0 -703
- package/dist/es5/papaparse/papaparse.js.map +0 -1
- package/dist/esm/bundle.js +0 -4
- package/dist/esm/bundle.js.map +0 -1
- package/dist/esm/csv-loader.js.map +0 -1
- package/dist/esm/csv-writer.js.map +0 -1
- package/dist/esm/index.js +0 -3
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/lib/encoders/encode-csv.js.map +0 -1
- package/dist/esm/papaparse/async-iterator-streamer.js.map +0 -1
- package/dist/esm/papaparse/papaparse.js.map +0 -1
- package/dist/src/bundle.d.ts +0 -2
- package/dist/src/bundle.d.ts.map +0 -1
- package/dist/src/csv-loader.d.ts.map +0 -1
- package/dist/src/csv-writer.d.ts.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/lib/encoders/encode-csv.d.ts.map +0 -1
- package/dist/src/papaparse/async-iterator-streamer.d.ts.map +0 -1
- package/dist/src/papaparse/papaparse.d.ts.map +0 -1
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/src/bundle.ts +0 -4
- /package/dist/{src/csv-loader.d.ts → csv-loader.d.ts} +0 -0
- /package/dist/{src/csv-writer.d.ts → csv-writer.d.ts} +0 -0
- /package/dist/{src/index.d.ts → index.d.ts} +0 -0
- /package/dist/{src/lib → lib}/encoders/encode-csv.d.ts +0 -0
- /package/dist/{esm/lib → lib}/encoders/encode-csv.js +0 -0
- /package/dist/{src/papaparse → papaparse}/async-iterator-streamer.d.ts +0 -0
- /package/dist/{src/papaparse → papaparse}/papaparse.d.ts +0 -0
- /package/dist/{esm/papaparse → papaparse}/papaparse.js +0 -0
|
@@ -1,77 +1,90 @@
|
|
|
1
|
-
(()
|
|
1
|
+
(function webpackUniversalModuleDefinition(root, factory) {
|
|
2
|
+
if (typeof exports === 'object' && typeof module === 'object')
|
|
3
|
+
module.exports = factory();
|
|
4
|
+
else if (typeof define === 'function' && define.amd) define([], factory);
|
|
5
|
+
else if (typeof exports === 'object') exports['loader'] = factory();
|
|
6
|
+
else root['loader'] = factory();})(globalThis, function () {
|
|
7
|
+
"use strict";
|
|
8
|
+
var __exports__ = (() => {
|
|
2
9
|
var __defProp = Object.defineProperty;
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
|
|
6
|
-
};
|
|
7
|
-
var __commonJS = (cb, mod) => function __require() {
|
|
8
|
-
return mod || (0, cb[Object.keys(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
9
|
-
};
|
|
10
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
11
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
12
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
10
13
|
var __export = (target, all) => {
|
|
11
|
-
__markAsModule(target);
|
|
12
14
|
for (var name in all)
|
|
13
15
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
14
16
|
};
|
|
17
|
+
var __copyProps = (to, from, except, desc) => {
|
|
18
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
19
|
+
for (let key of __getOwnPropNames(from))
|
|
20
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
21
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
22
|
+
}
|
|
23
|
+
return to;
|
|
24
|
+
};
|
|
25
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
26
|
+
|
|
27
|
+
// src/index.ts
|
|
28
|
+
var src_exports = {};
|
|
29
|
+
__export(src_exports, {
|
|
30
|
+
CSVLoader: () => CSVLoader,
|
|
31
|
+
CSVWriter: () => CSVWriter
|
|
32
|
+
});
|
|
15
33
|
|
|
16
34
|
// ../schema/src/lib/table/batches/base-table-batch-aggregator.ts
|
|
17
|
-
var DEFAULT_ROW_COUNT
|
|
18
|
-
var
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
this._headers = [];
|
|
31
|
-
for (const key in schema) {
|
|
32
|
-
this._headers[schema[key].index] = schema[key].name;
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
rowCount() {
|
|
37
|
-
return this.length;
|
|
38
|
-
}
|
|
39
|
-
addArrayRow(row, cursor) {
|
|
40
|
-
if (Number.isFinite(cursor)) {
|
|
41
|
-
this.cursor = cursor;
|
|
42
|
-
}
|
|
43
|
-
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
44
|
-
this.rows[this.length] = row;
|
|
45
|
-
this.length++;
|
|
46
|
-
}
|
|
47
|
-
addObjectRow(row, cursor) {
|
|
48
|
-
if (Number.isFinite(cursor)) {
|
|
49
|
-
this.cursor = cursor;
|
|
50
|
-
}
|
|
51
|
-
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
52
|
-
this.rows[this.length] = row;
|
|
53
|
-
this.length++;
|
|
54
|
-
}
|
|
55
|
-
getBatch() {
|
|
56
|
-
let rows = this.rows;
|
|
57
|
-
if (!rows) {
|
|
58
|
-
return null;
|
|
59
|
-
}
|
|
60
|
-
rows = rows.slice(0, this.length);
|
|
61
|
-
this.rows = null;
|
|
62
|
-
const batch = {
|
|
63
|
-
shape: this.options.shape,
|
|
64
|
-
batchType: "data",
|
|
65
|
-
data: rows,
|
|
66
|
-
length: this.length,
|
|
67
|
-
schema: this.schema,
|
|
68
|
-
cursor: this.cursor
|
|
69
|
-
};
|
|
70
|
-
return batch;
|
|
35
|
+
var DEFAULT_ROW_COUNT = 100;
|
|
36
|
+
var BaseTableBatchAggregator = class {
|
|
37
|
+
length = 0;
|
|
38
|
+
rows = null;
|
|
39
|
+
cursor = 0;
|
|
40
|
+
_headers = [];
|
|
41
|
+
constructor(schema, options) {
|
|
42
|
+
this.options = options;
|
|
43
|
+
this.schema = schema;
|
|
44
|
+
if (!Array.isArray(schema)) {
|
|
45
|
+
this._headers = [];
|
|
46
|
+
for (const key in schema) {
|
|
47
|
+
this._headers[schema[key].index] = schema[key].name;
|
|
71
48
|
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
rowCount() {
|
|
52
|
+
return this.length;
|
|
53
|
+
}
|
|
54
|
+
addArrayRow(row, cursor) {
|
|
55
|
+
if (Number.isFinite(cursor)) {
|
|
56
|
+
this.cursor = cursor;
|
|
57
|
+
}
|
|
58
|
+
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
59
|
+
this.rows[this.length] = row;
|
|
60
|
+
this.length++;
|
|
61
|
+
}
|
|
62
|
+
addObjectRow(row, cursor) {
|
|
63
|
+
if (Number.isFinite(cursor)) {
|
|
64
|
+
this.cursor = cursor;
|
|
65
|
+
}
|
|
66
|
+
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
67
|
+
this.rows[this.length] = row;
|
|
68
|
+
this.length++;
|
|
69
|
+
}
|
|
70
|
+
getBatch() {
|
|
71
|
+
let rows = this.rows;
|
|
72
|
+
if (!rows) {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
rows = rows.slice(0, this.length);
|
|
76
|
+
this.rows = null;
|
|
77
|
+
const batch = {
|
|
78
|
+
shape: this.options.shape,
|
|
79
|
+
batchType: "data",
|
|
80
|
+
data: rows,
|
|
81
|
+
length: this.length,
|
|
82
|
+
schema: this.schema,
|
|
83
|
+
cursor: this.cursor
|
|
72
84
|
};
|
|
85
|
+
return batch;
|
|
73
86
|
}
|
|
74
|
-
}
|
|
87
|
+
};
|
|
75
88
|
|
|
76
89
|
// ../schema/src/lib/table/simple-table/row-utils.ts
|
|
77
90
|
function convertToObjectRow(arrayRow, headers) {
|
|
@@ -100,306 +113,286 @@
|
|
|
100
113
|
}
|
|
101
114
|
return arrayRow;
|
|
102
115
|
}
|
|
103
|
-
var init_row_utils = __esm({
|
|
104
|
-
"../schema/src/lib/table/simple-table/row-utils.ts"() {
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
116
|
|
|
108
117
|
// ../schema/src/lib/table/batches/row-table-batch-aggregator.ts
|
|
109
|
-
var DEFAULT_ROW_COUNT2
|
|
110
|
-
var
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
this.schema = schema;
|
|
123
|
-
if (!Array.isArray(schema)) {
|
|
124
|
-
this._headers = [];
|
|
125
|
-
for (const key in schema) {
|
|
126
|
-
this._headers[schema[key].index] = schema[key].name;
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
rowCount() {
|
|
131
|
-
return this.length;
|
|
132
|
-
}
|
|
133
|
-
addArrayRow(row, cursor) {
|
|
134
|
-
if (Number.isFinite(cursor)) {
|
|
135
|
-
this.cursor = cursor;
|
|
136
|
-
}
|
|
137
|
-
switch (this.options.shape) {
|
|
138
|
-
case "object-row-table":
|
|
139
|
-
const rowObject = convertToObjectRow(row, this._headers);
|
|
140
|
-
this.addObjectRow(rowObject, cursor);
|
|
141
|
-
break;
|
|
142
|
-
case "array-row-table":
|
|
143
|
-
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT2);
|
|
144
|
-
this.arrayRows[this.length] = row;
|
|
145
|
-
this.length++;
|
|
146
|
-
break;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
addObjectRow(row, cursor) {
|
|
150
|
-
if (Number.isFinite(cursor)) {
|
|
151
|
-
this.cursor = cursor;
|
|
152
|
-
}
|
|
153
|
-
switch (this.options.shape) {
|
|
154
|
-
case "array-row-table":
|
|
155
|
-
const rowArray = convertToArrayRow(row, this._headers);
|
|
156
|
-
this.addArrayRow(rowArray, cursor);
|
|
157
|
-
break;
|
|
158
|
-
case "object-row-table":
|
|
159
|
-
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT2);
|
|
160
|
-
this.objectRows[this.length] = row;
|
|
161
|
-
this.length++;
|
|
162
|
-
break;
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
getBatch() {
|
|
166
|
-
let rows = this.arrayRows || this.objectRows;
|
|
167
|
-
if (!rows) {
|
|
168
|
-
return null;
|
|
169
|
-
}
|
|
170
|
-
rows = rows.slice(0, this.length);
|
|
171
|
-
this.arrayRows = null;
|
|
172
|
-
this.objectRows = null;
|
|
173
|
-
return {
|
|
174
|
-
shape: this.options.shape,
|
|
175
|
-
batchType: "data",
|
|
176
|
-
data: rows,
|
|
177
|
-
length: this.length,
|
|
178
|
-
schema: this.schema,
|
|
179
|
-
cursor: this.cursor
|
|
180
|
-
};
|
|
118
|
+
var DEFAULT_ROW_COUNT2 = 100;
|
|
119
|
+
var RowTableBatchAggregator = class {
|
|
120
|
+
length = 0;
|
|
121
|
+
objectRows = null;
|
|
122
|
+
arrayRows = null;
|
|
123
|
+
cursor = 0;
|
|
124
|
+
_headers = [];
|
|
125
|
+
constructor(schema, options) {
|
|
126
|
+
this.options = options;
|
|
127
|
+
this.schema = schema;
|
|
128
|
+
if (!Array.isArray(schema)) {
|
|
129
|
+
this._headers = [];
|
|
130
|
+
for (const key in schema) {
|
|
131
|
+
this._headers[schema[key].index] = schema[key].name;
|
|
181
132
|
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
rowCount() {
|
|
136
|
+
return this.length;
|
|
137
|
+
}
|
|
138
|
+
addArrayRow(row, cursor) {
|
|
139
|
+
if (Number.isFinite(cursor)) {
|
|
140
|
+
this.cursor = cursor;
|
|
141
|
+
}
|
|
142
|
+
switch (this.options.shape) {
|
|
143
|
+
case "object-row-table":
|
|
144
|
+
const rowObject = convertToObjectRow(row, this._headers);
|
|
145
|
+
this.addObjectRow(rowObject, cursor);
|
|
146
|
+
break;
|
|
147
|
+
case "array-row-table":
|
|
148
|
+
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT2);
|
|
149
|
+
this.arrayRows[this.length] = row;
|
|
150
|
+
this.length++;
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
addObjectRow(row, cursor) {
|
|
155
|
+
if (Number.isFinite(cursor)) {
|
|
156
|
+
this.cursor = cursor;
|
|
157
|
+
}
|
|
158
|
+
switch (this.options.shape) {
|
|
159
|
+
case "array-row-table":
|
|
160
|
+
const rowArray = convertToArrayRow(row, this._headers);
|
|
161
|
+
this.addArrayRow(rowArray, cursor);
|
|
162
|
+
break;
|
|
163
|
+
case "object-row-table":
|
|
164
|
+
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT2);
|
|
165
|
+
this.objectRows[this.length] = row;
|
|
166
|
+
this.length++;
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
getBatch() {
|
|
171
|
+
let rows = this.arrayRows || this.objectRows;
|
|
172
|
+
if (!rows) {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
rows = rows.slice(0, this.length);
|
|
176
|
+
this.arrayRows = null;
|
|
177
|
+
this.objectRows = null;
|
|
178
|
+
return {
|
|
179
|
+
shape: this.options.shape,
|
|
180
|
+
batchType: "data",
|
|
181
|
+
data: rows,
|
|
182
|
+
length: this.length,
|
|
183
|
+
schema: this.schema,
|
|
184
|
+
cursor: this.cursor
|
|
182
185
|
};
|
|
183
186
|
}
|
|
184
|
-
}
|
|
187
|
+
};
|
|
185
188
|
|
|
186
189
|
// ../schema/src/lib/table/batches/columnar-table-batch-aggregator.ts
|
|
187
|
-
var DEFAULT_ROW_COUNT3
|
|
188
|
-
var
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
for (const fieldName in this.schema) {
|
|
222
|
-
const field = this.schema[fieldName];
|
|
223
|
-
columns[field.name] = this.columns[field.index];
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
this.columns = {};
|
|
227
|
-
const batch = {
|
|
228
|
-
shape: "columnar-table",
|
|
229
|
-
batchType: "data",
|
|
230
|
-
data: columns,
|
|
231
|
-
schema: this.schema,
|
|
232
|
-
length: this.length
|
|
233
|
-
};
|
|
234
|
-
return batch;
|
|
235
|
-
}
|
|
236
|
-
_reallocateColumns() {
|
|
237
|
-
if (this.length < this.allocated) {
|
|
238
|
-
return;
|
|
239
|
-
}
|
|
240
|
-
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT3;
|
|
241
|
-
this.columns = {};
|
|
242
|
-
for (const fieldName in this.schema) {
|
|
243
|
-
const field = this.schema[fieldName];
|
|
244
|
-
const ArrayType = field.type || Float32Array;
|
|
245
|
-
const oldColumn = this.columns[field.index];
|
|
246
|
-
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
247
|
-
const typedArray = new ArrayType(this.allocated);
|
|
248
|
-
typedArray.set(oldColumn);
|
|
249
|
-
this.columns[field.index] = typedArray;
|
|
250
|
-
} else if (oldColumn) {
|
|
251
|
-
oldColumn.length = this.allocated;
|
|
252
|
-
this.columns[field.index] = oldColumn;
|
|
253
|
-
} else {
|
|
254
|
-
this.columns[field.index] = new ArrayType(this.allocated);
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
_pruneColumns() {
|
|
259
|
-
for (const [columnName, column] of Object.entries(this.columns)) {
|
|
260
|
-
this.columns[columnName] = column.slice(0, this.length);
|
|
261
|
-
}
|
|
190
|
+
var DEFAULT_ROW_COUNT3 = 100;
|
|
191
|
+
var ColumnarTableBatchAggregator = class {
|
|
192
|
+
length = 0;
|
|
193
|
+
allocated = 0;
|
|
194
|
+
columns = {};
|
|
195
|
+
constructor(schema, options) {
|
|
196
|
+
this.schema = schema;
|
|
197
|
+
this._reallocateColumns();
|
|
198
|
+
}
|
|
199
|
+
rowCount() {
|
|
200
|
+
return this.length;
|
|
201
|
+
}
|
|
202
|
+
addArrayRow(row) {
|
|
203
|
+
this._reallocateColumns();
|
|
204
|
+
let i = 0;
|
|
205
|
+
for (const fieldName in this.columns) {
|
|
206
|
+
this.columns[fieldName][this.length] = row[i++];
|
|
207
|
+
}
|
|
208
|
+
this.length++;
|
|
209
|
+
}
|
|
210
|
+
addObjectRow(row) {
|
|
211
|
+
this._reallocateColumns();
|
|
212
|
+
for (const fieldName in row) {
|
|
213
|
+
this.columns[fieldName][this.length] = row[fieldName];
|
|
214
|
+
}
|
|
215
|
+
this.length++;
|
|
216
|
+
}
|
|
217
|
+
getBatch() {
|
|
218
|
+
this._pruneColumns();
|
|
219
|
+
const columns = Array.isArray(this.schema) ? this.columns : {};
|
|
220
|
+
if (!Array.isArray(this.schema)) {
|
|
221
|
+
for (const fieldName in this.schema) {
|
|
222
|
+
const field = this.schema[fieldName];
|
|
223
|
+
columns[field.name] = this.columns[field.index];
|
|
262
224
|
}
|
|
225
|
+
}
|
|
226
|
+
this.columns = {};
|
|
227
|
+
const batch = {
|
|
228
|
+
shape: "columnar-table",
|
|
229
|
+
batchType: "data",
|
|
230
|
+
data: columns,
|
|
231
|
+
schema: this.schema,
|
|
232
|
+
length: this.length
|
|
263
233
|
};
|
|
234
|
+
return batch;
|
|
264
235
|
}
|
|
265
|
-
|
|
236
|
+
_reallocateColumns() {
|
|
237
|
+
if (this.length < this.allocated) {
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT3;
|
|
241
|
+
this.columns = {};
|
|
242
|
+
for (const fieldName in this.schema) {
|
|
243
|
+
const field = this.schema[fieldName];
|
|
244
|
+
const ArrayType = field.type || Float32Array;
|
|
245
|
+
const oldColumn = this.columns[field.index];
|
|
246
|
+
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
247
|
+
const typedArray = new ArrayType(this.allocated);
|
|
248
|
+
typedArray.set(oldColumn);
|
|
249
|
+
this.columns[field.index] = typedArray;
|
|
250
|
+
} else if (oldColumn) {
|
|
251
|
+
oldColumn.length = this.allocated;
|
|
252
|
+
this.columns[field.index] = oldColumn;
|
|
253
|
+
} else {
|
|
254
|
+
this.columns[field.index] = new ArrayType(this.allocated);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
_pruneColumns() {
|
|
259
|
+
for (const [columnName, column] of Object.entries(this.columns)) {
|
|
260
|
+
this.columns[columnName] = column.slice(0, this.length);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
};
|
|
266
264
|
|
|
267
265
|
// ../schema/src/lib/table/batches/table-batch-builder.ts
|
|
268
|
-
var DEFAULT_OPTIONS
|
|
269
|
-
|
|
270
|
-
"
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
266
|
+
var DEFAULT_OPTIONS = {
|
|
267
|
+
shape: "array-row-table",
|
|
268
|
+
batchSize: "auto",
|
|
269
|
+
batchDebounceMs: 0,
|
|
270
|
+
limit: 0,
|
|
271
|
+
_limitMB: 0
|
|
272
|
+
};
|
|
273
|
+
var ERR_MESSAGE = "TableBatchBuilder";
|
|
274
|
+
var TableBatchBuilder = class {
|
|
275
|
+
aggregator = null;
|
|
276
|
+
batchCount = 0;
|
|
277
|
+
bytesUsed = 0;
|
|
278
|
+
isChunkComplete = false;
|
|
279
|
+
lastBatchEmittedMs = Date.now();
|
|
280
|
+
totalLength = 0;
|
|
281
|
+
totalBytes = 0;
|
|
282
|
+
rowBytes = 0;
|
|
283
|
+
constructor(schema, options) {
|
|
284
|
+
this.schema = schema;
|
|
285
|
+
this.options = {
|
|
286
|
+
...DEFAULT_OPTIONS,
|
|
287
|
+
...options
|
|
280
288
|
};
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
289
|
+
}
|
|
290
|
+
limitReached() {
|
|
291
|
+
if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
|
|
292
|
+
return true;
|
|
293
|
+
}
|
|
294
|
+
if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
|
|
295
|
+
return true;
|
|
296
|
+
}
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
addRow(row) {
|
|
300
|
+
if (this.limitReached()) {
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
this.totalLength++;
|
|
304
|
+
this.rowBytes = this.rowBytes || this._estimateRowMB(row);
|
|
305
|
+
this.totalBytes += this.rowBytes;
|
|
306
|
+
if (Array.isArray(row)) {
|
|
307
|
+
this.addArrayRow(row);
|
|
308
|
+
} else {
|
|
309
|
+
this.addObjectRow(row);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
addArrayRow(row) {
|
|
313
|
+
if (!this.aggregator) {
|
|
314
|
+
const TableBatchType = this._getTableBatchType();
|
|
315
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
316
|
+
}
|
|
317
|
+
this.aggregator.addArrayRow(row);
|
|
318
|
+
}
|
|
319
|
+
addObjectRow(row) {
|
|
320
|
+
if (!this.aggregator) {
|
|
321
|
+
const TableBatchType = this._getTableBatchType();
|
|
322
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
323
|
+
}
|
|
324
|
+
this.aggregator.addObjectRow(row);
|
|
325
|
+
}
|
|
326
|
+
chunkComplete(chunk) {
|
|
327
|
+
if (chunk instanceof ArrayBuffer) {
|
|
328
|
+
this.bytesUsed += chunk.byteLength;
|
|
329
|
+
}
|
|
330
|
+
if (typeof chunk === "string") {
|
|
331
|
+
this.bytesUsed += chunk.length;
|
|
332
|
+
}
|
|
333
|
+
this.isChunkComplete = true;
|
|
334
|
+
}
|
|
335
|
+
getFullBatch(options) {
|
|
336
|
+
return this._isFull() ? this._getBatch(options) : null;
|
|
337
|
+
}
|
|
338
|
+
getFinalBatch(options) {
|
|
339
|
+
return this._getBatch(options);
|
|
340
|
+
}
|
|
341
|
+
_estimateRowMB(row) {
|
|
342
|
+
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
343
|
+
}
|
|
344
|
+
_isFull() {
|
|
345
|
+
if (!this.aggregator || this.aggregator.rowCount() === 0) {
|
|
346
|
+
return false;
|
|
347
|
+
}
|
|
348
|
+
if (this.options.batchSize === "auto") {
|
|
349
|
+
if (!this.isChunkComplete) {
|
|
302
350
|
return false;
|
|
303
351
|
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
return this._isFull() ? this._getBatch(options) : null;
|
|
342
|
-
}
|
|
343
|
-
getFinalBatch(options) {
|
|
344
|
-
return this._getBatch(options);
|
|
345
|
-
}
|
|
346
|
-
_estimateRowMB(row) {
|
|
347
|
-
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
348
|
-
}
|
|
349
|
-
_isFull() {
|
|
350
|
-
if (!this.aggregator || this.aggregator.rowCount() === 0) {
|
|
351
|
-
return false;
|
|
352
|
-
}
|
|
353
|
-
if (this.options.batchSize === "auto") {
|
|
354
|
-
if (!this.isChunkComplete) {
|
|
355
|
-
return false;
|
|
356
|
-
}
|
|
357
|
-
} else if (this.options.batchSize > this.aggregator.rowCount()) {
|
|
358
|
-
return false;
|
|
359
|
-
}
|
|
360
|
-
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
|
|
361
|
-
return false;
|
|
362
|
-
}
|
|
363
|
-
this.isChunkComplete = false;
|
|
364
|
-
this.lastBatchEmittedMs = Date.now();
|
|
365
|
-
return true;
|
|
366
|
-
}
|
|
367
|
-
_getBatch(options) {
|
|
368
|
-
if (!this.aggregator) {
|
|
369
|
-
return null;
|
|
370
|
-
}
|
|
371
|
-
if (options?.bytesUsed) {
|
|
372
|
-
this.bytesUsed = options.bytesUsed;
|
|
373
|
-
}
|
|
374
|
-
const normalizedBatch = this.aggregator.getBatch();
|
|
375
|
-
normalizedBatch.count = this.batchCount;
|
|
376
|
-
normalizedBatch.bytesUsed = this.bytesUsed;
|
|
377
|
-
Object.assign(normalizedBatch, options);
|
|
378
|
-
this.batchCount++;
|
|
379
|
-
this.aggregator = null;
|
|
380
|
-
return normalizedBatch;
|
|
381
|
-
}
|
|
382
|
-
_getTableBatchType() {
|
|
383
|
-
switch (this.options.shape) {
|
|
384
|
-
case "row-table":
|
|
385
|
-
return BaseTableBatchAggregator;
|
|
386
|
-
case "array-row-table":
|
|
387
|
-
case "object-row-table":
|
|
388
|
-
return RowTableBatchAggregator;
|
|
389
|
-
case "columnar-table":
|
|
390
|
-
return ColumnarTableBatchAggregator;
|
|
391
|
-
case "arrow-table":
|
|
392
|
-
if (!TableBatchBuilder.ArrowBatch) {
|
|
393
|
-
throw new Error(ERR_MESSAGE);
|
|
394
|
-
}
|
|
395
|
-
return TableBatchBuilder.ArrowBatch;
|
|
396
|
-
default:
|
|
397
|
-
throw new Error(ERR_MESSAGE);
|
|
352
|
+
} else if (this.options.batchSize > this.aggregator.rowCount()) {
|
|
353
|
+
return false;
|
|
354
|
+
}
|
|
355
|
+
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
this.isChunkComplete = false;
|
|
359
|
+
this.lastBatchEmittedMs = Date.now();
|
|
360
|
+
return true;
|
|
361
|
+
}
|
|
362
|
+
_getBatch(options) {
|
|
363
|
+
if (!this.aggregator) {
|
|
364
|
+
return null;
|
|
365
|
+
}
|
|
366
|
+
if (options?.bytesUsed) {
|
|
367
|
+
this.bytesUsed = options.bytesUsed;
|
|
368
|
+
}
|
|
369
|
+
const normalizedBatch = this.aggregator.getBatch();
|
|
370
|
+
normalizedBatch.count = this.batchCount;
|
|
371
|
+
normalizedBatch.bytesUsed = this.bytesUsed;
|
|
372
|
+
Object.assign(normalizedBatch, options);
|
|
373
|
+
this.batchCount++;
|
|
374
|
+
this.aggregator = null;
|
|
375
|
+
return normalizedBatch;
|
|
376
|
+
}
|
|
377
|
+
_getTableBatchType() {
|
|
378
|
+
switch (this.options.shape) {
|
|
379
|
+
case "row-table":
|
|
380
|
+
return BaseTableBatchAggregator;
|
|
381
|
+
case "array-row-table":
|
|
382
|
+
case "object-row-table":
|
|
383
|
+
return RowTableBatchAggregator;
|
|
384
|
+
case "columnar-table":
|
|
385
|
+
return ColumnarTableBatchAggregator;
|
|
386
|
+
case "arrow-table":
|
|
387
|
+
if (!TableBatchBuilder.ArrowBatch) {
|
|
388
|
+
throw new Error(ERR_MESSAGE);
|
|
398
389
|
}
|
|
399
|
-
|
|
400
|
-
|
|
390
|
+
return TableBatchBuilder.ArrowBatch;
|
|
391
|
+
default:
|
|
392
|
+
throw new Error(ERR_MESSAGE);
|
|
393
|
+
}
|
|
401
394
|
}
|
|
402
|
-
}
|
|
395
|
+
};
|
|
403
396
|
|
|
404
397
|
// ../schema/src/lib/table/simple-table/table-accessors.ts
|
|
405
398
|
function getTableLength(table) {
|
|
@@ -501,89 +494,100 @@
|
|
|
501
494
|
yield getTableRowAsArray(table, rowIndex, target);
|
|
502
495
|
}
|
|
503
496
|
}
|
|
504
|
-
var init_table_accessors = __esm({
|
|
505
|
-
"../schema/src/lib/table/simple-table/table-accessors.ts"() {
|
|
506
|
-
}
|
|
507
|
-
});
|
|
508
497
|
|
|
509
498
|
// ../schema/src/lib/utils/async-queue.ts
|
|
510
|
-
var ArrayQueue
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
throw new Error("Illegal internal state");
|
|
543
|
-
}
|
|
544
|
-
const settler = this._settlers.dequeue();
|
|
545
|
-
if (value instanceof Error) {
|
|
546
|
-
settler.reject(value);
|
|
547
|
-
} else {
|
|
548
|
-
settler.resolve({ value });
|
|
549
|
-
}
|
|
550
|
-
} else {
|
|
551
|
-
this._values.enqueue(value);
|
|
552
|
-
}
|
|
499
|
+
var ArrayQueue = class extends Array {
|
|
500
|
+
enqueue(value) {
|
|
501
|
+
return this.push(value);
|
|
502
|
+
}
|
|
503
|
+
dequeue() {
|
|
504
|
+
return this.shift();
|
|
505
|
+
}
|
|
506
|
+
};
|
|
507
|
+
var AsyncQueue = class {
|
|
508
|
+
constructor() {
|
|
509
|
+
this._values = new ArrayQueue();
|
|
510
|
+
this._settlers = new ArrayQueue();
|
|
511
|
+
this._closed = false;
|
|
512
|
+
}
|
|
513
|
+
close() {
|
|
514
|
+
while (this._settlers.length > 0) {
|
|
515
|
+
this._settlers.dequeue().resolve({
|
|
516
|
+
done: true
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
this._closed = true;
|
|
520
|
+
}
|
|
521
|
+
[Symbol.asyncIterator]() {
|
|
522
|
+
return this;
|
|
523
|
+
}
|
|
524
|
+
enqueue(value) {
|
|
525
|
+
if (this._closed) {
|
|
526
|
+
throw new Error("Closed");
|
|
527
|
+
}
|
|
528
|
+
if (this._settlers.length > 0) {
|
|
529
|
+
if (this._values.length > 0) {
|
|
530
|
+
throw new Error("Illegal internal state");
|
|
553
531
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
return Promise.resolve({ value });
|
|
561
|
-
}
|
|
562
|
-
if (this._closed) {
|
|
563
|
-
if (this._settlers.length > 0) {
|
|
564
|
-
throw new Error("Illegal internal state");
|
|
565
|
-
}
|
|
566
|
-
return Promise.resolve({ done: true });
|
|
567
|
-
}
|
|
568
|
-
return new Promise((resolve, reject) => {
|
|
569
|
-
this._settlers.enqueue({ resolve, reject });
|
|
532
|
+
const settler = this._settlers.dequeue();
|
|
533
|
+
if (value instanceof Error) {
|
|
534
|
+
settler.reject(value);
|
|
535
|
+
} else {
|
|
536
|
+
settler.resolve({
|
|
537
|
+
value
|
|
570
538
|
});
|
|
571
539
|
}
|
|
572
|
-
}
|
|
540
|
+
} else {
|
|
541
|
+
this._values.enqueue(value);
|
|
542
|
+
}
|
|
573
543
|
}
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
544
|
+
next() {
|
|
545
|
+
if (this._values.length > 0) {
|
|
546
|
+
const value = this._values.dequeue();
|
|
547
|
+
if (value instanceof Error) {
|
|
548
|
+
return Promise.reject(value);
|
|
549
|
+
}
|
|
550
|
+
return Promise.resolve({
|
|
551
|
+
value
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
if (this._closed) {
|
|
555
|
+
if (this._settlers.length > 0) {
|
|
556
|
+
throw new Error("Illegal internal state");
|
|
557
|
+
}
|
|
558
|
+
return Promise.resolve({
|
|
559
|
+
done: true
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
return new Promise((resolve, reject) => {
|
|
563
|
+
this._settlers.enqueue({
|
|
564
|
+
resolve,
|
|
565
|
+
reject
|
|
566
|
+
});
|
|
567
|
+
});
|
|
583
568
|
}
|
|
584
|
-
}
|
|
569
|
+
};
|
|
585
570
|
|
|
586
571
|
// src/papaparse/papaparse.ts
|
|
572
|
+
var BYTE_ORDER_MARK = "\uFEFF";
|
|
573
|
+
var Papa = {
|
|
574
|
+
parse: CsvToJson,
|
|
575
|
+
unparse: JsonToCsv,
|
|
576
|
+
RECORD_SEP: String.fromCharCode(30),
|
|
577
|
+
UNIT_SEP: String.fromCharCode(31),
|
|
578
|
+
BYTE_ORDER_MARK,
|
|
579
|
+
BAD_DELIMITERS: ["\r", "\n", '"', BYTE_ORDER_MARK],
|
|
580
|
+
WORKERS_SUPPORTED: false,
|
|
581
|
+
NODE_STREAM_INPUT: 1,
|
|
582
|
+
LocalChunkSize: 1024 * 1024 * 10,
|
|
583
|
+
RemoteChunkSize: 1024 * 1024 * 5,
|
|
584
|
+
DefaultDelimiter: ",",
|
|
585
|
+
Parser,
|
|
586
|
+
ParserHandle,
|
|
587
|
+
ChunkStreamer,
|
|
588
|
+
StringStreamer
|
|
589
|
+
};
|
|
590
|
+
var papaparse_default = Papa;
|
|
587
591
|
function CsvToJson(_input, _config, UserDefinedStreamer) {
|
|
588
592
|
_config = _config || {};
|
|
589
593
|
var dynamicTyping = _config.dynamicTyping || false;
|
|
@@ -838,6 +842,8 @@
|
|
|
838
842
|
return this.parseChunk(chunk);
|
|
839
843
|
};
|
|
840
844
|
}
|
|
845
|
+
StringStreamer.prototype = Object.create(StringStreamer.prototype);
|
|
846
|
+
StringStreamer.prototype.constructor = StringStreamer;
|
|
841
847
|
function ParserHandle(_config) {
|
|
842
848
|
var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
|
|
843
849
|
var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
|
|
@@ -898,7 +904,15 @@
|
|
|
898
904
|
_parser = new Parser(parserConfig);
|
|
899
905
|
_results = _parser.parse(_input, baseIndex, ignoreLastRow);
|
|
900
906
|
processResults();
|
|
901
|
-
return _paused ? {
|
|
907
|
+
return _paused ? {
|
|
908
|
+
meta: {
|
|
909
|
+
paused: true
|
|
910
|
+
}
|
|
911
|
+
} : _results || {
|
|
912
|
+
meta: {
|
|
913
|
+
paused: false
|
|
914
|
+
}
|
|
915
|
+
};
|
|
902
916
|
};
|
|
903
917
|
this.paused = function() {
|
|
904
918
|
return _paused;
|
|
@@ -1333,34 +1347,11 @@
|
|
|
1333
1347
|
function isFunction(func) {
|
|
1334
1348
|
return typeof func === "function";
|
|
1335
1349
|
}
|
|
1336
|
-
var BYTE_ORDER_MARK, Papa, papaparse_default;
|
|
1337
|
-
var init_papaparse = __esm({
|
|
1338
|
-
"src/papaparse/papaparse.ts"() {
|
|
1339
|
-
BYTE_ORDER_MARK = "\uFEFF";
|
|
1340
|
-
Papa = {
|
|
1341
|
-
parse: CsvToJson,
|
|
1342
|
-
unparse: JsonToCsv,
|
|
1343
|
-
RECORD_SEP: String.fromCharCode(30),
|
|
1344
|
-
UNIT_SEP: String.fromCharCode(31),
|
|
1345
|
-
BYTE_ORDER_MARK,
|
|
1346
|
-
BAD_DELIMITERS: ["\r", "\n", '"', BYTE_ORDER_MARK],
|
|
1347
|
-
WORKERS_SUPPORTED: false,
|
|
1348
|
-
NODE_STREAM_INPUT: 1,
|
|
1349
|
-
LocalChunkSize: 1024 * 1024 * 10,
|
|
1350
|
-
RemoteChunkSize: 1024 * 1024 * 5,
|
|
1351
|
-
DefaultDelimiter: ",",
|
|
1352
|
-
Parser,
|
|
1353
|
-
ParserHandle,
|
|
1354
|
-
ChunkStreamer,
|
|
1355
|
-
StringStreamer
|
|
1356
|
-
};
|
|
1357
|
-
papaparse_default = Papa;
|
|
1358
|
-
StringStreamer.prototype = Object.create(StringStreamer.prototype);
|
|
1359
|
-
StringStreamer.prototype.constructor = StringStreamer;
|
|
1360
|
-
}
|
|
1361
|
-
});
|
|
1362
1350
|
|
|
1363
1351
|
// src/papaparse/async-iterator-streamer.ts
|
|
1352
|
+
var {
|
|
1353
|
+
ChunkStreamer: ChunkStreamer2
|
|
1354
|
+
} = papaparse_default;
|
|
1364
1355
|
function AsyncIteratorStreamer(config) {
|
|
1365
1356
|
config = config || {};
|
|
1366
1357
|
ChunkStreamer2.call(this, config);
|
|
@@ -1380,22 +1371,48 @@
|
|
|
1380
1371
|
this._nextChunk = function nextChunk() {
|
|
1381
1372
|
};
|
|
1382
1373
|
this.getStringChunk = function(chunk) {
|
|
1383
|
-
return typeof chunk === "string" ? chunk : this.textDecoder.decode(chunk, {
|
|
1374
|
+
return typeof chunk === "string" ? chunk : this.textDecoder.decode(chunk, {
|
|
1375
|
+
stream: true
|
|
1376
|
+
});
|
|
1384
1377
|
};
|
|
1385
1378
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
"src/papaparse/async-iterator-streamer.ts"() {
|
|
1389
|
-
init_papaparse();
|
|
1390
|
-
({ ChunkStreamer: ChunkStreamer2 } = papaparse_default);
|
|
1391
|
-
AsyncIteratorStreamer.prototype = Object.create(ChunkStreamer2.prototype);
|
|
1392
|
-
AsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;
|
|
1393
|
-
}
|
|
1394
|
-
});
|
|
1379
|
+
AsyncIteratorStreamer.prototype = Object.create(ChunkStreamer2.prototype);
|
|
1380
|
+
AsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;
|
|
1395
1381
|
|
|
1396
1382
|
// src/csv-loader.ts
|
|
1383
|
+
var VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
|
|
1384
|
+
var DEFAULT_CSV_SHAPE = "object-row-table";
|
|
1385
|
+
var CSVLoader = {
|
|
1386
|
+
id: "csv",
|
|
1387
|
+
module: "csv",
|
|
1388
|
+
name: "CSV",
|
|
1389
|
+
version: VERSION,
|
|
1390
|
+
extensions: ["csv", "tsv", "dsv"],
|
|
1391
|
+
mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"],
|
|
1392
|
+
category: "table",
|
|
1393
|
+
parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
|
|
1394
|
+
parseText: (text, options) => parseCSV(text, options),
|
|
1395
|
+
parseInBatches: parseCSVInBatches,
|
|
1396
|
+
options: {
|
|
1397
|
+
csv: {
|
|
1398
|
+
shape: DEFAULT_CSV_SHAPE,
|
|
1399
|
+
optimizeMemoryUsage: false,
|
|
1400
|
+
header: "auto",
|
|
1401
|
+
columnPrefix: "column",
|
|
1402
|
+
quoteChar: '"',
|
|
1403
|
+
escapeChar: '"',
|
|
1404
|
+
dynamicTyping: true,
|
|
1405
|
+
comments: false,
|
|
1406
|
+
skipEmptyLines: true,
|
|
1407
|
+
delimitersToGuess: [",", " ", "|", ";"]
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
};
|
|
1397
1411
|
async function parseCSV(csvText, options) {
|
|
1398
|
-
const csvOptions = {
|
|
1412
|
+
const csvOptions = {
|
|
1413
|
+
...CSVLoader.options.csv,
|
|
1414
|
+
...options?.csv
|
|
1415
|
+
};
|
|
1399
1416
|
const firstRow = readFirstRow(csvText);
|
|
1400
1417
|
const header = csvOptions.header === "auto" ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
|
|
1401
1418
|
const parseWithHeader = header;
|
|
@@ -1428,11 +1445,16 @@
|
|
|
1428
1445
|
}
|
|
1429
1446
|
}
|
|
1430
1447
|
function parseCSVInBatches(asyncIterator, options) {
|
|
1431
|
-
options = {
|
|
1448
|
+
options = {
|
|
1449
|
+
...options
|
|
1450
|
+
};
|
|
1432
1451
|
if (options.batchSize === "auto") {
|
|
1433
1452
|
options.batchSize = 4e3;
|
|
1434
1453
|
}
|
|
1435
|
-
const csvOptions = {
|
|
1454
|
+
const csvOptions = {
|
|
1455
|
+
...CSVLoader.options.csv,
|
|
1456
|
+
...options?.csv
|
|
1457
|
+
};
|
|
1436
1458
|
const asyncQueue = new AsyncQueue();
|
|
1437
1459
|
let isFirstRow = true;
|
|
1438
1460
|
let headerRow = null;
|
|
@@ -1477,7 +1499,9 @@
|
|
|
1477
1499
|
});
|
|
1478
1500
|
try {
|
|
1479
1501
|
tableBatchBuilder.addRow(row);
|
|
1480
|
-
const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({
|
|
1502
|
+
const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({
|
|
1503
|
+
bytesUsed
|
|
1504
|
+
});
|
|
1481
1505
|
if (batch) {
|
|
1482
1506
|
asyncQueue.enqueue(batch);
|
|
1483
1507
|
}
|
|
@@ -1488,7 +1512,9 @@
|
|
|
1488
1512
|
complete(results) {
|
|
1489
1513
|
try {
|
|
1490
1514
|
const bytesUsed = results.meta.cursor;
|
|
1491
|
-
const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({
|
|
1515
|
+
const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({
|
|
1516
|
+
bytesUsed
|
|
1517
|
+
});
|
|
1492
1518
|
if (batch) {
|
|
1493
1519
|
asyncQueue.enqueue(batch);
|
|
1494
1520
|
}
|
|
@@ -1513,7 +1539,7 @@
|
|
|
1513
1539
|
return result.data[0];
|
|
1514
1540
|
}
|
|
1515
1541
|
function duplicateColumnTransformer() {
|
|
1516
|
-
const observedColumns = new Set();
|
|
1542
|
+
const observedColumns = /* @__PURE__ */ new Set();
|
|
1517
1543
|
return (col) => {
|
|
1518
1544
|
let colName = col;
|
|
1519
1545
|
let counter = 1;
|
|
@@ -1540,53 +1566,30 @@
|
|
|
1540
1566
|
switch (typeof value) {
|
|
1541
1567
|
case "number":
|
|
1542
1568
|
case "boolean":
|
|
1543
|
-
schema[columnName] = {
|
|
1569
|
+
schema[columnName] = {
|
|
1570
|
+
name: String(columnName),
|
|
1571
|
+
index: i,
|
|
1572
|
+
type: Float32Array
|
|
1573
|
+
};
|
|
1544
1574
|
break;
|
|
1545
1575
|
case "string":
|
|
1546
1576
|
default:
|
|
1547
|
-
schema[columnName] = {
|
|
1577
|
+
schema[columnName] = {
|
|
1578
|
+
name: String(columnName),
|
|
1579
|
+
index: i,
|
|
1580
|
+
type: Array
|
|
1581
|
+
};
|
|
1548
1582
|
}
|
|
1549
1583
|
}
|
|
1550
1584
|
return schema;
|
|
1551
1585
|
}
|
|
1552
|
-
var VERSION, DEFAULT_CSV_SHAPE, CSVLoader;
|
|
1553
|
-
var init_csv_loader = __esm({
|
|
1554
|
-
"src/csv-loader.ts"() {
|
|
1555
|
-
init_src();
|
|
1556
|
-
init_papaparse();
|
|
1557
|
-
init_async_iterator_streamer();
|
|
1558
|
-
VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
|
|
1559
|
-
DEFAULT_CSV_SHAPE = "object-row-table";
|
|
1560
|
-
CSVLoader = {
|
|
1561
|
-
id: "csv",
|
|
1562
|
-
module: "csv",
|
|
1563
|
-
name: "CSV",
|
|
1564
|
-
version: VERSION,
|
|
1565
|
-
extensions: ["csv", "tsv", "dsv"],
|
|
1566
|
-
mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"],
|
|
1567
|
-
category: "table",
|
|
1568
|
-
parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
|
|
1569
|
-
parseText: (text, options) => parseCSV(text, options),
|
|
1570
|
-
parseInBatches: parseCSVInBatches,
|
|
1571
|
-
options: {
|
|
1572
|
-
csv: {
|
|
1573
|
-
shape: DEFAULT_CSV_SHAPE,
|
|
1574
|
-
optimizeMemoryUsage: false,
|
|
1575
|
-
header: "auto",
|
|
1576
|
-
columnPrefix: "column",
|
|
1577
|
-
quoteChar: '"',
|
|
1578
|
-
escapeChar: '"',
|
|
1579
|
-
dynamicTyping: true,
|
|
1580
|
-
comments: false,
|
|
1581
|
-
skipEmptyLines: true,
|
|
1582
|
-
delimitersToGuess: [",", " ", "|", ";"]
|
|
1583
|
-
}
|
|
1584
|
-
}
|
|
1585
|
-
};
|
|
1586
|
-
}
|
|
1587
|
-
});
|
|
1588
1586
|
|
|
1589
1587
|
// ../../node_modules/d3-dsv/src/dsv.js
|
|
1588
|
+
var EOL = {};
|
|
1589
|
+
var EOF = {};
|
|
1590
|
+
var QUOTE = 34;
|
|
1591
|
+
var NEWLINE = 10;
|
|
1592
|
+
var RETURN = 13;
|
|
1590
1593
|
function objectConverter(columns) {
|
|
1591
1594
|
return new Function("d", "return {" + columns.map(function(name, i) {
|
|
1592
1595
|
return JSON.stringify(name) + ": d[" + i + '] || ""';
|
|
@@ -1599,7 +1602,7 @@
|
|
|
1599
1602
|
};
|
|
1600
1603
|
}
|
|
1601
1604
|
function inferColumns(rows) {
|
|
1602
|
-
var columnSet = Object.create(null), columns = [];
|
|
1605
|
+
var columnSet = /* @__PURE__ */ Object.create(null), columns = [];
|
|
1603
1606
|
rows.forEach(function(row) {
|
|
1604
1607
|
for (var column in row) {
|
|
1605
1608
|
if (!(column in columnSet)) {
|
|
@@ -1716,42 +1719,23 @@
|
|
|
1716
1719
|
formatValue
|
|
1717
1720
|
};
|
|
1718
1721
|
}
|
|
1719
|
-
var EOL, EOF, QUOTE, NEWLINE, RETURN;
|
|
1720
|
-
var init_dsv = __esm({
|
|
1721
|
-
"../../node_modules/d3-dsv/src/dsv.js"() {
|
|
1722
|
-
EOL = {};
|
|
1723
|
-
EOF = {};
|
|
1724
|
-
QUOTE = 34;
|
|
1725
|
-
NEWLINE = 10;
|
|
1726
|
-
RETURN = 13;
|
|
1727
|
-
}
|
|
1728
|
-
});
|
|
1729
1722
|
|
|
1730
1723
|
// ../../node_modules/d3-dsv/src/csv.js
|
|
1731
|
-
var csv
|
|
1732
|
-
var
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
csvFormatBody = csv.formatBody;
|
|
1740
|
-
csvFormatRows = csv.formatRows;
|
|
1741
|
-
csvFormatRow = csv.formatRow;
|
|
1742
|
-
csvFormatValue = csv.formatValue;
|
|
1743
|
-
}
|
|
1744
|
-
});
|
|
1745
|
-
|
|
1746
|
-
// ../../node_modules/d3-dsv/src/index.js
|
|
1747
|
-
var init_src2 = __esm({
|
|
1748
|
-
"../../node_modules/d3-dsv/src/index.js"() {
|
|
1749
|
-
init_csv();
|
|
1750
|
-
}
|
|
1751
|
-
});
|
|
1724
|
+
var csv = dsv_default(",");
|
|
1725
|
+
var csvParse = csv.parse;
|
|
1726
|
+
var csvParseRows = csv.parseRows;
|
|
1727
|
+
var csvFormat = csv.format;
|
|
1728
|
+
var csvFormatBody = csv.formatBody;
|
|
1729
|
+
var csvFormatRows = csv.formatRows;
|
|
1730
|
+
var csvFormatRow = csv.formatRow;
|
|
1731
|
+
var csvFormatValue = csv.formatValue;
|
|
1752
1732
|
|
|
1753
1733
|
// src/lib/encoders/encode-csv.ts
|
|
1754
|
-
function encodeTableAsCSV(table, options = {
|
|
1734
|
+
function encodeTableAsCSV(table, options = {
|
|
1735
|
+
csv: {
|
|
1736
|
+
useDisplayNames: true
|
|
1737
|
+
}
|
|
1738
|
+
}) {
|
|
1755
1739
|
const useDisplayNames = options.useDisplayNames || options.csv?.useDisplayNames;
|
|
1756
1740
|
const fields = table.schema?.fields || [];
|
|
1757
1741
|
const columnNames = fields.map((f) => {
|
|
@@ -1769,73 +1753,38 @@
|
|
|
1769
1753
|
}
|
|
1770
1754
|
return csvFormatRows(formattedData);
|
|
1771
1755
|
}
|
|
1772
|
-
var preformatFieldValue
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
init_src();
|
|
1776
|
-
init_src2();
|
|
1777
|
-
preformatFieldValue = (value) => {
|
|
1778
|
-
if (value === null || value === void 0) {
|
|
1779
|
-
return null;
|
|
1780
|
-
}
|
|
1781
|
-
if (value instanceof Date) {
|
|
1782
|
-
return value.toISOString();
|
|
1783
|
-
}
|
|
1784
|
-
if (typeof value === "object") {
|
|
1785
|
-
return JSON.stringify(value);
|
|
1786
|
-
}
|
|
1787
|
-
return String(value);
|
|
1788
|
-
};
|
|
1756
|
+
var preformatFieldValue = (value) => {
|
|
1757
|
+
if (value === null || value === void 0) {
|
|
1758
|
+
return null;
|
|
1789
1759
|
}
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
// src/csv-writer.ts
|
|
1793
|
-
var CSVWriter;
|
|
1794
|
-
var init_csv_writer = __esm({
|
|
1795
|
-
"src/csv-writer.ts"() {
|
|
1796
|
-
init_encode_csv();
|
|
1797
|
-
CSVWriter = {
|
|
1798
|
-
id: "csv",
|
|
1799
|
-
version: "latest",
|
|
1800
|
-
module: "csv",
|
|
1801
|
-
name: "CSV",
|
|
1802
|
-
extensions: ["csv"],
|
|
1803
|
-
mimeTypes: ["text/csv"],
|
|
1804
|
-
options: {
|
|
1805
|
-
csv: {
|
|
1806
|
-
useDisplayNames: false
|
|
1807
|
-
},
|
|
1808
|
-
useDisplayNames: false
|
|
1809
|
-
},
|
|
1810
|
-
text: true,
|
|
1811
|
-
encode: async (table, options) => new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
|
|
1812
|
-
encodeTextSync: (table, options) => encodeTableAsCSV(table, options)
|
|
1813
|
-
};
|
|
1760
|
+
if (value instanceof Date) {
|
|
1761
|
+
return value.toISOString();
|
|
1814
1762
|
}
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
// src/index.ts
|
|
1818
|
-
var src_exports = {};
|
|
1819
|
-
__export(src_exports, {
|
|
1820
|
-
CSVLoader: () => CSVLoader,
|
|
1821
|
-
CSVWriter: () => CSVWriter
|
|
1822
|
-
});
|
|
1823
|
-
var init_src3 = __esm({
|
|
1824
|
-
"src/index.ts"() {
|
|
1825
|
-
init_csv_loader();
|
|
1826
|
-
init_csv_writer();
|
|
1763
|
+
if (typeof value === "object") {
|
|
1764
|
+
return JSON.stringify(value);
|
|
1827
1765
|
}
|
|
1828
|
-
|
|
1766
|
+
return String(value);
|
|
1767
|
+
};
|
|
1829
1768
|
|
|
1830
|
-
// src/
|
|
1831
|
-
var
|
|
1832
|
-
"
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1769
|
+
// src/csv-writer.ts
|
|
1770
|
+
var CSVWriter = {
|
|
1771
|
+
id: "csv",
|
|
1772
|
+
version: "latest",
|
|
1773
|
+
module: "csv",
|
|
1774
|
+
name: "CSV",
|
|
1775
|
+
extensions: ["csv"],
|
|
1776
|
+
mimeTypes: ["text/csv"],
|
|
1777
|
+
options: {
|
|
1778
|
+
csv: {
|
|
1779
|
+
useDisplayNames: false
|
|
1780
|
+
},
|
|
1781
|
+
useDisplayNames: false
|
|
1782
|
+
},
|
|
1783
|
+
text: true,
|
|
1784
|
+
encode: async (table, options) => new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
|
|
1785
|
+
encodeTextSync: (table, options) => encodeTableAsCSV(table, options)
|
|
1786
|
+
};
|
|
1787
|
+
return __toCommonJS(src_exports);
|
|
1839
1788
|
})();
|
|
1840
1789
|
/* @license
|
|
1841
1790
|
Papa Parse
|
|
@@ -1843,3 +1792,5 @@ v5.0.0-beta.0
|
|
|
1843
1792
|
https://github.com/mholt/PapaParse
|
|
1844
1793
|
License: MIT
|
|
1845
1794
|
*/
|
|
1795
|
+
return __exports__;
|
|
1796
|
+
});
|