@loaders.gl/csv 4.3.0-alpha.8 → 4.3.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/csv-loader.d.ts.map +1 -1
- package/dist/csv-loader.js +1 -2
- package/dist/csv-writer.d.ts +0 -2
- package/dist/csv-writer.d.ts.map +1 -1
- package/dist/dist.dev.js +304 -311
- package/dist/dist.min.js +12 -12
- package/dist/index.cjs +285 -292
- package/dist/index.cjs.map +3 -3
- package/dist/papaparse/async-iterator-streamer.d.ts +28 -4
- package/dist/papaparse/async-iterator-streamer.d.ts.map +1 -1
- package/dist/papaparse/async-iterator-streamer.js +11 -12
- package/dist/papaparse/papaparse.d.ts +108 -12
- package/dist/papaparse/papaparse.d.ts.map +1 -1
- package/dist/papaparse/papaparse.js +274 -314
- package/package.json +4 -4
- package/src/csv-loader.ts +0 -1
- package/src/csv-writer.ts +0 -2
- package/src/papaparse/async-iterator-streamer.ts +11 -14
- package/src/papaparse/papaparse.ts +330 -342
package/dist/index.cjs
CHANGED
|
@@ -30,32 +30,7 @@ var import_schema = require("@loaders.gl/schema");
|
|
|
30
30
|
|
|
31
31
|
// dist/papaparse/papaparse.js
|
|
32
32
|
var BYTE_ORDER_MARK = "\uFEFF";
|
|
33
|
-
|
|
34
|
-
parse: CsvToJson,
|
|
35
|
-
unparse: JsonToCsv,
|
|
36
|
-
RECORD_SEP: String.fromCharCode(30),
|
|
37
|
-
UNIT_SEP: String.fromCharCode(31),
|
|
38
|
-
BYTE_ORDER_MARK,
|
|
39
|
-
BAD_DELIMITERS: ["\r", "\n", '"', BYTE_ORDER_MARK],
|
|
40
|
-
WORKERS_SUPPORTED: false,
|
|
41
|
-
// !IS_WORKER && !!globalThis.Worker
|
|
42
|
-
NODE_STREAM_INPUT: 1,
|
|
43
|
-
// Configurable chunk sizes for local and remote files, respectively
|
|
44
|
-
LocalChunkSize: 1024 * 1024 * 10,
|
|
45
|
-
// 10 M,
|
|
46
|
-
RemoteChunkSize: 1024 * 1024 * 5,
|
|
47
|
-
// 5 M,
|
|
48
|
-
DefaultDelimiter: ",",
|
|
49
|
-
// Used if not specified and detection fail,
|
|
50
|
-
// Exposed for testing and development only
|
|
51
|
-
Parser,
|
|
52
|
-
ParserHandle,
|
|
53
|
-
// BEGIN FORK
|
|
54
|
-
ChunkStreamer,
|
|
55
|
-
StringStreamer
|
|
56
|
-
};
|
|
57
|
-
var papaparse_default = Papa;
|
|
58
|
-
function CsvToJson(_input, _config, UserDefinedStreamer) {
|
|
33
|
+
function CsvToJson(_input, _config = {}, Streamer = StringStreamer) {
|
|
59
34
|
_config = _config || {};
|
|
60
35
|
var dynamicTyping = _config.dynamicTyping || false;
|
|
61
36
|
if (isFunction(dynamicTyping)) {
|
|
@@ -64,31 +39,7 @@ function CsvToJson(_input, _config, UserDefinedStreamer) {
|
|
|
64
39
|
}
|
|
65
40
|
_config.dynamicTyping = dynamicTyping;
|
|
66
41
|
_config.transform = isFunction(_config.transform) ? _config.transform : false;
|
|
67
|
-
|
|
68
|
-
var w = newWorker();
|
|
69
|
-
w.userStep = _config.step;
|
|
70
|
-
w.userChunk = _config.chunk;
|
|
71
|
-
w.userComplete = _config.complete;
|
|
72
|
-
w.userError = _config.error;
|
|
73
|
-
_config.step = isFunction(_config.step);
|
|
74
|
-
_config.chunk = isFunction(_config.chunk);
|
|
75
|
-
_config.complete = isFunction(_config.complete);
|
|
76
|
-
_config.error = isFunction(_config.error);
|
|
77
|
-
delete _config.worker;
|
|
78
|
-
w.postMessage({
|
|
79
|
-
input: _input,
|
|
80
|
-
config: _config,
|
|
81
|
-
workerId: w.id
|
|
82
|
-
});
|
|
83
|
-
return;
|
|
84
|
-
}
|
|
85
|
-
var streamer = null;
|
|
86
|
-
if (typeof _input === "string") {
|
|
87
|
-
streamer = new StringStreamer(_config);
|
|
88
|
-
}
|
|
89
|
-
if (!streamer) {
|
|
90
|
-
streamer = new UserDefinedStreamer(_config);
|
|
91
|
-
}
|
|
42
|
+
var streamer = new Streamer(_config);
|
|
92
43
|
return streamer.stream(_input);
|
|
93
44
|
}
|
|
94
45
|
function JsonToCsv(_input, _config) {
|
|
@@ -108,7 +59,7 @@ function JsonToCsv(_input, _config) {
|
|
|
108
59
|
if (!_input.length || Array.isArray(_input[0]))
|
|
109
60
|
return serialize(null, _input, _skipEmptyLines);
|
|
110
61
|
else if (typeof _input[0] === "object")
|
|
111
|
-
return serialize(_columns ||
|
|
62
|
+
return serialize(_columns || Object.keys(_input[0]), _input, _skipEmptyLines);
|
|
112
63
|
} else if (typeof _input === "object") {
|
|
113
64
|
if (typeof _input.data === "string")
|
|
114
65
|
_input.data = JSON.parse(_input.data);
|
|
@@ -116,7 +67,7 @@ function JsonToCsv(_input, _config) {
|
|
|
116
67
|
if (!_input.fields)
|
|
117
68
|
_input.fields = _input.meta && _input.meta.fields;
|
|
118
69
|
if (!_input.fields)
|
|
119
|
-
_input.fields = Array.isArray(_input.data[0]) ? _input.fields :
|
|
70
|
+
_input.fields = Array.isArray(_input.data[0]) ? _input.fields : Object.keys(_input.data[0]);
|
|
120
71
|
if (!Array.isArray(_input.data[0]) && typeof _input.data[0] !== "object")
|
|
121
72
|
_input.data = [_input.data];
|
|
122
73
|
}
|
|
@@ -150,14 +101,6 @@ function JsonToCsv(_input, _config) {
|
|
|
150
101
|
_escapedQuote = _config.escapeChar + _quoteChar;
|
|
151
102
|
}
|
|
152
103
|
}
|
|
153
|
-
function objectKeys(obj) {
|
|
154
|
-
if (typeof obj !== "object")
|
|
155
|
-
return [];
|
|
156
|
-
var keys = [];
|
|
157
|
-
for (var key in obj)
|
|
158
|
-
keys.push(key);
|
|
159
|
-
return keys;
|
|
160
|
-
}
|
|
161
104
|
function serialize(fields, data, skipEmptyLines) {
|
|
162
105
|
var csv = "";
|
|
163
106
|
if (typeof fields === "string")
|
|
@@ -220,24 +163,33 @@ function JsonToCsv(_input, _config) {
|
|
|
220
163
|
return false;
|
|
221
164
|
}
|
|
222
165
|
}
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
166
|
+
var ChunkStreamer = class {
|
|
167
|
+
_handle;
|
|
168
|
+
_config;
|
|
169
|
+
_finished = false;
|
|
170
|
+
_completed = false;
|
|
171
|
+
_input = null;
|
|
172
|
+
_baseIndex = 0;
|
|
173
|
+
_partialLine = "";
|
|
174
|
+
_rowCount = 0;
|
|
175
|
+
_start = 0;
|
|
176
|
+
isFirstChunk = true;
|
|
177
|
+
_completeResults = {
|
|
235
178
|
data: [],
|
|
236
179
|
errors: [],
|
|
237
180
|
meta: {}
|
|
238
181
|
};
|
|
239
|
-
|
|
240
|
-
|
|
182
|
+
constructor(config) {
|
|
183
|
+
var configCopy = { ...config };
|
|
184
|
+
configCopy.chunkSize = parseInt(configCopy.chunkSize);
|
|
185
|
+
if (!config.step && !config.chunk) {
|
|
186
|
+
configCopy.chunkSize = null;
|
|
187
|
+
}
|
|
188
|
+
this._handle = new ParserHandle(configCopy);
|
|
189
|
+
this._handle.streamer = this;
|
|
190
|
+
this._config = configCopy;
|
|
191
|
+
}
|
|
192
|
+
parseChunk(chunk, isFakeChunk) {
|
|
241
193
|
if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) {
|
|
242
194
|
var modifiedChunk = this._config.beforeFirstChunk(chunk);
|
|
243
195
|
if (modifiedChunk !== void 0)
|
|
@@ -273,173 +225,181 @@ function ChunkStreamer(config) {
|
|
|
273
225
|
this._config.complete(this._completeResults, this._input);
|
|
274
226
|
this._completed = true;
|
|
275
227
|
}
|
|
276
|
-
if (!finishedIncludingPreview && (!results || !results.meta.paused))
|
|
277
|
-
this._nextChunk();
|
|
278
228
|
return results;
|
|
279
|
-
}
|
|
280
|
-
|
|
229
|
+
}
|
|
230
|
+
_sendError(error) {
|
|
281
231
|
if (isFunction(this._config.error))
|
|
282
232
|
this._config.error(error);
|
|
283
|
-
};
|
|
284
|
-
function replaceConfig(config2) {
|
|
285
|
-
var configCopy = copy(config2);
|
|
286
|
-
configCopy.chunkSize = parseInt(configCopy.chunkSize);
|
|
287
|
-
if (!config2.step && !config2.chunk)
|
|
288
|
-
configCopy.chunkSize = null;
|
|
289
|
-
this._handle = new ParserHandle(configCopy);
|
|
290
|
-
this._handle.streamer = this;
|
|
291
|
-
this._config = configCopy;
|
|
292
233
|
}
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
234
|
+
};
|
|
235
|
+
var StringStreamer = class extends ChunkStreamer {
|
|
236
|
+
remaining;
|
|
237
|
+
constructor(config = {}) {
|
|
238
|
+
super(config);
|
|
239
|
+
}
|
|
240
|
+
stream(s) {
|
|
241
|
+
this.remaining = s;
|
|
300
242
|
return this._nextChunk();
|
|
301
|
-
}
|
|
302
|
-
|
|
243
|
+
}
|
|
244
|
+
_nextChunk() {
|
|
303
245
|
if (this._finished)
|
|
304
246
|
return;
|
|
305
247
|
var size = this._config.chunkSize;
|
|
306
|
-
var chunk = size ? remaining.substr(0, size) : remaining;
|
|
307
|
-
remaining = size ? remaining.substr(size) : "";
|
|
308
|
-
this._finished = !remaining;
|
|
248
|
+
var chunk = size ? this.remaining.substr(0, size) : this.remaining;
|
|
249
|
+
this.remaining = size ? this.remaining.substr(size) : "";
|
|
250
|
+
this._finished = !this.remaining;
|
|
309
251
|
return this.parseChunk(chunk);
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
252
|
+
}
|
|
253
|
+
};
|
|
254
|
+
var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
|
|
255
|
+
var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
|
|
256
|
+
var ParserHandle = class {
|
|
257
|
+
_config;
|
|
258
|
+
/** Number of times step was called (number of rows parsed) */
|
|
259
|
+
_stepCounter = 0;
|
|
260
|
+
/** Number of rows that have been parsed so far */
|
|
261
|
+
_rowCounter = 0;
|
|
262
|
+
/** The input being parsed */
|
|
263
|
+
_input;
|
|
264
|
+
/** The core parser being used */
|
|
265
|
+
_parser;
|
|
266
|
+
/** Whether we are paused or not */
|
|
267
|
+
_paused = false;
|
|
268
|
+
/** Whether the parser has aborted or not */
|
|
269
|
+
_aborted = false;
|
|
270
|
+
/** Temporary state between delimiter detection and processing results */
|
|
271
|
+
_delimiterError = false;
|
|
272
|
+
/** Fields are from the header row of the input, if there is one */
|
|
273
|
+
_fields = [];
|
|
274
|
+
/** The last results returned from the parser */
|
|
275
|
+
_results = {
|
|
328
276
|
data: [],
|
|
329
277
|
errors: [],
|
|
330
278
|
meta: {}
|
|
331
279
|
};
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
280
|
+
constructor(_config) {
|
|
281
|
+
if (isFunction(_config.step)) {
|
|
282
|
+
var userStep = _config.step;
|
|
283
|
+
_config.step = (results) => {
|
|
284
|
+
this._results = results;
|
|
285
|
+
if (this.needsHeaderRow()) {
|
|
286
|
+
this.processResults();
|
|
287
|
+
} else {
|
|
288
|
+
this.processResults();
|
|
289
|
+
if (!this._results.data || this._results.data.length === 0)
|
|
290
|
+
return;
|
|
291
|
+
this._stepCounter += results.data.length;
|
|
292
|
+
if (_config.preview && this._stepCounter > _config.preview) {
|
|
293
|
+
this._parser.abort();
|
|
294
|
+
} else {
|
|
295
|
+
userStep(this._results, this);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
this._config = _config;
|
|
349
301
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
302
|
+
/**
|
|
303
|
+
* Parses input. Most users won't need, and shouldn't mess with, the baseIndex
|
|
304
|
+
* and ignoreLastRow parameters. They are used by streamers (wrapper functions)
|
|
305
|
+
* when an input comes in multiple chunks, like from a file.
|
|
306
|
+
*/
|
|
307
|
+
parse(input, baseIndex, ignoreLastRow) {
|
|
308
|
+
var quoteChar = this._config.quoteChar || '"';
|
|
309
|
+
if (!this._config.newline)
|
|
310
|
+
this._config.newline = guessLineEndings(input, quoteChar);
|
|
311
|
+
this._delimiterError = false;
|
|
312
|
+
if (!this._config.delimiter) {
|
|
313
|
+
var delimGuess = this.guessDelimiter(input, this._config.newline, this._config.skipEmptyLines, this._config.comments, this._config.delimitersToGuess);
|
|
314
|
+
if (delimGuess.successful) {
|
|
315
|
+
this._config.delimiter = delimGuess.bestDelimiter;
|
|
316
|
+
} else {
|
|
317
|
+
this._delimiterError = true;
|
|
318
|
+
this._config.delimiter = Papa.DefaultDelimiter;
|
|
362
319
|
}
|
|
363
|
-
_results.meta.delimiter = _config.delimiter;
|
|
364
|
-
} else if (isFunction(_config.delimiter)) {
|
|
365
|
-
_config.delimiter = _config.delimiter(input);
|
|
366
|
-
_results.meta.delimiter = _config.delimiter;
|
|
320
|
+
this._results.meta.delimiter = this._config.delimiter;
|
|
321
|
+
} else if (isFunction(this._config.delimiter)) {
|
|
322
|
+
this._config.delimiter = this._config.delimiter(input);
|
|
323
|
+
this._results.meta.delimiter = this._config.delimiter;
|
|
367
324
|
}
|
|
368
|
-
var parserConfig = copy(_config);
|
|
369
|
-
if (_config.preview && _config.header)
|
|
325
|
+
var parserConfig = copy(this._config);
|
|
326
|
+
if (this._config.preview && this._config.header)
|
|
370
327
|
parserConfig.preview++;
|
|
371
|
-
_input = input;
|
|
372
|
-
_parser = new Parser(parserConfig);
|
|
373
|
-
_results = _parser.parse(_input, baseIndex, ignoreLastRow);
|
|
374
|
-
processResults();
|
|
375
|
-
return _paused ? { meta: { paused: true } } : _results || { meta: { paused: false } };
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
return _paused;
|
|
379
|
-
};
|
|
380
|
-
this.pause = function() {
|
|
381
|
-
_paused = true;
|
|
382
|
-
_parser.abort();
|
|
383
|
-
_input = _input.substr(_parser.getCharIndex());
|
|
384
|
-
};
|
|
385
|
-
this.resume = function() {
|
|
386
|
-
_paused = false;
|
|
387
|
-
self.streamer.parseChunk(_input, true);
|
|
388
|
-
};
|
|
389
|
-
this.aborted = function() {
|
|
390
|
-
return _aborted;
|
|
391
|
-
};
|
|
392
|
-
this.abort = function() {
|
|
393
|
-
_aborted = true;
|
|
394
|
-
_parser.abort();
|
|
395
|
-
_results.meta.aborted = true;
|
|
396
|
-
if (isFunction(_config.complete))
|
|
397
|
-
_config.complete(_results);
|
|
398
|
-
_input = "";
|
|
399
|
-
};
|
|
400
|
-
function testEmptyLine(s) {
|
|
401
|
-
return _config.skipEmptyLines === "greedy" ? s.join("").trim() === "" : s.length === 1 && s[0].length === 0;
|
|
328
|
+
this._input = input;
|
|
329
|
+
this._parser = new Parser(parserConfig);
|
|
330
|
+
this._results = this._parser.parse(this._input, baseIndex, ignoreLastRow);
|
|
331
|
+
this.processResults();
|
|
332
|
+
return this._paused ? { meta: { paused: true } } : this._results || { meta: { paused: false } };
|
|
333
|
+
}
|
|
334
|
+
paused() {
|
|
335
|
+
return this._paused;
|
|
402
336
|
}
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
337
|
+
pause() {
|
|
338
|
+
this._paused = true;
|
|
339
|
+
this._parser.abort();
|
|
340
|
+
this._input = this._input.substr(this._parser.getCharIndex());
|
|
341
|
+
}
|
|
342
|
+
resume() {
|
|
343
|
+
this._paused = false;
|
|
344
|
+
this.streamer.parseChunk(this._input, true);
|
|
345
|
+
}
|
|
346
|
+
aborted() {
|
|
347
|
+
return this._aborted;
|
|
348
|
+
}
|
|
349
|
+
abort() {
|
|
350
|
+
this._aborted = true;
|
|
351
|
+
this._parser.abort();
|
|
352
|
+
this._results.meta.aborted = true;
|
|
353
|
+
if (isFunction(this._config.complete)) {
|
|
354
|
+
this._config.complete(this._results);
|
|
355
|
+
}
|
|
356
|
+
this._input = "";
|
|
357
|
+
}
|
|
358
|
+
testEmptyLine(s) {
|
|
359
|
+
return this._config.skipEmptyLines === "greedy" ? s.join("").trim() === "" : s.length === 1 && s[0].length === 0;
|
|
360
|
+
}
|
|
361
|
+
processResults() {
|
|
362
|
+
if (this._results && this._delimiterError) {
|
|
363
|
+
this.addError("Delimiter", "UndetectableDelimiter", "Unable to auto-detect delimiting character; defaulted to '" + Papa.DefaultDelimiter + "'");
|
|
364
|
+
this._delimiterError = false;
|
|
407
365
|
}
|
|
408
|
-
if (_config.skipEmptyLines) {
|
|
409
|
-
for (var i = 0; i < _results.data.length; i++)
|
|
410
|
-
if (testEmptyLine(_results.data[i]))
|
|
411
|
-
_results.data.splice(i--, 1);
|
|
366
|
+
if (this._config.skipEmptyLines) {
|
|
367
|
+
for (var i = 0; i < this._results.data.length; i++)
|
|
368
|
+
if (this.testEmptyLine(this._results.data[i]))
|
|
369
|
+
this._results.data.splice(i--, 1);
|
|
412
370
|
}
|
|
413
|
-
if (needsHeaderRow())
|
|
414
|
-
fillHeaderFields();
|
|
415
|
-
|
|
371
|
+
if (this.needsHeaderRow()) {
|
|
372
|
+
this.fillHeaderFields();
|
|
373
|
+
}
|
|
374
|
+
return this.applyHeaderAndDynamicTypingAndTransformation();
|
|
416
375
|
}
|
|
417
|
-
|
|
418
|
-
return _config.header && _fields.length === 0;
|
|
376
|
+
needsHeaderRow() {
|
|
377
|
+
return this._config.header && this._fields.length === 0;
|
|
419
378
|
}
|
|
420
|
-
|
|
421
|
-
if (!_results)
|
|
379
|
+
fillHeaderFields() {
|
|
380
|
+
if (!this._results)
|
|
422
381
|
return;
|
|
423
|
-
|
|
424
|
-
if (isFunction(_config.transformHeader))
|
|
425
|
-
header = _config.transformHeader(header);
|
|
426
|
-
_fields.push(header);
|
|
382
|
+
const addHeder = (header) => {
|
|
383
|
+
if (isFunction(this._config.transformHeader))
|
|
384
|
+
header = this._config.transformHeader(header);
|
|
385
|
+
this._fields.push(header);
|
|
386
|
+
};
|
|
387
|
+
if (Array.isArray(this._results.data[0])) {
|
|
388
|
+
for (var i = 0; this.needsHeaderRow() && i < this._results.data.length; i++)
|
|
389
|
+
this._results.data[i].forEach(addHeder);
|
|
390
|
+
this._results.data.splice(0, 1);
|
|
391
|
+
} else {
|
|
392
|
+
this._results.data.forEach(addHeder);
|
|
427
393
|
}
|
|
428
|
-
if (Array.isArray(_results.data[0])) {
|
|
429
|
-
for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
|
|
430
|
-
_results.data[i].forEach(addHeder);
|
|
431
|
-
_results.data.splice(0, 1);
|
|
432
|
-
} else
|
|
433
|
-
_results.data.forEach(addHeder);
|
|
434
394
|
}
|
|
435
|
-
|
|
436
|
-
if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === void 0) {
|
|
437
|
-
_config.dynamicTyping[field] = _config.dynamicTypingFunction(field);
|
|
395
|
+
shouldApplyDynamicTyping(field) {
|
|
396
|
+
if (this._config.dynamicTypingFunction && this._config.dynamicTyping[field] === void 0) {
|
|
397
|
+
this._config.dynamicTyping[field] = this._config.dynamicTypingFunction(field);
|
|
438
398
|
}
|
|
439
|
-
return (_config.dynamicTyping[field] || _config.dynamicTyping) === true;
|
|
399
|
+
return (this._config.dynamicTyping[field] || this._config.dynamicTyping) === true;
|
|
440
400
|
}
|
|
441
|
-
|
|
442
|
-
if (shouldApplyDynamicTyping(field)) {
|
|
401
|
+
parseDynamic(field, value) {
|
|
402
|
+
if (this.shouldApplyDynamicTyping(field)) {
|
|
443
403
|
if (value === "true" || value === "TRUE")
|
|
444
404
|
return true;
|
|
445
405
|
else if (value === "false" || value === "FALSE")
|
|
@@ -453,46 +413,48 @@ function ParserHandle(_config) {
|
|
|
453
413
|
}
|
|
454
414
|
return value;
|
|
455
415
|
}
|
|
456
|
-
|
|
457
|
-
if (!_results || !_results.data || !_config.header && !_config.dynamicTyping && !_config.transform)
|
|
458
|
-
return _results;
|
|
459
|
-
function processRow(rowSource, i) {
|
|
460
|
-
var row = _config.header ? {} : [];
|
|
461
|
-
var j;
|
|
462
|
-
for (j = 0; j < rowSource.length; j++) {
|
|
463
|
-
var field = j;
|
|
464
|
-
var value = rowSource[j];
|
|
465
|
-
if (_config.header)
|
|
466
|
-
field = j >= _fields.length ? "__parsed_extra" : _fields[j];
|
|
467
|
-
if (_config.transform)
|
|
468
|
-
value = _config.transform(value, field);
|
|
469
|
-
value = parseDynamic(field, value);
|
|
470
|
-
if (field === "__parsed_extra") {
|
|
471
|
-
row[field] = row[field] || [];
|
|
472
|
-
row[field].push(value);
|
|
473
|
-
} else
|
|
474
|
-
row[field] = value;
|
|
475
|
-
}
|
|
476
|
-
if (_config.header) {
|
|
477
|
-
if (j > _fields.length)
|
|
478
|
-
addError("FieldMismatch", "TooManyFields", "Too many fields: expected " + _fields.length + " fields but parsed " + j, _rowCounter + i);
|
|
479
|
-
else if (j < _fields.length)
|
|
480
|
-
addError("FieldMismatch", "TooFewFields", "Too few fields: expected " + _fields.length + " fields but parsed " + j, _rowCounter + i);
|
|
481
|
-
}
|
|
482
|
-
return row;
|
|
416
|
+
applyHeaderAndDynamicTypingAndTransformation() {
|
|
417
|
+
if (!this._results || !this._results.data || !this._config.header && !this._config.dynamicTyping && !this._config.transform) {
|
|
418
|
+
return this._results;
|
|
483
419
|
}
|
|
484
420
|
var incrementBy = 1;
|
|
485
|
-
if (!_results.data[0] || Array.isArray(_results.data[0])) {
|
|
486
|
-
_results.data = _results.data.map(processRow);
|
|
487
|
-
incrementBy = _results.data.length;
|
|
488
|
-
} else
|
|
489
|
-
_results.data = processRow(_results.data, 0);
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
421
|
+
if (!this._results.data[0] || Array.isArray(this._results.data[0])) {
|
|
422
|
+
this._results.data = this._results.data.map(this.processRow.bind(this));
|
|
423
|
+
incrementBy = this._results.data.length;
|
|
424
|
+
} else {
|
|
425
|
+
this._results.data = this.processRow(this._results.data, 0);
|
|
426
|
+
}
|
|
427
|
+
if (this._config.header && this._results.meta)
|
|
428
|
+
this._results.meta.fields = this._fields;
|
|
429
|
+
this._rowCounter += incrementBy;
|
|
430
|
+
return this._results;
|
|
494
431
|
}
|
|
495
|
-
|
|
432
|
+
processRow(rowSource, i) {
|
|
433
|
+
var row = this._config.header ? {} : [];
|
|
434
|
+
var j;
|
|
435
|
+
for (j = 0; j < rowSource.length; j++) {
|
|
436
|
+
var field = j;
|
|
437
|
+
var value = rowSource[j];
|
|
438
|
+
if (this._config.header)
|
|
439
|
+
field = j >= this._fields.length ? "__parsed_extra" : this._fields[j];
|
|
440
|
+
if (this._config.transform)
|
|
441
|
+
value = this._config.transform(value, field);
|
|
442
|
+
value = this.parseDynamic(field, value);
|
|
443
|
+
if (field === "__parsed_extra") {
|
|
444
|
+
row[field] = row[field] || [];
|
|
445
|
+
row[field].push(value);
|
|
446
|
+
} else
|
|
447
|
+
row[field] = value;
|
|
448
|
+
}
|
|
449
|
+
if (this._config.header) {
|
|
450
|
+
if (j > this._fields.length)
|
|
451
|
+
this.addError("FieldMismatch", "TooManyFields", "Too many fields: expected " + this._fields.length + " fields but parsed " + j, this._rowCounter + i);
|
|
452
|
+
else if (j < this._fields.length)
|
|
453
|
+
this.addError("FieldMismatch", "TooFewFields", "Too few fields: expected " + this._fields.length + " fields but parsed " + j, this._rowCounter + i);
|
|
454
|
+
}
|
|
455
|
+
return row;
|
|
456
|
+
}
|
|
457
|
+
guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) {
|
|
496
458
|
var bestDelim, bestDelta, fieldCountPrevRow;
|
|
497
459
|
delimitersToGuess = delimitersToGuess || [",", " ", "|", ";", Papa.RECORD_SEP, Papa.UNIT_SEP];
|
|
498
460
|
for (var i = 0; i < delimitersToGuess.length; i++) {
|
|
@@ -506,7 +468,7 @@ function ParserHandle(_config) {
|
|
|
506
468
|
preview: 10
|
|
507
469
|
}).parse(input);
|
|
508
470
|
for (var j = 0; j < preview.data.length; j++) {
|
|
509
|
-
if (skipEmptyLines && testEmptyLine(preview.data[j])) {
|
|
471
|
+
if (skipEmptyLines && this.testEmptyLine(preview.data[j])) {
|
|
510
472
|
emptyLinesCount++;
|
|
511
473
|
continue;
|
|
512
474
|
}
|
|
@@ -527,36 +489,36 @@ function ParserHandle(_config) {
|
|
|
527
489
|
bestDelim = delim;
|
|
528
490
|
}
|
|
529
491
|
}
|
|
530
|
-
_config.delimiter = bestDelim;
|
|
492
|
+
this._config.delimiter = bestDelim;
|
|
531
493
|
return {
|
|
532
494
|
successful: !!bestDelim,
|
|
533
495
|
bestDelimiter: bestDelim
|
|
534
496
|
};
|
|
535
497
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
var re = new RegExp(escapeRegExp(quoteChar) + "([^]*?)" + escapeRegExp(quoteChar), "gm");
|
|
539
|
-
input = input.replace(re, "");
|
|
540
|
-
var r = input.split("\r");
|
|
541
|
-
var n = input.split("\n");
|
|
542
|
-
var nAppearsFirst = n.length > 1 && n[0].length < r[0].length;
|
|
543
|
-
if (r.length === 1 || nAppearsFirst)
|
|
544
|
-
return "\n";
|
|
545
|
-
var numWithN = 0;
|
|
546
|
-
for (var i = 0; i < r.length; i++) {
|
|
547
|
-
if (r[i][0] === "\n")
|
|
548
|
-
numWithN++;
|
|
549
|
-
}
|
|
550
|
-
return numWithN >= r.length / 2 ? "\r\n" : "\r";
|
|
551
|
-
}
|
|
552
|
-
function addError(type, code, msg, row) {
|
|
553
|
-
_results.errors.push({
|
|
498
|
+
addError(type, code, msg, row) {
|
|
499
|
+
this._results.errors.push({
|
|
554
500
|
type,
|
|
555
501
|
code,
|
|
556
502
|
message: msg,
|
|
557
503
|
row
|
|
558
504
|
});
|
|
559
505
|
}
|
|
506
|
+
};
|
|
507
|
+
function guessLineEndings(input, quoteChar) {
|
|
508
|
+
input = input.substr(0, 1024 * 1024);
|
|
509
|
+
var re = new RegExp(escapeRegExp(quoteChar) + "([^]*?)" + escapeRegExp(quoteChar), "gm");
|
|
510
|
+
input = input.replace(re, "");
|
|
511
|
+
var r = input.split("\r");
|
|
512
|
+
var n = input.split("\n");
|
|
513
|
+
var nAppearsFirst = n.length > 1 && n[0].length < r[0].length;
|
|
514
|
+
if (r.length === 1 || nAppearsFirst)
|
|
515
|
+
return "\n";
|
|
516
|
+
var numWithN = 0;
|
|
517
|
+
for (var i = 0; i < r.length; i++) {
|
|
518
|
+
if (r[i][0] === "\n")
|
|
519
|
+
numWithN++;
|
|
520
|
+
}
|
|
521
|
+
return numWithN >= r.length / 2 ? "\r\n" : "\r";
|
|
560
522
|
}
|
|
561
523
|
function escapeRegExp(string) {
|
|
562
524
|
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
@@ -603,22 +565,22 @@ function Parser(config) {
|
|
|
603
565
|
if (fastMode || fastMode !== false && input.indexOf(quoteChar) === -1) {
|
|
604
566
|
var rows = input.split(newline);
|
|
605
567
|
for (var i = 0; i < rows.length; i++) {
|
|
606
|
-
|
|
607
|
-
cursor +=
|
|
568
|
+
const row2 = rows[i];
|
|
569
|
+
cursor += row2.length;
|
|
608
570
|
if (i !== rows.length - 1)
|
|
609
571
|
cursor += newline.length;
|
|
610
572
|
else if (ignoreLastRow)
|
|
611
573
|
return returnable();
|
|
612
|
-
if (comments &&
|
|
574
|
+
if (comments && row2.substr(0, commentsLen) === comments)
|
|
613
575
|
continue;
|
|
614
576
|
if (stepIsFunction) {
|
|
615
577
|
data = [];
|
|
616
|
-
pushRow(
|
|
578
|
+
pushRow(row2.split(delim));
|
|
617
579
|
doStep();
|
|
618
580
|
if (aborted)
|
|
619
581
|
return returnable();
|
|
620
582
|
} else
|
|
621
|
-
pushRow(
|
|
583
|
+
pushRow(row2.split(delim));
|
|
622
584
|
if (preview && i >= preview) {
|
|
623
585
|
data = data.slice(0, preview);
|
|
624
586
|
return returnable(true);
|
|
@@ -809,14 +771,47 @@ function copy(obj) {
|
|
|
809
771
|
function isFunction(func) {
|
|
810
772
|
return typeof func === "function";
|
|
811
773
|
}
|
|
774
|
+
var Papa = {
|
|
775
|
+
parse: CsvToJson,
|
|
776
|
+
unparse: JsonToCsv,
|
|
777
|
+
RECORD_SEP: String.fromCharCode(30),
|
|
778
|
+
UNIT_SEP: String.fromCharCode(31),
|
|
779
|
+
BYTE_ORDER_MARK,
|
|
780
|
+
BAD_DELIMITERS: ["\r", "\n", '"', BYTE_ORDER_MARK],
|
|
781
|
+
WORKERS_SUPPORTED: false,
|
|
782
|
+
// !IS_WORKER && !!globalThis.Worker
|
|
783
|
+
NODE_STREAM_INPUT: 1,
|
|
784
|
+
// Configurable chunk sizes for local and remote files, respectively
|
|
785
|
+
LocalChunkSize: 1024 * 1024 * 10,
|
|
786
|
+
// 10 M,
|
|
787
|
+
RemoteChunkSize: 1024 * 1024 * 5,
|
|
788
|
+
// 5 M,
|
|
789
|
+
DefaultDelimiter: ",",
|
|
790
|
+
// Used if not specified and detection fail,
|
|
791
|
+
// Exposed for testing and development only
|
|
792
|
+
Parser,
|
|
793
|
+
ParserHandle,
|
|
794
|
+
// BEGIN FORK
|
|
795
|
+
ChunkStreamer
|
|
796
|
+
};
|
|
797
|
+
var papaparse_default = Papa;
|
|
812
798
|
|
|
813
799
|
// dist/papaparse/async-iterator-streamer.js
|
|
814
800
|
var { ChunkStreamer: ChunkStreamer2 } = papaparse_default;
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
801
|
+
var AsyncIteratorStreamer = class extends ChunkStreamer2 {
|
|
802
|
+
textDecoder = new TextDecoder(this._config.encoding);
|
|
803
|
+
constructor(config = {}) {
|
|
804
|
+
super(config);
|
|
805
|
+
}
|
|
806
|
+
// Implement ChunkStreamer base class methods
|
|
807
|
+
// this.pause = function() {
|
|
808
|
+
// ChunkStreamer.prototype.pause.apply(this, arguments);
|
|
809
|
+
// };
|
|
810
|
+
// this.resume = function() {
|
|
811
|
+
// ChunkStreamer.prototype.resume.apply(this, arguments);
|
|
812
|
+
// this._input.resume();
|
|
813
|
+
// };
|
|
814
|
+
async stream(asyncIterator) {
|
|
820
815
|
this._input = asyncIterator;
|
|
821
816
|
try {
|
|
822
817
|
for await (const chunk of asyncIterator) {
|
|
@@ -827,18 +822,17 @@ function AsyncIteratorStreamer(config) {
|
|
|
827
822
|
} catch (error) {
|
|
828
823
|
this._sendError(error);
|
|
829
824
|
}
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
}
|
|
833
|
-
|
|
825
|
+
}
|
|
826
|
+
_nextChunk() {
|
|
827
|
+
}
|
|
828
|
+
// HELPER METHODS
|
|
829
|
+
getStringChunk(chunk) {
|
|
834
830
|
return typeof chunk === "string" ? chunk : this.textDecoder.decode(chunk, { stream: true });
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
AsyncIteratorStreamer.prototype = Object.create(ChunkStreamer2.prototype);
|
|
838
|
-
AsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;
|
|
831
|
+
}
|
|
832
|
+
};
|
|
839
833
|
|
|
840
834
|
// dist/csv-loader.js
|
|
841
|
-
var VERSION = true ? "4.3.0-
|
|
835
|
+
var VERSION = true ? "4.3.0-beta.1" : "latest";
|
|
842
836
|
var DEFAULT_CSV_SHAPE = "object-row-table";
|
|
843
837
|
var CSVLoader = {
|
|
844
838
|
dataType: null,
|
|
@@ -1008,7 +1002,6 @@ function isHeaderRow(row) {
|
|
|
1008
1002
|
}
|
|
1009
1003
|
function readFirstRow(csvText) {
|
|
1010
1004
|
const result = papaparse_default.parse(csvText, {
|
|
1011
|
-
download: false,
|
|
1012
1005
|
dynamicTyping: true,
|
|
1013
1006
|
preview: 1
|
|
1014
1007
|
});
|