@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/cjs/data-processor.js +674 -0
  2. package/dist/cjs/data-processor.js.map +1 -0
  3. package/dist/cjs/export/dataset-to-geojson.js +138 -0
  4. package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
  5. package/dist/cjs/export/geopackage-writer.js +385 -0
  6. package/dist/cjs/export/geopackage-writer.js.map +1 -0
  7. package/dist/cjs/export/geoparquet-writer.js +351 -0
  8. package/dist/cjs/export/geoparquet-writer.js.map +1 -0
  9. package/dist/cjs/export/kml-writer.js +347 -0
  10. package/dist/cjs/export/kml-writer.js.map +1 -0
  11. package/dist/cjs/export/shapefile-writer.js +708 -0
  12. package/dist/cjs/export/shapefile-writer.js.map +1 -0
  13. package/dist/cjs/export/sql-js.js +95 -0
  14. package/dist/cjs/export/sql-js.js.map +1 -0
  15. package/dist/cjs/export/vector-export.js +187 -0
  16. package/dist/cjs/export/vector-export.js.map +1 -0
  17. package/dist/cjs/file-handler.js +997 -0
  18. package/dist/cjs/file-handler.js.map +1 -0
  19. package/dist/cjs/index.js +114 -0
  20. package/dist/cjs/index.js.map +1 -0
  21. package/dist/cjs/kepler-csv-loader.js +163 -0
  22. package/dist/cjs/kepler-csv-loader.js.map +1 -0
  23. package/dist/cjs/loader-registry.js +542 -0
  24. package/dist/cjs/loader-registry.js.map +1 -0
  25. package/dist/cjs/remote-file.js +675 -0
  26. package/dist/cjs/remote-file.js.map +1 -0
  27. package/dist/cjs/shapefile-files.js +281 -0
  28. package/dist/cjs/shapefile-files.js.map +1 -0
  29. package/dist/cjs/types.js +6 -0
  30. package/dist/cjs/types.js.map +1 -0
  31. package/dist/data-processor.d.ts +30 -16
  32. package/dist/esm/data-processor.js +655 -0
  33. package/dist/esm/data-processor.js.map +1 -0
  34. package/dist/esm/export/dataset-to-geojson.js +132 -0
  35. package/dist/esm/export/dataset-to-geojson.js.map +1 -0
  36. package/dist/esm/export/geopackage-writer.js +381 -0
  37. package/dist/esm/export/geopackage-writer.js.map +1 -0
  38. package/dist/esm/export/geoparquet-writer.js +336 -0
  39. package/dist/esm/export/geoparquet-writer.js.map +1 -0
  40. package/dist/esm/export/kml-writer.js +341 -0
  41. package/dist/esm/export/kml-writer.js.map +1 -0
  42. package/dist/esm/export/shapefile-writer.js +701 -0
  43. package/dist/esm/export/shapefile-writer.js.map +1 -0
  44. package/dist/esm/export/sql-js.js +80 -0
  45. package/dist/esm/export/sql-js.js.map +1 -0
  46. package/dist/esm/export/vector-export.js +176 -0
  47. package/dist/esm/export/vector-export.js.map +1 -0
  48. package/dist/esm/file-handler.js +972 -0
  49. package/dist/esm/file-handler.js.map +1 -0
  50. package/dist/esm/index.js +12 -0
  51. package/dist/esm/index.js.map +1 -0
  52. package/dist/esm/kepler-csv-loader.js +152 -0
  53. package/dist/esm/kepler-csv-loader.js.map +1 -0
  54. package/dist/esm/loader-registry.js +510 -0
  55. package/dist/esm/loader-registry.js.map +1 -0
  56. package/dist/esm/remote-file.js +664 -0
  57. package/dist/esm/remote-file.js.map +1 -0
  58. package/dist/esm/shapefile-files.js +264 -0
  59. package/dist/esm/shapefile-files.js.map +1 -0
  60. package/dist/esm/types.js +3 -0
  61. package/dist/esm/types.js.map +1 -0
  62. package/dist/export/dataset-to-geojson.d.ts +12 -0
  63. package/dist/export/geopackage-writer.d.ts +28 -0
  64. package/dist/export/geoparquet-writer.d.ts +7 -0
  65. package/dist/export/kml-writer.d.ts +13 -0
  66. package/dist/export/shapefile-writer.d.ts +28 -0
  67. package/dist/export/sql-js.d.ts +8 -0
  68. package/dist/export/vector-export.d.ts +22 -0
  69. package/dist/file-handler.d.ts +55 -8
  70. package/dist/index.d.ts +6 -0
  71. package/dist/kepler-csv-loader.d.ts +12 -0
  72. package/dist/loader-registry.d.ts +29 -0
  73. package/dist/remote-file.d.ts +79 -0
  74. package/dist/shapefile-files.d.ts +39 -0
  75. package/dist/types.d.ts +13 -0
  76. package/package.json +44 -26
  77. package/dist/data-processor.js +0 -637
  78. package/dist/file-handler.js +0 -409
  79. package/dist/index.js +0 -39
  80. package/dist/types.js +0 -6
@@ -0,0 +1,972 @@
1
+ import _toConsumableArray from "@babel/runtime/helpers/esm/toConsumableArray";
2
+ import _asyncToGenerator from "@babel/runtime/helpers/esm/asyncToGenerator";
3
+ import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
4
+ import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
5
+ import _slicedToArray from "@babel/runtime/helpers/esm/slicedToArray";
6
+ import _awaitAsyncGenerator from "@babel/runtime/helpers/esm/awaitAsyncGenerator";
7
+ import _wrapAsyncGenerator from "@babel/runtime/helpers/esm/wrapAsyncGenerator";
8
+ var _excluded = ["geoArrowBinaryData"],
9
+ _excluded2 = ["progressiveData"];
10
+ import _regeneratorRuntime from "@babel/runtime/regenerator";
11
+ function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
12
+ function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
13
+ function _asyncIterator(r) { var n, t, o, e = 2; for ("undefined" != typeof Symbol && (t = Symbol.asyncIterator, o = Symbol.iterator); e--;) { if (t && null != (n = r[t])) return n.call(r); if (o && null != (n = r[o])) return new AsyncFromSyncIterator(n.call(r)); t = "@@asyncIterator", o = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
14
+ function AsyncFromSyncIterator(r) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var n = r.done; return Promise.resolve(r.value).then(function (r) { return { value: r, done: n }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(r) { this.s = r, this.n = r.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, "return": function _return(r) { var n = this.s["return"]; return void 0 === n ? Promise.resolve({ value: r, done: !0 }) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); }, "throw": function _throw(r) { var n = this.s["return"]; return void 0 === n ? Promise.reject(r) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(r); }
15
+ // SPDX-License-Identifier: MIT
16
+ // Copyright contributors to the kepler.gl project
17
+
18
+ import { parseInBatches } from '@loaders.gl/core';
19
+ import { getBinaryGeometriesFromArrow } from '@loaders.gl/gis';
20
+ import * as arrow from 'apache-arrow';
21
+ import { hardClone, parseGeoArrowOnWorker } from '@loaders.gl/arrow';
22
+ import { isPlainObject, generateHashIdFromString, getApplicationConfig, getError, isArrowTable, convertStructToFixedSizeList } from '@kepler.gl/utils';
23
+ import { generateHashId } from '@kepler.gl/common-utils';
24
+ import { DATASET_FORMATS, DatasetType, GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY, REMOTE_FILE_EXTENSIONS } from '@kepler.gl/constants';
25
+ import { KeplerTable } from '@kepler.gl/table';
26
+ import { processArrowBatches, processGeojson, processKeplerglJSON, processRowObject } from './data-processor';
27
+ import { getKeplerLoaders, isKeplerFileFormatAccepted } from './loader-registry';
28
+ import { createCompanionFetch, getDroppedFileExtension, isZipFileName, unzipShapefileArchive } from './shapefile-files';
29
+ var BATCH_TYPE = {
30
+ METADATA: 'metadata',
31
+ PARTIAL_RESULT: 'partial-result',
32
+ FINAL_RESULT: 'final-result'
33
+ };
34
+ var CSV_LOADER_OPTIONS = {
35
+ shape: 'object-row-table',
36
+ dynamicTyping: false // not working for now
37
+ };
38
+ var ARROW_LOADER_OPTIONS = {
39
+ shape: 'arrow-table',
40
+ batchDebounceMs: 10 // time to delay between batches, for incremental loading
41
+ };
42
+ var PARQUET_LOADER_OPTIONS = {
43
+ shape: 'arrow-table'
44
+ };
45
+ var JSON_LOADER_OPTIONS = {
46
+ shape: 'object-row-table',
47
+ // instruct loaders.gl on what json paths to stream
48
+ jsonpaths: ['$',
49
+ // JSON Row array
50
+ '$.features',
51
+ // GeoJSON
52
+ '$.datasets' // KeplerGL JSON
53
+ ]
54
+ };
55
+ var KML_LOADER_OPTIONS = {
56
+ shape: 'geojson-table'
57
+ };
58
+ var GIS_TABLE_LOADER_OPTIONS = {
59
+ shape: 'geojson-table'
60
+ };
61
+ var WORKER_GEOARROW_ENCODINGS = new Set([GEOARROW_EXTENSIONS.POINT, GEOARROW_EXTENSIONS.LINESTRING, GEOARROW_EXTENSIONS.POLYGON, GEOARROW_EXTENSIONS.MULTIPOINT, GEOARROW_EXTENSIONS.MULTILINESTRING, GEOARROW_EXTENSIONS.MULTIPOLYGON]);
62
+ var EXCEL_LOADER_OPTIONS = {
63
+ shape: 'object-row-table'
64
+ };
65
+ export { isArrowTable };
66
+ function combineGeoArrowWorkerChunks(parsed) {
67
+ var combined = {};
68
+ Object.entries(parsed).forEach(function (_ref) {
69
+ var _ref2 = _slicedToArray(_ref, 2),
70
+ columnIndex = _ref2[0],
71
+ chunks = _ref2[1];
72
+ var ordered = Object.entries(chunks).map(function (_ref3) {
73
+ var _ref4 = _slicedToArray(_ref3, 2),
74
+ chunkIndex = _ref4[0],
75
+ data = _ref4[1];
76
+ return {
77
+ chunkIndex: Number(chunkIndex),
78
+ data: data
79
+ };
80
+ }).sort(function (a, b) {
81
+ return a.chunkIndex - b.chunkIndex;
82
+ });
83
+ if (!ordered.length) {
84
+ return;
85
+ }
86
+ var featureTypes = {
87
+ point: false,
88
+ line: false,
89
+ polygon: false
90
+ };
91
+ var bounds = [Infinity, Infinity, -Infinity, -Infinity];
92
+ var binaryGeometries = [];
93
+ var meanCenters = [];
94
+ var hasMeanCenters = true;
95
+ ordered.forEach(function (_ref5) {
96
+ var data = _ref5.data;
97
+ data.binaryGeometries.forEach(function (geometry) {
98
+ return binaryGeometries.push(geometry);
99
+ });
100
+ featureTypes.point || (featureTypes.point = data.featureTypes.point);
101
+ featureTypes.line || (featureTypes.line = data.featureTypes.line);
102
+ featureTypes.polygon || (featureTypes.polygon = data.featureTypes.polygon);
103
+ bounds[0] = Math.min(bounds[0], data.bounds[0]);
104
+ bounds[1] = Math.min(bounds[1], data.bounds[1]);
105
+ bounds[2] = Math.max(bounds[2], data.bounds[2]);
106
+ bounds[3] = Math.max(bounds[3], data.bounds[3]);
107
+ if (data.meanCenters) {
108
+ data.meanCenters.forEach(function (center) {
109
+ return meanCenters.push(center);
110
+ });
111
+ } else {
112
+ hasMeanCenters = false;
113
+ }
114
+ });
115
+ combined[Number(columnIndex)] = {
116
+ 0: _objectSpread({
117
+ binaryGeometries: binaryGeometries,
118
+ featureTypes: featureTypes,
119
+ bounds: bounds
120
+ }, hasMeanCenters ? {
121
+ meanCenters: meanCenters
122
+ } : {})
123
+ };
124
+ });
125
+ return Object.keys(combined).length ? combined : undefined;
126
+ }
127
+ function geoArrowWorkerChunksThrough(parsed, lastChunkIndex) {
128
+ if (!parsed) {
129
+ return undefined;
130
+ }
131
+ var selected = {};
132
+ Object.entries(parsed).forEach(function (_ref6) {
133
+ var _ref7 = _slicedToArray(_ref6, 2),
134
+ columnIndex = _ref7[0],
135
+ chunks = _ref7[1];
136
+ var selectedChunks = Object.fromEntries(Object.entries(chunks).filter(function (_ref8) {
137
+ var _ref9 = _slicedToArray(_ref8, 1),
138
+ chunkIndex = _ref9[0];
139
+ return Number(chunkIndex) <= lastChunkIndex;
140
+ }));
141
+ if (Object.keys(selectedChunks).length) {
142
+ selected[Number(columnIndex)] = selectedChunks;
143
+ }
144
+ });
145
+ return Object.keys(selected).length ? selected : undefined;
146
+ }
147
+ function createProgressiveArrowData(source, finalData, geoArrowBinaryData) {
148
+ var _source$cols;
149
+ var chunks = (_source$cols = source.cols) === null || _source$cols === void 0 || (_source$cols = _source$cols[0]) === null || _source$cols === void 0 ? void 0 : _source$cols.data;
150
+ if (!Array.isArray(chunks) || chunks.length < 2 || !source.cols) {
151
+ return undefined;
152
+ }
153
+ var _sourceCache = source.geoArrowBinaryData,
154
+ sourceWithoutCache = _objectWithoutProperties(source, _excluded);
155
+ var snapshots = [];
156
+ var rowCount = 0;
157
+ // The final snapshot uses the normally processed (and usually compacted)
158
+ // table. Intermediate snapshots retain record-batch boundaries.
159
+ for (var chunkIndex = 0; chunkIndex < chunks.length - 1; chunkIndex++) {
160
+ rowCount += chunks[chunkIndex].length;
161
+ var cols = source.cols.map(function (column) {
162
+ return column.slice(0, rowCount);
163
+ });
164
+ var snapshotCache = geoArrowWorkerChunksThrough(geoArrowBinaryData, chunkIndex);
165
+ snapshots.push(_objectSpread(_objectSpread({}, sourceWithoutCache), {}, {
166
+ cols: cols
167
+ }, snapshotCache ? {
168
+ geoArrowBinaryData: snapshotCache
169
+ } : {}));
170
+ }
171
+ snapshots.push(finalData);
172
+ return snapshots;
173
+ }
174
+
175
+ /**
176
+ * Convert native GeoArrow chunks to deck.gl binary features without blocking
177
+ * the main thread. WKB/WKT remain on the synchronous fallback because the
178
+ * loaders.gl triangulation worker only supports native GeoArrow encodings.
179
+ */
180
+ export function parseGeoArrowBatchesOnWorkers(_x1) {
181
+ return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
182
+ }
183
+
184
+ /**
185
+ * check if data is an ArrowData object, which is an array of RecordBatch
186
+ * @param data - object to check
187
+ * @returns {boolean} - true if data is an ArrowData object type guarded
188
+ */
189
+ function _parseGeoArrowBatchesOnWorkers() {
190
+ _parseGeoArrowBatchesOnWorkers = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee3(data) {
191
+ var options,
192
+ parseOnWorker,
193
+ jobs,
194
+ parsed,
195
+ results,
196
+ useFallback,
197
+ _args3 = arguments;
198
+ return _regeneratorRuntime.wrap(function _callee3$(_context3) {
199
+ while (1) switch (_context3.prev = _context3.next) {
200
+ case 0:
201
+ options = _args3.length > 1 && _args3[1] !== undefined ? _args3[1] : {};
202
+ parseOnWorker = options.parseOnWorker || parseGeoArrowOnWorker;
203
+ if (!(!options.parseOnWorker && typeof globalThis.Worker === 'undefined')) {
204
+ _context3.next = 4;
205
+ break;
206
+ }
207
+ return _context3.abrupt("return", undefined);
208
+ case 4:
209
+ jobs = [];
210
+ data.fields.forEach(function (field, columnIndex) {
211
+ var _field$metadata, _field$metadata$get, _data$cols, _geoColumn$data;
212
+ var encoding = (_field$metadata = field.metadata) === null || _field$metadata === void 0 || (_field$metadata$get = _field$metadata.get) === null || _field$metadata$get === void 0 ? void 0 : _field$metadata$get.call(_field$metadata, GEOARROW_METADATA_KEY);
213
+ var geoColumn = (_data$cols = data.cols) === null || _data$cols === void 0 ? void 0 : _data$cols[columnIndex];
214
+ if (!WORKER_GEOARROW_ENCODINGS.has(encoding) || !(geoColumn !== null && geoColumn !== void 0 && (_geoColumn$data = geoColumn.data) !== null && _geoColumn$data !== void 0 && _geoColumn$data.length)) {
215
+ return;
216
+ }
217
+ var chunkIndexes = options.latestOnly ? [geoColumn.data.length - 1] : geoColumn.data.map(function (_, index) {
218
+ return index;
219
+ });
220
+ chunkIndexes.forEach(function (chunkIndex) {
221
+ var geometryChunk = encoding === GEOARROW_EXTENSIONS.POINT ? convertStructToFixedSizeList(geoColumn.data[chunkIndex]) : geoColumn.data[chunkIndex];
222
+ var chunkCopy = hardClone(geometryChunk, true);
223
+ var chunkOffset = geoColumn.data.slice(0, chunkIndex).reduce(function (offset, chunk) {
224
+ return offset + chunk.length;
225
+ }, 0);
226
+ var chunkType = chunkCopy.type;
227
+ var transferableType = _objectSpread(_objectSpread({}, chunkType), {}, {
228
+ typeId: chunkType.typeId
229
+ }, chunkType.listSize !== undefined ? {
230
+ listSize: chunkType.listSize
231
+ } : {});
232
+ var sourceData = {
233
+ operation: 'parse-geoarrow',
234
+ chunkData: {
235
+ type: transferableType,
236
+ offset: chunkCopy.offset,
237
+ length: chunkCopy.length,
238
+ nullCount: chunkCopy.nullCount,
239
+ buffers: chunkCopy.buffers,
240
+ children: chunkCopy.children,
241
+ dictionary: chunkCopy.dictionary
242
+ },
243
+ chunkIndex: chunkIndex,
244
+ chunkOffset: chunkOffset,
245
+ geometryEncoding: encoding,
246
+ calculateMeanCenters: true,
247
+ triangle: true
248
+ };
249
+ jobs.push({
250
+ columnIndex: columnIndex,
251
+ chunkIndex: chunkIndex,
252
+ result: parseOnWorker(sourceData),
253
+ fallback: function fallback() {
254
+ return getBinaryGeometriesFromArrow(
255
+ // Use the original buffers: the worker copy may be transferred.
256
+ // A local Data wrapper also accepts chunks from another Arrow runtime.
257
+ arrow.makeVector(new arrow.Data(geometryChunk.type, geometryChunk.offset, geometryChunk.length, geometryChunk.nullCount, geometryChunk.buffers, geometryChunk.children, geometryChunk.dictionary)), sourceData.geometryEncoding, {
258
+ chunkOffset: chunkOffset,
259
+ calculateMeanCenters: true,
260
+ triangulate: true
261
+ });
262
+ }
263
+ });
264
+ });
265
+ });
266
+ if (jobs.length) {
267
+ _context3.next = 8;
268
+ break;
269
+ }
270
+ return _context3.abrupt("return", undefined);
271
+ case 8:
272
+ parsed = {};
273
+ _context3.next = 11;
274
+ return Promise.allSettled(jobs.map(function (job) {
275
+ return job.result;
276
+ }));
277
+ case 11:
278
+ results = _context3.sent;
279
+ // A partial cache can look complete after compaction merges surviving chunks
280
+ // into one. Reparse every selected original batch if acceleration failed.
281
+ // Using the originals also avoids parsing padding in newly compacted buffers.
282
+ useFallback = results.some(function (result) {
283
+ return result.status !== 'fulfilled' || !result.value.binaryDataFromGeoArrow;
284
+ });
285
+ results.forEach(function (result, index) {
286
+ var _jobs$index = jobs[index],
287
+ columnIndex = _jobs$index.columnIndex,
288
+ chunkIndex = _jobs$index.chunkIndex,
289
+ fallback = _jobs$index.fallback;
290
+ var binaryData = useFallback ? fallback() : result.status === 'fulfilled' ? result.value.binaryDataFromGeoArrow : undefined;
291
+ if (!binaryData) {
292
+ return;
293
+ }
294
+ parsed[columnIndex] = parsed[columnIndex] || {};
295
+ parsed[columnIndex][chunkIndex] = binaryData;
296
+ });
297
+ if (Object.keys(parsed).length) {
298
+ _context3.next = 16;
299
+ break;
300
+ }
301
+ return _context3.abrupt("return", undefined);
302
+ case 16:
303
+ return _context3.abrupt("return", options.targetChunkCount === 1 && !options.latestOnly ? combineGeoArrowWorkerChunks(parsed) : parsed);
304
+ case 17:
305
+ case "end":
306
+ return _context3.stop();
307
+ }
308
+ }, _callee3);
309
+ }));
310
+ return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
311
+ }
312
+ export function isArrowData(data) {
313
+ return Array.isArray(data) && Boolean(data.length && data[0].data && data[0].schema);
314
+ }
315
+ export function isGeoJson(json) {
316
+ // json can be feature collection
317
+ // or single feature
318
+ return isPlainObject(json) && (isFeature(json) || isFeatureCollection(json));
319
+ }
320
+ export function isFeature(json) {
321
+ return isPlainObject(json) && json.type === 'Feature' && Boolean(json.geometry);
322
+ }
323
+ export function isFeatureCollection(json) {
324
+ return isPlainObject(json) && json.type === 'FeatureCollection' && Boolean(json.features);
325
+ }
326
+ export function isGeoJsonFeatureArray(json) {
327
+ return Array.isArray(json) && json.length > 0 && isFeature(json[0]);
328
+ }
329
+
330
+ /**
331
+ * Normalize loaders.gl GIS output (Feature arrays, geojson-table, object-row
332
+ * tables of Features) into a GeoJSON Feature or FeatureCollection Kepler can process.
333
+ */
334
+ export function getGeoJsonFromLoaderResult(data) {
335
+ if (isGeoJson(data)) {
336
+ return data;
337
+ }
338
+ if (isGeoJsonFeatureArray(data)) {
339
+ return {
340
+ type: 'FeatureCollection',
341
+ features: data
342
+ };
343
+ }
344
+ if (!isPlainObject(data)) {
345
+ return null;
346
+ }
347
+ if (data.shape === 'geojson-table' && Array.isArray(data.features)) {
348
+ return {
349
+ type: 'FeatureCollection',
350
+ features: data.features
351
+ };
352
+ }
353
+ if (data.shape === 'object-row-table' || data.shape === 'row-table') {
354
+ if (isGeoJsonFeatureArray(data.data)) {
355
+ return {
356
+ type: 'FeatureCollection',
357
+ features: data.data
358
+ };
359
+ }
360
+ // Empty spreadsheet tables stay rows, not empty GeoJSON.
361
+ return null;
362
+ }
363
+ // ShapefileLoader parseInBatches (v3 shape) yields `{data: Feature[]}`,
364
+ // including empty files as `{data: []}`.
365
+ if (Array.isArray(data.data) && (data.data.length === 0 || isGeoJsonFeatureArray(data.data))) {
366
+ return {
367
+ type: 'FeatureCollection',
368
+ features: data.data
369
+ };
370
+ }
371
+ return null;
372
+ }
373
+ function getBatchRows(batch) {
374
+ if (Array.isArray(batch === null || batch === void 0 ? void 0 : batch.features) && (batch.shape === 'geojson-table' || batch.type === 'FeatureCollection')) {
375
+ return batch.features;
376
+ }
377
+ var batchData = isArrowTable(batch.data) ? batch.data.batches : batch.data;
378
+ return Array.isArray(batchData) ? batchData : null;
379
+ }
380
+
381
+ /**
382
+ * Reconstruct the loader table/GIS identity from the current batch. Empty
383
+ * shapefile (`{data: []}`) and spreadsheet (`object-row-table`) batches must
384
+ * not be flattened to a bare `[]`, which processFileData cannot classify.
385
+ *
386
+ * JSON streaming first yields an empty `object-row-table` `partial-result`
387
+ * (with `jsonpath` / `container`). That is a placeholder, not the dataset.
388
+ */
389
+ function getAggregatedLoaderResult(batch, rows, current) {
390
+ if (batch.batchType === BATCH_TYPE.METADATA || batch.batchType === BATCH_TYPE.PARTIAL_RESULT || batch.batchType === BATCH_TYPE.FINAL_RESULT || batch.jsonpath || batch.container) {
391
+ return current;
392
+ }
393
+ if (batch.shape === 'geojson-table') {
394
+ return {
395
+ type: 'FeatureCollection',
396
+ features: _toConsumableArray(rows)
397
+ };
398
+ }
399
+ // Non-empty CSV/Excel stay a flat row array (existing readBatch contract).
400
+ if (rows.length > 0) {
401
+ return current;
402
+ }
403
+ if (batch.shape === 'object-row-table' || batch.shape === 'row-table') {
404
+ return {
405
+ shape: batch.shape,
406
+ data: []
407
+ };
408
+ }
409
+ // ShapefileLoader parseInBatches always yields v3 `{header, data: Feature[]}`.
410
+ if (batch.header != null && Array.isArray(batch.data)) {
411
+ return {
412
+ data: []
413
+ };
414
+ }
415
+ return current;
416
+ }
417
+ export function isRowObject(json) {
418
+ return Array.isArray(json) && isPlainObject(json[0]);
419
+ }
420
+ export function isKeplerGlMap(json) {
421
+ return Boolean(isPlainObject(json) && json.datasets && json.config && json.info && isPlainObject(json.info) && json.info.app === 'kepler.gl');
422
+ }
423
+ export function makeProgressIterator(_x, _x2) {
424
+ return _makeProgressIterator.apply(this, arguments);
425
+ }
426
+
427
+ // eslint-disable-next-line complexity
428
+ function _makeProgressIterator() {
429
+ _makeProgressIterator = _wrapAsyncGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee(asyncIterator, info) {
430
+ var rowCount, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch, rowCountInBatch, percent, progress;
431
+ return _regeneratorRuntime.wrap(function _callee$(_context) {
432
+ while (1) switch (_context.prev = _context.next) {
433
+ case 0:
434
+ rowCount = 0;
435
+ _iteratorAbruptCompletion = false;
436
+ _didIteratorError = false;
437
+ _context.prev = 3;
438
+ _iterator = _asyncIterator(asyncIterator);
439
+ case 5:
440
+ _context.next = 7;
441
+ return _awaitAsyncGenerator(_iterator.next());
442
+ case 7:
443
+ if (!(_iteratorAbruptCompletion = !(_step = _context.sent).done)) {
444
+ _context.next = 18;
445
+ break;
446
+ }
447
+ batch = _step.value;
448
+ // the length could be stored in `batch.length` for arrow batch
449
+ rowCountInBatch = batch.data && (batch.data.length || batch.length) || 0;
450
+ rowCount += rowCountInBatch;
451
+ percent = Number.isFinite(batch.bytesUsed) ? batch.bytesUsed / info.size : null; // Update progress object
452
+ progress = _objectSpread({
453
+ rowCount: rowCount,
454
+ rowCountInBatch: rowCountInBatch
455
+ }, Number.isFinite(percent) ? {
456
+ percent: percent
457
+ } : {});
458
+ _context.next = 15;
459
+ return _objectSpread(_objectSpread({}, batch), {}, {
460
+ progress: progress
461
+ });
462
+ case 15:
463
+ _iteratorAbruptCompletion = false;
464
+ _context.next = 5;
465
+ break;
466
+ case 18:
467
+ _context.next = 24;
468
+ break;
469
+ case 20:
470
+ _context.prev = 20;
471
+ _context.t0 = _context["catch"](3);
472
+ _didIteratorError = true;
473
+ _iteratorError = _context.t0;
474
+ case 24:
475
+ _context.prev = 24;
476
+ _context.prev = 25;
477
+ if (!(_iteratorAbruptCompletion && _iterator["return"] != null)) {
478
+ _context.next = 29;
479
+ break;
480
+ }
481
+ _context.next = 29;
482
+ return _awaitAsyncGenerator(_iterator["return"]());
483
+ case 29:
484
+ _context.prev = 29;
485
+ if (!_didIteratorError) {
486
+ _context.next = 32;
487
+ break;
488
+ }
489
+ throw _iteratorError;
490
+ case 32:
491
+ return _context.finish(29);
492
+ case 33:
493
+ return _context.finish(24);
494
+ case 34:
495
+ case "end":
496
+ return _context.stop();
497
+ }
498
+ }, _callee, null, [[3, 20, 24, 34], [25,, 29, 33]]);
499
+ }));
500
+ return _makeProgressIterator.apply(this, arguments);
501
+ }
502
+ export function getPersistedRemoteFormat(keplerFormat, fileName) {
503
+ var _fileName$match;
504
+ if (keplerFormat && keplerFormat !== 'auto' && Object.prototype.hasOwnProperty.call(REMOTE_FILE_EXTENSIONS, keplerFormat)) {
505
+ return keplerFormat;
506
+ }
507
+ var ext = fileName === null || fileName === void 0 || (_fileName$match = fileName.match(/\.([a-z0-9]+)$/i)) === null || _fileName$match === void 0 || (_fileName$match = _fileName$match[1]) === null || _fileName$match === void 0 ? void 0 : _fileName$match.toLowerCase();
508
+ if (!ext) {
509
+ return undefined;
510
+ }
511
+ if (Object.prototype.hasOwnProperty.call(REMOTE_FILE_EXTENSIONS, ext)) {
512
+ return ext;
513
+ }
514
+ var extensionAliases = {
515
+ jsonl: 'ndjson',
516
+ ndgeojson: 'geojsonl',
517
+ ldgeojson: 'geojsonl',
518
+ xls: 'xlsx',
519
+ xlsm: 'xlsx',
520
+ xlsb: 'xlsx',
521
+ shapefile: 'shp',
522
+ excel: 'xlsx',
523
+ flatgeobuf: 'fgb'
524
+ };
525
+ return extensionAliases[ext];
526
+ }
527
+ export function readBatch(_x3, _x4, _x5, _x6, _x7, _x8, _x9, _x0) {
528
+ return _readBatch.apply(this, arguments);
529
+ }
530
+ function _readBatch() {
531
+ _readBatch = _wrapAsyncGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee2(asyncIterator, fileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, fileSize) {
532
+ var result, batches, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, batch, _yield$_awaitAsyncGen, _JSONPath, streamingPath, batchRows, i;
533
+ return _regeneratorRuntime.wrap(function _callee2$(_context2) {
534
+ while (1) switch (_context2.prev = _context2.next) {
535
+ case 0:
536
+ result = null;
537
+ batches = [];
538
+ _iteratorAbruptCompletion2 = false;
539
+ _didIteratorError2 = false;
540
+ _context2.prev = 4;
541
+ _iterator2 = _asyncIterator(asyncIterator);
542
+ case 6:
543
+ _context2.next = 8;
544
+ return _awaitAsyncGenerator(_iterator2.next());
545
+ case 8:
546
+ if (!(_iteratorAbruptCompletion2 = !(_step2 = _context2.sent).done)) {
547
+ _context2.next = 32;
548
+ break;
549
+ }
550
+ batch = _step2.value;
551
+ if (!(batch.batchType === BATCH_TYPE.FINAL_RESULT)) {
552
+ _context2.next = 24;
553
+ break;
554
+ }
555
+ if (batch.container) {
556
+ result = _objectSpread({}, batch.container);
557
+ }
558
+ // Set the streamed data correctly is Batch json path is set
559
+ // and the path streamed is not the top level object (jsonpath = '$')
560
+ if (!(batch.jsonpath && batch.jsonpath.length > 1)) {
561
+ _context2.next = 21;
562
+ break;
563
+ }
564
+ _context2.next = 15;
565
+ return _awaitAsyncGenerator(import('@loaders.gl/json'));
566
+ case 15:
567
+ _yield$_awaitAsyncGen = _context2.sent;
568
+ _JSONPath = _yield$_awaitAsyncGen._JSONPath;
569
+ streamingPath = new _JSONPath(batch.jsonpath);
570
+ streamingPath.setFieldAtPath(result, batches);
571
+ _context2.next = 22;
572
+ break;
573
+ case 21:
574
+ if (batch.jsonpath && batch.jsonpath.length === 1) {
575
+ // The streamed object is a ROW JSON-batch (jsonpath = '$')
576
+ // row objects
577
+ result = batches;
578
+ }
579
+ case 22:
580
+ _context2.next = 27;
581
+ break;
582
+ case 24:
583
+ batchRows = getBatchRows(batch);
584
+ if (batchRows) {
585
+ for (i = 0; i < batchRows.length; i++) {
586
+ batches.push(batchRows[i]);
587
+ }
588
+ }
589
+ result = getAggregatedLoaderResult(batch, batches, result);
590
+ case 27:
591
+ _context2.next = 29;
592
+ return _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({}, batch), batch.schema ? {
593
+ headers: batch.schema.fields ? batch.schema.fields.map(function (f) {
594
+ return f.name;
595
+ }) : Object.keys(batch.schema)
596
+ } : {}), {}, {
597
+ fileName: fileName
598
+ }, typeof fileSize === 'number' ? {
599
+ fileSize: fileSize
600
+ } : {}), {}, {
601
+ // if dataset is CSV, data is set to the raw batches
602
+ data: result ? result : batches
603
+ }, sourceUrl ? {
604
+ sourceUrl: sourceUrl
605
+ } : {}), keplerFormat ? {
606
+ keplerFormat: keplerFormat
607
+ } : {}), typeof refreshIntervalMs === 'number' && refreshIntervalMs > 0 ? {
608
+ refreshIntervalMs: refreshIntervalMs
609
+ } : {}), etag ? {
610
+ etag: etag
611
+ } : {}), lastModified ? {
612
+ lastModified: lastModified
613
+ } : {});
614
+ case 29:
615
+ _iteratorAbruptCompletion2 = false;
616
+ _context2.next = 6;
617
+ break;
618
+ case 32:
619
+ _context2.next = 38;
620
+ break;
621
+ case 34:
622
+ _context2.prev = 34;
623
+ _context2.t0 = _context2["catch"](4);
624
+ _didIteratorError2 = true;
625
+ _iteratorError2 = _context2.t0;
626
+ case 38:
627
+ _context2.prev = 38;
628
+ _context2.prev = 39;
629
+ if (!(_iteratorAbruptCompletion2 && _iterator2["return"] != null)) {
630
+ _context2.next = 43;
631
+ break;
632
+ }
633
+ _context2.next = 43;
634
+ return _awaitAsyncGenerator(_iterator2["return"]());
635
+ case 43:
636
+ _context2.prev = 43;
637
+ if (!_didIteratorError2) {
638
+ _context2.next = 46;
639
+ break;
640
+ }
641
+ throw _iteratorError2;
642
+ case 46:
643
+ return _context2.finish(43);
644
+ case 47:
645
+ return _context2.finish(38);
646
+ case 48:
647
+ case "end":
648
+ return _context2.stop();
649
+ }
650
+ }, _callee2, null, [[4, 34, 38, 48], [39,, 43, 47]]);
651
+ }));
652
+ return _readBatch.apply(this, arguments);
653
+ }
654
+ export function readFileInBatches(_x10) {
655
+ return _readFileInBatches.apply(this, arguments);
656
+ }
657
+ function _readFileInBatches() {
658
+ _readFileInBatches = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee4(_ref0) {
659
+ var file, _ref0$loaders, loaders, _ref0$loadOptions, loadOptions, companionFiles, fileName, displayFileName, unzipped, shapefile, hasExtension, mimeType, isShapefile, companionFetch, batchIterator, progressIterator, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified;
660
+ return _regeneratorRuntime.wrap(function _callee4$(_context4) {
661
+ while (1) switch (_context4.prev = _context4.next) {
662
+ case 0:
663
+ file = _ref0.file, _ref0$loaders = _ref0.loaders, loaders = _ref0$loaders === void 0 ? [] : _ref0$loaders, _ref0$loadOptions = _ref0.loadOptions, loadOptions = _ref0$loadOptions === void 0 ? {} : _ref0$loadOptions, companionFiles = _ref0.companionFiles, fileName = _ref0.fileName;
664
+ displayFileName = fileName || file.name; // Local drops expand mixed zips in the load queue before this runs. A `.zip`
665
+ // that still arrives here is a shapefile archive (remote URL, or a direct call).
666
+ if (!(isZipFileName(file.name) && isKeplerFileFormatAccepted('shp'))) {
667
+ _context4.next = 10;
668
+ break;
669
+ }
670
+ _context4.next = 5;
671
+ return unzipShapefileArchive(file);
672
+ case 5:
673
+ unzipped = _context4.sent;
674
+ shapefile = unzipped.find(function (entry) {
675
+ return getDroppedFileExtension(entry.name) === 'shp';
676
+ });
677
+ if (shapefile) {
678
+ _context4.next = 9;
679
+ break;
680
+ }
681
+ throw new Error('Zip archive does not contain a shapefile (.shp)');
682
+ case 9:
683
+ return _context4.abrupt("return", readFileInBatches({
684
+ file: shapefile,
685
+ loaders: loaders,
686
+ loadOptions: loadOptions,
687
+ companionFiles: [].concat(_toConsumableArray(unzipped), _toConsumableArray(companionFiles || [])),
688
+ fileName: displayFileName
689
+ }));
690
+ case 10:
691
+ _context4.next = 12;
692
+ return getKeplerLoaders(file, loaders);
693
+ case 12:
694
+ loaders = _context4.sent;
695
+ hasExtension = /\.[a-z0-9]+$/i.test(file.name);
696
+ mimeType = !hasExtension && file.type ? file.type : undefined; // Shapefile sidecars are resolved via a custom fetch that cannot be cloned
697
+ // into a worker. Other formats keep loaders.gl worker parsing.
698
+ isShapefile = getDroppedFileExtension(file.name) === 'shp';
699
+ companionFetch = isShapefile ? createCompanionFetch(companionFiles) : undefined;
700
+ loadOptions = _objectSpread(_objectSpread(_objectSpread(_objectSpread({
701
+ csv: CSV_LOADER_OPTIONS,
702
+ arrow: ARROW_LOADER_OPTIONS,
703
+ json: JSON_LOADER_OPTIONS,
704
+ parquet: PARQUET_LOADER_OPTIONS,
705
+ kml: KML_LOADER_OPTIONS,
706
+ gpx: KML_LOADER_OPTIONS,
707
+ tcx: KML_LOADER_OPTIONS,
708
+ shapefile: GIS_TABLE_LOADER_OPTIONS,
709
+ shp: {
710
+ _maxDimensions: 2
711
+ },
712
+ excel: EXCEL_LOADER_OPTIONS,
713
+ flatgeobuf: GIS_TABLE_LOADER_OPTIONS,
714
+ gis: {
715
+ reproject: true,
716
+ _targetCrs: 'WGS84'
717
+ },
718
+ metadata: true
719
+ }, isShapefile ? {
720
+ worker: false
721
+ } : {}), mimeType ? {
722
+ mimeType: mimeType
723
+ } : {}), companionFetch ? {
724
+ fetch: companionFetch
725
+ } : {}), loadOptions);
726
+ _context4.next = 20;
727
+ return parseInBatches(file, loaders, loadOptions);
728
+ case 20:
729
+ batchIterator = _context4.sent;
730
+ progressIterator = makeProgressIterator(batchIterator, {
731
+ size: file.size
732
+ });
733
+ sourceUrl = file.keplerSourceUrl;
734
+ keplerFormat = file.keplerFormat;
735
+ refreshIntervalMs = file.keplerRefreshIntervalMs;
736
+ etag = file.keplerEtag;
737
+ lastModified = file.keplerLastModified;
738
+ return _context4.abrupt("return", readBatch(progressIterator, displayFileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, file.size));
739
+ case 28:
740
+ case "end":
741
+ return _context4.stop();
742
+ }
743
+ }, _callee4);
744
+ }));
745
+ return _readFileInBatches.apply(this, arguments);
746
+ }
747
+ export function processFileData(_x11) {
748
+ return _processFileData.apply(this, arguments);
749
+ }
750
+
751
+ /**
752
+ * Expand staged files into cumulative payloads for one redraw per Arrow batch.
753
+ * Non-progressive files are present only in the first payload.
754
+ */
755
+ function _processFileData() {
756
+ _processFileData = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee5(_ref1) {
757
+ var _getApplicationConfig;
758
+ var content, fileCache, fileName, data, format, processor, id, table, processorResult, geojsonData, tableRows, _result, result, progressiveData, _result$cols$0$data$l, _result$cols$, latestOnly, targetChunkCount, workerSource, sourceTable, cols, workerGeoArrowBinaryData, geoArrowBinaryData, sourceUrl, remoteFormat, fileSize, metadata;
759
+ return _regeneratorRuntime.wrap(function _callee5$(_context5) {
760
+ while (1) switch (_context5.prev = _context5.next) {
761
+ case 0:
762
+ content = _ref1.content, fileCache = _ref1.fileCache;
763
+ fileName = content.fileName, data = content.data;
764
+ // Hash the source URL when present so two remote files that share a filename
765
+ // (or a local file and a remote URL with the same name) do not collide and
766
+ // overwrite each other. The UI shows `label` (the filename), not this id.
767
+ id = generateHashIdFromString(content.sourceUrl || fileName); // decide on which table class to use based on application config
768
+ table = (_getApplicationConfig = getApplicationConfig().table) !== null && _getApplicationConfig !== void 0 ? _getApplicationConfig : KeplerTable;
769
+ if (typeof table.getFileProcessor === 'function') {
770
+ // use custom processors from table class
771
+ processorResult = table.getFileProcessor(data);
772
+ format = processorResult.format;
773
+ processor = processorResult.processor;
774
+ } else {
775
+ // use default processors
776
+ geojsonData = getGeoJsonFromLoaderResult(data);
777
+ if (isArrowData(data)) {
778
+ format = DATASET_FORMATS.arrow;
779
+ processor = processArrowBatches;
780
+ } else if (isKeplerGlMap(data)) {
781
+ format = DATASET_FORMATS.keplergl;
782
+ processor = processKeplerglJSON;
783
+ } else if (geojsonData) {
784
+ format = DATASET_FORMATS.geojson;
785
+ processor = function processor() {
786
+ return processGeojson(geojsonData);
787
+ };
788
+ } else if (isRowObject(data)) {
789
+ // csv file goes here
790
+ format = DATASET_FORMATS.row;
791
+ processor = processRowObject;
792
+ } else if (isPlainObject(data)) {
793
+ tableRows = data.data;
794
+ if ((data.shape === 'object-row-table' || data.shape === 'row-table') && Array.isArray(tableRows)) {
795
+ format = DATASET_FORMATS.row;
796
+ processor = function processor() {
797
+ return processRowObject(tableRows);
798
+ };
799
+ }
800
+ }
801
+ }
802
+ if (!(format && processor)) {
803
+ _context5.next = 40;
804
+ break;
805
+ }
806
+ _context5.prev = 6;
807
+ if (!(format === DATASET_FORMATS.arrow && content.skipArrowCompact)) {
808
+ _context5.next = 13;
809
+ break;
810
+ }
811
+ _context5.next = 10;
812
+ return processArrowBatches(data, {
813
+ compact: false
814
+ });
815
+ case 10:
816
+ _context5.t0 = _context5.sent;
817
+ _context5.next = 16;
818
+ break;
819
+ case 13:
820
+ _context5.next = 15;
821
+ return processor(data);
822
+ case 15:
823
+ _context5.t0 = _context5.sent;
824
+ case 16:
825
+ result = _context5.t0;
826
+ _context5.next = 22;
827
+ break;
828
+ case 19:
829
+ _context5.prev = 19;
830
+ _context5.t1 = _context5["catch"](6);
831
+ throw new Error("Can not process uploaded file, ".concat(getError(_context5.t1)));
832
+ case 22:
833
+ if (!(format === DATASET_FORMATS.arrow && (_result = result) !== null && _result !== void 0 && _result.cols)) {
834
+ _context5.next = 33;
835
+ break;
836
+ }
837
+ latestOnly = Boolean(content.skipArrowCompact);
838
+ targetChunkCount = (_result$cols$0$data$l = (_result$cols$ = result.cols[0]) === null || _result$cols$ === void 0 || (_result$cols$ = _result$cols$.data) === null || _result$cols$ === void 0 ? void 0 : _result$cols$.length) !== null && _result$cols$0$data$l !== void 0 ? _result$cols$0$data$l : 0;
839
+ workerSource = result; // Final file processing compacts Arrow batches to keep the number of deck.gl
840
+ // sublayers low. Schedule worker jobs from the original batches first so
841
+ // geometry conversion and triangulation can still run in parallel, then
842
+ // combine their output to match the single compacted table chunk.
843
+ if (!latestOnly && Array.isArray(data) && data.length > targetChunkCount) {
844
+ sourceTable = new arrow.Table(data);
845
+ cols = _toConsumableArray(Array(sourceTable.numCols).keys()).map(function (i) {
846
+ return sourceTable.getChildAt(i);
847
+ });
848
+ workerSource = _objectSpread(_objectSpread({}, result), {}, {
849
+ cols: cols
850
+ });
851
+ }
852
+ _context5.next = 29;
853
+ return parseGeoArrowBatchesOnWorkers(workerSource, {
854
+ latestOnly: latestOnly
855
+ });
856
+ case 29:
857
+ workerGeoArrowBinaryData = _context5.sent;
858
+ geoArrowBinaryData = targetChunkCount === 1 && !latestOnly && workerGeoArrowBinaryData ? combineGeoArrowWorkerChunks(workerGeoArrowBinaryData) : workerGeoArrowBinaryData;
859
+ if (geoArrowBinaryData) {
860
+ result = _objectSpread(_objectSpread({}, result), {}, {
861
+ geoArrowBinaryData: geoArrowBinaryData
862
+ });
863
+ }
864
+ if (content.prepareProgressiveReplay && !latestOnly) {
865
+ progressiveData = createProgressiveArrowData(workerSource, result, workerGeoArrowBinaryData);
866
+ }
867
+ case 33:
868
+ sourceUrl = content.sourceUrl;
869
+ remoteFormat = getPersistedRemoteFormat(content.keplerFormat, content.fileName);
870
+ fileSize = typeof content.fileSize === 'number' ? content.fileSize : undefined;
871
+ metadata = _objectSpread(_objectSpread({}, sourceUrl ? _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({
872
+ source: sourceUrl
873
+ }, remoteFormat ? {
874
+ sourceFormat: remoteFormat
875
+ } : {}), typeof content.refreshIntervalMs === 'number' && content.refreshIntervalMs > 0 ? {
876
+ refreshIntervalMs: content.refreshIntervalMs
877
+ } : {}), typeof content.etag === 'string' && content.etag ? {
878
+ etag: content.etag
879
+ } : {}), typeof content.lastModified === 'string' && content.lastModified ? {
880
+ lastModified: content.lastModified
881
+ } : {}), {}, {
882
+ lastFetchedAt: Date.now()
883
+ }) : {}), fileSize != null ? {
884
+ size: fileSize
885
+ } : {});
886
+ return _context5.abrupt("return", [].concat(_toConsumableArray(fileCache), [_objectSpread(_objectSpread({
887
+ data: result
888
+ }, progressiveData ? {
889
+ progressiveData: progressiveData
890
+ } : {}), {}, {
891
+ info: _objectSpread({
892
+ id: id,
893
+ label: content.fileName,
894
+ format: format
895
+ }, sourceUrl ? {
896
+ type: DatasetType.EXTERNALLY_HOSTED
897
+ } : {})
898
+ }, sourceUrl || fileSize != null ? {
899
+ metadata: metadata
900
+ } : {})]));
901
+ case 40:
902
+ throw new Error('Can not process uploaded file, unknown file format');
903
+ case 41:
904
+ case "end":
905
+ return _context5.stop();
906
+ }
907
+ }, _callee5, null, [[6, 19]]);
908
+ }));
909
+ return _processFileData.apply(this, arguments);
910
+ }
911
+ export function fileCacheToProgressiveSteps(fileCache) {
912
+ var numberOfSteps = Math.max.apply(Math, [1].concat(_toConsumableArray(fileCache.map(function (file) {
913
+ var _file$progressiveData;
914
+ return ((_file$progressiveData = file.progressiveData) === null || _file$progressiveData === void 0 ? void 0 : _file$progressiveData.length) || 1;
915
+ }))));
916
+ return Array.from({
917
+ length: numberOfSteps
918
+ }, function (_, stepIndex) {
919
+ return fileCache.flatMap(function (file) {
920
+ var progressiveData = file.progressiveData,
921
+ fileWithoutProgressiveData = _objectWithoutProperties(file, _excluded2);
922
+ if (!progressiveData) {
923
+ return stepIndex === 0 ? [fileWithoutProgressiveData] : [];
924
+ }
925
+ var data = progressiveData[stepIndex];
926
+ return data ? [_objectSpread(_objectSpread({}, fileWithoutProgressiveData), {}, {
927
+ data: data
928
+ })] : [];
929
+ });
930
+ }).filter(function (step) {
931
+ return step.length;
932
+ });
933
+ }
934
+ export function filesToDataPayload(fileCache, options) {
935
+ // seperate out files which could be a single datasets. or a keplergl map json
936
+ var collection = fileCache.reduce(function (accu, file) {
937
+ var data = file.data,
938
+ info = file.info;
939
+ if ((info === null || info === void 0 ? void 0 : info.format) === DATASET_FORMATS.keplergl) {
940
+ // if file contains a single kepler map dataset & config
941
+ accu.keplerMaps.push(_objectSpread(_objectSpread({}, data), {}, {
942
+ options: _objectSpread({
943
+ centerMap: !(data.config && data.config.mapState)
944
+ }, options || {})
945
+ }));
946
+ } else if (DATASET_FORMATS[info === null || info === void 0 ? void 0 : info.format]) {
947
+ // if file contains only data
948
+ var newDataset = _objectSpread({
949
+ data: data,
950
+ info: _objectSpread({
951
+ id: (info === null || info === void 0 ? void 0 : info.id) || generateHashId(4)
952
+ }, info || {})
953
+ }, file.metadata ? {
954
+ metadata: file.metadata
955
+ } : {});
956
+ accu.datasets.push(newDataset);
957
+ }
958
+ return accu;
959
+ }, {
960
+ datasets: [],
961
+ keplerMaps: []
962
+ });
963
+
964
+ // add kepler map first with config
965
+ // add datasets later in one add data call
966
+ return collection.keplerMaps.concat(_objectSpread({
967
+ datasets: collection.datasets
968
+ }, options && collection.datasets.length ? {
969
+ options: options
970
+ } : {}));
971
+ }
972
+ //# sourceMappingURL=file-handler.js.map