@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/cjs/data-processor.js +674 -0
  2. package/dist/cjs/data-processor.js.map +1 -0
  3. package/dist/cjs/export/dataset-to-geojson.js +138 -0
  4. package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
  5. package/dist/cjs/export/geopackage-writer.js +385 -0
  6. package/dist/cjs/export/geopackage-writer.js.map +1 -0
  7. package/dist/cjs/export/geoparquet-writer.js +351 -0
  8. package/dist/cjs/export/geoparquet-writer.js.map +1 -0
  9. package/dist/cjs/export/kml-writer.js +347 -0
  10. package/dist/cjs/export/kml-writer.js.map +1 -0
  11. package/dist/cjs/export/shapefile-writer.js +708 -0
  12. package/dist/cjs/export/shapefile-writer.js.map +1 -0
  13. package/dist/cjs/export/sql-js.js +95 -0
  14. package/dist/cjs/export/sql-js.js.map +1 -0
  15. package/dist/cjs/export/vector-export.js +187 -0
  16. package/dist/cjs/export/vector-export.js.map +1 -0
  17. package/dist/cjs/file-handler.js +997 -0
  18. package/dist/cjs/file-handler.js.map +1 -0
  19. package/dist/cjs/index.js +114 -0
  20. package/dist/cjs/index.js.map +1 -0
  21. package/dist/cjs/kepler-csv-loader.js +163 -0
  22. package/dist/cjs/kepler-csv-loader.js.map +1 -0
  23. package/dist/cjs/loader-registry.js +542 -0
  24. package/dist/cjs/loader-registry.js.map +1 -0
  25. package/dist/cjs/remote-file.js +675 -0
  26. package/dist/cjs/remote-file.js.map +1 -0
  27. package/dist/cjs/shapefile-files.js +281 -0
  28. package/dist/cjs/shapefile-files.js.map +1 -0
  29. package/dist/cjs/types.js +6 -0
  30. package/dist/cjs/types.js.map +1 -0
  31. package/dist/data-processor.d.ts +30 -16
  32. package/dist/esm/data-processor.js +655 -0
  33. package/dist/esm/data-processor.js.map +1 -0
  34. package/dist/esm/export/dataset-to-geojson.js +132 -0
  35. package/dist/esm/export/dataset-to-geojson.js.map +1 -0
  36. package/dist/esm/export/geopackage-writer.js +381 -0
  37. package/dist/esm/export/geopackage-writer.js.map +1 -0
  38. package/dist/esm/export/geoparquet-writer.js +336 -0
  39. package/dist/esm/export/geoparquet-writer.js.map +1 -0
  40. package/dist/esm/export/kml-writer.js +341 -0
  41. package/dist/esm/export/kml-writer.js.map +1 -0
  42. package/dist/esm/export/shapefile-writer.js +701 -0
  43. package/dist/esm/export/shapefile-writer.js.map +1 -0
  44. package/dist/esm/export/sql-js.js +80 -0
  45. package/dist/esm/export/sql-js.js.map +1 -0
  46. package/dist/esm/export/vector-export.js +176 -0
  47. package/dist/esm/export/vector-export.js.map +1 -0
  48. package/dist/esm/file-handler.js +972 -0
  49. package/dist/esm/file-handler.js.map +1 -0
  50. package/dist/esm/index.js +12 -0
  51. package/dist/esm/index.js.map +1 -0
  52. package/dist/esm/kepler-csv-loader.js +152 -0
  53. package/dist/esm/kepler-csv-loader.js.map +1 -0
  54. package/dist/esm/loader-registry.js +510 -0
  55. package/dist/esm/loader-registry.js.map +1 -0
  56. package/dist/esm/remote-file.js +664 -0
  57. package/dist/esm/remote-file.js.map +1 -0
  58. package/dist/esm/shapefile-files.js +264 -0
  59. package/dist/esm/shapefile-files.js.map +1 -0
  60. package/dist/esm/types.js +3 -0
  61. package/dist/esm/types.js.map +1 -0
  62. package/dist/export/dataset-to-geojson.d.ts +12 -0
  63. package/dist/export/geopackage-writer.d.ts +28 -0
  64. package/dist/export/geoparquet-writer.d.ts +7 -0
  65. package/dist/export/kml-writer.d.ts +13 -0
  66. package/dist/export/shapefile-writer.d.ts +28 -0
  67. package/dist/export/sql-js.d.ts +8 -0
  68. package/dist/export/vector-export.d.ts +22 -0
  69. package/dist/file-handler.d.ts +55 -8
  70. package/dist/index.d.ts +6 -0
  71. package/dist/kepler-csv-loader.d.ts +12 -0
  72. package/dist/loader-registry.d.ts +29 -0
  73. package/dist/remote-file.d.ts +79 -0
  74. package/dist/shapefile-files.d.ts +39 -0
  75. package/dist/types.d.ts +13 -0
  76. package/package.json +44 -26
  77. package/dist/data-processor.js +0 -637
  78. package/dist/file-handler.js +0 -409
  79. package/dist/index.js +0 -39
  80. package/dist/types.js +0 -6
@@ -0,0 +1,997 @@
1
+ "use strict";
2
+
3
+ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
+ var _typeof = require("@babel/runtime/helpers/typeof");
5
+ Object.defineProperty(exports, "__esModule", {
6
+ value: true
7
+ });
8
+ exports.fileCacheToProgressiveSteps = fileCacheToProgressiveSteps;
9
+ exports.filesToDataPayload = filesToDataPayload;
10
+ exports.getGeoJsonFromLoaderResult = getGeoJsonFromLoaderResult;
11
+ exports.getPersistedRemoteFormat = getPersistedRemoteFormat;
12
+ exports.isArrowData = isArrowData;
13
+ Object.defineProperty(exports, "isArrowTable", {
14
+ enumerable: true,
15
+ get: function get() {
16
+ return _utils.isArrowTable;
17
+ }
18
+ });
19
+ exports.isFeature = isFeature;
20
+ exports.isFeatureCollection = isFeatureCollection;
21
+ exports.isGeoJson = isGeoJson;
22
+ exports.isGeoJsonFeatureArray = isGeoJsonFeatureArray;
23
+ exports.isKeplerGlMap = isKeplerGlMap;
24
+ exports.isRowObject = isRowObject;
25
+ exports.makeProgressIterator = makeProgressIterator;
26
+ exports.parseGeoArrowBatchesOnWorkers = parseGeoArrowBatchesOnWorkers;
27
+ exports.processFileData = processFileData;
28
+ exports.readBatch = readBatch;
29
+ exports.readFileInBatches = readFileInBatches;
30
+ var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
31
+ var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
32
+ var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
33
+ var _objectWithoutProperties2 = _interopRequireDefault(require("@babel/runtime/helpers/objectWithoutProperties"));
34
+ var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
35
+ var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
36
+ var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
37
+ var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
38
+ var _core = require("@loaders.gl/core");
39
+ var _gis = require("@loaders.gl/gis");
40
+ var arrow = _interopRequireWildcard(require("apache-arrow"));
41
+ var _arrow = require("@loaders.gl/arrow");
42
+ var _utils = require("@kepler.gl/utils");
43
+ var _commonUtils = require("@kepler.gl/common-utils");
44
+ var _constants = require("@kepler.gl/constants");
45
+ var _table = require("@kepler.gl/table");
46
+ var _dataProcessor = require("./data-processor");
47
+ var _loaderRegistry = require("./loader-registry");
48
+ var _shapefileFiles = require("./shapefile-files");
49
+ var _excluded = ["geoArrowBinaryData"],
50
+ _excluded2 = ["progressiveData"];
51
+ function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function _interopRequireWildcard(e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, "default": e }; if (null === e || "object" != _typeof(e) && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (var _t in e) "default" !== _t && {}.hasOwnProperty.call(e, _t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, _t)) && (i.get || i.set) ? o(f, _t, i) : f[_t] = e[_t]); return f; })(e, t); }
52
+ function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
53
+ function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { (0, _defineProperty2["default"])(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
54
+ function _asyncIterator(r) { var n, t, o, e = 2; for ("undefined" != typeof Symbol && (t = Symbol.asyncIterator, o = Symbol.iterator); e--;) { if (t && null != (n = r[t])) return n.call(r); if (o && null != (n = r[o])) return new AsyncFromSyncIterator(n.call(r)); t = "@@asyncIterator", o = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
55
+ function AsyncFromSyncIterator(r) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var n = r.done; return Promise.resolve(r.value).then(function (r) { return { value: r, done: n }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(r) { this.s = r, this.n = r.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, "return": function _return(r) { var n = this.s["return"]; return void 0 === n ? Promise.resolve({ value: r, done: !0 }) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); }, "throw": function _throw(r) { var n = this.s["return"]; return void 0 === n ? Promise.reject(r) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(r); } // SPDX-License-Identifier: MIT
56
+ // Copyright contributors to the kepler.gl project
57
+ var BATCH_TYPE = {
58
+ METADATA: 'metadata',
59
+ PARTIAL_RESULT: 'partial-result',
60
+ FINAL_RESULT: 'final-result'
61
+ };
62
+ var CSV_LOADER_OPTIONS = {
63
+ shape: 'object-row-table',
64
+ dynamicTyping: false // not working for now
65
+ };
66
+ var ARROW_LOADER_OPTIONS = {
67
+ shape: 'arrow-table',
68
+ batchDebounceMs: 10 // time to delay between batches, for incremental loading
69
+ };
70
+ var PARQUET_LOADER_OPTIONS = {
71
+ shape: 'arrow-table'
72
+ };
73
+ var JSON_LOADER_OPTIONS = {
74
+ shape: 'object-row-table',
75
+ // instruct loaders.gl on what json paths to stream
76
+ jsonpaths: ['$',
77
+ // JSON Row array
78
+ '$.features',
79
+ // GeoJSON
80
+ '$.datasets' // KeplerGL JSON
81
+ ]
82
+ };
83
+ var KML_LOADER_OPTIONS = {
84
+ shape: 'geojson-table'
85
+ };
86
+ var GIS_TABLE_LOADER_OPTIONS = {
87
+ shape: 'geojson-table'
88
+ };
89
+ var WORKER_GEOARROW_ENCODINGS = new Set([_constants.GEOARROW_EXTENSIONS.POINT, _constants.GEOARROW_EXTENSIONS.LINESTRING, _constants.GEOARROW_EXTENSIONS.POLYGON, _constants.GEOARROW_EXTENSIONS.MULTIPOINT, _constants.GEOARROW_EXTENSIONS.MULTILINESTRING, _constants.GEOARROW_EXTENSIONS.MULTIPOLYGON]);
90
+ var EXCEL_LOADER_OPTIONS = {
91
+ shape: 'object-row-table'
92
+ };
93
+ function combineGeoArrowWorkerChunks(parsed) {
94
+ var combined = {};
95
+ Object.entries(parsed).forEach(function (_ref) {
96
+ var _ref2 = (0, _slicedToArray2["default"])(_ref, 2),
97
+ columnIndex = _ref2[0],
98
+ chunks = _ref2[1];
99
+ var ordered = Object.entries(chunks).map(function (_ref3) {
100
+ var _ref4 = (0, _slicedToArray2["default"])(_ref3, 2),
101
+ chunkIndex = _ref4[0],
102
+ data = _ref4[1];
103
+ return {
104
+ chunkIndex: Number(chunkIndex),
105
+ data: data
106
+ };
107
+ }).sort(function (a, b) {
108
+ return a.chunkIndex - b.chunkIndex;
109
+ });
110
+ if (!ordered.length) {
111
+ return;
112
+ }
113
+ var featureTypes = {
114
+ point: false,
115
+ line: false,
116
+ polygon: false
117
+ };
118
+ var bounds = [Infinity, Infinity, -Infinity, -Infinity];
119
+ var binaryGeometries = [];
120
+ var meanCenters = [];
121
+ var hasMeanCenters = true;
122
+ ordered.forEach(function (_ref5) {
123
+ var data = _ref5.data;
124
+ data.binaryGeometries.forEach(function (geometry) {
125
+ return binaryGeometries.push(geometry);
126
+ });
127
+ featureTypes.point || (featureTypes.point = data.featureTypes.point);
128
+ featureTypes.line || (featureTypes.line = data.featureTypes.line);
129
+ featureTypes.polygon || (featureTypes.polygon = data.featureTypes.polygon);
130
+ bounds[0] = Math.min(bounds[0], data.bounds[0]);
131
+ bounds[1] = Math.min(bounds[1], data.bounds[1]);
132
+ bounds[2] = Math.max(bounds[2], data.bounds[2]);
133
+ bounds[3] = Math.max(bounds[3], data.bounds[3]);
134
+ if (data.meanCenters) {
135
+ data.meanCenters.forEach(function (center) {
136
+ return meanCenters.push(center);
137
+ });
138
+ } else {
139
+ hasMeanCenters = false;
140
+ }
141
+ });
142
+ combined[Number(columnIndex)] = {
143
+ 0: _objectSpread({
144
+ binaryGeometries: binaryGeometries,
145
+ featureTypes: featureTypes,
146
+ bounds: bounds
147
+ }, hasMeanCenters ? {
148
+ meanCenters: meanCenters
149
+ } : {})
150
+ };
151
+ });
152
+ return Object.keys(combined).length ? combined : undefined;
153
+ }
154
+ function geoArrowWorkerChunksThrough(parsed, lastChunkIndex) {
155
+ if (!parsed) {
156
+ return undefined;
157
+ }
158
+ var selected = {};
159
+ Object.entries(parsed).forEach(function (_ref6) {
160
+ var _ref7 = (0, _slicedToArray2["default"])(_ref6, 2),
161
+ columnIndex = _ref7[0],
162
+ chunks = _ref7[1];
163
+ var selectedChunks = Object.fromEntries(Object.entries(chunks).filter(function (_ref8) {
164
+ var _ref9 = (0, _slicedToArray2["default"])(_ref8, 1),
165
+ chunkIndex = _ref9[0];
166
+ return Number(chunkIndex) <= lastChunkIndex;
167
+ }));
168
+ if (Object.keys(selectedChunks).length) {
169
+ selected[Number(columnIndex)] = selectedChunks;
170
+ }
171
+ });
172
+ return Object.keys(selected).length ? selected : undefined;
173
+ }
174
+ function createProgressiveArrowData(source, finalData, geoArrowBinaryData) {
175
+ var _source$cols;
176
+ var chunks = (_source$cols = source.cols) === null || _source$cols === void 0 || (_source$cols = _source$cols[0]) === null || _source$cols === void 0 ? void 0 : _source$cols.data;
177
+ if (!Array.isArray(chunks) || chunks.length < 2 || !source.cols) {
178
+ return undefined;
179
+ }
180
+ var _sourceCache = source.geoArrowBinaryData,
181
+ sourceWithoutCache = (0, _objectWithoutProperties2["default"])(source, _excluded);
182
+ var snapshots = [];
183
+ var rowCount = 0;
184
+ // The final snapshot uses the normally processed (and usually compacted)
185
+ // table. Intermediate snapshots retain record-batch boundaries.
186
+ for (var chunkIndex = 0; chunkIndex < chunks.length - 1; chunkIndex++) {
187
+ rowCount += chunks[chunkIndex].length;
188
+ var cols = source.cols.map(function (column) {
189
+ return column.slice(0, rowCount);
190
+ });
191
+ var snapshotCache = geoArrowWorkerChunksThrough(geoArrowBinaryData, chunkIndex);
192
+ snapshots.push(_objectSpread(_objectSpread({}, sourceWithoutCache), {}, {
193
+ cols: cols
194
+ }, snapshotCache ? {
195
+ geoArrowBinaryData: snapshotCache
196
+ } : {}));
197
+ }
198
+ snapshots.push(finalData);
199
+ return snapshots;
200
+ }
201
+
202
+ /**
203
+ * Convert native GeoArrow chunks to deck.gl binary features without blocking
204
+ * the main thread. WKB/WKT remain on the synchronous fallback because the
205
+ * loaders.gl triangulation worker only supports native GeoArrow encodings.
206
+ */
207
+ function parseGeoArrowBatchesOnWorkers(_x1) {
208
+ return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
209
+ }
210
+ /**
211
+ * check if data is an ArrowData object, which is an array of RecordBatch
212
+ * @param data - object to check
213
+ * @returns {boolean} - true if data is an ArrowData object type guarded
214
+ */
215
+ function _parseGeoArrowBatchesOnWorkers() {
216
+ _parseGeoArrowBatchesOnWorkers = (0, _asyncToGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee3(data) {
217
+ var options,
218
+ parseOnWorker,
219
+ jobs,
220
+ parsed,
221
+ results,
222
+ useFallback,
223
+ _args3 = arguments;
224
+ return _regenerator["default"].wrap(function _callee3$(_context3) {
225
+ while (1) switch (_context3.prev = _context3.next) {
226
+ case 0:
227
+ options = _args3.length > 1 && _args3[1] !== undefined ? _args3[1] : {};
228
+ parseOnWorker = options.parseOnWorker || _arrow.parseGeoArrowOnWorker;
229
+ if (!(!options.parseOnWorker && typeof globalThis.Worker === 'undefined')) {
230
+ _context3.next = 4;
231
+ break;
232
+ }
233
+ return _context3.abrupt("return", undefined);
234
+ case 4:
235
+ jobs = [];
236
+ data.fields.forEach(function (field, columnIndex) {
237
+ var _field$metadata, _field$metadata$get, _data$cols, _geoColumn$data;
238
+ var encoding = (_field$metadata = field.metadata) === null || _field$metadata === void 0 || (_field$metadata$get = _field$metadata.get) === null || _field$metadata$get === void 0 ? void 0 : _field$metadata$get.call(_field$metadata, _constants.GEOARROW_METADATA_KEY);
239
+ var geoColumn = (_data$cols = data.cols) === null || _data$cols === void 0 ? void 0 : _data$cols[columnIndex];
240
+ if (!WORKER_GEOARROW_ENCODINGS.has(encoding) || !(geoColumn !== null && geoColumn !== void 0 && (_geoColumn$data = geoColumn.data) !== null && _geoColumn$data !== void 0 && _geoColumn$data.length)) {
241
+ return;
242
+ }
243
+ var chunkIndexes = options.latestOnly ? [geoColumn.data.length - 1] : geoColumn.data.map(function (_, index) {
244
+ return index;
245
+ });
246
+ chunkIndexes.forEach(function (chunkIndex) {
247
+ var geometryChunk = encoding === _constants.GEOARROW_EXTENSIONS.POINT ? (0, _utils.convertStructToFixedSizeList)(geoColumn.data[chunkIndex]) : geoColumn.data[chunkIndex];
248
+ var chunkCopy = (0, _arrow.hardClone)(geometryChunk, true);
249
+ var chunkOffset = geoColumn.data.slice(0, chunkIndex).reduce(function (offset, chunk) {
250
+ return offset + chunk.length;
251
+ }, 0);
252
+ var chunkType = chunkCopy.type;
253
+ var transferableType = _objectSpread(_objectSpread({}, chunkType), {}, {
254
+ typeId: chunkType.typeId
255
+ }, chunkType.listSize !== undefined ? {
256
+ listSize: chunkType.listSize
257
+ } : {});
258
+ var sourceData = {
259
+ operation: 'parse-geoarrow',
260
+ chunkData: {
261
+ type: transferableType,
262
+ offset: chunkCopy.offset,
263
+ length: chunkCopy.length,
264
+ nullCount: chunkCopy.nullCount,
265
+ buffers: chunkCopy.buffers,
266
+ children: chunkCopy.children,
267
+ dictionary: chunkCopy.dictionary
268
+ },
269
+ chunkIndex: chunkIndex,
270
+ chunkOffset: chunkOffset,
271
+ geometryEncoding: encoding,
272
+ calculateMeanCenters: true,
273
+ triangle: true
274
+ };
275
+ jobs.push({
276
+ columnIndex: columnIndex,
277
+ chunkIndex: chunkIndex,
278
+ result: parseOnWorker(sourceData),
279
+ fallback: function fallback() {
280
+ return (0, _gis.getBinaryGeometriesFromArrow)(
281
+ // Use the original buffers: the worker copy may be transferred.
282
+ // A local Data wrapper also accepts chunks from another Arrow runtime.
283
+ arrow.makeVector(new arrow.Data(geometryChunk.type, geometryChunk.offset, geometryChunk.length, geometryChunk.nullCount, geometryChunk.buffers, geometryChunk.children, geometryChunk.dictionary)), sourceData.geometryEncoding, {
284
+ chunkOffset: chunkOffset,
285
+ calculateMeanCenters: true,
286
+ triangulate: true
287
+ });
288
+ }
289
+ });
290
+ });
291
+ });
292
+ if (jobs.length) {
293
+ _context3.next = 8;
294
+ break;
295
+ }
296
+ return _context3.abrupt("return", undefined);
297
+ case 8:
298
+ parsed = {};
299
+ _context3.next = 11;
300
+ return Promise.allSettled(jobs.map(function (job) {
301
+ return job.result;
302
+ }));
303
+ case 11:
304
+ results = _context3.sent;
305
+ // A partial cache can look complete after compaction merges surviving chunks
306
+ // into one. Reparse every selected original batch if acceleration failed.
307
+ // Using the originals also avoids parsing padding in newly compacted buffers.
308
+ useFallback = results.some(function (result) {
309
+ return result.status !== 'fulfilled' || !result.value.binaryDataFromGeoArrow;
310
+ });
311
+ results.forEach(function (result, index) {
312
+ var _jobs$index = jobs[index],
313
+ columnIndex = _jobs$index.columnIndex,
314
+ chunkIndex = _jobs$index.chunkIndex,
315
+ fallback = _jobs$index.fallback;
316
+ var binaryData = useFallback ? fallback() : result.status === 'fulfilled' ? result.value.binaryDataFromGeoArrow : undefined;
317
+ if (!binaryData) {
318
+ return;
319
+ }
320
+ parsed[columnIndex] = parsed[columnIndex] || {};
321
+ parsed[columnIndex][chunkIndex] = binaryData;
322
+ });
323
+ if (Object.keys(parsed).length) {
324
+ _context3.next = 16;
325
+ break;
326
+ }
327
+ return _context3.abrupt("return", undefined);
328
+ case 16:
329
+ return _context3.abrupt("return", options.targetChunkCount === 1 && !options.latestOnly ? combineGeoArrowWorkerChunks(parsed) : parsed);
330
+ case 17:
331
+ case "end":
332
+ return _context3.stop();
333
+ }
334
+ }, _callee3);
335
+ }));
336
+ return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
337
+ }
338
+ function isArrowData(data) {
339
+ return Array.isArray(data) && Boolean(data.length && data[0].data && data[0].schema);
340
+ }
341
+ function isGeoJson(json) {
342
+ // json can be feature collection
343
+ // or single feature
344
+ return (0, _utils.isPlainObject)(json) && (isFeature(json) || isFeatureCollection(json));
345
+ }
346
+ function isFeature(json) {
347
+ return (0, _utils.isPlainObject)(json) && json.type === 'Feature' && Boolean(json.geometry);
348
+ }
349
+ function isFeatureCollection(json) {
350
+ return (0, _utils.isPlainObject)(json) && json.type === 'FeatureCollection' && Boolean(json.features);
351
+ }
352
+ function isGeoJsonFeatureArray(json) {
353
+ return Array.isArray(json) && json.length > 0 && isFeature(json[0]);
354
+ }
355
+
356
+ /**
357
+ * Normalize loaders.gl GIS output (Feature arrays, geojson-table, object-row
358
+ * tables of Features) into a GeoJSON Feature or FeatureCollection Kepler can process.
359
+ */
360
+ function getGeoJsonFromLoaderResult(data) {
361
+ if (isGeoJson(data)) {
362
+ return data;
363
+ }
364
+ if (isGeoJsonFeatureArray(data)) {
365
+ return {
366
+ type: 'FeatureCollection',
367
+ features: data
368
+ };
369
+ }
370
+ if (!(0, _utils.isPlainObject)(data)) {
371
+ return null;
372
+ }
373
+ if (data.shape === 'geojson-table' && Array.isArray(data.features)) {
374
+ return {
375
+ type: 'FeatureCollection',
376
+ features: data.features
377
+ };
378
+ }
379
+ if (data.shape === 'object-row-table' || data.shape === 'row-table') {
380
+ if (isGeoJsonFeatureArray(data.data)) {
381
+ return {
382
+ type: 'FeatureCollection',
383
+ features: data.data
384
+ };
385
+ }
386
+ // Empty spreadsheet tables stay rows, not empty GeoJSON.
387
+ return null;
388
+ }
389
+ // ShapefileLoader parseInBatches (v3 shape) yields `{data: Feature[]}`,
390
+ // including empty files as `{data: []}`.
391
+ if (Array.isArray(data.data) && (data.data.length === 0 || isGeoJsonFeatureArray(data.data))) {
392
+ return {
393
+ type: 'FeatureCollection',
394
+ features: data.data
395
+ };
396
+ }
397
+ return null;
398
+ }
399
+ function getBatchRows(batch) {
400
+ if (Array.isArray(batch === null || batch === void 0 ? void 0 : batch.features) && (batch.shape === 'geojson-table' || batch.type === 'FeatureCollection')) {
401
+ return batch.features;
402
+ }
403
+ var batchData = (0, _utils.isArrowTable)(batch.data) ? batch.data.batches : batch.data;
404
+ return Array.isArray(batchData) ? batchData : null;
405
+ }
406
+
407
+ /**
408
+ * Reconstruct the loader table/GIS identity from the current batch. Empty
409
+ * shapefile (`{data: []}`) and spreadsheet (`object-row-table`) batches must
410
+ * not be flattened to a bare `[]`, which processFileData cannot classify.
411
+ *
412
+ * JSON streaming first yields an empty `object-row-table` `partial-result`
413
+ * (with `jsonpath` / `container`). That is a placeholder, not the dataset.
414
+ */
415
+ function getAggregatedLoaderResult(batch, rows, current) {
416
+ if (batch.batchType === BATCH_TYPE.METADATA || batch.batchType === BATCH_TYPE.PARTIAL_RESULT || batch.batchType === BATCH_TYPE.FINAL_RESULT || batch.jsonpath || batch.container) {
417
+ return current;
418
+ }
419
+ if (batch.shape === 'geojson-table') {
420
+ return {
421
+ type: 'FeatureCollection',
422
+ features: (0, _toConsumableArray2["default"])(rows)
423
+ };
424
+ }
425
+ // Non-empty CSV/Excel stay a flat row array (existing readBatch contract).
426
+ if (rows.length > 0) {
427
+ return current;
428
+ }
429
+ if (batch.shape === 'object-row-table' || batch.shape === 'row-table') {
430
+ return {
431
+ shape: batch.shape,
432
+ data: []
433
+ };
434
+ }
435
+ // ShapefileLoader parseInBatches always yields v3 `{header, data: Feature[]}`.
436
+ if (batch.header != null && Array.isArray(batch.data)) {
437
+ return {
438
+ data: []
439
+ };
440
+ }
441
+ return current;
442
+ }
443
+ function isRowObject(json) {
444
+ return Array.isArray(json) && (0, _utils.isPlainObject)(json[0]);
445
+ }
446
+ function isKeplerGlMap(json) {
447
+ return Boolean((0, _utils.isPlainObject)(json) && json.datasets && json.config && json.info && (0, _utils.isPlainObject)(json.info) && json.info.app === 'kepler.gl');
448
+ }
449
+ function makeProgressIterator(_x, _x2) {
450
+ return _makeProgressIterator.apply(this, arguments);
451
+ } // eslint-disable-next-line complexity
452
+ function _makeProgressIterator() {
453
+ _makeProgressIterator = (0, _wrapAsyncGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee(asyncIterator, info) {
454
+ var rowCount, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch, rowCountInBatch, percent, progress;
455
+ return _regenerator["default"].wrap(function _callee$(_context) {
456
+ while (1) switch (_context.prev = _context.next) {
457
+ case 0:
458
+ rowCount = 0;
459
+ _iteratorAbruptCompletion = false;
460
+ _didIteratorError = false;
461
+ _context.prev = 3;
462
+ _iterator = _asyncIterator(asyncIterator);
463
+ case 5:
464
+ _context.next = 7;
465
+ return (0, _awaitAsyncGenerator2["default"])(_iterator.next());
466
+ case 7:
467
+ if (!(_iteratorAbruptCompletion = !(_step = _context.sent).done)) {
468
+ _context.next = 18;
469
+ break;
470
+ }
471
+ batch = _step.value;
472
+ // the length could be stored in `batch.length` for arrow batch
473
+ rowCountInBatch = batch.data && (batch.data.length || batch.length) || 0;
474
+ rowCount += rowCountInBatch;
475
+ percent = Number.isFinite(batch.bytesUsed) ? batch.bytesUsed / info.size : null; // Update progress object
476
+ progress = _objectSpread({
477
+ rowCount: rowCount,
478
+ rowCountInBatch: rowCountInBatch
479
+ }, Number.isFinite(percent) ? {
480
+ percent: percent
481
+ } : {});
482
+ _context.next = 15;
483
+ return _objectSpread(_objectSpread({}, batch), {}, {
484
+ progress: progress
485
+ });
486
+ case 15:
487
+ _iteratorAbruptCompletion = false;
488
+ _context.next = 5;
489
+ break;
490
+ case 18:
491
+ _context.next = 24;
492
+ break;
493
+ case 20:
494
+ _context.prev = 20;
495
+ _context.t0 = _context["catch"](3);
496
+ _didIteratorError = true;
497
+ _iteratorError = _context.t0;
498
+ case 24:
499
+ _context.prev = 24;
500
+ _context.prev = 25;
501
+ if (!(_iteratorAbruptCompletion && _iterator["return"] != null)) {
502
+ _context.next = 29;
503
+ break;
504
+ }
505
+ _context.next = 29;
506
+ return (0, _awaitAsyncGenerator2["default"])(_iterator["return"]());
507
+ case 29:
508
+ _context.prev = 29;
509
+ if (!_didIteratorError) {
510
+ _context.next = 32;
511
+ break;
512
+ }
513
+ throw _iteratorError;
514
+ case 32:
515
+ return _context.finish(29);
516
+ case 33:
517
+ return _context.finish(24);
518
+ case 34:
519
+ case "end":
520
+ return _context.stop();
521
+ }
522
+ }, _callee, null, [[3, 20, 24, 34], [25,, 29, 33]]);
523
+ }));
524
+ return _makeProgressIterator.apply(this, arguments);
525
+ }
526
+ function getPersistedRemoteFormat(keplerFormat, fileName) {
527
+ var _fileName$match;
528
+ if (keplerFormat && keplerFormat !== 'auto' && Object.prototype.hasOwnProperty.call(_constants.REMOTE_FILE_EXTENSIONS, keplerFormat)) {
529
+ return keplerFormat;
530
+ }
531
+ var ext = fileName === null || fileName === void 0 || (_fileName$match = fileName.match(/\.([a-z0-9]+)$/i)) === null || _fileName$match === void 0 || (_fileName$match = _fileName$match[1]) === null || _fileName$match === void 0 ? void 0 : _fileName$match.toLowerCase();
532
+ if (!ext) {
533
+ return undefined;
534
+ }
535
+ if (Object.prototype.hasOwnProperty.call(_constants.REMOTE_FILE_EXTENSIONS, ext)) {
536
+ return ext;
537
+ }
538
+ var extensionAliases = {
539
+ jsonl: 'ndjson',
540
+ ndgeojson: 'geojsonl',
541
+ ldgeojson: 'geojsonl',
542
+ xls: 'xlsx',
543
+ xlsm: 'xlsx',
544
+ xlsb: 'xlsx',
545
+ shapefile: 'shp',
546
+ excel: 'xlsx',
547
+ flatgeobuf: 'fgb'
548
+ };
549
+ return extensionAliases[ext];
550
+ }
551
+ function readBatch(_x3, _x4, _x5, _x6, _x7, _x8, _x9, _x0) {
552
+ return _readBatch.apply(this, arguments);
553
+ }
554
+ function _readBatch() {
555
+ _readBatch = (0, _wrapAsyncGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee2(asyncIterator, fileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, fileSize) {
556
+ var result, batches, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, batch, _yield$_awaitAsyncGen, _JSONPath, streamingPath, batchRows, i;
557
+ return _regenerator["default"].wrap(function _callee2$(_context2) {
558
+ while (1) switch (_context2.prev = _context2.next) {
559
+ case 0:
560
+ result = null;
561
+ batches = [];
562
+ _iteratorAbruptCompletion2 = false;
563
+ _didIteratorError2 = false;
564
+ _context2.prev = 4;
565
+ _iterator2 = _asyncIterator(asyncIterator);
566
+ case 6:
567
+ _context2.next = 8;
568
+ return (0, _awaitAsyncGenerator2["default"])(_iterator2.next());
569
+ case 8:
570
+ if (!(_iteratorAbruptCompletion2 = !(_step2 = _context2.sent).done)) {
571
+ _context2.next = 32;
572
+ break;
573
+ }
574
+ batch = _step2.value;
575
+ if (!(batch.batchType === BATCH_TYPE.FINAL_RESULT)) {
576
+ _context2.next = 24;
577
+ break;
578
+ }
579
+ if (batch.container) {
580
+ result = _objectSpread({}, batch.container);
581
+ }
582
+ // Set the streamed data correctly is Batch json path is set
583
+ // and the path streamed is not the top level object (jsonpath = '$')
584
+ if (!(batch.jsonpath && batch.jsonpath.length > 1)) {
585
+ _context2.next = 21;
586
+ break;
587
+ }
588
+ _context2.next = 15;
589
+ return (0, _awaitAsyncGenerator2["default"])(Promise.resolve().then(function () {
590
+ return _interopRequireWildcard(require('@loaders.gl/json'));
591
+ }));
592
+ case 15:
593
+ _yield$_awaitAsyncGen = _context2.sent;
594
+ _JSONPath = _yield$_awaitAsyncGen._JSONPath;
595
+ streamingPath = new _JSONPath(batch.jsonpath);
596
+ streamingPath.setFieldAtPath(result, batches);
597
+ _context2.next = 22;
598
+ break;
599
+ case 21:
600
+ if (batch.jsonpath && batch.jsonpath.length === 1) {
601
+ // The streamed object is a ROW JSON-batch (jsonpath = '$')
602
+ // row objects
603
+ result = batches;
604
+ }
605
+ case 22:
606
+ _context2.next = 27;
607
+ break;
608
+ case 24:
609
+ batchRows = getBatchRows(batch);
610
+ if (batchRows) {
611
+ for (i = 0; i < batchRows.length; i++) {
612
+ batches.push(batchRows[i]);
613
+ }
614
+ }
615
+ result = getAggregatedLoaderResult(batch, batches, result);
616
+ case 27:
617
+ _context2.next = 29;
618
+ return _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({}, batch), batch.schema ? {
619
+ headers: batch.schema.fields ? batch.schema.fields.map(function (f) {
620
+ return f.name;
621
+ }) : Object.keys(batch.schema)
622
+ } : {}), {}, {
623
+ fileName: fileName
624
+ }, typeof fileSize === 'number' ? {
625
+ fileSize: fileSize
626
+ } : {}), {}, {
627
+ // if dataset is CSV, data is set to the raw batches
628
+ data: result ? result : batches
629
+ }, sourceUrl ? {
630
+ sourceUrl: sourceUrl
631
+ } : {}), keplerFormat ? {
632
+ keplerFormat: keplerFormat
633
+ } : {}), typeof refreshIntervalMs === 'number' && refreshIntervalMs > 0 ? {
634
+ refreshIntervalMs: refreshIntervalMs
635
+ } : {}), etag ? {
636
+ etag: etag
637
+ } : {}), lastModified ? {
638
+ lastModified: lastModified
639
+ } : {});
640
+ case 29:
641
+ _iteratorAbruptCompletion2 = false;
642
+ _context2.next = 6;
643
+ break;
644
+ case 32:
645
+ _context2.next = 38;
646
+ break;
647
+ case 34:
648
+ _context2.prev = 34;
649
+ _context2.t0 = _context2["catch"](4);
650
+ _didIteratorError2 = true;
651
+ _iteratorError2 = _context2.t0;
652
+ case 38:
653
+ _context2.prev = 38;
654
+ _context2.prev = 39;
655
+ if (!(_iteratorAbruptCompletion2 && _iterator2["return"] != null)) {
656
+ _context2.next = 43;
657
+ break;
658
+ }
659
+ _context2.next = 43;
660
+ return (0, _awaitAsyncGenerator2["default"])(_iterator2["return"]());
661
+ case 43:
662
+ _context2.prev = 43;
663
+ if (!_didIteratorError2) {
664
+ _context2.next = 46;
665
+ break;
666
+ }
667
+ throw _iteratorError2;
668
+ case 46:
669
+ return _context2.finish(43);
670
+ case 47:
671
+ return _context2.finish(38);
672
+ case 48:
673
+ case "end":
674
+ return _context2.stop();
675
+ }
676
+ }, _callee2, null, [[4, 34, 38, 48], [39,, 43, 47]]);
677
+ }));
678
+ return _readBatch.apply(this, arguments);
679
+ }
680
+ function readFileInBatches(_x10) {
681
+ return _readFileInBatches.apply(this, arguments);
682
+ }
683
+ function _readFileInBatches() {
684
+ _readFileInBatches = (0, _asyncToGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee4(_ref0) {
685
+ var file, _ref0$loaders, loaders, _ref0$loadOptions, loadOptions, companionFiles, fileName, displayFileName, unzipped, shapefile, hasExtension, mimeType, isShapefile, companionFetch, batchIterator, progressIterator, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified;
686
+ return _regenerator["default"].wrap(function _callee4$(_context4) {
687
+ while (1) switch (_context4.prev = _context4.next) {
688
+ case 0:
689
+ file = _ref0.file, _ref0$loaders = _ref0.loaders, loaders = _ref0$loaders === void 0 ? [] : _ref0$loaders, _ref0$loadOptions = _ref0.loadOptions, loadOptions = _ref0$loadOptions === void 0 ? {} : _ref0$loadOptions, companionFiles = _ref0.companionFiles, fileName = _ref0.fileName;
690
+ displayFileName = fileName || file.name; // Local drops expand mixed zips in the load queue before this runs. A `.zip`
691
+ // that still arrives here is a shapefile archive (remote URL, or a direct call).
692
+ if (!((0, _shapefileFiles.isZipFileName)(file.name) && (0, _loaderRegistry.isKeplerFileFormatAccepted)('shp'))) {
693
+ _context4.next = 10;
694
+ break;
695
+ }
696
+ _context4.next = 5;
697
+ return (0, _shapefileFiles.unzipShapefileArchive)(file);
698
+ case 5:
699
+ unzipped = _context4.sent;
700
+ shapefile = unzipped.find(function (entry) {
701
+ return (0, _shapefileFiles.getDroppedFileExtension)(entry.name) === 'shp';
702
+ });
703
+ if (shapefile) {
704
+ _context4.next = 9;
705
+ break;
706
+ }
707
+ throw new Error('Zip archive does not contain a shapefile (.shp)');
708
+ case 9:
709
+ return _context4.abrupt("return", readFileInBatches({
710
+ file: shapefile,
711
+ loaders: loaders,
712
+ loadOptions: loadOptions,
713
+ companionFiles: [].concat((0, _toConsumableArray2["default"])(unzipped), (0, _toConsumableArray2["default"])(companionFiles || [])),
714
+ fileName: displayFileName
715
+ }));
716
+ case 10:
717
+ _context4.next = 12;
718
+ return (0, _loaderRegistry.getKeplerLoaders)(file, loaders);
719
+ case 12:
720
+ loaders = _context4.sent;
721
+ hasExtension = /\.[a-z0-9]+$/i.test(file.name);
722
+ mimeType = !hasExtension && file.type ? file.type : undefined; // Shapefile sidecars are resolved via a custom fetch that cannot be cloned
723
+ // into a worker. Other formats keep loaders.gl worker parsing.
724
+ isShapefile = (0, _shapefileFiles.getDroppedFileExtension)(file.name) === 'shp';
725
+ companionFetch = isShapefile ? (0, _shapefileFiles.createCompanionFetch)(companionFiles) : undefined;
726
+ loadOptions = _objectSpread(_objectSpread(_objectSpread(_objectSpread({
727
+ csv: CSV_LOADER_OPTIONS,
728
+ arrow: ARROW_LOADER_OPTIONS,
729
+ json: JSON_LOADER_OPTIONS,
730
+ parquet: PARQUET_LOADER_OPTIONS,
731
+ kml: KML_LOADER_OPTIONS,
732
+ gpx: KML_LOADER_OPTIONS,
733
+ tcx: KML_LOADER_OPTIONS,
734
+ shapefile: GIS_TABLE_LOADER_OPTIONS,
735
+ shp: {
736
+ _maxDimensions: 2
737
+ },
738
+ excel: EXCEL_LOADER_OPTIONS,
739
+ flatgeobuf: GIS_TABLE_LOADER_OPTIONS,
740
+ gis: {
741
+ reproject: true,
742
+ _targetCrs: 'WGS84'
743
+ },
744
+ metadata: true
745
+ }, isShapefile ? {
746
+ worker: false
747
+ } : {}), mimeType ? {
748
+ mimeType: mimeType
749
+ } : {}), companionFetch ? {
750
+ fetch: companionFetch
751
+ } : {}), loadOptions);
752
+ _context4.next = 20;
753
+ return (0, _core.parseInBatches)(file, loaders, loadOptions);
754
+ case 20:
755
+ batchIterator = _context4.sent;
756
+ progressIterator = makeProgressIterator(batchIterator, {
757
+ size: file.size
758
+ });
759
+ sourceUrl = file.keplerSourceUrl;
760
+ keplerFormat = file.keplerFormat;
761
+ refreshIntervalMs = file.keplerRefreshIntervalMs;
762
+ etag = file.keplerEtag;
763
+ lastModified = file.keplerLastModified;
764
+ return _context4.abrupt("return", readBatch(progressIterator, displayFileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, file.size));
765
+ case 28:
766
+ case "end":
767
+ return _context4.stop();
768
+ }
769
+ }, _callee4);
770
+ }));
771
+ return _readFileInBatches.apply(this, arguments);
772
+ }
773
+ function processFileData(_x11) {
774
+ return _processFileData.apply(this, arguments);
775
+ }
776
+ /**
777
+ * Expand staged files into cumulative payloads for one redraw per Arrow batch.
778
+ * Non-progressive files are present only in the first payload.
779
+ */
780
+ function _processFileData() {
781
+ _processFileData = (0, _asyncToGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee5(_ref1) {
782
+ var _getApplicationConfig;
783
+ var content, fileCache, fileName, data, format, processor, id, table, processorResult, geojsonData, tableRows, _result, result, progressiveData, _result$cols$0$data$l, _result$cols$, latestOnly, targetChunkCount, workerSource, sourceTable, cols, workerGeoArrowBinaryData, geoArrowBinaryData, sourceUrl, remoteFormat, fileSize, metadata;
784
+ return _regenerator["default"].wrap(function _callee5$(_context5) {
785
+ while (1) switch (_context5.prev = _context5.next) {
786
+ case 0:
787
+ content = _ref1.content, fileCache = _ref1.fileCache;
788
+ fileName = content.fileName, data = content.data;
789
+ // Hash the source URL when present so two remote files that share a filename
790
+ // (or a local file and a remote URL with the same name) do not collide and
791
+ // overwrite each other. The UI shows `label` (the filename), not this id.
792
+ id = (0, _utils.generateHashIdFromString)(content.sourceUrl || fileName); // decide on which table class to use based on application config
793
+ table = (_getApplicationConfig = (0, _utils.getApplicationConfig)().table) !== null && _getApplicationConfig !== void 0 ? _getApplicationConfig : _table.KeplerTable;
794
+ if (typeof table.getFileProcessor === 'function') {
795
+ // use custom processors from table class
796
+ processorResult = table.getFileProcessor(data);
797
+ format = processorResult.format;
798
+ processor = processorResult.processor;
799
+ } else {
800
+ // use default processors
801
+ geojsonData = getGeoJsonFromLoaderResult(data);
802
+ if (isArrowData(data)) {
803
+ format = _constants.DATASET_FORMATS.arrow;
804
+ processor = _dataProcessor.processArrowBatches;
805
+ } else if (isKeplerGlMap(data)) {
806
+ format = _constants.DATASET_FORMATS.keplergl;
807
+ processor = _dataProcessor.processKeplerglJSON;
808
+ } else if (geojsonData) {
809
+ format = _constants.DATASET_FORMATS.geojson;
810
+ processor = function processor() {
811
+ return (0, _dataProcessor.processGeojson)(geojsonData);
812
+ };
813
+ } else if (isRowObject(data)) {
814
+ // csv file goes here
815
+ format = _constants.DATASET_FORMATS.row;
816
+ processor = _dataProcessor.processRowObject;
817
+ } else if ((0, _utils.isPlainObject)(data)) {
818
+ tableRows = data.data;
819
+ if ((data.shape === 'object-row-table' || data.shape === 'row-table') && Array.isArray(tableRows)) {
820
+ format = _constants.DATASET_FORMATS.row;
821
+ processor = function processor() {
822
+ return (0, _dataProcessor.processRowObject)(tableRows);
823
+ };
824
+ }
825
+ }
826
+ }
827
+ if (!(format && processor)) {
828
+ _context5.next = 40;
829
+ break;
830
+ }
831
+ _context5.prev = 6;
832
+ if (!(format === _constants.DATASET_FORMATS.arrow && content.skipArrowCompact)) {
833
+ _context5.next = 13;
834
+ break;
835
+ }
836
+ _context5.next = 10;
837
+ return (0, _dataProcessor.processArrowBatches)(data, {
838
+ compact: false
839
+ });
840
+ case 10:
841
+ _context5.t0 = _context5.sent;
842
+ _context5.next = 16;
843
+ break;
844
+ case 13:
845
+ _context5.next = 15;
846
+ return processor(data);
847
+ case 15:
848
+ _context5.t0 = _context5.sent;
849
+ case 16:
850
+ result = _context5.t0;
851
+ _context5.next = 22;
852
+ break;
853
+ case 19:
854
+ _context5.prev = 19;
855
+ _context5.t1 = _context5["catch"](6);
856
+ throw new Error("Can not process uploaded file, ".concat((0, _utils.getError)(_context5.t1)));
857
+ case 22:
858
+ if (!(format === _constants.DATASET_FORMATS.arrow && (_result = result) !== null && _result !== void 0 && _result.cols)) {
859
+ _context5.next = 33;
860
+ break;
861
+ }
862
+ latestOnly = Boolean(content.skipArrowCompact);
863
+ targetChunkCount = (_result$cols$0$data$l = (_result$cols$ = result.cols[0]) === null || _result$cols$ === void 0 || (_result$cols$ = _result$cols$.data) === null || _result$cols$ === void 0 ? void 0 : _result$cols$.length) !== null && _result$cols$0$data$l !== void 0 ? _result$cols$0$data$l : 0;
864
+ workerSource = result; // Final file processing compacts Arrow batches to keep the number of deck.gl
865
+ // sublayers low. Schedule worker jobs from the original batches first so
866
+ // geometry conversion and triangulation can still run in parallel, then
867
+ // combine their output to match the single compacted table chunk.
868
+ if (!latestOnly && Array.isArray(data) && data.length > targetChunkCount) {
869
+ sourceTable = new arrow.Table(data);
870
+ cols = (0, _toConsumableArray2["default"])(Array(sourceTable.numCols).keys()).map(function (i) {
871
+ return sourceTable.getChildAt(i);
872
+ });
873
+ workerSource = _objectSpread(_objectSpread({}, result), {}, {
874
+ cols: cols
875
+ });
876
+ }
877
+ _context5.next = 29;
878
+ return parseGeoArrowBatchesOnWorkers(workerSource, {
879
+ latestOnly: latestOnly
880
+ });
881
+ case 29:
882
+ workerGeoArrowBinaryData = _context5.sent;
883
+ geoArrowBinaryData = targetChunkCount === 1 && !latestOnly && workerGeoArrowBinaryData ? combineGeoArrowWorkerChunks(workerGeoArrowBinaryData) : workerGeoArrowBinaryData;
884
+ if (geoArrowBinaryData) {
885
+ result = _objectSpread(_objectSpread({}, result), {}, {
886
+ geoArrowBinaryData: geoArrowBinaryData
887
+ });
888
+ }
889
+ if (content.prepareProgressiveReplay && !latestOnly) {
890
+ progressiveData = createProgressiveArrowData(workerSource, result, workerGeoArrowBinaryData);
891
+ }
892
+ case 33:
893
+ sourceUrl = content.sourceUrl;
894
+ remoteFormat = getPersistedRemoteFormat(content.keplerFormat, content.fileName);
895
+ fileSize = typeof content.fileSize === 'number' ? content.fileSize : undefined;
896
+ metadata = _objectSpread(_objectSpread({}, sourceUrl ? _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({
897
+ source: sourceUrl
898
+ }, remoteFormat ? {
899
+ sourceFormat: remoteFormat
900
+ } : {}), typeof content.refreshIntervalMs === 'number' && content.refreshIntervalMs > 0 ? {
901
+ refreshIntervalMs: content.refreshIntervalMs
902
+ } : {}), typeof content.etag === 'string' && content.etag ? {
903
+ etag: content.etag
904
+ } : {}), typeof content.lastModified === 'string' && content.lastModified ? {
905
+ lastModified: content.lastModified
906
+ } : {}), {}, {
907
+ lastFetchedAt: Date.now()
908
+ }) : {}), fileSize != null ? {
909
+ size: fileSize
910
+ } : {});
911
+ return _context5.abrupt("return", [].concat((0, _toConsumableArray2["default"])(fileCache), [_objectSpread(_objectSpread({
912
+ data: result
913
+ }, progressiveData ? {
914
+ progressiveData: progressiveData
915
+ } : {}), {}, {
916
+ info: _objectSpread({
917
+ id: id,
918
+ label: content.fileName,
919
+ format: format
920
+ }, sourceUrl ? {
921
+ type: _constants.DatasetType.EXTERNALLY_HOSTED
922
+ } : {})
923
+ }, sourceUrl || fileSize != null ? {
924
+ metadata: metadata
925
+ } : {})]));
926
+ case 40:
927
+ throw new Error('Can not process uploaded file, unknown file format');
928
+ case 41:
929
+ case "end":
930
+ return _context5.stop();
931
+ }
932
+ }, _callee5, null, [[6, 19]]);
933
+ }));
934
+ return _processFileData.apply(this, arguments);
935
+ }
936
+ function fileCacheToProgressiveSteps(fileCache) {
937
+ var numberOfSteps = Math.max.apply(Math, [1].concat((0, _toConsumableArray2["default"])(fileCache.map(function (file) {
938
+ var _file$progressiveData;
939
+ return ((_file$progressiveData = file.progressiveData) === null || _file$progressiveData === void 0 ? void 0 : _file$progressiveData.length) || 1;
940
+ }))));
941
+ return Array.from({
942
+ length: numberOfSteps
943
+ }, function (_, stepIndex) {
944
+ return fileCache.flatMap(function (file) {
945
+ var progressiveData = file.progressiveData,
946
+ fileWithoutProgressiveData = (0, _objectWithoutProperties2["default"])(file, _excluded2);
947
+ if (!progressiveData) {
948
+ return stepIndex === 0 ? [fileWithoutProgressiveData] : [];
949
+ }
950
+ var data = progressiveData[stepIndex];
951
+ return data ? [_objectSpread(_objectSpread({}, fileWithoutProgressiveData), {}, {
952
+ data: data
953
+ })] : [];
954
+ });
955
+ }).filter(function (step) {
956
+ return step.length;
957
+ });
958
+ }
959
+ function filesToDataPayload(fileCache, options) {
960
+ // seperate out files which could be a single datasets. or a keplergl map json
961
+ var collection = fileCache.reduce(function (accu, file) {
962
+ var data = file.data,
963
+ info = file.info;
964
+ if ((info === null || info === void 0 ? void 0 : info.format) === _constants.DATASET_FORMATS.keplergl) {
965
+ // if file contains a single kepler map dataset & config
966
+ accu.keplerMaps.push(_objectSpread(_objectSpread({}, data), {}, {
967
+ options: _objectSpread({
968
+ centerMap: !(data.config && data.config.mapState)
969
+ }, options || {})
970
+ }));
971
+ } else if (_constants.DATASET_FORMATS[info === null || info === void 0 ? void 0 : info.format]) {
972
+ // if file contains only data
973
+ var newDataset = _objectSpread({
974
+ data: data,
975
+ info: _objectSpread({
976
+ id: (info === null || info === void 0 ? void 0 : info.id) || (0, _commonUtils.generateHashId)(4)
977
+ }, info || {})
978
+ }, file.metadata ? {
979
+ metadata: file.metadata
980
+ } : {});
981
+ accu.datasets.push(newDataset);
982
+ }
983
+ return accu;
984
+ }, {
985
+ datasets: [],
986
+ keplerMaps: []
987
+ });
988
+
989
+ // add kepler map first with config
990
+ // add datasets later in one add data call
991
+ return collection.keplerMaps.concat(_objectSpread({
992
+ datasets: collection.datasets
993
+ }, options && collection.datasets.length ? {
994
+ options: options
995
+ } : {}));
996
+ }
997
+ //# sourceMappingURL=file-handler.js.map