@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/data-processor.js +674 -0
- package/dist/cjs/data-processor.js.map +1 -0
- package/dist/cjs/export/dataset-to-geojson.js +138 -0
- package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
- package/dist/cjs/export/geopackage-writer.js +385 -0
- package/dist/cjs/export/geopackage-writer.js.map +1 -0
- package/dist/cjs/export/geoparquet-writer.js +351 -0
- package/dist/cjs/export/geoparquet-writer.js.map +1 -0
- package/dist/cjs/export/kml-writer.js +347 -0
- package/dist/cjs/export/kml-writer.js.map +1 -0
- package/dist/cjs/export/shapefile-writer.js +708 -0
- package/dist/cjs/export/shapefile-writer.js.map +1 -0
- package/dist/cjs/export/sql-js.js +95 -0
- package/dist/cjs/export/sql-js.js.map +1 -0
- package/dist/cjs/export/vector-export.js +187 -0
- package/dist/cjs/export/vector-export.js.map +1 -0
- package/dist/cjs/file-handler.js +997 -0
- package/dist/cjs/file-handler.js.map +1 -0
- package/dist/cjs/index.js +114 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/kepler-csv-loader.js +163 -0
- package/dist/cjs/kepler-csv-loader.js.map +1 -0
- package/dist/cjs/loader-registry.js +542 -0
- package/dist/cjs/loader-registry.js.map +1 -0
- package/dist/cjs/remote-file.js +675 -0
- package/dist/cjs/remote-file.js.map +1 -0
- package/dist/cjs/shapefile-files.js +281 -0
- package/dist/cjs/shapefile-files.js.map +1 -0
- package/dist/cjs/types.js +6 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/data-processor.d.ts +30 -16
- package/dist/esm/data-processor.js +655 -0
- package/dist/esm/data-processor.js.map +1 -0
- package/dist/esm/export/dataset-to-geojson.js +132 -0
- package/dist/esm/export/dataset-to-geojson.js.map +1 -0
- package/dist/esm/export/geopackage-writer.js +381 -0
- package/dist/esm/export/geopackage-writer.js.map +1 -0
- package/dist/esm/export/geoparquet-writer.js +336 -0
- package/dist/esm/export/geoparquet-writer.js.map +1 -0
- package/dist/esm/export/kml-writer.js +341 -0
- package/dist/esm/export/kml-writer.js.map +1 -0
- package/dist/esm/export/shapefile-writer.js +701 -0
- package/dist/esm/export/shapefile-writer.js.map +1 -0
- package/dist/esm/export/sql-js.js +80 -0
- package/dist/esm/export/sql-js.js.map +1 -0
- package/dist/esm/export/vector-export.js +176 -0
- package/dist/esm/export/vector-export.js.map +1 -0
- package/dist/esm/file-handler.js +972 -0
- package/dist/esm/file-handler.js.map +1 -0
- package/dist/esm/index.js +12 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/kepler-csv-loader.js +152 -0
- package/dist/esm/kepler-csv-loader.js.map +1 -0
- package/dist/esm/loader-registry.js +510 -0
- package/dist/esm/loader-registry.js.map +1 -0
- package/dist/esm/remote-file.js +664 -0
- package/dist/esm/remote-file.js.map +1 -0
- package/dist/esm/shapefile-files.js +264 -0
- package/dist/esm/shapefile-files.js.map +1 -0
- package/dist/esm/types.js +3 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/export/dataset-to-geojson.d.ts +12 -0
- package/dist/export/geopackage-writer.d.ts +28 -0
- package/dist/export/geoparquet-writer.d.ts +7 -0
- package/dist/export/kml-writer.d.ts +13 -0
- package/dist/export/shapefile-writer.d.ts +28 -0
- package/dist/export/sql-js.d.ts +8 -0
- package/dist/export/vector-export.d.ts +22 -0
- package/dist/file-handler.d.ts +55 -8
- package/dist/index.d.ts +6 -0
- package/dist/kepler-csv-loader.d.ts +12 -0
- package/dist/loader-registry.d.ts +29 -0
- package/dist/remote-file.d.ts +79 -0
- package/dist/shapefile-files.d.ts +39 -0
- package/dist/types.d.ts +13 -0
- package/package.json +44 -26
- package/dist/data-processor.js +0 -637
- package/dist/file-handler.js +0 -409
- package/dist/index.js +0 -39
- package/dist/types.js +0 -6
|
@@ -0,0 +1,972 @@
|
|
|
1
|
+
import _toConsumableArray from "@babel/runtime/helpers/esm/toConsumableArray";
|
|
2
|
+
import _asyncToGenerator from "@babel/runtime/helpers/esm/asyncToGenerator";
|
|
3
|
+
import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
|
|
4
|
+
import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
|
|
5
|
+
import _slicedToArray from "@babel/runtime/helpers/esm/slicedToArray";
|
|
6
|
+
import _awaitAsyncGenerator from "@babel/runtime/helpers/esm/awaitAsyncGenerator";
|
|
7
|
+
import _wrapAsyncGenerator from "@babel/runtime/helpers/esm/wrapAsyncGenerator";
|
|
8
|
+
var _excluded = ["geoArrowBinaryData"],
|
|
9
|
+
_excluded2 = ["progressiveData"];
|
|
10
|
+
import _regeneratorRuntime from "@babel/runtime/regenerator";
|
|
11
|
+
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
|
|
12
|
+
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
|
|
13
|
+
function _asyncIterator(r) { var n, t, o, e = 2; for ("undefined" != typeof Symbol && (t = Symbol.asyncIterator, o = Symbol.iterator); e--;) { if (t && null != (n = r[t])) return n.call(r); if (o && null != (n = r[o])) return new AsyncFromSyncIterator(n.call(r)); t = "@@asyncIterator", o = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
|
|
14
|
+
function AsyncFromSyncIterator(r) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var n = r.done; return Promise.resolve(r.value).then(function (r) { return { value: r, done: n }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(r) { this.s = r, this.n = r.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, "return": function _return(r) { var n = this.s["return"]; return void 0 === n ? Promise.resolve({ value: r, done: !0 }) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); }, "throw": function _throw(r) { var n = this.s["return"]; return void 0 === n ? Promise.reject(r) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(r); }
|
|
15
|
+
// SPDX-License-Identifier: MIT
|
|
16
|
+
// Copyright contributors to the kepler.gl project
|
|
17
|
+
|
|
18
|
+
import { parseInBatches } from '@loaders.gl/core';
|
|
19
|
+
import { getBinaryGeometriesFromArrow } from '@loaders.gl/gis';
|
|
20
|
+
import * as arrow from 'apache-arrow';
|
|
21
|
+
import { hardClone, parseGeoArrowOnWorker } from '@loaders.gl/arrow';
|
|
22
|
+
import { isPlainObject, generateHashIdFromString, getApplicationConfig, getError, isArrowTable, convertStructToFixedSizeList } from '@kepler.gl/utils';
|
|
23
|
+
import { generateHashId } from '@kepler.gl/common-utils';
|
|
24
|
+
import { DATASET_FORMATS, DatasetType, GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY, REMOTE_FILE_EXTENSIONS } from '@kepler.gl/constants';
|
|
25
|
+
import { KeplerTable } from '@kepler.gl/table';
|
|
26
|
+
import { processArrowBatches, processGeojson, processKeplerglJSON, processRowObject } from './data-processor';
|
|
27
|
+
import { getKeplerLoaders, isKeplerFileFormatAccepted } from './loader-registry';
|
|
28
|
+
import { createCompanionFetch, getDroppedFileExtension, isZipFileName, unzipShapefileArchive } from './shapefile-files';
|
|
29
|
+
var BATCH_TYPE = {
|
|
30
|
+
METADATA: 'metadata',
|
|
31
|
+
PARTIAL_RESULT: 'partial-result',
|
|
32
|
+
FINAL_RESULT: 'final-result'
|
|
33
|
+
};
|
|
34
|
+
var CSV_LOADER_OPTIONS = {
|
|
35
|
+
shape: 'object-row-table',
|
|
36
|
+
dynamicTyping: false // not working for now
|
|
37
|
+
};
|
|
38
|
+
var ARROW_LOADER_OPTIONS = {
|
|
39
|
+
shape: 'arrow-table',
|
|
40
|
+
batchDebounceMs: 10 // time to delay between batches, for incremental loading
|
|
41
|
+
};
|
|
42
|
+
var PARQUET_LOADER_OPTIONS = {
|
|
43
|
+
shape: 'arrow-table'
|
|
44
|
+
};
|
|
45
|
+
var JSON_LOADER_OPTIONS = {
|
|
46
|
+
shape: 'object-row-table',
|
|
47
|
+
// instruct loaders.gl on what json paths to stream
|
|
48
|
+
jsonpaths: ['$',
|
|
49
|
+
// JSON Row array
|
|
50
|
+
'$.features',
|
|
51
|
+
// GeoJSON
|
|
52
|
+
'$.datasets' // KeplerGL JSON
|
|
53
|
+
]
|
|
54
|
+
};
|
|
55
|
+
var KML_LOADER_OPTIONS = {
|
|
56
|
+
shape: 'geojson-table'
|
|
57
|
+
};
|
|
58
|
+
var GIS_TABLE_LOADER_OPTIONS = {
|
|
59
|
+
shape: 'geojson-table'
|
|
60
|
+
};
|
|
61
|
+
var WORKER_GEOARROW_ENCODINGS = new Set([GEOARROW_EXTENSIONS.POINT, GEOARROW_EXTENSIONS.LINESTRING, GEOARROW_EXTENSIONS.POLYGON, GEOARROW_EXTENSIONS.MULTIPOINT, GEOARROW_EXTENSIONS.MULTILINESTRING, GEOARROW_EXTENSIONS.MULTIPOLYGON]);
|
|
62
|
+
var EXCEL_LOADER_OPTIONS = {
|
|
63
|
+
shape: 'object-row-table'
|
|
64
|
+
};
|
|
65
|
+
export { isArrowTable };
|
|
66
|
+
function combineGeoArrowWorkerChunks(parsed) {
|
|
67
|
+
var combined = {};
|
|
68
|
+
Object.entries(parsed).forEach(function (_ref) {
|
|
69
|
+
var _ref2 = _slicedToArray(_ref, 2),
|
|
70
|
+
columnIndex = _ref2[0],
|
|
71
|
+
chunks = _ref2[1];
|
|
72
|
+
var ordered = Object.entries(chunks).map(function (_ref3) {
|
|
73
|
+
var _ref4 = _slicedToArray(_ref3, 2),
|
|
74
|
+
chunkIndex = _ref4[0],
|
|
75
|
+
data = _ref4[1];
|
|
76
|
+
return {
|
|
77
|
+
chunkIndex: Number(chunkIndex),
|
|
78
|
+
data: data
|
|
79
|
+
};
|
|
80
|
+
}).sort(function (a, b) {
|
|
81
|
+
return a.chunkIndex - b.chunkIndex;
|
|
82
|
+
});
|
|
83
|
+
if (!ordered.length) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
var featureTypes = {
|
|
87
|
+
point: false,
|
|
88
|
+
line: false,
|
|
89
|
+
polygon: false
|
|
90
|
+
};
|
|
91
|
+
var bounds = [Infinity, Infinity, -Infinity, -Infinity];
|
|
92
|
+
var binaryGeometries = [];
|
|
93
|
+
var meanCenters = [];
|
|
94
|
+
var hasMeanCenters = true;
|
|
95
|
+
ordered.forEach(function (_ref5) {
|
|
96
|
+
var data = _ref5.data;
|
|
97
|
+
data.binaryGeometries.forEach(function (geometry) {
|
|
98
|
+
return binaryGeometries.push(geometry);
|
|
99
|
+
});
|
|
100
|
+
featureTypes.point || (featureTypes.point = data.featureTypes.point);
|
|
101
|
+
featureTypes.line || (featureTypes.line = data.featureTypes.line);
|
|
102
|
+
featureTypes.polygon || (featureTypes.polygon = data.featureTypes.polygon);
|
|
103
|
+
bounds[0] = Math.min(bounds[0], data.bounds[0]);
|
|
104
|
+
bounds[1] = Math.min(bounds[1], data.bounds[1]);
|
|
105
|
+
bounds[2] = Math.max(bounds[2], data.bounds[2]);
|
|
106
|
+
bounds[3] = Math.max(bounds[3], data.bounds[3]);
|
|
107
|
+
if (data.meanCenters) {
|
|
108
|
+
data.meanCenters.forEach(function (center) {
|
|
109
|
+
return meanCenters.push(center);
|
|
110
|
+
});
|
|
111
|
+
} else {
|
|
112
|
+
hasMeanCenters = false;
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
combined[Number(columnIndex)] = {
|
|
116
|
+
0: _objectSpread({
|
|
117
|
+
binaryGeometries: binaryGeometries,
|
|
118
|
+
featureTypes: featureTypes,
|
|
119
|
+
bounds: bounds
|
|
120
|
+
}, hasMeanCenters ? {
|
|
121
|
+
meanCenters: meanCenters
|
|
122
|
+
} : {})
|
|
123
|
+
};
|
|
124
|
+
});
|
|
125
|
+
return Object.keys(combined).length ? combined : undefined;
|
|
126
|
+
}
|
|
127
|
+
function geoArrowWorkerChunksThrough(parsed, lastChunkIndex) {
|
|
128
|
+
if (!parsed) {
|
|
129
|
+
return undefined;
|
|
130
|
+
}
|
|
131
|
+
var selected = {};
|
|
132
|
+
Object.entries(parsed).forEach(function (_ref6) {
|
|
133
|
+
var _ref7 = _slicedToArray(_ref6, 2),
|
|
134
|
+
columnIndex = _ref7[0],
|
|
135
|
+
chunks = _ref7[1];
|
|
136
|
+
var selectedChunks = Object.fromEntries(Object.entries(chunks).filter(function (_ref8) {
|
|
137
|
+
var _ref9 = _slicedToArray(_ref8, 1),
|
|
138
|
+
chunkIndex = _ref9[0];
|
|
139
|
+
return Number(chunkIndex) <= lastChunkIndex;
|
|
140
|
+
}));
|
|
141
|
+
if (Object.keys(selectedChunks).length) {
|
|
142
|
+
selected[Number(columnIndex)] = selectedChunks;
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
return Object.keys(selected).length ? selected : undefined;
|
|
146
|
+
}
|
|
147
|
+
function createProgressiveArrowData(source, finalData, geoArrowBinaryData) {
|
|
148
|
+
var _source$cols;
|
|
149
|
+
var chunks = (_source$cols = source.cols) === null || _source$cols === void 0 || (_source$cols = _source$cols[0]) === null || _source$cols === void 0 ? void 0 : _source$cols.data;
|
|
150
|
+
if (!Array.isArray(chunks) || chunks.length < 2 || !source.cols) {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
var _sourceCache = source.geoArrowBinaryData,
|
|
154
|
+
sourceWithoutCache = _objectWithoutProperties(source, _excluded);
|
|
155
|
+
var snapshots = [];
|
|
156
|
+
var rowCount = 0;
|
|
157
|
+
// The final snapshot uses the normally processed (and usually compacted)
|
|
158
|
+
// table. Intermediate snapshots retain record-batch boundaries.
|
|
159
|
+
for (var chunkIndex = 0; chunkIndex < chunks.length - 1; chunkIndex++) {
|
|
160
|
+
rowCount += chunks[chunkIndex].length;
|
|
161
|
+
var cols = source.cols.map(function (column) {
|
|
162
|
+
return column.slice(0, rowCount);
|
|
163
|
+
});
|
|
164
|
+
var snapshotCache = geoArrowWorkerChunksThrough(geoArrowBinaryData, chunkIndex);
|
|
165
|
+
snapshots.push(_objectSpread(_objectSpread({}, sourceWithoutCache), {}, {
|
|
166
|
+
cols: cols
|
|
167
|
+
}, snapshotCache ? {
|
|
168
|
+
geoArrowBinaryData: snapshotCache
|
|
169
|
+
} : {}));
|
|
170
|
+
}
|
|
171
|
+
snapshots.push(finalData);
|
|
172
|
+
return snapshots;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Convert native GeoArrow chunks to deck.gl binary features without blocking
|
|
177
|
+
* the main thread. WKB/WKT remain on the synchronous fallback because the
|
|
178
|
+
* loaders.gl triangulation worker only supports native GeoArrow encodings.
|
|
179
|
+
*/
|
|
180
|
+
export function parseGeoArrowBatchesOnWorkers(_x1) {
|
|
181
|
+
return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* check if data is an ArrowData object, which is an array of RecordBatch
|
|
186
|
+
* @param data - object to check
|
|
187
|
+
* @returns {boolean} - true if data is an ArrowData object type guarded
|
|
188
|
+
*/
|
|
189
|
+
function _parseGeoArrowBatchesOnWorkers() {
|
|
190
|
+
_parseGeoArrowBatchesOnWorkers = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee3(data) {
|
|
191
|
+
var options,
|
|
192
|
+
parseOnWorker,
|
|
193
|
+
jobs,
|
|
194
|
+
parsed,
|
|
195
|
+
results,
|
|
196
|
+
useFallback,
|
|
197
|
+
_args3 = arguments;
|
|
198
|
+
return _regeneratorRuntime.wrap(function _callee3$(_context3) {
|
|
199
|
+
while (1) switch (_context3.prev = _context3.next) {
|
|
200
|
+
case 0:
|
|
201
|
+
options = _args3.length > 1 && _args3[1] !== undefined ? _args3[1] : {};
|
|
202
|
+
parseOnWorker = options.parseOnWorker || parseGeoArrowOnWorker;
|
|
203
|
+
if (!(!options.parseOnWorker && typeof globalThis.Worker === 'undefined')) {
|
|
204
|
+
_context3.next = 4;
|
|
205
|
+
break;
|
|
206
|
+
}
|
|
207
|
+
return _context3.abrupt("return", undefined);
|
|
208
|
+
case 4:
|
|
209
|
+
jobs = [];
|
|
210
|
+
data.fields.forEach(function (field, columnIndex) {
|
|
211
|
+
var _field$metadata, _field$metadata$get, _data$cols, _geoColumn$data;
|
|
212
|
+
var encoding = (_field$metadata = field.metadata) === null || _field$metadata === void 0 || (_field$metadata$get = _field$metadata.get) === null || _field$metadata$get === void 0 ? void 0 : _field$metadata$get.call(_field$metadata, GEOARROW_METADATA_KEY);
|
|
213
|
+
var geoColumn = (_data$cols = data.cols) === null || _data$cols === void 0 ? void 0 : _data$cols[columnIndex];
|
|
214
|
+
if (!WORKER_GEOARROW_ENCODINGS.has(encoding) || !(geoColumn !== null && geoColumn !== void 0 && (_geoColumn$data = geoColumn.data) !== null && _geoColumn$data !== void 0 && _geoColumn$data.length)) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
var chunkIndexes = options.latestOnly ? [geoColumn.data.length - 1] : geoColumn.data.map(function (_, index) {
|
|
218
|
+
return index;
|
|
219
|
+
});
|
|
220
|
+
chunkIndexes.forEach(function (chunkIndex) {
|
|
221
|
+
var geometryChunk = encoding === GEOARROW_EXTENSIONS.POINT ? convertStructToFixedSizeList(geoColumn.data[chunkIndex]) : geoColumn.data[chunkIndex];
|
|
222
|
+
var chunkCopy = hardClone(geometryChunk, true);
|
|
223
|
+
var chunkOffset = geoColumn.data.slice(0, chunkIndex).reduce(function (offset, chunk) {
|
|
224
|
+
return offset + chunk.length;
|
|
225
|
+
}, 0);
|
|
226
|
+
var chunkType = chunkCopy.type;
|
|
227
|
+
var transferableType = _objectSpread(_objectSpread({}, chunkType), {}, {
|
|
228
|
+
typeId: chunkType.typeId
|
|
229
|
+
}, chunkType.listSize !== undefined ? {
|
|
230
|
+
listSize: chunkType.listSize
|
|
231
|
+
} : {});
|
|
232
|
+
var sourceData = {
|
|
233
|
+
operation: 'parse-geoarrow',
|
|
234
|
+
chunkData: {
|
|
235
|
+
type: transferableType,
|
|
236
|
+
offset: chunkCopy.offset,
|
|
237
|
+
length: chunkCopy.length,
|
|
238
|
+
nullCount: chunkCopy.nullCount,
|
|
239
|
+
buffers: chunkCopy.buffers,
|
|
240
|
+
children: chunkCopy.children,
|
|
241
|
+
dictionary: chunkCopy.dictionary
|
|
242
|
+
},
|
|
243
|
+
chunkIndex: chunkIndex,
|
|
244
|
+
chunkOffset: chunkOffset,
|
|
245
|
+
geometryEncoding: encoding,
|
|
246
|
+
calculateMeanCenters: true,
|
|
247
|
+
triangle: true
|
|
248
|
+
};
|
|
249
|
+
jobs.push({
|
|
250
|
+
columnIndex: columnIndex,
|
|
251
|
+
chunkIndex: chunkIndex,
|
|
252
|
+
result: parseOnWorker(sourceData),
|
|
253
|
+
fallback: function fallback() {
|
|
254
|
+
return getBinaryGeometriesFromArrow(
|
|
255
|
+
// Use the original buffers: the worker copy may be transferred.
|
|
256
|
+
// A local Data wrapper also accepts chunks from another Arrow runtime.
|
|
257
|
+
arrow.makeVector(new arrow.Data(geometryChunk.type, geometryChunk.offset, geometryChunk.length, geometryChunk.nullCount, geometryChunk.buffers, geometryChunk.children, geometryChunk.dictionary)), sourceData.geometryEncoding, {
|
|
258
|
+
chunkOffset: chunkOffset,
|
|
259
|
+
calculateMeanCenters: true,
|
|
260
|
+
triangulate: true
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
if (jobs.length) {
|
|
267
|
+
_context3.next = 8;
|
|
268
|
+
break;
|
|
269
|
+
}
|
|
270
|
+
return _context3.abrupt("return", undefined);
|
|
271
|
+
case 8:
|
|
272
|
+
parsed = {};
|
|
273
|
+
_context3.next = 11;
|
|
274
|
+
return Promise.allSettled(jobs.map(function (job) {
|
|
275
|
+
return job.result;
|
|
276
|
+
}));
|
|
277
|
+
case 11:
|
|
278
|
+
results = _context3.sent;
|
|
279
|
+
// A partial cache can look complete after compaction merges surviving chunks
|
|
280
|
+
// into one. Reparse every selected original batch if acceleration failed.
|
|
281
|
+
// Using the originals also avoids parsing padding in newly compacted buffers.
|
|
282
|
+
useFallback = results.some(function (result) {
|
|
283
|
+
return result.status !== 'fulfilled' || !result.value.binaryDataFromGeoArrow;
|
|
284
|
+
});
|
|
285
|
+
results.forEach(function (result, index) {
|
|
286
|
+
var _jobs$index = jobs[index],
|
|
287
|
+
columnIndex = _jobs$index.columnIndex,
|
|
288
|
+
chunkIndex = _jobs$index.chunkIndex,
|
|
289
|
+
fallback = _jobs$index.fallback;
|
|
290
|
+
var binaryData = useFallback ? fallback() : result.status === 'fulfilled' ? result.value.binaryDataFromGeoArrow : undefined;
|
|
291
|
+
if (!binaryData) {
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
parsed[columnIndex] = parsed[columnIndex] || {};
|
|
295
|
+
parsed[columnIndex][chunkIndex] = binaryData;
|
|
296
|
+
});
|
|
297
|
+
if (Object.keys(parsed).length) {
|
|
298
|
+
_context3.next = 16;
|
|
299
|
+
break;
|
|
300
|
+
}
|
|
301
|
+
return _context3.abrupt("return", undefined);
|
|
302
|
+
case 16:
|
|
303
|
+
return _context3.abrupt("return", options.targetChunkCount === 1 && !options.latestOnly ? combineGeoArrowWorkerChunks(parsed) : parsed);
|
|
304
|
+
case 17:
|
|
305
|
+
case "end":
|
|
306
|
+
return _context3.stop();
|
|
307
|
+
}
|
|
308
|
+
}, _callee3);
|
|
309
|
+
}));
|
|
310
|
+
return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
|
|
311
|
+
}
|
|
312
|
+
export function isArrowData(data) {
|
|
313
|
+
return Array.isArray(data) && Boolean(data.length && data[0].data && data[0].schema);
|
|
314
|
+
}
|
|
315
|
+
export function isGeoJson(json) {
|
|
316
|
+
// json can be feature collection
|
|
317
|
+
// or single feature
|
|
318
|
+
return isPlainObject(json) && (isFeature(json) || isFeatureCollection(json));
|
|
319
|
+
}
|
|
320
|
+
export function isFeature(json) {
|
|
321
|
+
return isPlainObject(json) && json.type === 'Feature' && Boolean(json.geometry);
|
|
322
|
+
}
|
|
323
|
+
export function isFeatureCollection(json) {
|
|
324
|
+
return isPlainObject(json) && json.type === 'FeatureCollection' && Boolean(json.features);
|
|
325
|
+
}
|
|
326
|
+
export function isGeoJsonFeatureArray(json) {
|
|
327
|
+
return Array.isArray(json) && json.length > 0 && isFeature(json[0]);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Normalize loaders.gl GIS output (Feature arrays, geojson-table, object-row
|
|
332
|
+
* tables of Features) into a GeoJSON Feature or FeatureCollection Kepler can process.
|
|
333
|
+
*/
|
|
334
|
+
export function getGeoJsonFromLoaderResult(data) {
|
|
335
|
+
if (isGeoJson(data)) {
|
|
336
|
+
return data;
|
|
337
|
+
}
|
|
338
|
+
if (isGeoJsonFeatureArray(data)) {
|
|
339
|
+
return {
|
|
340
|
+
type: 'FeatureCollection',
|
|
341
|
+
features: data
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
if (!isPlainObject(data)) {
|
|
345
|
+
return null;
|
|
346
|
+
}
|
|
347
|
+
if (data.shape === 'geojson-table' && Array.isArray(data.features)) {
|
|
348
|
+
return {
|
|
349
|
+
type: 'FeatureCollection',
|
|
350
|
+
features: data.features
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
if (data.shape === 'object-row-table' || data.shape === 'row-table') {
|
|
354
|
+
if (isGeoJsonFeatureArray(data.data)) {
|
|
355
|
+
return {
|
|
356
|
+
type: 'FeatureCollection',
|
|
357
|
+
features: data.data
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
// Empty spreadsheet tables stay rows, not empty GeoJSON.
|
|
361
|
+
return null;
|
|
362
|
+
}
|
|
363
|
+
// ShapefileLoader parseInBatches (v3 shape) yields `{data: Feature[]}`,
|
|
364
|
+
// including empty files as `{data: []}`.
|
|
365
|
+
if (Array.isArray(data.data) && (data.data.length === 0 || isGeoJsonFeatureArray(data.data))) {
|
|
366
|
+
return {
|
|
367
|
+
type: 'FeatureCollection',
|
|
368
|
+
features: data.data
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
return null;
|
|
372
|
+
}
|
|
373
|
+
function getBatchRows(batch) {
|
|
374
|
+
if (Array.isArray(batch === null || batch === void 0 ? void 0 : batch.features) && (batch.shape === 'geojson-table' || batch.type === 'FeatureCollection')) {
|
|
375
|
+
return batch.features;
|
|
376
|
+
}
|
|
377
|
+
var batchData = isArrowTable(batch.data) ? batch.data.batches : batch.data;
|
|
378
|
+
return Array.isArray(batchData) ? batchData : null;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Reconstruct the loader table/GIS identity from the current batch. Empty
|
|
383
|
+
* shapefile (`{data: []}`) and spreadsheet (`object-row-table`) batches must
|
|
384
|
+
* not be flattened to a bare `[]`, which processFileData cannot classify.
|
|
385
|
+
*
|
|
386
|
+
* JSON streaming first yields an empty `object-row-table` `partial-result`
|
|
387
|
+
* (with `jsonpath` / `container`). That is a placeholder, not the dataset.
|
|
388
|
+
*/
|
|
389
|
+
function getAggregatedLoaderResult(batch, rows, current) {
|
|
390
|
+
if (batch.batchType === BATCH_TYPE.METADATA || batch.batchType === BATCH_TYPE.PARTIAL_RESULT || batch.batchType === BATCH_TYPE.FINAL_RESULT || batch.jsonpath || batch.container) {
|
|
391
|
+
return current;
|
|
392
|
+
}
|
|
393
|
+
if (batch.shape === 'geojson-table') {
|
|
394
|
+
return {
|
|
395
|
+
type: 'FeatureCollection',
|
|
396
|
+
features: _toConsumableArray(rows)
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
// Non-empty CSV/Excel stay a flat row array (existing readBatch contract).
|
|
400
|
+
if (rows.length > 0) {
|
|
401
|
+
return current;
|
|
402
|
+
}
|
|
403
|
+
if (batch.shape === 'object-row-table' || batch.shape === 'row-table') {
|
|
404
|
+
return {
|
|
405
|
+
shape: batch.shape,
|
|
406
|
+
data: []
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
// ShapefileLoader parseInBatches always yields v3 `{header, data: Feature[]}`.
|
|
410
|
+
if (batch.header != null && Array.isArray(batch.data)) {
|
|
411
|
+
return {
|
|
412
|
+
data: []
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
return current;
|
|
416
|
+
}
|
|
417
|
+
export function isRowObject(json) {
|
|
418
|
+
return Array.isArray(json) && isPlainObject(json[0]);
|
|
419
|
+
}
|
|
420
|
+
export function isKeplerGlMap(json) {
|
|
421
|
+
return Boolean(isPlainObject(json) && json.datasets && json.config && json.info && isPlainObject(json.info) && json.info.app === 'kepler.gl');
|
|
422
|
+
}
|
|
423
|
+
export function makeProgressIterator(_x, _x2) {
|
|
424
|
+
return _makeProgressIterator.apply(this, arguments);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// eslint-disable-next-line complexity
|
|
428
|
+
function _makeProgressIterator() {
|
|
429
|
+
_makeProgressIterator = _wrapAsyncGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee(asyncIterator, info) {
|
|
430
|
+
var rowCount, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch, rowCountInBatch, percent, progress;
|
|
431
|
+
return _regeneratorRuntime.wrap(function _callee$(_context) {
|
|
432
|
+
while (1) switch (_context.prev = _context.next) {
|
|
433
|
+
case 0:
|
|
434
|
+
rowCount = 0;
|
|
435
|
+
_iteratorAbruptCompletion = false;
|
|
436
|
+
_didIteratorError = false;
|
|
437
|
+
_context.prev = 3;
|
|
438
|
+
_iterator = _asyncIterator(asyncIterator);
|
|
439
|
+
case 5:
|
|
440
|
+
_context.next = 7;
|
|
441
|
+
return _awaitAsyncGenerator(_iterator.next());
|
|
442
|
+
case 7:
|
|
443
|
+
if (!(_iteratorAbruptCompletion = !(_step = _context.sent).done)) {
|
|
444
|
+
_context.next = 18;
|
|
445
|
+
break;
|
|
446
|
+
}
|
|
447
|
+
batch = _step.value;
|
|
448
|
+
// the length could be stored in `batch.length` for arrow batch
|
|
449
|
+
rowCountInBatch = batch.data && (batch.data.length || batch.length) || 0;
|
|
450
|
+
rowCount += rowCountInBatch;
|
|
451
|
+
percent = Number.isFinite(batch.bytesUsed) ? batch.bytesUsed / info.size : null; // Update progress object
|
|
452
|
+
progress = _objectSpread({
|
|
453
|
+
rowCount: rowCount,
|
|
454
|
+
rowCountInBatch: rowCountInBatch
|
|
455
|
+
}, Number.isFinite(percent) ? {
|
|
456
|
+
percent: percent
|
|
457
|
+
} : {});
|
|
458
|
+
_context.next = 15;
|
|
459
|
+
return _objectSpread(_objectSpread({}, batch), {}, {
|
|
460
|
+
progress: progress
|
|
461
|
+
});
|
|
462
|
+
case 15:
|
|
463
|
+
_iteratorAbruptCompletion = false;
|
|
464
|
+
_context.next = 5;
|
|
465
|
+
break;
|
|
466
|
+
case 18:
|
|
467
|
+
_context.next = 24;
|
|
468
|
+
break;
|
|
469
|
+
case 20:
|
|
470
|
+
_context.prev = 20;
|
|
471
|
+
_context.t0 = _context["catch"](3);
|
|
472
|
+
_didIteratorError = true;
|
|
473
|
+
_iteratorError = _context.t0;
|
|
474
|
+
case 24:
|
|
475
|
+
_context.prev = 24;
|
|
476
|
+
_context.prev = 25;
|
|
477
|
+
if (!(_iteratorAbruptCompletion && _iterator["return"] != null)) {
|
|
478
|
+
_context.next = 29;
|
|
479
|
+
break;
|
|
480
|
+
}
|
|
481
|
+
_context.next = 29;
|
|
482
|
+
return _awaitAsyncGenerator(_iterator["return"]());
|
|
483
|
+
case 29:
|
|
484
|
+
_context.prev = 29;
|
|
485
|
+
if (!_didIteratorError) {
|
|
486
|
+
_context.next = 32;
|
|
487
|
+
break;
|
|
488
|
+
}
|
|
489
|
+
throw _iteratorError;
|
|
490
|
+
case 32:
|
|
491
|
+
return _context.finish(29);
|
|
492
|
+
case 33:
|
|
493
|
+
return _context.finish(24);
|
|
494
|
+
case 34:
|
|
495
|
+
case "end":
|
|
496
|
+
return _context.stop();
|
|
497
|
+
}
|
|
498
|
+
}, _callee, null, [[3, 20, 24, 34], [25,, 29, 33]]);
|
|
499
|
+
}));
|
|
500
|
+
return _makeProgressIterator.apply(this, arguments);
|
|
501
|
+
}
|
|
502
|
+
export function getPersistedRemoteFormat(keplerFormat, fileName) {
|
|
503
|
+
var _fileName$match;
|
|
504
|
+
if (keplerFormat && keplerFormat !== 'auto' && Object.prototype.hasOwnProperty.call(REMOTE_FILE_EXTENSIONS, keplerFormat)) {
|
|
505
|
+
return keplerFormat;
|
|
506
|
+
}
|
|
507
|
+
var ext = fileName === null || fileName === void 0 || (_fileName$match = fileName.match(/\.([a-z0-9]+)$/i)) === null || _fileName$match === void 0 || (_fileName$match = _fileName$match[1]) === null || _fileName$match === void 0 ? void 0 : _fileName$match.toLowerCase();
|
|
508
|
+
if (!ext) {
|
|
509
|
+
return undefined;
|
|
510
|
+
}
|
|
511
|
+
if (Object.prototype.hasOwnProperty.call(REMOTE_FILE_EXTENSIONS, ext)) {
|
|
512
|
+
return ext;
|
|
513
|
+
}
|
|
514
|
+
var extensionAliases = {
|
|
515
|
+
jsonl: 'ndjson',
|
|
516
|
+
ndgeojson: 'geojsonl',
|
|
517
|
+
ldgeojson: 'geojsonl',
|
|
518
|
+
xls: 'xlsx',
|
|
519
|
+
xlsm: 'xlsx',
|
|
520
|
+
xlsb: 'xlsx',
|
|
521
|
+
shapefile: 'shp',
|
|
522
|
+
excel: 'xlsx',
|
|
523
|
+
flatgeobuf: 'fgb'
|
|
524
|
+
};
|
|
525
|
+
return extensionAliases[ext];
|
|
526
|
+
}
|
|
527
|
+
export function readBatch(_x3, _x4, _x5, _x6, _x7, _x8, _x9, _x0) {
|
|
528
|
+
return _readBatch.apply(this, arguments);
|
|
529
|
+
}
|
|
530
|
+
function _readBatch() {
|
|
531
|
+
_readBatch = _wrapAsyncGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee2(asyncIterator, fileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, fileSize) {
|
|
532
|
+
var result, batches, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, batch, _yield$_awaitAsyncGen, _JSONPath, streamingPath, batchRows, i;
|
|
533
|
+
return _regeneratorRuntime.wrap(function _callee2$(_context2) {
|
|
534
|
+
while (1) switch (_context2.prev = _context2.next) {
|
|
535
|
+
case 0:
|
|
536
|
+
result = null;
|
|
537
|
+
batches = [];
|
|
538
|
+
_iteratorAbruptCompletion2 = false;
|
|
539
|
+
_didIteratorError2 = false;
|
|
540
|
+
_context2.prev = 4;
|
|
541
|
+
_iterator2 = _asyncIterator(asyncIterator);
|
|
542
|
+
case 6:
|
|
543
|
+
_context2.next = 8;
|
|
544
|
+
return _awaitAsyncGenerator(_iterator2.next());
|
|
545
|
+
case 8:
|
|
546
|
+
if (!(_iteratorAbruptCompletion2 = !(_step2 = _context2.sent).done)) {
|
|
547
|
+
_context2.next = 32;
|
|
548
|
+
break;
|
|
549
|
+
}
|
|
550
|
+
batch = _step2.value;
|
|
551
|
+
if (!(batch.batchType === BATCH_TYPE.FINAL_RESULT)) {
|
|
552
|
+
_context2.next = 24;
|
|
553
|
+
break;
|
|
554
|
+
}
|
|
555
|
+
if (batch.container) {
|
|
556
|
+
result = _objectSpread({}, batch.container);
|
|
557
|
+
}
|
|
558
|
+
// Set the streamed data correctly is Batch json path is set
|
|
559
|
+
// and the path streamed is not the top level object (jsonpath = '$')
|
|
560
|
+
if (!(batch.jsonpath && batch.jsonpath.length > 1)) {
|
|
561
|
+
_context2.next = 21;
|
|
562
|
+
break;
|
|
563
|
+
}
|
|
564
|
+
_context2.next = 15;
|
|
565
|
+
return _awaitAsyncGenerator(import('@loaders.gl/json'));
|
|
566
|
+
case 15:
|
|
567
|
+
_yield$_awaitAsyncGen = _context2.sent;
|
|
568
|
+
_JSONPath = _yield$_awaitAsyncGen._JSONPath;
|
|
569
|
+
streamingPath = new _JSONPath(batch.jsonpath);
|
|
570
|
+
streamingPath.setFieldAtPath(result, batches);
|
|
571
|
+
_context2.next = 22;
|
|
572
|
+
break;
|
|
573
|
+
case 21:
|
|
574
|
+
if (batch.jsonpath && batch.jsonpath.length === 1) {
|
|
575
|
+
// The streamed object is a ROW JSON-batch (jsonpath = '$')
|
|
576
|
+
// row objects
|
|
577
|
+
result = batches;
|
|
578
|
+
}
|
|
579
|
+
case 22:
|
|
580
|
+
_context2.next = 27;
|
|
581
|
+
break;
|
|
582
|
+
case 24:
|
|
583
|
+
batchRows = getBatchRows(batch);
|
|
584
|
+
if (batchRows) {
|
|
585
|
+
for (i = 0; i < batchRows.length; i++) {
|
|
586
|
+
batches.push(batchRows[i]);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
result = getAggregatedLoaderResult(batch, batches, result);
|
|
590
|
+
case 27:
|
|
591
|
+
_context2.next = 29;
|
|
592
|
+
return _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({}, batch), batch.schema ? {
|
|
593
|
+
headers: batch.schema.fields ? batch.schema.fields.map(function (f) {
|
|
594
|
+
return f.name;
|
|
595
|
+
}) : Object.keys(batch.schema)
|
|
596
|
+
} : {}), {}, {
|
|
597
|
+
fileName: fileName
|
|
598
|
+
}, typeof fileSize === 'number' ? {
|
|
599
|
+
fileSize: fileSize
|
|
600
|
+
} : {}), {}, {
|
|
601
|
+
// if dataset is CSV, data is set to the raw batches
|
|
602
|
+
data: result ? result : batches
|
|
603
|
+
}, sourceUrl ? {
|
|
604
|
+
sourceUrl: sourceUrl
|
|
605
|
+
} : {}), keplerFormat ? {
|
|
606
|
+
keplerFormat: keplerFormat
|
|
607
|
+
} : {}), typeof refreshIntervalMs === 'number' && refreshIntervalMs > 0 ? {
|
|
608
|
+
refreshIntervalMs: refreshIntervalMs
|
|
609
|
+
} : {}), etag ? {
|
|
610
|
+
etag: etag
|
|
611
|
+
} : {}), lastModified ? {
|
|
612
|
+
lastModified: lastModified
|
|
613
|
+
} : {});
|
|
614
|
+
case 29:
|
|
615
|
+
_iteratorAbruptCompletion2 = false;
|
|
616
|
+
_context2.next = 6;
|
|
617
|
+
break;
|
|
618
|
+
case 32:
|
|
619
|
+
_context2.next = 38;
|
|
620
|
+
break;
|
|
621
|
+
case 34:
|
|
622
|
+
_context2.prev = 34;
|
|
623
|
+
_context2.t0 = _context2["catch"](4);
|
|
624
|
+
_didIteratorError2 = true;
|
|
625
|
+
_iteratorError2 = _context2.t0;
|
|
626
|
+
case 38:
|
|
627
|
+
_context2.prev = 38;
|
|
628
|
+
_context2.prev = 39;
|
|
629
|
+
if (!(_iteratorAbruptCompletion2 && _iterator2["return"] != null)) {
|
|
630
|
+
_context2.next = 43;
|
|
631
|
+
break;
|
|
632
|
+
}
|
|
633
|
+
_context2.next = 43;
|
|
634
|
+
return _awaitAsyncGenerator(_iterator2["return"]());
|
|
635
|
+
case 43:
|
|
636
|
+
_context2.prev = 43;
|
|
637
|
+
if (!_didIteratorError2) {
|
|
638
|
+
_context2.next = 46;
|
|
639
|
+
break;
|
|
640
|
+
}
|
|
641
|
+
throw _iteratorError2;
|
|
642
|
+
case 46:
|
|
643
|
+
return _context2.finish(43);
|
|
644
|
+
case 47:
|
|
645
|
+
return _context2.finish(38);
|
|
646
|
+
case 48:
|
|
647
|
+
case "end":
|
|
648
|
+
return _context2.stop();
|
|
649
|
+
}
|
|
650
|
+
}, _callee2, null, [[4, 34, 38, 48], [39,, 43, 47]]);
|
|
651
|
+
}));
|
|
652
|
+
return _readBatch.apply(this, arguments);
|
|
653
|
+
}
|
|
654
|
+
export function readFileInBatches(_x10) {
|
|
655
|
+
return _readFileInBatches.apply(this, arguments);
|
|
656
|
+
}
|
|
657
|
+
function _readFileInBatches() {
|
|
658
|
+
_readFileInBatches = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee4(_ref0) {
|
|
659
|
+
var file, _ref0$loaders, loaders, _ref0$loadOptions, loadOptions, companionFiles, fileName, displayFileName, unzipped, shapefile, hasExtension, mimeType, isShapefile, companionFetch, batchIterator, progressIterator, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified;
|
|
660
|
+
return _regeneratorRuntime.wrap(function _callee4$(_context4) {
|
|
661
|
+
while (1) switch (_context4.prev = _context4.next) {
|
|
662
|
+
case 0:
|
|
663
|
+
file = _ref0.file, _ref0$loaders = _ref0.loaders, loaders = _ref0$loaders === void 0 ? [] : _ref0$loaders, _ref0$loadOptions = _ref0.loadOptions, loadOptions = _ref0$loadOptions === void 0 ? {} : _ref0$loadOptions, companionFiles = _ref0.companionFiles, fileName = _ref0.fileName;
|
|
664
|
+
displayFileName = fileName || file.name; // Local drops expand mixed zips in the load queue before this runs. A `.zip`
|
|
665
|
+
// that still arrives here is a shapefile archive (remote URL, or a direct call).
|
|
666
|
+
if (!(isZipFileName(file.name) && isKeplerFileFormatAccepted('shp'))) {
|
|
667
|
+
_context4.next = 10;
|
|
668
|
+
break;
|
|
669
|
+
}
|
|
670
|
+
_context4.next = 5;
|
|
671
|
+
return unzipShapefileArchive(file);
|
|
672
|
+
case 5:
|
|
673
|
+
unzipped = _context4.sent;
|
|
674
|
+
shapefile = unzipped.find(function (entry) {
|
|
675
|
+
return getDroppedFileExtension(entry.name) === 'shp';
|
|
676
|
+
});
|
|
677
|
+
if (shapefile) {
|
|
678
|
+
_context4.next = 9;
|
|
679
|
+
break;
|
|
680
|
+
}
|
|
681
|
+
throw new Error('Zip archive does not contain a shapefile (.shp)');
|
|
682
|
+
case 9:
|
|
683
|
+
return _context4.abrupt("return", readFileInBatches({
|
|
684
|
+
file: shapefile,
|
|
685
|
+
loaders: loaders,
|
|
686
|
+
loadOptions: loadOptions,
|
|
687
|
+
companionFiles: [].concat(_toConsumableArray(unzipped), _toConsumableArray(companionFiles || [])),
|
|
688
|
+
fileName: displayFileName
|
|
689
|
+
}));
|
|
690
|
+
case 10:
|
|
691
|
+
_context4.next = 12;
|
|
692
|
+
return getKeplerLoaders(file, loaders);
|
|
693
|
+
case 12:
|
|
694
|
+
loaders = _context4.sent;
|
|
695
|
+
hasExtension = /\.[a-z0-9]+$/i.test(file.name);
|
|
696
|
+
mimeType = !hasExtension && file.type ? file.type : undefined; // Shapefile sidecars are resolved via a custom fetch that cannot be cloned
|
|
697
|
+
// into a worker. Other formats keep loaders.gl worker parsing.
|
|
698
|
+
isShapefile = getDroppedFileExtension(file.name) === 'shp';
|
|
699
|
+
companionFetch = isShapefile ? createCompanionFetch(companionFiles) : undefined;
|
|
700
|
+
loadOptions = _objectSpread(_objectSpread(_objectSpread(_objectSpread({
|
|
701
|
+
csv: CSV_LOADER_OPTIONS,
|
|
702
|
+
arrow: ARROW_LOADER_OPTIONS,
|
|
703
|
+
json: JSON_LOADER_OPTIONS,
|
|
704
|
+
parquet: PARQUET_LOADER_OPTIONS,
|
|
705
|
+
kml: KML_LOADER_OPTIONS,
|
|
706
|
+
gpx: KML_LOADER_OPTIONS,
|
|
707
|
+
tcx: KML_LOADER_OPTIONS,
|
|
708
|
+
shapefile: GIS_TABLE_LOADER_OPTIONS,
|
|
709
|
+
shp: {
|
|
710
|
+
_maxDimensions: 2
|
|
711
|
+
},
|
|
712
|
+
excel: EXCEL_LOADER_OPTIONS,
|
|
713
|
+
flatgeobuf: GIS_TABLE_LOADER_OPTIONS,
|
|
714
|
+
gis: {
|
|
715
|
+
reproject: true,
|
|
716
|
+
_targetCrs: 'WGS84'
|
|
717
|
+
},
|
|
718
|
+
metadata: true
|
|
719
|
+
}, isShapefile ? {
|
|
720
|
+
worker: false
|
|
721
|
+
} : {}), mimeType ? {
|
|
722
|
+
mimeType: mimeType
|
|
723
|
+
} : {}), companionFetch ? {
|
|
724
|
+
fetch: companionFetch
|
|
725
|
+
} : {}), loadOptions);
|
|
726
|
+
_context4.next = 20;
|
|
727
|
+
return parseInBatches(file, loaders, loadOptions);
|
|
728
|
+
case 20:
|
|
729
|
+
batchIterator = _context4.sent;
|
|
730
|
+
progressIterator = makeProgressIterator(batchIterator, {
|
|
731
|
+
size: file.size
|
|
732
|
+
});
|
|
733
|
+
sourceUrl = file.keplerSourceUrl;
|
|
734
|
+
keplerFormat = file.keplerFormat;
|
|
735
|
+
refreshIntervalMs = file.keplerRefreshIntervalMs;
|
|
736
|
+
etag = file.keplerEtag;
|
|
737
|
+
lastModified = file.keplerLastModified;
|
|
738
|
+
return _context4.abrupt("return", readBatch(progressIterator, displayFileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, file.size));
|
|
739
|
+
case 28:
|
|
740
|
+
case "end":
|
|
741
|
+
return _context4.stop();
|
|
742
|
+
}
|
|
743
|
+
}, _callee4);
|
|
744
|
+
}));
|
|
745
|
+
return _readFileInBatches.apply(this, arguments);
|
|
746
|
+
}
|
|
747
|
+
export function processFileData(_x11) {
|
|
748
|
+
return _processFileData.apply(this, arguments);
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* Expand staged files into cumulative payloads for one redraw per Arrow batch.
|
|
753
|
+
* Non-progressive files are present only in the first payload.
|
|
754
|
+
*/
|
|
755
|
+
function _processFileData() {
|
|
756
|
+
_processFileData = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee5(_ref1) {
|
|
757
|
+
var _getApplicationConfig;
|
|
758
|
+
var content, fileCache, fileName, data, format, processor, id, table, processorResult, geojsonData, tableRows, _result, result, progressiveData, _result$cols$0$data$l, _result$cols$, latestOnly, targetChunkCount, workerSource, sourceTable, cols, workerGeoArrowBinaryData, geoArrowBinaryData, sourceUrl, remoteFormat, fileSize, metadata;
|
|
759
|
+
return _regeneratorRuntime.wrap(function _callee5$(_context5) {
|
|
760
|
+
while (1) switch (_context5.prev = _context5.next) {
|
|
761
|
+
case 0:
|
|
762
|
+
content = _ref1.content, fileCache = _ref1.fileCache;
|
|
763
|
+
fileName = content.fileName, data = content.data;
|
|
764
|
+
// Hash the source URL when present so two remote files that share a filename
|
|
765
|
+
// (or a local file and a remote URL with the same name) do not collide and
|
|
766
|
+
// overwrite each other. The UI shows `label` (the filename), not this id.
|
|
767
|
+
id = generateHashIdFromString(content.sourceUrl || fileName); // decide on which table class to use based on application config
|
|
768
|
+
table = (_getApplicationConfig = getApplicationConfig().table) !== null && _getApplicationConfig !== void 0 ? _getApplicationConfig : KeplerTable;
|
|
769
|
+
if (typeof table.getFileProcessor === 'function') {
|
|
770
|
+
// use custom processors from table class
|
|
771
|
+
processorResult = table.getFileProcessor(data);
|
|
772
|
+
format = processorResult.format;
|
|
773
|
+
processor = processorResult.processor;
|
|
774
|
+
} else {
|
|
775
|
+
// use default processors
|
|
776
|
+
geojsonData = getGeoJsonFromLoaderResult(data);
|
|
777
|
+
if (isArrowData(data)) {
|
|
778
|
+
format = DATASET_FORMATS.arrow;
|
|
779
|
+
processor = processArrowBatches;
|
|
780
|
+
} else if (isKeplerGlMap(data)) {
|
|
781
|
+
format = DATASET_FORMATS.keplergl;
|
|
782
|
+
processor = processKeplerglJSON;
|
|
783
|
+
} else if (geojsonData) {
|
|
784
|
+
format = DATASET_FORMATS.geojson;
|
|
785
|
+
processor = function processor() {
|
|
786
|
+
return processGeojson(geojsonData);
|
|
787
|
+
};
|
|
788
|
+
} else if (isRowObject(data)) {
|
|
789
|
+
// csv file goes here
|
|
790
|
+
format = DATASET_FORMATS.row;
|
|
791
|
+
processor = processRowObject;
|
|
792
|
+
} else if (isPlainObject(data)) {
|
|
793
|
+
tableRows = data.data;
|
|
794
|
+
if ((data.shape === 'object-row-table' || data.shape === 'row-table') && Array.isArray(tableRows)) {
|
|
795
|
+
format = DATASET_FORMATS.row;
|
|
796
|
+
processor = function processor() {
|
|
797
|
+
return processRowObject(tableRows);
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
if (!(format && processor)) {
|
|
803
|
+
_context5.next = 40;
|
|
804
|
+
break;
|
|
805
|
+
}
|
|
806
|
+
_context5.prev = 6;
|
|
807
|
+
if (!(format === DATASET_FORMATS.arrow && content.skipArrowCompact)) {
|
|
808
|
+
_context5.next = 13;
|
|
809
|
+
break;
|
|
810
|
+
}
|
|
811
|
+
_context5.next = 10;
|
|
812
|
+
return processArrowBatches(data, {
|
|
813
|
+
compact: false
|
|
814
|
+
});
|
|
815
|
+
case 10:
|
|
816
|
+
_context5.t0 = _context5.sent;
|
|
817
|
+
_context5.next = 16;
|
|
818
|
+
break;
|
|
819
|
+
case 13:
|
|
820
|
+
_context5.next = 15;
|
|
821
|
+
return processor(data);
|
|
822
|
+
case 15:
|
|
823
|
+
_context5.t0 = _context5.sent;
|
|
824
|
+
case 16:
|
|
825
|
+
result = _context5.t0;
|
|
826
|
+
_context5.next = 22;
|
|
827
|
+
break;
|
|
828
|
+
case 19:
|
|
829
|
+
_context5.prev = 19;
|
|
830
|
+
_context5.t1 = _context5["catch"](6);
|
|
831
|
+
throw new Error("Can not process uploaded file, ".concat(getError(_context5.t1)));
|
|
832
|
+
case 22:
|
|
833
|
+
if (!(format === DATASET_FORMATS.arrow && (_result = result) !== null && _result !== void 0 && _result.cols)) {
|
|
834
|
+
_context5.next = 33;
|
|
835
|
+
break;
|
|
836
|
+
}
|
|
837
|
+
latestOnly = Boolean(content.skipArrowCompact);
|
|
838
|
+
targetChunkCount = (_result$cols$0$data$l = (_result$cols$ = result.cols[0]) === null || _result$cols$ === void 0 || (_result$cols$ = _result$cols$.data) === null || _result$cols$ === void 0 ? void 0 : _result$cols$.length) !== null && _result$cols$0$data$l !== void 0 ? _result$cols$0$data$l : 0;
|
|
839
|
+
workerSource = result; // Final file processing compacts Arrow batches to keep the number of deck.gl
|
|
840
|
+
// sublayers low. Schedule worker jobs from the original batches first so
|
|
841
|
+
// geometry conversion and triangulation can still run in parallel, then
|
|
842
|
+
// combine their output to match the single compacted table chunk.
|
|
843
|
+
if (!latestOnly && Array.isArray(data) && data.length > targetChunkCount) {
|
|
844
|
+
sourceTable = new arrow.Table(data);
|
|
845
|
+
cols = _toConsumableArray(Array(sourceTable.numCols).keys()).map(function (i) {
|
|
846
|
+
return sourceTable.getChildAt(i);
|
|
847
|
+
});
|
|
848
|
+
workerSource = _objectSpread(_objectSpread({}, result), {}, {
|
|
849
|
+
cols: cols
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
_context5.next = 29;
|
|
853
|
+
return parseGeoArrowBatchesOnWorkers(workerSource, {
|
|
854
|
+
latestOnly: latestOnly
|
|
855
|
+
});
|
|
856
|
+
case 29:
|
|
857
|
+
workerGeoArrowBinaryData = _context5.sent;
|
|
858
|
+
geoArrowBinaryData = targetChunkCount === 1 && !latestOnly && workerGeoArrowBinaryData ? combineGeoArrowWorkerChunks(workerGeoArrowBinaryData) : workerGeoArrowBinaryData;
|
|
859
|
+
if (geoArrowBinaryData) {
|
|
860
|
+
result = _objectSpread(_objectSpread({}, result), {}, {
|
|
861
|
+
geoArrowBinaryData: geoArrowBinaryData
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
if (content.prepareProgressiveReplay && !latestOnly) {
|
|
865
|
+
progressiveData = createProgressiveArrowData(workerSource, result, workerGeoArrowBinaryData);
|
|
866
|
+
}
|
|
867
|
+
case 33:
|
|
868
|
+
sourceUrl = content.sourceUrl;
|
|
869
|
+
remoteFormat = getPersistedRemoteFormat(content.keplerFormat, content.fileName);
|
|
870
|
+
fileSize = typeof content.fileSize === 'number' ? content.fileSize : undefined;
|
|
871
|
+
metadata = _objectSpread(_objectSpread({}, sourceUrl ? _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({
|
|
872
|
+
source: sourceUrl
|
|
873
|
+
}, remoteFormat ? {
|
|
874
|
+
sourceFormat: remoteFormat
|
|
875
|
+
} : {}), typeof content.refreshIntervalMs === 'number' && content.refreshIntervalMs > 0 ? {
|
|
876
|
+
refreshIntervalMs: content.refreshIntervalMs
|
|
877
|
+
} : {}), typeof content.etag === 'string' && content.etag ? {
|
|
878
|
+
etag: content.etag
|
|
879
|
+
} : {}), typeof content.lastModified === 'string' && content.lastModified ? {
|
|
880
|
+
lastModified: content.lastModified
|
|
881
|
+
} : {}), {}, {
|
|
882
|
+
lastFetchedAt: Date.now()
|
|
883
|
+
}) : {}), fileSize != null ? {
|
|
884
|
+
size: fileSize
|
|
885
|
+
} : {});
|
|
886
|
+
return _context5.abrupt("return", [].concat(_toConsumableArray(fileCache), [_objectSpread(_objectSpread({
|
|
887
|
+
data: result
|
|
888
|
+
}, progressiveData ? {
|
|
889
|
+
progressiveData: progressiveData
|
|
890
|
+
} : {}), {}, {
|
|
891
|
+
info: _objectSpread({
|
|
892
|
+
id: id,
|
|
893
|
+
label: content.fileName,
|
|
894
|
+
format: format
|
|
895
|
+
}, sourceUrl ? {
|
|
896
|
+
type: DatasetType.EXTERNALLY_HOSTED
|
|
897
|
+
} : {})
|
|
898
|
+
}, sourceUrl || fileSize != null ? {
|
|
899
|
+
metadata: metadata
|
|
900
|
+
} : {})]));
|
|
901
|
+
case 40:
|
|
902
|
+
throw new Error('Can not process uploaded file, unknown file format');
|
|
903
|
+
case 41:
|
|
904
|
+
case "end":
|
|
905
|
+
return _context5.stop();
|
|
906
|
+
}
|
|
907
|
+
}, _callee5, null, [[6, 19]]);
|
|
908
|
+
}));
|
|
909
|
+
return _processFileData.apply(this, arguments);
|
|
910
|
+
}
|
|
911
|
+
export function fileCacheToProgressiveSteps(fileCache) {
|
|
912
|
+
var numberOfSteps = Math.max.apply(Math, [1].concat(_toConsumableArray(fileCache.map(function (file) {
|
|
913
|
+
var _file$progressiveData;
|
|
914
|
+
return ((_file$progressiveData = file.progressiveData) === null || _file$progressiveData === void 0 ? void 0 : _file$progressiveData.length) || 1;
|
|
915
|
+
}))));
|
|
916
|
+
return Array.from({
|
|
917
|
+
length: numberOfSteps
|
|
918
|
+
}, function (_, stepIndex) {
|
|
919
|
+
return fileCache.flatMap(function (file) {
|
|
920
|
+
var progressiveData = file.progressiveData,
|
|
921
|
+
fileWithoutProgressiveData = _objectWithoutProperties(file, _excluded2);
|
|
922
|
+
if (!progressiveData) {
|
|
923
|
+
return stepIndex === 0 ? [fileWithoutProgressiveData] : [];
|
|
924
|
+
}
|
|
925
|
+
var data = progressiveData[stepIndex];
|
|
926
|
+
return data ? [_objectSpread(_objectSpread({}, fileWithoutProgressiveData), {}, {
|
|
927
|
+
data: data
|
|
928
|
+
})] : [];
|
|
929
|
+
});
|
|
930
|
+
}).filter(function (step) {
|
|
931
|
+
return step.length;
|
|
932
|
+
});
|
|
933
|
+
}
|
|
934
|
+
export function filesToDataPayload(fileCache, options) {
|
|
935
|
+
// seperate out files which could be a single datasets. or a keplergl map json
|
|
936
|
+
var collection = fileCache.reduce(function (accu, file) {
|
|
937
|
+
var data = file.data,
|
|
938
|
+
info = file.info;
|
|
939
|
+
if ((info === null || info === void 0 ? void 0 : info.format) === DATASET_FORMATS.keplergl) {
|
|
940
|
+
// if file contains a single kepler map dataset & config
|
|
941
|
+
accu.keplerMaps.push(_objectSpread(_objectSpread({}, data), {}, {
|
|
942
|
+
options: _objectSpread({
|
|
943
|
+
centerMap: !(data.config && data.config.mapState)
|
|
944
|
+
}, options || {})
|
|
945
|
+
}));
|
|
946
|
+
} else if (DATASET_FORMATS[info === null || info === void 0 ? void 0 : info.format]) {
|
|
947
|
+
// if file contains only data
|
|
948
|
+
var newDataset = _objectSpread({
|
|
949
|
+
data: data,
|
|
950
|
+
info: _objectSpread({
|
|
951
|
+
id: (info === null || info === void 0 ? void 0 : info.id) || generateHashId(4)
|
|
952
|
+
}, info || {})
|
|
953
|
+
}, file.metadata ? {
|
|
954
|
+
metadata: file.metadata
|
|
955
|
+
} : {});
|
|
956
|
+
accu.datasets.push(newDataset);
|
|
957
|
+
}
|
|
958
|
+
return accu;
|
|
959
|
+
}, {
|
|
960
|
+
datasets: [],
|
|
961
|
+
keplerMaps: []
|
|
962
|
+
});
|
|
963
|
+
|
|
964
|
+
// add kepler map first with config
|
|
965
|
+
// add datasets later in one add data call
|
|
966
|
+
return collection.keplerMaps.concat(_objectSpread({
|
|
967
|
+
datasets: collection.datasets
|
|
968
|
+
}, options && collection.datasets.length ? {
|
|
969
|
+
options: options
|
|
970
|
+
} : {}));
|
|
971
|
+
}
|
|
972
|
+
//# sourceMappingURL=file-handler.js.map
|