@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/data-processor.js +674 -0
- package/dist/cjs/data-processor.js.map +1 -0
- package/dist/cjs/export/dataset-to-geojson.js +138 -0
- package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
- package/dist/cjs/export/geopackage-writer.js +385 -0
- package/dist/cjs/export/geopackage-writer.js.map +1 -0
- package/dist/cjs/export/geoparquet-writer.js +351 -0
- package/dist/cjs/export/geoparquet-writer.js.map +1 -0
- package/dist/cjs/export/kml-writer.js +347 -0
- package/dist/cjs/export/kml-writer.js.map +1 -0
- package/dist/cjs/export/shapefile-writer.js +708 -0
- package/dist/cjs/export/shapefile-writer.js.map +1 -0
- package/dist/cjs/export/sql-js.js +95 -0
- package/dist/cjs/export/sql-js.js.map +1 -0
- package/dist/cjs/export/vector-export.js +187 -0
- package/dist/cjs/export/vector-export.js.map +1 -0
- package/dist/cjs/file-handler.js +997 -0
- package/dist/cjs/file-handler.js.map +1 -0
- package/dist/cjs/index.js +114 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/kepler-csv-loader.js +163 -0
- package/dist/cjs/kepler-csv-loader.js.map +1 -0
- package/dist/cjs/loader-registry.js +542 -0
- package/dist/cjs/loader-registry.js.map +1 -0
- package/dist/cjs/remote-file.js +675 -0
- package/dist/cjs/remote-file.js.map +1 -0
- package/dist/cjs/shapefile-files.js +281 -0
- package/dist/cjs/shapefile-files.js.map +1 -0
- package/dist/cjs/types.js +6 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/data-processor.d.ts +30 -16
- package/dist/esm/data-processor.js +655 -0
- package/dist/esm/data-processor.js.map +1 -0
- package/dist/esm/export/dataset-to-geojson.js +132 -0
- package/dist/esm/export/dataset-to-geojson.js.map +1 -0
- package/dist/esm/export/geopackage-writer.js +381 -0
- package/dist/esm/export/geopackage-writer.js.map +1 -0
- package/dist/esm/export/geoparquet-writer.js +336 -0
- package/dist/esm/export/geoparquet-writer.js.map +1 -0
- package/dist/esm/export/kml-writer.js +341 -0
- package/dist/esm/export/kml-writer.js.map +1 -0
- package/dist/esm/export/shapefile-writer.js +701 -0
- package/dist/esm/export/shapefile-writer.js.map +1 -0
- package/dist/esm/export/sql-js.js +80 -0
- package/dist/esm/export/sql-js.js.map +1 -0
- package/dist/esm/export/vector-export.js +176 -0
- package/dist/esm/export/vector-export.js.map +1 -0
- package/dist/esm/file-handler.js +972 -0
- package/dist/esm/file-handler.js.map +1 -0
- package/dist/esm/index.js +12 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/kepler-csv-loader.js +152 -0
- package/dist/esm/kepler-csv-loader.js.map +1 -0
- package/dist/esm/loader-registry.js +510 -0
- package/dist/esm/loader-registry.js.map +1 -0
- package/dist/esm/remote-file.js +664 -0
- package/dist/esm/remote-file.js.map +1 -0
- package/dist/esm/shapefile-files.js +264 -0
- package/dist/esm/shapefile-files.js.map +1 -0
- package/dist/esm/types.js +3 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/export/dataset-to-geojson.d.ts +12 -0
- package/dist/export/geopackage-writer.d.ts +28 -0
- package/dist/export/geoparquet-writer.d.ts +7 -0
- package/dist/export/kml-writer.d.ts +13 -0
- package/dist/export/shapefile-writer.d.ts +28 -0
- package/dist/export/sql-js.d.ts +8 -0
- package/dist/export/vector-export.d.ts +22 -0
- package/dist/file-handler.d.ts +55 -8
- package/dist/index.d.ts +6 -0
- package/dist/kepler-csv-loader.d.ts +12 -0
- package/dist/loader-registry.d.ts +29 -0
- package/dist/remote-file.d.ts +79 -0
- package/dist/shapefile-files.d.ts +39 -0
- package/dist/types.d.ts +13 -0
- package/package.json +44 -26
- package/dist/data-processor.js +0 -637
- package/dist/file-handler.js +0 -409
- package/dist/index.js +0 -39
- package/dist/types.js +0 -6
|
@@ -0,0 +1,997 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
+
var _typeof = require("@babel/runtime/helpers/typeof");
|
|
5
|
+
Object.defineProperty(exports, "__esModule", {
|
|
6
|
+
value: true
|
|
7
|
+
});
|
|
8
|
+
exports.fileCacheToProgressiveSteps = fileCacheToProgressiveSteps;
|
|
9
|
+
exports.filesToDataPayload = filesToDataPayload;
|
|
10
|
+
exports.getGeoJsonFromLoaderResult = getGeoJsonFromLoaderResult;
|
|
11
|
+
exports.getPersistedRemoteFormat = getPersistedRemoteFormat;
|
|
12
|
+
exports.isArrowData = isArrowData;
|
|
13
|
+
Object.defineProperty(exports, "isArrowTable", {
|
|
14
|
+
enumerable: true,
|
|
15
|
+
get: function get() {
|
|
16
|
+
return _utils.isArrowTable;
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
exports.isFeature = isFeature;
|
|
20
|
+
exports.isFeatureCollection = isFeatureCollection;
|
|
21
|
+
exports.isGeoJson = isGeoJson;
|
|
22
|
+
exports.isGeoJsonFeatureArray = isGeoJsonFeatureArray;
|
|
23
|
+
exports.isKeplerGlMap = isKeplerGlMap;
|
|
24
|
+
exports.isRowObject = isRowObject;
|
|
25
|
+
exports.makeProgressIterator = makeProgressIterator;
|
|
26
|
+
exports.parseGeoArrowBatchesOnWorkers = parseGeoArrowBatchesOnWorkers;
|
|
27
|
+
exports.processFileData = processFileData;
|
|
28
|
+
exports.readBatch = readBatch;
|
|
29
|
+
exports.readFileInBatches = readFileInBatches;
|
|
30
|
+
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
|
|
31
|
+
var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
|
|
32
|
+
var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
|
|
33
|
+
var _objectWithoutProperties2 = _interopRequireDefault(require("@babel/runtime/helpers/objectWithoutProperties"));
|
|
34
|
+
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
35
|
+
var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
|
|
36
|
+
var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
|
|
37
|
+
var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
|
|
38
|
+
var _core = require("@loaders.gl/core");
|
|
39
|
+
var _gis = require("@loaders.gl/gis");
|
|
40
|
+
var arrow = _interopRequireWildcard(require("apache-arrow"));
|
|
41
|
+
var _arrow = require("@loaders.gl/arrow");
|
|
42
|
+
var _utils = require("@kepler.gl/utils");
|
|
43
|
+
var _commonUtils = require("@kepler.gl/common-utils");
|
|
44
|
+
var _constants = require("@kepler.gl/constants");
|
|
45
|
+
var _table = require("@kepler.gl/table");
|
|
46
|
+
var _dataProcessor = require("./data-processor");
|
|
47
|
+
var _loaderRegistry = require("./loader-registry");
|
|
48
|
+
var _shapefileFiles = require("./shapefile-files");
|
|
49
|
+
var _excluded = ["geoArrowBinaryData"],
|
|
50
|
+
_excluded2 = ["progressiveData"];
|
|
51
|
+
function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function _interopRequireWildcard(e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, "default": e }; if (null === e || "object" != _typeof(e) && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (var _t in e) "default" !== _t && {}.hasOwnProperty.call(e, _t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, _t)) && (i.get || i.set) ? o(f, _t, i) : f[_t] = e[_t]); return f; })(e, t); }
|
|
52
|
+
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
|
|
53
|
+
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { (0, _defineProperty2["default"])(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
|
|
54
|
+
function _asyncIterator(r) { var n, t, o, e = 2; for ("undefined" != typeof Symbol && (t = Symbol.asyncIterator, o = Symbol.iterator); e--;) { if (t && null != (n = r[t])) return n.call(r); if (o && null != (n = r[o])) return new AsyncFromSyncIterator(n.call(r)); t = "@@asyncIterator", o = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
|
|
55
|
+
function AsyncFromSyncIterator(r) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var n = r.done; return Promise.resolve(r.value).then(function (r) { return { value: r, done: n }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(r) { this.s = r, this.n = r.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, "return": function _return(r) { var n = this.s["return"]; return void 0 === n ? Promise.resolve({ value: r, done: !0 }) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); }, "throw": function _throw(r) { var n = this.s["return"]; return void 0 === n ? Promise.reject(r) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(r); } // SPDX-License-Identifier: MIT
|
|
56
|
+
// Copyright contributors to the kepler.gl project
|
|
57
|
+
var BATCH_TYPE = {
|
|
58
|
+
METADATA: 'metadata',
|
|
59
|
+
PARTIAL_RESULT: 'partial-result',
|
|
60
|
+
FINAL_RESULT: 'final-result'
|
|
61
|
+
};
|
|
62
|
+
var CSV_LOADER_OPTIONS = {
|
|
63
|
+
shape: 'object-row-table',
|
|
64
|
+
dynamicTyping: false // not working for now
|
|
65
|
+
};
|
|
66
|
+
var ARROW_LOADER_OPTIONS = {
|
|
67
|
+
shape: 'arrow-table',
|
|
68
|
+
batchDebounceMs: 10 // time to delay between batches, for incremental loading
|
|
69
|
+
};
|
|
70
|
+
var PARQUET_LOADER_OPTIONS = {
|
|
71
|
+
shape: 'arrow-table'
|
|
72
|
+
};
|
|
73
|
+
var JSON_LOADER_OPTIONS = {
|
|
74
|
+
shape: 'object-row-table',
|
|
75
|
+
// instruct loaders.gl on what json paths to stream
|
|
76
|
+
jsonpaths: ['$',
|
|
77
|
+
// JSON Row array
|
|
78
|
+
'$.features',
|
|
79
|
+
// GeoJSON
|
|
80
|
+
'$.datasets' // KeplerGL JSON
|
|
81
|
+
]
|
|
82
|
+
};
|
|
83
|
+
var KML_LOADER_OPTIONS = {
|
|
84
|
+
shape: 'geojson-table'
|
|
85
|
+
};
|
|
86
|
+
var GIS_TABLE_LOADER_OPTIONS = {
|
|
87
|
+
shape: 'geojson-table'
|
|
88
|
+
};
|
|
89
|
+
var WORKER_GEOARROW_ENCODINGS = new Set([_constants.GEOARROW_EXTENSIONS.POINT, _constants.GEOARROW_EXTENSIONS.LINESTRING, _constants.GEOARROW_EXTENSIONS.POLYGON, _constants.GEOARROW_EXTENSIONS.MULTIPOINT, _constants.GEOARROW_EXTENSIONS.MULTILINESTRING, _constants.GEOARROW_EXTENSIONS.MULTIPOLYGON]);
|
|
90
|
+
var EXCEL_LOADER_OPTIONS = {
|
|
91
|
+
shape: 'object-row-table'
|
|
92
|
+
};
|
|
93
|
+
function combineGeoArrowWorkerChunks(parsed) {
|
|
94
|
+
var combined = {};
|
|
95
|
+
Object.entries(parsed).forEach(function (_ref) {
|
|
96
|
+
var _ref2 = (0, _slicedToArray2["default"])(_ref, 2),
|
|
97
|
+
columnIndex = _ref2[0],
|
|
98
|
+
chunks = _ref2[1];
|
|
99
|
+
var ordered = Object.entries(chunks).map(function (_ref3) {
|
|
100
|
+
var _ref4 = (0, _slicedToArray2["default"])(_ref3, 2),
|
|
101
|
+
chunkIndex = _ref4[0],
|
|
102
|
+
data = _ref4[1];
|
|
103
|
+
return {
|
|
104
|
+
chunkIndex: Number(chunkIndex),
|
|
105
|
+
data: data
|
|
106
|
+
};
|
|
107
|
+
}).sort(function (a, b) {
|
|
108
|
+
return a.chunkIndex - b.chunkIndex;
|
|
109
|
+
});
|
|
110
|
+
if (!ordered.length) {
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
var featureTypes = {
|
|
114
|
+
point: false,
|
|
115
|
+
line: false,
|
|
116
|
+
polygon: false
|
|
117
|
+
};
|
|
118
|
+
var bounds = [Infinity, Infinity, -Infinity, -Infinity];
|
|
119
|
+
var binaryGeometries = [];
|
|
120
|
+
var meanCenters = [];
|
|
121
|
+
var hasMeanCenters = true;
|
|
122
|
+
ordered.forEach(function (_ref5) {
|
|
123
|
+
var data = _ref5.data;
|
|
124
|
+
data.binaryGeometries.forEach(function (geometry) {
|
|
125
|
+
return binaryGeometries.push(geometry);
|
|
126
|
+
});
|
|
127
|
+
featureTypes.point || (featureTypes.point = data.featureTypes.point);
|
|
128
|
+
featureTypes.line || (featureTypes.line = data.featureTypes.line);
|
|
129
|
+
featureTypes.polygon || (featureTypes.polygon = data.featureTypes.polygon);
|
|
130
|
+
bounds[0] = Math.min(bounds[0], data.bounds[0]);
|
|
131
|
+
bounds[1] = Math.min(bounds[1], data.bounds[1]);
|
|
132
|
+
bounds[2] = Math.max(bounds[2], data.bounds[2]);
|
|
133
|
+
bounds[3] = Math.max(bounds[3], data.bounds[3]);
|
|
134
|
+
if (data.meanCenters) {
|
|
135
|
+
data.meanCenters.forEach(function (center) {
|
|
136
|
+
return meanCenters.push(center);
|
|
137
|
+
});
|
|
138
|
+
} else {
|
|
139
|
+
hasMeanCenters = false;
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
combined[Number(columnIndex)] = {
|
|
143
|
+
0: _objectSpread({
|
|
144
|
+
binaryGeometries: binaryGeometries,
|
|
145
|
+
featureTypes: featureTypes,
|
|
146
|
+
bounds: bounds
|
|
147
|
+
}, hasMeanCenters ? {
|
|
148
|
+
meanCenters: meanCenters
|
|
149
|
+
} : {})
|
|
150
|
+
};
|
|
151
|
+
});
|
|
152
|
+
return Object.keys(combined).length ? combined : undefined;
|
|
153
|
+
}
|
|
154
|
+
function geoArrowWorkerChunksThrough(parsed, lastChunkIndex) {
|
|
155
|
+
if (!parsed) {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
var selected = {};
|
|
159
|
+
Object.entries(parsed).forEach(function (_ref6) {
|
|
160
|
+
var _ref7 = (0, _slicedToArray2["default"])(_ref6, 2),
|
|
161
|
+
columnIndex = _ref7[0],
|
|
162
|
+
chunks = _ref7[1];
|
|
163
|
+
var selectedChunks = Object.fromEntries(Object.entries(chunks).filter(function (_ref8) {
|
|
164
|
+
var _ref9 = (0, _slicedToArray2["default"])(_ref8, 1),
|
|
165
|
+
chunkIndex = _ref9[0];
|
|
166
|
+
return Number(chunkIndex) <= lastChunkIndex;
|
|
167
|
+
}));
|
|
168
|
+
if (Object.keys(selectedChunks).length) {
|
|
169
|
+
selected[Number(columnIndex)] = selectedChunks;
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
return Object.keys(selected).length ? selected : undefined;
|
|
173
|
+
}
|
|
174
|
+
function createProgressiveArrowData(source, finalData, geoArrowBinaryData) {
|
|
175
|
+
var _source$cols;
|
|
176
|
+
var chunks = (_source$cols = source.cols) === null || _source$cols === void 0 || (_source$cols = _source$cols[0]) === null || _source$cols === void 0 ? void 0 : _source$cols.data;
|
|
177
|
+
if (!Array.isArray(chunks) || chunks.length < 2 || !source.cols) {
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
var _sourceCache = source.geoArrowBinaryData,
|
|
181
|
+
sourceWithoutCache = (0, _objectWithoutProperties2["default"])(source, _excluded);
|
|
182
|
+
var snapshots = [];
|
|
183
|
+
var rowCount = 0;
|
|
184
|
+
// The final snapshot uses the normally processed (and usually compacted)
|
|
185
|
+
// table. Intermediate snapshots retain record-batch boundaries.
|
|
186
|
+
for (var chunkIndex = 0; chunkIndex < chunks.length - 1; chunkIndex++) {
|
|
187
|
+
rowCount += chunks[chunkIndex].length;
|
|
188
|
+
var cols = source.cols.map(function (column) {
|
|
189
|
+
return column.slice(0, rowCount);
|
|
190
|
+
});
|
|
191
|
+
var snapshotCache = geoArrowWorkerChunksThrough(geoArrowBinaryData, chunkIndex);
|
|
192
|
+
snapshots.push(_objectSpread(_objectSpread({}, sourceWithoutCache), {}, {
|
|
193
|
+
cols: cols
|
|
194
|
+
}, snapshotCache ? {
|
|
195
|
+
geoArrowBinaryData: snapshotCache
|
|
196
|
+
} : {}));
|
|
197
|
+
}
|
|
198
|
+
snapshots.push(finalData);
|
|
199
|
+
return snapshots;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Convert native GeoArrow chunks to deck.gl binary features without blocking
|
|
204
|
+
* the main thread. WKB/WKT remain on the synchronous fallback because the
|
|
205
|
+
* loaders.gl triangulation worker only supports native GeoArrow encodings.
|
|
206
|
+
*/
|
|
207
|
+
function parseGeoArrowBatchesOnWorkers(_x1) {
|
|
208
|
+
return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* check if data is an ArrowData object, which is an array of RecordBatch
|
|
212
|
+
* @param data - object to check
|
|
213
|
+
* @returns {boolean} - true if data is an ArrowData object type guarded
|
|
214
|
+
*/
|
|
215
|
+
function _parseGeoArrowBatchesOnWorkers() {
|
|
216
|
+
_parseGeoArrowBatchesOnWorkers = (0, _asyncToGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee3(data) {
|
|
217
|
+
var options,
|
|
218
|
+
parseOnWorker,
|
|
219
|
+
jobs,
|
|
220
|
+
parsed,
|
|
221
|
+
results,
|
|
222
|
+
useFallback,
|
|
223
|
+
_args3 = arguments;
|
|
224
|
+
return _regenerator["default"].wrap(function _callee3$(_context3) {
|
|
225
|
+
while (1) switch (_context3.prev = _context3.next) {
|
|
226
|
+
case 0:
|
|
227
|
+
options = _args3.length > 1 && _args3[1] !== undefined ? _args3[1] : {};
|
|
228
|
+
parseOnWorker = options.parseOnWorker || _arrow.parseGeoArrowOnWorker;
|
|
229
|
+
if (!(!options.parseOnWorker && typeof globalThis.Worker === 'undefined')) {
|
|
230
|
+
_context3.next = 4;
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
return _context3.abrupt("return", undefined);
|
|
234
|
+
case 4:
|
|
235
|
+
jobs = [];
|
|
236
|
+
data.fields.forEach(function (field, columnIndex) {
|
|
237
|
+
var _field$metadata, _field$metadata$get, _data$cols, _geoColumn$data;
|
|
238
|
+
var encoding = (_field$metadata = field.metadata) === null || _field$metadata === void 0 || (_field$metadata$get = _field$metadata.get) === null || _field$metadata$get === void 0 ? void 0 : _field$metadata$get.call(_field$metadata, _constants.GEOARROW_METADATA_KEY);
|
|
239
|
+
var geoColumn = (_data$cols = data.cols) === null || _data$cols === void 0 ? void 0 : _data$cols[columnIndex];
|
|
240
|
+
if (!WORKER_GEOARROW_ENCODINGS.has(encoding) || !(geoColumn !== null && geoColumn !== void 0 && (_geoColumn$data = geoColumn.data) !== null && _geoColumn$data !== void 0 && _geoColumn$data.length)) {
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
var chunkIndexes = options.latestOnly ? [geoColumn.data.length - 1] : geoColumn.data.map(function (_, index) {
|
|
244
|
+
return index;
|
|
245
|
+
});
|
|
246
|
+
chunkIndexes.forEach(function (chunkIndex) {
|
|
247
|
+
var geometryChunk = encoding === _constants.GEOARROW_EXTENSIONS.POINT ? (0, _utils.convertStructToFixedSizeList)(geoColumn.data[chunkIndex]) : geoColumn.data[chunkIndex];
|
|
248
|
+
var chunkCopy = (0, _arrow.hardClone)(geometryChunk, true);
|
|
249
|
+
var chunkOffset = geoColumn.data.slice(0, chunkIndex).reduce(function (offset, chunk) {
|
|
250
|
+
return offset + chunk.length;
|
|
251
|
+
}, 0);
|
|
252
|
+
var chunkType = chunkCopy.type;
|
|
253
|
+
var transferableType = _objectSpread(_objectSpread({}, chunkType), {}, {
|
|
254
|
+
typeId: chunkType.typeId
|
|
255
|
+
}, chunkType.listSize !== undefined ? {
|
|
256
|
+
listSize: chunkType.listSize
|
|
257
|
+
} : {});
|
|
258
|
+
var sourceData = {
|
|
259
|
+
operation: 'parse-geoarrow',
|
|
260
|
+
chunkData: {
|
|
261
|
+
type: transferableType,
|
|
262
|
+
offset: chunkCopy.offset,
|
|
263
|
+
length: chunkCopy.length,
|
|
264
|
+
nullCount: chunkCopy.nullCount,
|
|
265
|
+
buffers: chunkCopy.buffers,
|
|
266
|
+
children: chunkCopy.children,
|
|
267
|
+
dictionary: chunkCopy.dictionary
|
|
268
|
+
},
|
|
269
|
+
chunkIndex: chunkIndex,
|
|
270
|
+
chunkOffset: chunkOffset,
|
|
271
|
+
geometryEncoding: encoding,
|
|
272
|
+
calculateMeanCenters: true,
|
|
273
|
+
triangle: true
|
|
274
|
+
};
|
|
275
|
+
jobs.push({
|
|
276
|
+
columnIndex: columnIndex,
|
|
277
|
+
chunkIndex: chunkIndex,
|
|
278
|
+
result: parseOnWorker(sourceData),
|
|
279
|
+
fallback: function fallback() {
|
|
280
|
+
return (0, _gis.getBinaryGeometriesFromArrow)(
|
|
281
|
+
// Use the original buffers: the worker copy may be transferred.
|
|
282
|
+
// A local Data wrapper also accepts chunks from another Arrow runtime.
|
|
283
|
+
arrow.makeVector(new arrow.Data(geometryChunk.type, geometryChunk.offset, geometryChunk.length, geometryChunk.nullCount, geometryChunk.buffers, geometryChunk.children, geometryChunk.dictionary)), sourceData.geometryEncoding, {
|
|
284
|
+
chunkOffset: chunkOffset,
|
|
285
|
+
calculateMeanCenters: true,
|
|
286
|
+
triangulate: true
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
});
|
|
290
|
+
});
|
|
291
|
+
});
|
|
292
|
+
if (jobs.length) {
|
|
293
|
+
_context3.next = 8;
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
return _context3.abrupt("return", undefined);
|
|
297
|
+
case 8:
|
|
298
|
+
parsed = {};
|
|
299
|
+
_context3.next = 11;
|
|
300
|
+
return Promise.allSettled(jobs.map(function (job) {
|
|
301
|
+
return job.result;
|
|
302
|
+
}));
|
|
303
|
+
case 11:
|
|
304
|
+
results = _context3.sent;
|
|
305
|
+
// A partial cache can look complete after compaction merges surviving chunks
|
|
306
|
+
// into one. Reparse every selected original batch if acceleration failed.
|
|
307
|
+
// Using the originals also avoids parsing padding in newly compacted buffers.
|
|
308
|
+
useFallback = results.some(function (result) {
|
|
309
|
+
return result.status !== 'fulfilled' || !result.value.binaryDataFromGeoArrow;
|
|
310
|
+
});
|
|
311
|
+
results.forEach(function (result, index) {
|
|
312
|
+
var _jobs$index = jobs[index],
|
|
313
|
+
columnIndex = _jobs$index.columnIndex,
|
|
314
|
+
chunkIndex = _jobs$index.chunkIndex,
|
|
315
|
+
fallback = _jobs$index.fallback;
|
|
316
|
+
var binaryData = useFallback ? fallback() : result.status === 'fulfilled' ? result.value.binaryDataFromGeoArrow : undefined;
|
|
317
|
+
if (!binaryData) {
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
parsed[columnIndex] = parsed[columnIndex] || {};
|
|
321
|
+
parsed[columnIndex][chunkIndex] = binaryData;
|
|
322
|
+
});
|
|
323
|
+
if (Object.keys(parsed).length) {
|
|
324
|
+
_context3.next = 16;
|
|
325
|
+
break;
|
|
326
|
+
}
|
|
327
|
+
return _context3.abrupt("return", undefined);
|
|
328
|
+
case 16:
|
|
329
|
+
return _context3.abrupt("return", options.targetChunkCount === 1 && !options.latestOnly ? combineGeoArrowWorkerChunks(parsed) : parsed);
|
|
330
|
+
case 17:
|
|
331
|
+
case "end":
|
|
332
|
+
return _context3.stop();
|
|
333
|
+
}
|
|
334
|
+
}, _callee3);
|
|
335
|
+
}));
|
|
336
|
+
return _parseGeoArrowBatchesOnWorkers.apply(this, arguments);
|
|
337
|
+
}
|
|
338
|
+
function isArrowData(data) {
|
|
339
|
+
return Array.isArray(data) && Boolean(data.length && data[0].data && data[0].schema);
|
|
340
|
+
}
|
|
341
|
+
function isGeoJson(json) {
|
|
342
|
+
// json can be feature collection
|
|
343
|
+
// or single feature
|
|
344
|
+
return (0, _utils.isPlainObject)(json) && (isFeature(json) || isFeatureCollection(json));
|
|
345
|
+
}
|
|
346
|
+
function isFeature(json) {
|
|
347
|
+
return (0, _utils.isPlainObject)(json) && json.type === 'Feature' && Boolean(json.geometry);
|
|
348
|
+
}
|
|
349
|
+
function isFeatureCollection(json) {
|
|
350
|
+
return (0, _utils.isPlainObject)(json) && json.type === 'FeatureCollection' && Boolean(json.features);
|
|
351
|
+
}
|
|
352
|
+
function isGeoJsonFeatureArray(json) {
|
|
353
|
+
return Array.isArray(json) && json.length > 0 && isFeature(json[0]);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Normalize loaders.gl GIS output (Feature arrays, geojson-table, object-row
|
|
358
|
+
* tables of Features) into a GeoJSON Feature or FeatureCollection Kepler can process.
|
|
359
|
+
*/
|
|
360
|
+
function getGeoJsonFromLoaderResult(data) {
|
|
361
|
+
if (isGeoJson(data)) {
|
|
362
|
+
return data;
|
|
363
|
+
}
|
|
364
|
+
if (isGeoJsonFeatureArray(data)) {
|
|
365
|
+
return {
|
|
366
|
+
type: 'FeatureCollection',
|
|
367
|
+
features: data
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
if (!(0, _utils.isPlainObject)(data)) {
|
|
371
|
+
return null;
|
|
372
|
+
}
|
|
373
|
+
if (data.shape === 'geojson-table' && Array.isArray(data.features)) {
|
|
374
|
+
return {
|
|
375
|
+
type: 'FeatureCollection',
|
|
376
|
+
features: data.features
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
if (data.shape === 'object-row-table' || data.shape === 'row-table') {
|
|
380
|
+
if (isGeoJsonFeatureArray(data.data)) {
|
|
381
|
+
return {
|
|
382
|
+
type: 'FeatureCollection',
|
|
383
|
+
features: data.data
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
// Empty spreadsheet tables stay rows, not empty GeoJSON.
|
|
387
|
+
return null;
|
|
388
|
+
}
|
|
389
|
+
// ShapefileLoader parseInBatches (v3 shape) yields `{data: Feature[]}`,
|
|
390
|
+
// including empty files as `{data: []}`.
|
|
391
|
+
if (Array.isArray(data.data) && (data.data.length === 0 || isGeoJsonFeatureArray(data.data))) {
|
|
392
|
+
return {
|
|
393
|
+
type: 'FeatureCollection',
|
|
394
|
+
features: data.data
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
return null;
|
|
398
|
+
}
|
|
399
|
+
function getBatchRows(batch) {
|
|
400
|
+
if (Array.isArray(batch === null || batch === void 0 ? void 0 : batch.features) && (batch.shape === 'geojson-table' || batch.type === 'FeatureCollection')) {
|
|
401
|
+
return batch.features;
|
|
402
|
+
}
|
|
403
|
+
var batchData = (0, _utils.isArrowTable)(batch.data) ? batch.data.batches : batch.data;
|
|
404
|
+
return Array.isArray(batchData) ? batchData : null;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Reconstruct the loader table/GIS identity from the current batch. Empty
|
|
409
|
+
* shapefile (`{data: []}`) and spreadsheet (`object-row-table`) batches must
|
|
410
|
+
* not be flattened to a bare `[]`, which processFileData cannot classify.
|
|
411
|
+
*
|
|
412
|
+
* JSON streaming first yields an empty `object-row-table` `partial-result`
|
|
413
|
+
* (with `jsonpath` / `container`). That is a placeholder, not the dataset.
|
|
414
|
+
*/
|
|
415
|
+
function getAggregatedLoaderResult(batch, rows, current) {
|
|
416
|
+
if (batch.batchType === BATCH_TYPE.METADATA || batch.batchType === BATCH_TYPE.PARTIAL_RESULT || batch.batchType === BATCH_TYPE.FINAL_RESULT || batch.jsonpath || batch.container) {
|
|
417
|
+
return current;
|
|
418
|
+
}
|
|
419
|
+
if (batch.shape === 'geojson-table') {
|
|
420
|
+
return {
|
|
421
|
+
type: 'FeatureCollection',
|
|
422
|
+
features: (0, _toConsumableArray2["default"])(rows)
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
// Non-empty CSV/Excel stay a flat row array (existing readBatch contract).
|
|
426
|
+
if (rows.length > 0) {
|
|
427
|
+
return current;
|
|
428
|
+
}
|
|
429
|
+
if (batch.shape === 'object-row-table' || batch.shape === 'row-table') {
|
|
430
|
+
return {
|
|
431
|
+
shape: batch.shape,
|
|
432
|
+
data: []
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
// ShapefileLoader parseInBatches always yields v3 `{header, data: Feature[]}`.
|
|
436
|
+
if (batch.header != null && Array.isArray(batch.data)) {
|
|
437
|
+
return {
|
|
438
|
+
data: []
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
return current;
|
|
442
|
+
}
|
|
443
|
+
function isRowObject(json) {
|
|
444
|
+
return Array.isArray(json) && (0, _utils.isPlainObject)(json[0]);
|
|
445
|
+
}
|
|
446
|
+
function isKeplerGlMap(json) {
|
|
447
|
+
return Boolean((0, _utils.isPlainObject)(json) && json.datasets && json.config && json.info && (0, _utils.isPlainObject)(json.info) && json.info.app === 'kepler.gl');
|
|
448
|
+
}
|
|
449
|
+
function makeProgressIterator(_x, _x2) {
|
|
450
|
+
return _makeProgressIterator.apply(this, arguments);
|
|
451
|
+
} // eslint-disable-next-line complexity
|
|
452
|
+
function _makeProgressIterator() {
|
|
453
|
+
_makeProgressIterator = (0, _wrapAsyncGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee(asyncIterator, info) {
|
|
454
|
+
var rowCount, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch, rowCountInBatch, percent, progress;
|
|
455
|
+
return _regenerator["default"].wrap(function _callee$(_context) {
|
|
456
|
+
while (1) switch (_context.prev = _context.next) {
|
|
457
|
+
case 0:
|
|
458
|
+
rowCount = 0;
|
|
459
|
+
_iteratorAbruptCompletion = false;
|
|
460
|
+
_didIteratorError = false;
|
|
461
|
+
_context.prev = 3;
|
|
462
|
+
_iterator = _asyncIterator(asyncIterator);
|
|
463
|
+
case 5:
|
|
464
|
+
_context.next = 7;
|
|
465
|
+
return (0, _awaitAsyncGenerator2["default"])(_iterator.next());
|
|
466
|
+
case 7:
|
|
467
|
+
if (!(_iteratorAbruptCompletion = !(_step = _context.sent).done)) {
|
|
468
|
+
_context.next = 18;
|
|
469
|
+
break;
|
|
470
|
+
}
|
|
471
|
+
batch = _step.value;
|
|
472
|
+
// the length could be stored in `batch.length` for arrow batch
|
|
473
|
+
rowCountInBatch = batch.data && (batch.data.length || batch.length) || 0;
|
|
474
|
+
rowCount += rowCountInBatch;
|
|
475
|
+
percent = Number.isFinite(batch.bytesUsed) ? batch.bytesUsed / info.size : null; // Update progress object
|
|
476
|
+
progress = _objectSpread({
|
|
477
|
+
rowCount: rowCount,
|
|
478
|
+
rowCountInBatch: rowCountInBatch
|
|
479
|
+
}, Number.isFinite(percent) ? {
|
|
480
|
+
percent: percent
|
|
481
|
+
} : {});
|
|
482
|
+
_context.next = 15;
|
|
483
|
+
return _objectSpread(_objectSpread({}, batch), {}, {
|
|
484
|
+
progress: progress
|
|
485
|
+
});
|
|
486
|
+
case 15:
|
|
487
|
+
_iteratorAbruptCompletion = false;
|
|
488
|
+
_context.next = 5;
|
|
489
|
+
break;
|
|
490
|
+
case 18:
|
|
491
|
+
_context.next = 24;
|
|
492
|
+
break;
|
|
493
|
+
case 20:
|
|
494
|
+
_context.prev = 20;
|
|
495
|
+
_context.t0 = _context["catch"](3);
|
|
496
|
+
_didIteratorError = true;
|
|
497
|
+
_iteratorError = _context.t0;
|
|
498
|
+
case 24:
|
|
499
|
+
_context.prev = 24;
|
|
500
|
+
_context.prev = 25;
|
|
501
|
+
if (!(_iteratorAbruptCompletion && _iterator["return"] != null)) {
|
|
502
|
+
_context.next = 29;
|
|
503
|
+
break;
|
|
504
|
+
}
|
|
505
|
+
_context.next = 29;
|
|
506
|
+
return (0, _awaitAsyncGenerator2["default"])(_iterator["return"]());
|
|
507
|
+
case 29:
|
|
508
|
+
_context.prev = 29;
|
|
509
|
+
if (!_didIteratorError) {
|
|
510
|
+
_context.next = 32;
|
|
511
|
+
break;
|
|
512
|
+
}
|
|
513
|
+
throw _iteratorError;
|
|
514
|
+
case 32:
|
|
515
|
+
return _context.finish(29);
|
|
516
|
+
case 33:
|
|
517
|
+
return _context.finish(24);
|
|
518
|
+
case 34:
|
|
519
|
+
case "end":
|
|
520
|
+
return _context.stop();
|
|
521
|
+
}
|
|
522
|
+
}, _callee, null, [[3, 20, 24, 34], [25,, 29, 33]]);
|
|
523
|
+
}));
|
|
524
|
+
return _makeProgressIterator.apply(this, arguments);
|
|
525
|
+
}
|
|
526
|
+
function getPersistedRemoteFormat(keplerFormat, fileName) {
|
|
527
|
+
var _fileName$match;
|
|
528
|
+
if (keplerFormat && keplerFormat !== 'auto' && Object.prototype.hasOwnProperty.call(_constants.REMOTE_FILE_EXTENSIONS, keplerFormat)) {
|
|
529
|
+
return keplerFormat;
|
|
530
|
+
}
|
|
531
|
+
var ext = fileName === null || fileName === void 0 || (_fileName$match = fileName.match(/\.([a-z0-9]+)$/i)) === null || _fileName$match === void 0 || (_fileName$match = _fileName$match[1]) === null || _fileName$match === void 0 ? void 0 : _fileName$match.toLowerCase();
|
|
532
|
+
if (!ext) {
|
|
533
|
+
return undefined;
|
|
534
|
+
}
|
|
535
|
+
if (Object.prototype.hasOwnProperty.call(_constants.REMOTE_FILE_EXTENSIONS, ext)) {
|
|
536
|
+
return ext;
|
|
537
|
+
}
|
|
538
|
+
var extensionAliases = {
|
|
539
|
+
jsonl: 'ndjson',
|
|
540
|
+
ndgeojson: 'geojsonl',
|
|
541
|
+
ldgeojson: 'geojsonl',
|
|
542
|
+
xls: 'xlsx',
|
|
543
|
+
xlsm: 'xlsx',
|
|
544
|
+
xlsb: 'xlsx',
|
|
545
|
+
shapefile: 'shp',
|
|
546
|
+
excel: 'xlsx',
|
|
547
|
+
flatgeobuf: 'fgb'
|
|
548
|
+
};
|
|
549
|
+
return extensionAliases[ext];
|
|
550
|
+
}
|
|
551
|
+
function readBatch(_x3, _x4, _x5, _x6, _x7, _x8, _x9, _x0) {
|
|
552
|
+
return _readBatch.apply(this, arguments);
|
|
553
|
+
}
|
|
554
|
+
function _readBatch() {
|
|
555
|
+
_readBatch = (0, _wrapAsyncGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee2(asyncIterator, fileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, fileSize) {
|
|
556
|
+
var result, batches, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, batch, _yield$_awaitAsyncGen, _JSONPath, streamingPath, batchRows, i;
|
|
557
|
+
return _regenerator["default"].wrap(function _callee2$(_context2) {
|
|
558
|
+
while (1) switch (_context2.prev = _context2.next) {
|
|
559
|
+
case 0:
|
|
560
|
+
result = null;
|
|
561
|
+
batches = [];
|
|
562
|
+
_iteratorAbruptCompletion2 = false;
|
|
563
|
+
_didIteratorError2 = false;
|
|
564
|
+
_context2.prev = 4;
|
|
565
|
+
_iterator2 = _asyncIterator(asyncIterator);
|
|
566
|
+
case 6:
|
|
567
|
+
_context2.next = 8;
|
|
568
|
+
return (0, _awaitAsyncGenerator2["default"])(_iterator2.next());
|
|
569
|
+
case 8:
|
|
570
|
+
if (!(_iteratorAbruptCompletion2 = !(_step2 = _context2.sent).done)) {
|
|
571
|
+
_context2.next = 32;
|
|
572
|
+
break;
|
|
573
|
+
}
|
|
574
|
+
batch = _step2.value;
|
|
575
|
+
if (!(batch.batchType === BATCH_TYPE.FINAL_RESULT)) {
|
|
576
|
+
_context2.next = 24;
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
579
|
+
if (batch.container) {
|
|
580
|
+
result = _objectSpread({}, batch.container);
|
|
581
|
+
}
|
|
582
|
+
// Set the streamed data correctly is Batch json path is set
|
|
583
|
+
// and the path streamed is not the top level object (jsonpath = '$')
|
|
584
|
+
if (!(batch.jsonpath && batch.jsonpath.length > 1)) {
|
|
585
|
+
_context2.next = 21;
|
|
586
|
+
break;
|
|
587
|
+
}
|
|
588
|
+
_context2.next = 15;
|
|
589
|
+
return (0, _awaitAsyncGenerator2["default"])(Promise.resolve().then(function () {
|
|
590
|
+
return _interopRequireWildcard(require('@loaders.gl/json'));
|
|
591
|
+
}));
|
|
592
|
+
case 15:
|
|
593
|
+
_yield$_awaitAsyncGen = _context2.sent;
|
|
594
|
+
_JSONPath = _yield$_awaitAsyncGen._JSONPath;
|
|
595
|
+
streamingPath = new _JSONPath(batch.jsonpath);
|
|
596
|
+
streamingPath.setFieldAtPath(result, batches);
|
|
597
|
+
_context2.next = 22;
|
|
598
|
+
break;
|
|
599
|
+
case 21:
|
|
600
|
+
if (batch.jsonpath && batch.jsonpath.length === 1) {
|
|
601
|
+
// The streamed object is a ROW JSON-batch (jsonpath = '$')
|
|
602
|
+
// row objects
|
|
603
|
+
result = batches;
|
|
604
|
+
}
|
|
605
|
+
case 22:
|
|
606
|
+
_context2.next = 27;
|
|
607
|
+
break;
|
|
608
|
+
case 24:
|
|
609
|
+
batchRows = getBatchRows(batch);
|
|
610
|
+
if (batchRows) {
|
|
611
|
+
for (i = 0; i < batchRows.length; i++) {
|
|
612
|
+
batches.push(batchRows[i]);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
result = getAggregatedLoaderResult(batch, batches, result);
|
|
616
|
+
case 27:
|
|
617
|
+
_context2.next = 29;
|
|
618
|
+
return _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({}, batch), batch.schema ? {
|
|
619
|
+
headers: batch.schema.fields ? batch.schema.fields.map(function (f) {
|
|
620
|
+
return f.name;
|
|
621
|
+
}) : Object.keys(batch.schema)
|
|
622
|
+
} : {}), {}, {
|
|
623
|
+
fileName: fileName
|
|
624
|
+
}, typeof fileSize === 'number' ? {
|
|
625
|
+
fileSize: fileSize
|
|
626
|
+
} : {}), {}, {
|
|
627
|
+
// if dataset is CSV, data is set to the raw batches
|
|
628
|
+
data: result ? result : batches
|
|
629
|
+
}, sourceUrl ? {
|
|
630
|
+
sourceUrl: sourceUrl
|
|
631
|
+
} : {}), keplerFormat ? {
|
|
632
|
+
keplerFormat: keplerFormat
|
|
633
|
+
} : {}), typeof refreshIntervalMs === 'number' && refreshIntervalMs > 0 ? {
|
|
634
|
+
refreshIntervalMs: refreshIntervalMs
|
|
635
|
+
} : {}), etag ? {
|
|
636
|
+
etag: etag
|
|
637
|
+
} : {}), lastModified ? {
|
|
638
|
+
lastModified: lastModified
|
|
639
|
+
} : {});
|
|
640
|
+
case 29:
|
|
641
|
+
_iteratorAbruptCompletion2 = false;
|
|
642
|
+
_context2.next = 6;
|
|
643
|
+
break;
|
|
644
|
+
case 32:
|
|
645
|
+
_context2.next = 38;
|
|
646
|
+
break;
|
|
647
|
+
case 34:
|
|
648
|
+
_context2.prev = 34;
|
|
649
|
+
_context2.t0 = _context2["catch"](4);
|
|
650
|
+
_didIteratorError2 = true;
|
|
651
|
+
_iteratorError2 = _context2.t0;
|
|
652
|
+
case 38:
|
|
653
|
+
_context2.prev = 38;
|
|
654
|
+
_context2.prev = 39;
|
|
655
|
+
if (!(_iteratorAbruptCompletion2 && _iterator2["return"] != null)) {
|
|
656
|
+
_context2.next = 43;
|
|
657
|
+
break;
|
|
658
|
+
}
|
|
659
|
+
_context2.next = 43;
|
|
660
|
+
return (0, _awaitAsyncGenerator2["default"])(_iterator2["return"]());
|
|
661
|
+
case 43:
|
|
662
|
+
_context2.prev = 43;
|
|
663
|
+
if (!_didIteratorError2) {
|
|
664
|
+
_context2.next = 46;
|
|
665
|
+
break;
|
|
666
|
+
}
|
|
667
|
+
throw _iteratorError2;
|
|
668
|
+
case 46:
|
|
669
|
+
return _context2.finish(43);
|
|
670
|
+
case 47:
|
|
671
|
+
return _context2.finish(38);
|
|
672
|
+
case 48:
|
|
673
|
+
case "end":
|
|
674
|
+
return _context2.stop();
|
|
675
|
+
}
|
|
676
|
+
}, _callee2, null, [[4, 34, 38, 48], [39,, 43, 47]]);
|
|
677
|
+
}));
|
|
678
|
+
return _readBatch.apply(this, arguments);
|
|
679
|
+
}
|
|
680
|
+
function readFileInBatches(_x10) {
|
|
681
|
+
return _readFileInBatches.apply(this, arguments);
|
|
682
|
+
}
|
|
683
|
+
function _readFileInBatches() {
|
|
684
|
+
_readFileInBatches = (0, _asyncToGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee4(_ref0) {
|
|
685
|
+
var file, _ref0$loaders, loaders, _ref0$loadOptions, loadOptions, companionFiles, fileName, displayFileName, unzipped, shapefile, hasExtension, mimeType, isShapefile, companionFetch, batchIterator, progressIterator, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified;
|
|
686
|
+
return _regenerator["default"].wrap(function _callee4$(_context4) {
|
|
687
|
+
while (1) switch (_context4.prev = _context4.next) {
|
|
688
|
+
case 0:
|
|
689
|
+
file = _ref0.file, _ref0$loaders = _ref0.loaders, loaders = _ref0$loaders === void 0 ? [] : _ref0$loaders, _ref0$loadOptions = _ref0.loadOptions, loadOptions = _ref0$loadOptions === void 0 ? {} : _ref0$loadOptions, companionFiles = _ref0.companionFiles, fileName = _ref0.fileName;
|
|
690
|
+
displayFileName = fileName || file.name; // Local drops expand mixed zips in the load queue before this runs. A `.zip`
|
|
691
|
+
// that still arrives here is a shapefile archive (remote URL, or a direct call).
|
|
692
|
+
if (!((0, _shapefileFiles.isZipFileName)(file.name) && (0, _loaderRegistry.isKeplerFileFormatAccepted)('shp'))) {
|
|
693
|
+
_context4.next = 10;
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
_context4.next = 5;
|
|
697
|
+
return (0, _shapefileFiles.unzipShapefileArchive)(file);
|
|
698
|
+
case 5:
|
|
699
|
+
unzipped = _context4.sent;
|
|
700
|
+
shapefile = unzipped.find(function (entry) {
|
|
701
|
+
return (0, _shapefileFiles.getDroppedFileExtension)(entry.name) === 'shp';
|
|
702
|
+
});
|
|
703
|
+
if (shapefile) {
|
|
704
|
+
_context4.next = 9;
|
|
705
|
+
break;
|
|
706
|
+
}
|
|
707
|
+
throw new Error('Zip archive does not contain a shapefile (.shp)');
|
|
708
|
+
case 9:
|
|
709
|
+
return _context4.abrupt("return", readFileInBatches({
|
|
710
|
+
file: shapefile,
|
|
711
|
+
loaders: loaders,
|
|
712
|
+
loadOptions: loadOptions,
|
|
713
|
+
companionFiles: [].concat((0, _toConsumableArray2["default"])(unzipped), (0, _toConsumableArray2["default"])(companionFiles || [])),
|
|
714
|
+
fileName: displayFileName
|
|
715
|
+
}));
|
|
716
|
+
case 10:
|
|
717
|
+
_context4.next = 12;
|
|
718
|
+
return (0, _loaderRegistry.getKeplerLoaders)(file, loaders);
|
|
719
|
+
case 12:
|
|
720
|
+
loaders = _context4.sent;
|
|
721
|
+
hasExtension = /\.[a-z0-9]+$/i.test(file.name);
|
|
722
|
+
mimeType = !hasExtension && file.type ? file.type : undefined; // Shapefile sidecars are resolved via a custom fetch that cannot be cloned
|
|
723
|
+
// into a worker. Other formats keep loaders.gl worker parsing.
|
|
724
|
+
isShapefile = (0, _shapefileFiles.getDroppedFileExtension)(file.name) === 'shp';
|
|
725
|
+
companionFetch = isShapefile ? (0, _shapefileFiles.createCompanionFetch)(companionFiles) : undefined;
|
|
726
|
+
loadOptions = _objectSpread(_objectSpread(_objectSpread(_objectSpread({
|
|
727
|
+
csv: CSV_LOADER_OPTIONS,
|
|
728
|
+
arrow: ARROW_LOADER_OPTIONS,
|
|
729
|
+
json: JSON_LOADER_OPTIONS,
|
|
730
|
+
parquet: PARQUET_LOADER_OPTIONS,
|
|
731
|
+
kml: KML_LOADER_OPTIONS,
|
|
732
|
+
gpx: KML_LOADER_OPTIONS,
|
|
733
|
+
tcx: KML_LOADER_OPTIONS,
|
|
734
|
+
shapefile: GIS_TABLE_LOADER_OPTIONS,
|
|
735
|
+
shp: {
|
|
736
|
+
_maxDimensions: 2
|
|
737
|
+
},
|
|
738
|
+
excel: EXCEL_LOADER_OPTIONS,
|
|
739
|
+
flatgeobuf: GIS_TABLE_LOADER_OPTIONS,
|
|
740
|
+
gis: {
|
|
741
|
+
reproject: true,
|
|
742
|
+
_targetCrs: 'WGS84'
|
|
743
|
+
},
|
|
744
|
+
metadata: true
|
|
745
|
+
}, isShapefile ? {
|
|
746
|
+
worker: false
|
|
747
|
+
} : {}), mimeType ? {
|
|
748
|
+
mimeType: mimeType
|
|
749
|
+
} : {}), companionFetch ? {
|
|
750
|
+
fetch: companionFetch
|
|
751
|
+
} : {}), loadOptions);
|
|
752
|
+
_context4.next = 20;
|
|
753
|
+
return (0, _core.parseInBatches)(file, loaders, loadOptions);
|
|
754
|
+
case 20:
|
|
755
|
+
batchIterator = _context4.sent;
|
|
756
|
+
progressIterator = makeProgressIterator(batchIterator, {
|
|
757
|
+
size: file.size
|
|
758
|
+
});
|
|
759
|
+
sourceUrl = file.keplerSourceUrl;
|
|
760
|
+
keplerFormat = file.keplerFormat;
|
|
761
|
+
refreshIntervalMs = file.keplerRefreshIntervalMs;
|
|
762
|
+
etag = file.keplerEtag;
|
|
763
|
+
lastModified = file.keplerLastModified;
|
|
764
|
+
return _context4.abrupt("return", readBatch(progressIterator, displayFileName, sourceUrl, keplerFormat, refreshIntervalMs, etag, lastModified, file.size));
|
|
765
|
+
case 28:
|
|
766
|
+
case "end":
|
|
767
|
+
return _context4.stop();
|
|
768
|
+
}
|
|
769
|
+
}, _callee4);
|
|
770
|
+
}));
|
|
771
|
+
return _readFileInBatches.apply(this, arguments);
|
|
772
|
+
}
|
|
773
|
+
function processFileData(_x11) {
|
|
774
|
+
return _processFileData.apply(this, arguments);
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Expand staged files into cumulative payloads for one redraw per Arrow batch.
|
|
778
|
+
* Non-progressive files are present only in the first payload.
|
|
779
|
+
*/
|
|
780
|
+
function _processFileData() {
|
|
781
|
+
_processFileData = (0, _asyncToGenerator2["default"])(/*#__PURE__*/_regenerator["default"].mark(function _callee5(_ref1) {
|
|
782
|
+
var _getApplicationConfig;
|
|
783
|
+
var content, fileCache, fileName, data, format, processor, id, table, processorResult, geojsonData, tableRows, _result, result, progressiveData, _result$cols$0$data$l, _result$cols$, latestOnly, targetChunkCount, workerSource, sourceTable, cols, workerGeoArrowBinaryData, geoArrowBinaryData, sourceUrl, remoteFormat, fileSize, metadata;
|
|
784
|
+
return _regenerator["default"].wrap(function _callee5$(_context5) {
|
|
785
|
+
while (1) switch (_context5.prev = _context5.next) {
|
|
786
|
+
case 0:
|
|
787
|
+
content = _ref1.content, fileCache = _ref1.fileCache;
|
|
788
|
+
fileName = content.fileName, data = content.data;
|
|
789
|
+
// Hash the source URL when present so two remote files that share a filename
|
|
790
|
+
// (or a local file and a remote URL with the same name) do not collide and
|
|
791
|
+
// overwrite each other. The UI shows `label` (the filename), not this id.
|
|
792
|
+
id = (0, _utils.generateHashIdFromString)(content.sourceUrl || fileName); // decide on which table class to use based on application config
|
|
793
|
+
table = (_getApplicationConfig = (0, _utils.getApplicationConfig)().table) !== null && _getApplicationConfig !== void 0 ? _getApplicationConfig : _table.KeplerTable;
|
|
794
|
+
if (typeof table.getFileProcessor === 'function') {
|
|
795
|
+
// use custom processors from table class
|
|
796
|
+
processorResult = table.getFileProcessor(data);
|
|
797
|
+
format = processorResult.format;
|
|
798
|
+
processor = processorResult.processor;
|
|
799
|
+
} else {
|
|
800
|
+
// use default processors
|
|
801
|
+
geojsonData = getGeoJsonFromLoaderResult(data);
|
|
802
|
+
if (isArrowData(data)) {
|
|
803
|
+
format = _constants.DATASET_FORMATS.arrow;
|
|
804
|
+
processor = _dataProcessor.processArrowBatches;
|
|
805
|
+
} else if (isKeplerGlMap(data)) {
|
|
806
|
+
format = _constants.DATASET_FORMATS.keplergl;
|
|
807
|
+
processor = _dataProcessor.processKeplerglJSON;
|
|
808
|
+
} else if (geojsonData) {
|
|
809
|
+
format = _constants.DATASET_FORMATS.geojson;
|
|
810
|
+
processor = function processor() {
|
|
811
|
+
return (0, _dataProcessor.processGeojson)(geojsonData);
|
|
812
|
+
};
|
|
813
|
+
} else if (isRowObject(data)) {
|
|
814
|
+
// csv file goes here
|
|
815
|
+
format = _constants.DATASET_FORMATS.row;
|
|
816
|
+
processor = _dataProcessor.processRowObject;
|
|
817
|
+
} else if ((0, _utils.isPlainObject)(data)) {
|
|
818
|
+
tableRows = data.data;
|
|
819
|
+
if ((data.shape === 'object-row-table' || data.shape === 'row-table') && Array.isArray(tableRows)) {
|
|
820
|
+
format = _constants.DATASET_FORMATS.row;
|
|
821
|
+
processor = function processor() {
|
|
822
|
+
return (0, _dataProcessor.processRowObject)(tableRows);
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
if (!(format && processor)) {
|
|
828
|
+
_context5.next = 40;
|
|
829
|
+
break;
|
|
830
|
+
}
|
|
831
|
+
_context5.prev = 6;
|
|
832
|
+
if (!(format === _constants.DATASET_FORMATS.arrow && content.skipArrowCompact)) {
|
|
833
|
+
_context5.next = 13;
|
|
834
|
+
break;
|
|
835
|
+
}
|
|
836
|
+
_context5.next = 10;
|
|
837
|
+
return (0, _dataProcessor.processArrowBatches)(data, {
|
|
838
|
+
compact: false
|
|
839
|
+
});
|
|
840
|
+
case 10:
|
|
841
|
+
_context5.t0 = _context5.sent;
|
|
842
|
+
_context5.next = 16;
|
|
843
|
+
break;
|
|
844
|
+
case 13:
|
|
845
|
+
_context5.next = 15;
|
|
846
|
+
return processor(data);
|
|
847
|
+
case 15:
|
|
848
|
+
_context5.t0 = _context5.sent;
|
|
849
|
+
case 16:
|
|
850
|
+
result = _context5.t0;
|
|
851
|
+
_context5.next = 22;
|
|
852
|
+
break;
|
|
853
|
+
case 19:
|
|
854
|
+
_context5.prev = 19;
|
|
855
|
+
_context5.t1 = _context5["catch"](6);
|
|
856
|
+
throw new Error("Can not process uploaded file, ".concat((0, _utils.getError)(_context5.t1)));
|
|
857
|
+
case 22:
|
|
858
|
+
if (!(format === _constants.DATASET_FORMATS.arrow && (_result = result) !== null && _result !== void 0 && _result.cols)) {
|
|
859
|
+
_context5.next = 33;
|
|
860
|
+
break;
|
|
861
|
+
}
|
|
862
|
+
latestOnly = Boolean(content.skipArrowCompact);
|
|
863
|
+
targetChunkCount = (_result$cols$0$data$l = (_result$cols$ = result.cols[0]) === null || _result$cols$ === void 0 || (_result$cols$ = _result$cols$.data) === null || _result$cols$ === void 0 ? void 0 : _result$cols$.length) !== null && _result$cols$0$data$l !== void 0 ? _result$cols$0$data$l : 0;
|
|
864
|
+
workerSource = result; // Final file processing compacts Arrow batches to keep the number of deck.gl
|
|
865
|
+
// sublayers low. Schedule worker jobs from the original batches first so
|
|
866
|
+
// geometry conversion and triangulation can still run in parallel, then
|
|
867
|
+
// combine their output to match the single compacted table chunk.
|
|
868
|
+
if (!latestOnly && Array.isArray(data) && data.length > targetChunkCount) {
|
|
869
|
+
sourceTable = new arrow.Table(data);
|
|
870
|
+
cols = (0, _toConsumableArray2["default"])(Array(sourceTable.numCols).keys()).map(function (i) {
|
|
871
|
+
return sourceTable.getChildAt(i);
|
|
872
|
+
});
|
|
873
|
+
workerSource = _objectSpread(_objectSpread({}, result), {}, {
|
|
874
|
+
cols: cols
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
_context5.next = 29;
|
|
878
|
+
return parseGeoArrowBatchesOnWorkers(workerSource, {
|
|
879
|
+
latestOnly: latestOnly
|
|
880
|
+
});
|
|
881
|
+
case 29:
|
|
882
|
+
workerGeoArrowBinaryData = _context5.sent;
|
|
883
|
+
geoArrowBinaryData = targetChunkCount === 1 && !latestOnly && workerGeoArrowBinaryData ? combineGeoArrowWorkerChunks(workerGeoArrowBinaryData) : workerGeoArrowBinaryData;
|
|
884
|
+
if (geoArrowBinaryData) {
|
|
885
|
+
result = _objectSpread(_objectSpread({}, result), {}, {
|
|
886
|
+
geoArrowBinaryData: geoArrowBinaryData
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
if (content.prepareProgressiveReplay && !latestOnly) {
|
|
890
|
+
progressiveData = createProgressiveArrowData(workerSource, result, workerGeoArrowBinaryData);
|
|
891
|
+
}
|
|
892
|
+
case 33:
|
|
893
|
+
sourceUrl = content.sourceUrl;
|
|
894
|
+
remoteFormat = getPersistedRemoteFormat(content.keplerFormat, content.fileName);
|
|
895
|
+
fileSize = typeof content.fileSize === 'number' ? content.fileSize : undefined;
|
|
896
|
+
metadata = _objectSpread(_objectSpread({}, sourceUrl ? _objectSpread(_objectSpread(_objectSpread(_objectSpread(_objectSpread({
|
|
897
|
+
source: sourceUrl
|
|
898
|
+
}, remoteFormat ? {
|
|
899
|
+
sourceFormat: remoteFormat
|
|
900
|
+
} : {}), typeof content.refreshIntervalMs === 'number' && content.refreshIntervalMs > 0 ? {
|
|
901
|
+
refreshIntervalMs: content.refreshIntervalMs
|
|
902
|
+
} : {}), typeof content.etag === 'string' && content.etag ? {
|
|
903
|
+
etag: content.etag
|
|
904
|
+
} : {}), typeof content.lastModified === 'string' && content.lastModified ? {
|
|
905
|
+
lastModified: content.lastModified
|
|
906
|
+
} : {}), {}, {
|
|
907
|
+
lastFetchedAt: Date.now()
|
|
908
|
+
}) : {}), fileSize != null ? {
|
|
909
|
+
size: fileSize
|
|
910
|
+
} : {});
|
|
911
|
+
return _context5.abrupt("return", [].concat((0, _toConsumableArray2["default"])(fileCache), [_objectSpread(_objectSpread({
|
|
912
|
+
data: result
|
|
913
|
+
}, progressiveData ? {
|
|
914
|
+
progressiveData: progressiveData
|
|
915
|
+
} : {}), {}, {
|
|
916
|
+
info: _objectSpread({
|
|
917
|
+
id: id,
|
|
918
|
+
label: content.fileName,
|
|
919
|
+
format: format
|
|
920
|
+
}, sourceUrl ? {
|
|
921
|
+
type: _constants.DatasetType.EXTERNALLY_HOSTED
|
|
922
|
+
} : {})
|
|
923
|
+
}, sourceUrl || fileSize != null ? {
|
|
924
|
+
metadata: metadata
|
|
925
|
+
} : {})]));
|
|
926
|
+
case 40:
|
|
927
|
+
throw new Error('Can not process uploaded file, unknown file format');
|
|
928
|
+
case 41:
|
|
929
|
+
case "end":
|
|
930
|
+
return _context5.stop();
|
|
931
|
+
}
|
|
932
|
+
}, _callee5, null, [[6, 19]]);
|
|
933
|
+
}));
|
|
934
|
+
return _processFileData.apply(this, arguments);
|
|
935
|
+
}
|
|
936
|
+
function fileCacheToProgressiveSteps(fileCache) {
|
|
937
|
+
var numberOfSteps = Math.max.apply(Math, [1].concat((0, _toConsumableArray2["default"])(fileCache.map(function (file) {
|
|
938
|
+
var _file$progressiveData;
|
|
939
|
+
return ((_file$progressiveData = file.progressiveData) === null || _file$progressiveData === void 0 ? void 0 : _file$progressiveData.length) || 1;
|
|
940
|
+
}))));
|
|
941
|
+
return Array.from({
|
|
942
|
+
length: numberOfSteps
|
|
943
|
+
}, function (_, stepIndex) {
|
|
944
|
+
return fileCache.flatMap(function (file) {
|
|
945
|
+
var progressiveData = file.progressiveData,
|
|
946
|
+
fileWithoutProgressiveData = (0, _objectWithoutProperties2["default"])(file, _excluded2);
|
|
947
|
+
if (!progressiveData) {
|
|
948
|
+
return stepIndex === 0 ? [fileWithoutProgressiveData] : [];
|
|
949
|
+
}
|
|
950
|
+
var data = progressiveData[stepIndex];
|
|
951
|
+
return data ? [_objectSpread(_objectSpread({}, fileWithoutProgressiveData), {}, {
|
|
952
|
+
data: data
|
|
953
|
+
})] : [];
|
|
954
|
+
});
|
|
955
|
+
}).filter(function (step) {
|
|
956
|
+
return step.length;
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
function filesToDataPayload(fileCache, options) {
|
|
960
|
+
// seperate out files which could be a single datasets. or a keplergl map json
|
|
961
|
+
var collection = fileCache.reduce(function (accu, file) {
|
|
962
|
+
var data = file.data,
|
|
963
|
+
info = file.info;
|
|
964
|
+
if ((info === null || info === void 0 ? void 0 : info.format) === _constants.DATASET_FORMATS.keplergl) {
|
|
965
|
+
// if file contains a single kepler map dataset & config
|
|
966
|
+
accu.keplerMaps.push(_objectSpread(_objectSpread({}, data), {}, {
|
|
967
|
+
options: _objectSpread({
|
|
968
|
+
centerMap: !(data.config && data.config.mapState)
|
|
969
|
+
}, options || {})
|
|
970
|
+
}));
|
|
971
|
+
} else if (_constants.DATASET_FORMATS[info === null || info === void 0 ? void 0 : info.format]) {
|
|
972
|
+
// if file contains only data
|
|
973
|
+
var newDataset = _objectSpread({
|
|
974
|
+
data: data,
|
|
975
|
+
info: _objectSpread({
|
|
976
|
+
id: (info === null || info === void 0 ? void 0 : info.id) || (0, _commonUtils.generateHashId)(4)
|
|
977
|
+
}, info || {})
|
|
978
|
+
}, file.metadata ? {
|
|
979
|
+
metadata: file.metadata
|
|
980
|
+
} : {});
|
|
981
|
+
accu.datasets.push(newDataset);
|
|
982
|
+
}
|
|
983
|
+
return accu;
|
|
984
|
+
}, {
|
|
985
|
+
datasets: [],
|
|
986
|
+
keplerMaps: []
|
|
987
|
+
});
|
|
988
|
+
|
|
989
|
+
// add kepler map first with config
|
|
990
|
+
// add datasets later in one add data call
|
|
991
|
+
return collection.keplerMaps.concat(_objectSpread({
|
|
992
|
+
datasets: collection.datasets
|
|
993
|
+
}, options && collection.datasets.length ? {
|
|
994
|
+
options: options
|
|
995
|
+
} : {}));
|
|
996
|
+
}
|
|
997
|
+
//# sourceMappingURL=file-handler.js.map
|