@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/data-processor.js +674 -0
- package/dist/cjs/data-processor.js.map +1 -0
- package/dist/cjs/export/dataset-to-geojson.js +138 -0
- package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
- package/dist/cjs/export/geopackage-writer.js +385 -0
- package/dist/cjs/export/geopackage-writer.js.map +1 -0
- package/dist/cjs/export/geoparquet-writer.js +351 -0
- package/dist/cjs/export/geoparquet-writer.js.map +1 -0
- package/dist/cjs/export/kml-writer.js +347 -0
- package/dist/cjs/export/kml-writer.js.map +1 -0
- package/dist/cjs/export/shapefile-writer.js +708 -0
- package/dist/cjs/export/shapefile-writer.js.map +1 -0
- package/dist/cjs/export/sql-js.js +95 -0
- package/dist/cjs/export/sql-js.js.map +1 -0
- package/dist/cjs/export/vector-export.js +187 -0
- package/dist/cjs/export/vector-export.js.map +1 -0
- package/dist/cjs/file-handler.js +997 -0
- package/dist/cjs/file-handler.js.map +1 -0
- package/dist/cjs/index.js +114 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/kepler-csv-loader.js +163 -0
- package/dist/cjs/kepler-csv-loader.js.map +1 -0
- package/dist/cjs/loader-registry.js +542 -0
- package/dist/cjs/loader-registry.js.map +1 -0
- package/dist/cjs/remote-file.js +675 -0
- package/dist/cjs/remote-file.js.map +1 -0
- package/dist/cjs/shapefile-files.js +281 -0
- package/dist/cjs/shapefile-files.js.map +1 -0
- package/dist/cjs/types.js +6 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/data-processor.d.ts +30 -16
- package/dist/esm/data-processor.js +655 -0
- package/dist/esm/data-processor.js.map +1 -0
- package/dist/esm/export/dataset-to-geojson.js +132 -0
- package/dist/esm/export/dataset-to-geojson.js.map +1 -0
- package/dist/esm/export/geopackage-writer.js +381 -0
- package/dist/esm/export/geopackage-writer.js.map +1 -0
- package/dist/esm/export/geoparquet-writer.js +336 -0
- package/dist/esm/export/geoparquet-writer.js.map +1 -0
- package/dist/esm/export/kml-writer.js +341 -0
- package/dist/esm/export/kml-writer.js.map +1 -0
- package/dist/esm/export/shapefile-writer.js +701 -0
- package/dist/esm/export/shapefile-writer.js.map +1 -0
- package/dist/esm/export/sql-js.js +80 -0
- package/dist/esm/export/sql-js.js.map +1 -0
- package/dist/esm/export/vector-export.js +176 -0
- package/dist/esm/export/vector-export.js.map +1 -0
- package/dist/esm/file-handler.js +972 -0
- package/dist/esm/file-handler.js.map +1 -0
- package/dist/esm/index.js +12 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/kepler-csv-loader.js +152 -0
- package/dist/esm/kepler-csv-loader.js.map +1 -0
- package/dist/esm/loader-registry.js +510 -0
- package/dist/esm/loader-registry.js.map +1 -0
- package/dist/esm/remote-file.js +664 -0
- package/dist/esm/remote-file.js.map +1 -0
- package/dist/esm/shapefile-files.js +264 -0
- package/dist/esm/shapefile-files.js.map +1 -0
- package/dist/esm/types.js +3 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/export/dataset-to-geojson.d.ts +12 -0
- package/dist/export/geopackage-writer.d.ts +28 -0
- package/dist/export/geoparquet-writer.d.ts +7 -0
- package/dist/export/kml-writer.d.ts +13 -0
- package/dist/export/shapefile-writer.d.ts +28 -0
- package/dist/export/sql-js.d.ts +8 -0
- package/dist/export/vector-export.d.ts +22 -0
- package/dist/file-handler.d.ts +55 -8
- package/dist/index.d.ts +6 -0
- package/dist/kepler-csv-loader.d.ts +12 -0
- package/dist/loader-registry.d.ts +29 -0
- package/dist/remote-file.d.ts +79 -0
- package/dist/shapefile-files.d.ts +39 -0
- package/dist/types.d.ts +13 -0
- package/package.json +44 -26
- package/dist/data-processor.js +0 -637
- package/dist/file-handler.js +0 -409
- package/dist/index.js +0 -39
- package/dist/types.js +0 -6
|
@@ -0,0 +1,664 @@
|
|
|
1
|
+
import _toConsumableArray from "@babel/runtime/helpers/esm/toConsumableArray";
|
|
2
|
+
import _typeof from "@babel/runtime/helpers/esm/typeof";
|
|
3
|
+
import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
|
|
4
|
+
import _asyncToGenerator from "@babel/runtime/helpers/esm/asyncToGenerator";
|
|
5
|
+
import _regeneratorRuntime from "@babel/runtime/regenerator";
|
|
6
|
+
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
|
|
7
|
+
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
|
|
8
|
+
function _asyncIterator(r) { var n, t, o, e = 2; for ("undefined" != typeof Symbol && (t = Symbol.asyncIterator, o = Symbol.iterator); e--;) { if (t && null != (n = r[t])) return n.call(r); if (o && null != (n = r[o])) return new AsyncFromSyncIterator(n.call(r)); t = "@@asyncIterator", o = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
|
|
9
|
+
function AsyncFromSyncIterator(r) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var n = r.done; return Promise.resolve(r.value).then(function (r) { return { value: r, done: n }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(r) { this.s = r, this.n = r.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, "return": function _return(r) { var n = this.s["return"]; return void 0 === n ? Promise.resolve({ value: r, done: !0 }) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); }, "throw": function _throw(r) { var n = this.s["return"]; return void 0 === n ? Promise.reject(r) : AsyncFromSyncIteratorContinuation(n.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(r); }
|
|
10
|
+
// SPDX-License-Identifier: MIT
|
|
11
|
+
// Copyright contributors to the kepler.gl project
|
|
12
|
+
|
|
13
|
+
import { parseUri } from '@kepler.gl/common-utils';
|
|
14
|
+
import { DATASET_FORMATS, DatasetType, MIME_TO_REMOTE_FILE_EXTENSION, REMOTE_FILE_EXTENSIONS, REMOTE_FILE_MIME_TYPES } from '@kepler.gl/constants';
|
|
15
|
+
import { generateHashIdFromString } from '@kepler.gl/utils';
|
|
16
|
+
import { getPersistedRemoteFormat, processFileData, readFileInBatches } from './file-handler';
|
|
17
|
+
import { KeplerCSVLoader } from './kepler-csv-loader';
|
|
18
|
+
function getPathFileName(url) {
|
|
19
|
+
try {
|
|
20
|
+
var parsed = new URL(url);
|
|
21
|
+
var segments = parsed.pathname.split('/').filter(Boolean);
|
|
22
|
+
var last = segments.length ? decodeURIComponent(segments[segments.length - 1]) : '';
|
|
23
|
+
return last.split('?')[0];
|
|
24
|
+
} catch (_unused) {
|
|
25
|
+
var _parseUri = parseUri(url),
|
|
26
|
+
file = _parseUri.file;
|
|
27
|
+
return decodeURIComponent(String(file || '')).split('?')[0];
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function getExtensionFromUrl(url) {
|
|
31
|
+
var file = getPathFileName(url);
|
|
32
|
+
var match = file.match(/\.([a-z0-9]+)$/i);
|
|
33
|
+
return match ? match[1].toLowerCase() : '';
|
|
34
|
+
}
|
|
35
|
+
export function getMimeTypeForFormat(format) {
|
|
36
|
+
if (!format || format === 'auto') {
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
return REMOTE_FILE_MIME_TYPES[format];
|
|
40
|
+
}
|
|
41
|
+
export function isRemoteDatasetUrl(url) {
|
|
42
|
+
try {
|
|
43
|
+
var _URL = new URL(url),
|
|
44
|
+
protocol = _URL.protocol;
|
|
45
|
+
return protocol === 'http:' || protocol === 'https:';
|
|
46
|
+
} catch (_unused2) {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Build a filename loaders.gl can use to pick a parser.
|
|
53
|
+
* Prefers an explicit format, then the URL path extension, then Content-Type.
|
|
54
|
+
*/
|
|
55
|
+
export function getFileNameForRemoteUrl(url, format, mimeType) {
|
|
56
|
+
var base = getPathFileName(url) || 'dataset';
|
|
57
|
+
var formatExt = format && format !== 'auto' ? REMOTE_FILE_EXTENSIONS[format] : undefined;
|
|
58
|
+
var urlExt = getExtensionFromUrl(url);
|
|
59
|
+
var mimeExt = mimeType ? MIME_TO_REMOTE_FILE_EXTENSION[mimeType.split(';')[0].trim().toLowerCase()] : undefined;
|
|
60
|
+
var ext = formatExt || urlExt || mimeExt;
|
|
61
|
+
if (!ext) {
|
|
62
|
+
return base;
|
|
63
|
+
}
|
|
64
|
+
if (/\.[a-z0-9]+$/i.test(base)) {
|
|
65
|
+
if (formatExt && !base.toLowerCase().endsWith(".".concat(formatExt))) {
|
|
66
|
+
return "".concat(base.replace(/\.[^.]+$/, ''), ".").concat(formatExt);
|
|
67
|
+
}
|
|
68
|
+
return base;
|
|
69
|
+
}
|
|
70
|
+
return "".concat(base, ".").concat(ext);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Fetch a remote dataset URL. Uses conditional headers for same-origin requests
|
|
75
|
+
* (or outside a browser) so unchanged files can skip a re-parse.
|
|
76
|
+
*/
|
|
77
|
+
export function fetchRemoteFile(_x) {
|
|
78
|
+
return _fetchRemoteFile.apply(this, arguments);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* A remote URL to hold until Add Data. Rows stay empty so adding it runs the
|
|
83
|
+
* externally hosted download instead of parsing a file in the modal.
|
|
84
|
+
*/
|
|
85
|
+
function _fetchRemoteFile() {
|
|
86
|
+
_fetchRemoteFile = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee(url) {
|
|
87
|
+
var options,
|
|
88
|
+
format,
|
|
89
|
+
onProgress,
|
|
90
|
+
etag,
|
|
91
|
+
lastModified,
|
|
92
|
+
bypassCache,
|
|
93
|
+
headers,
|
|
94
|
+
isCrossOrigin,
|
|
95
|
+
response,
|
|
96
|
+
_response$body,
|
|
97
|
+
_response$body$cancel,
|
|
98
|
+
status,
|
|
99
|
+
blob,
|
|
100
|
+
headerType,
|
|
101
|
+
mimeType,
|
|
102
|
+
fileName,
|
|
103
|
+
file,
|
|
104
|
+
responseEtag,
|
|
105
|
+
responseLastModified,
|
|
106
|
+
_args = arguments;
|
|
107
|
+
return _regeneratorRuntime.wrap(function _callee$(_context) {
|
|
108
|
+
while (1) switch (_context.prev = _context.next) {
|
|
109
|
+
case 0:
|
|
110
|
+
options = _args.length > 1 && _args[1] !== undefined ? _args[1] : {};
|
|
111
|
+
format = options.format, onProgress = options.onProgress, etag = options.etag, lastModified = options.lastModified, bypassCache = options.bypassCache;
|
|
112
|
+
if (isRemoteDatasetUrl(url)) {
|
|
113
|
+
_context.next = 4;
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
throw new Error('Remote dataset URL must use http or https');
|
|
117
|
+
case 4:
|
|
118
|
+
headers = {}; // Conditional headers trigger CORS preflights, which simple file servers may
|
|
119
|
+
// not support even when they allow cross-origin GET requests.
|
|
120
|
+
isCrossOrigin = typeof location !== 'undefined' && new URL(url).origin !== location.origin;
|
|
121
|
+
if (etag && !isCrossOrigin) {
|
|
122
|
+
headers['If-None-Match'] = etag;
|
|
123
|
+
}
|
|
124
|
+
if (lastModified && !isCrossOrigin) {
|
|
125
|
+
headers['If-Modified-Since'] = lastModified;
|
|
126
|
+
}
|
|
127
|
+
_context.next = 10;
|
|
128
|
+
return fetch(url, _objectSpread(_objectSpread({}, bypassCache ? {
|
|
129
|
+
cache: 'no-store'
|
|
130
|
+
} : {}), Object.keys(headers).length ? {
|
|
131
|
+
headers: headers
|
|
132
|
+
} : {}));
|
|
133
|
+
case 10:
|
|
134
|
+
response = _context.sent;
|
|
135
|
+
if (!(response.status === 304)) {
|
|
136
|
+
_context.next = 13;
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
139
|
+
return _context.abrupt("return", {
|
|
140
|
+
file: null,
|
|
141
|
+
notModified: true,
|
|
142
|
+
etag: response.headers.get('etag') || etag || undefined,
|
|
143
|
+
lastModified: response.headers.get('last-modified') || lastModified || undefined
|
|
144
|
+
});
|
|
145
|
+
case 13:
|
|
146
|
+
if (response.ok) {
|
|
147
|
+
_context.next = 17;
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
// Don't put the response body in the error: 404 HTML pages would dump into the UI.
|
|
151
|
+
(_response$body = response.body) === null || _response$body === void 0 || (_response$body$cancel = _response$body.cancel) === null || _response$body$cancel === void 0 || _response$body$cancel.call(_response$body)["catch"](function () {
|
|
152
|
+
return undefined;
|
|
153
|
+
});
|
|
154
|
+
status = response.statusText ? "".concat(response.status, " ").concat(response.statusText) : String(response.status);
|
|
155
|
+
throw new Error("Failed to fetch ".concat(url, " (").concat(status, ")"));
|
|
156
|
+
case 17:
|
|
157
|
+
_context.next = 19;
|
|
158
|
+
return readResponseBlob(response, onProgress);
|
|
159
|
+
case 19:
|
|
160
|
+
blob = _context.sent;
|
|
161
|
+
headerType = response.headers.get('content-type') || blob.type || '';
|
|
162
|
+
mimeType = getMimeTypeForFormat(format) || headerType.split(';')[0].trim();
|
|
163
|
+
fileName = getFileNameForRemoteUrl(url, format, mimeType);
|
|
164
|
+
file = new File([blob], fileName, mimeType ? {
|
|
165
|
+
type: mimeType
|
|
166
|
+
} : undefined);
|
|
167
|
+
file.keplerSourceUrl = url;
|
|
168
|
+
if (format && format !== 'auto') {
|
|
169
|
+
file.keplerFormat = format;
|
|
170
|
+
}
|
|
171
|
+
responseEtag = response.headers.get('etag') || undefined;
|
|
172
|
+
responseLastModified = response.headers.get('last-modified') || undefined;
|
|
173
|
+
if (responseEtag) {
|
|
174
|
+
file.keplerEtag = responseEtag;
|
|
175
|
+
}
|
|
176
|
+
if (responseLastModified) {
|
|
177
|
+
file.keplerLastModified = responseLastModified;
|
|
178
|
+
}
|
|
179
|
+
return _context.abrupt("return", {
|
|
180
|
+
file: file,
|
|
181
|
+
notModified: false,
|
|
182
|
+
etag: responseEtag,
|
|
183
|
+
lastModified: responseLastModified,
|
|
184
|
+
size: file.size
|
|
185
|
+
});
|
|
186
|
+
case 31:
|
|
187
|
+
case "end":
|
|
188
|
+
return _context.stop();
|
|
189
|
+
}
|
|
190
|
+
}, _callee);
|
|
191
|
+
}));
|
|
192
|
+
return _fetchRemoteFile.apply(this, arguments);
|
|
193
|
+
}
|
|
194
|
+
export function remoteDatasetFromUrl(url, format) {
|
|
195
|
+
if (!isRemoteDatasetUrl(url)) {
|
|
196
|
+
throw new Error('Remote dataset URL must use http or https');
|
|
197
|
+
}
|
|
198
|
+
var label = getFileNameForRemoteUrl(url, format);
|
|
199
|
+
var sourceFormat = getPersistedRemoteFormat(format && format !== 'auto' ? format : undefined, label);
|
|
200
|
+
return {
|
|
201
|
+
data: {
|
|
202
|
+
fields: [],
|
|
203
|
+
rows: []
|
|
204
|
+
},
|
|
205
|
+
info: {
|
|
206
|
+
id: generateHashIdFromString(url),
|
|
207
|
+
label: label,
|
|
208
|
+
format: DATASET_FORMATS.row,
|
|
209
|
+
type: DatasetType.EXTERNALLY_HOSTED
|
|
210
|
+
},
|
|
211
|
+
metadata: _objectSpread({
|
|
212
|
+
source: url
|
|
213
|
+
}, sourceFormat ? {
|
|
214
|
+
sourceFormat: sourceFormat
|
|
215
|
+
} : {})
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Fetch a remote dataset URL and wrap it as a File the existing loadFiles pipeline can parse.
|
|
221
|
+
* Sets File.type from an explicit format or the response Content-Type so extensionless
|
|
222
|
+
* URLs (SAS keys) still select the right loader.
|
|
223
|
+
*/
|
|
224
|
+
export function fetchRemoteFileAsKeplerFile(_x2, _x3, _x4) {
|
|
225
|
+
return _fetchRemoteFileAsKeplerFile.apply(this, arguments);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Two frames are enough for the processing indicator to paint. */
|
|
229
|
+
function _fetchRemoteFileAsKeplerFile() {
|
|
230
|
+
_fetchRemoteFileAsKeplerFile = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee2(url, format, onProgress) {
|
|
231
|
+
var result;
|
|
232
|
+
return _regeneratorRuntime.wrap(function _callee2$(_context2) {
|
|
233
|
+
while (1) switch (_context2.prev = _context2.next) {
|
|
234
|
+
case 0:
|
|
235
|
+
_context2.next = 2;
|
|
236
|
+
return fetchRemoteFile(url, {
|
|
237
|
+
format: format,
|
|
238
|
+
onProgress: onProgress
|
|
239
|
+
});
|
|
240
|
+
case 2:
|
|
241
|
+
result = _context2.sent;
|
|
242
|
+
if (result.file) {
|
|
243
|
+
_context2.next = 5;
|
|
244
|
+
break;
|
|
245
|
+
}
|
|
246
|
+
throw new Error("Failed to fetch ".concat(url, " (304 Not Modified)"));
|
|
247
|
+
case 5:
|
|
248
|
+
return _context2.abrupt("return", result.file);
|
|
249
|
+
case 6:
|
|
250
|
+
case "end":
|
|
251
|
+
return _context2.stop();
|
|
252
|
+
}
|
|
253
|
+
}, _callee2);
|
|
254
|
+
}));
|
|
255
|
+
return _fetchRemoteFileAsKeplerFile.apply(this, arguments);
|
|
256
|
+
}
|
|
257
|
+
var PAINT_WAIT_MS = 100;
|
|
258
|
+
function waitForNextPaint() {
|
|
259
|
+
// Hidden tabs suspend requestAnimationFrame. A download that finishes in the
|
|
260
|
+
// background would otherwise wait until the user comes back, and there is
|
|
261
|
+
// nothing to paint there.
|
|
262
|
+
if (typeof requestAnimationFrame !== 'function' || typeof document !== 'undefined' && document.hidden) {
|
|
263
|
+
return Promise.resolve();
|
|
264
|
+
}
|
|
265
|
+
return new Promise(function (resolve) {
|
|
266
|
+
var settled = false;
|
|
267
|
+
var frames = [];
|
|
268
|
+
var finish = function finish() {
|
|
269
|
+
if (settled) {
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
settled = true;
|
|
273
|
+
clearTimeout(timer);
|
|
274
|
+
if (typeof cancelAnimationFrame === 'function') {
|
|
275
|
+
frames.forEach(function (id) {
|
|
276
|
+
return cancelAnimationFrame(id);
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
resolve();
|
|
280
|
+
};
|
|
281
|
+
// The tab can be hidden after this wait starts. Don't sit on frames that will never run.
|
|
282
|
+
var timer = setTimeout(finish, PAINT_WAIT_MS);
|
|
283
|
+
frames.push(requestAnimationFrame(function () {
|
|
284
|
+
if (!settled) {
|
|
285
|
+
frames.push(requestAnimationFrame(finish));
|
|
286
|
+
}
|
|
287
|
+
}));
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
function readResponseBlob(_x5, _x6) {
|
|
291
|
+
return _readResponseBlob.apply(this, arguments);
|
|
292
|
+
}
|
|
293
|
+
function _readResponseBlob() {
|
|
294
|
+
_readResponseBlob = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee3(response, onProgress) {
|
|
295
|
+
var contentLength, total, PROGRESS_INTERVAL_MS, lastEmitAt, lastEmittedPercent, emit, blob, reader, chunks, loaded, _yield$reader$read, done, value;
|
|
296
|
+
return _regeneratorRuntime.wrap(function _callee3$(_context3) {
|
|
297
|
+
while (1) switch (_context3.prev = _context3.next) {
|
|
298
|
+
case 0:
|
|
299
|
+
contentLength = Number(response.headers.get('content-length'));
|
|
300
|
+
total = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : undefined;
|
|
301
|
+
PROGRESS_INTERVAL_MS = 80;
|
|
302
|
+
lastEmitAt = 0;
|
|
303
|
+
lastEmittedPercent = -1;
|
|
304
|
+
emit = function emit(loaded) {
|
|
305
|
+
var complete = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
|
|
306
|
+
var percent = complete ? 1 : total ? Math.min(1, loaded / total) : 0;
|
|
307
|
+
var rounded = Math.round(percent * 100);
|
|
308
|
+
var now = typeof performance !== 'undefined' ? performance.now() : Date.now();
|
|
309
|
+
if (!complete && loaded !== 0 && rounded === lastEmittedPercent && now - lastEmitAt < PROGRESS_INTERVAL_MS) {
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
lastEmitAt = now;
|
|
313
|
+
lastEmittedPercent = rounded;
|
|
314
|
+
onProgress === null || onProgress === void 0 || onProgress({
|
|
315
|
+
loaded: loaded,
|
|
316
|
+
total: total,
|
|
317
|
+
percent: percent
|
|
318
|
+
});
|
|
319
|
+
};
|
|
320
|
+
if (response.body) {
|
|
321
|
+
_context3.next = 12;
|
|
322
|
+
break;
|
|
323
|
+
}
|
|
324
|
+
_context3.next = 9;
|
|
325
|
+
return response.blob();
|
|
326
|
+
case 9:
|
|
327
|
+
blob = _context3.sent;
|
|
328
|
+
emit(blob.size, true);
|
|
329
|
+
return _context3.abrupt("return", blob);
|
|
330
|
+
case 12:
|
|
331
|
+
reader = response.body.getReader();
|
|
332
|
+
chunks = [];
|
|
333
|
+
loaded = 0;
|
|
334
|
+
emit(0);
|
|
335
|
+
case 16:
|
|
336
|
+
if (!true) {
|
|
337
|
+
_context3.next = 29;
|
|
338
|
+
break;
|
|
339
|
+
}
|
|
340
|
+
_context3.next = 19;
|
|
341
|
+
return reader.read();
|
|
342
|
+
case 19:
|
|
343
|
+
_yield$reader$read = _context3.sent;
|
|
344
|
+
done = _yield$reader$read.done;
|
|
345
|
+
value = _yield$reader$read.value;
|
|
346
|
+
if (!done) {
|
|
347
|
+
_context3.next = 24;
|
|
348
|
+
break;
|
|
349
|
+
}
|
|
350
|
+
return _context3.abrupt("break", 29);
|
|
351
|
+
case 24:
|
|
352
|
+
chunks.push(value);
|
|
353
|
+
loaded += value.byteLength;
|
|
354
|
+
emit(loaded);
|
|
355
|
+
_context3.next = 16;
|
|
356
|
+
break;
|
|
357
|
+
case 29:
|
|
358
|
+
emit(loaded, true);
|
|
359
|
+
return _context3.abrupt("return", new Blob(chunks));
|
|
360
|
+
case 31:
|
|
361
|
+
case "end":
|
|
362
|
+
return _context3.stop();
|
|
363
|
+
}
|
|
364
|
+
}, _callee3);
|
|
365
|
+
}));
|
|
366
|
+
return _readResponseBlob.apply(this, arguments);
|
|
367
|
+
}
|
|
368
|
+
function isTabularProcessorResult(data) {
|
|
369
|
+
return Boolean(data && _typeof(data) === 'object' && Array.isArray(data === null || data === void 0 ? void 0 : data.fields) && Array.isArray(data === null || data === void 0 ? void 0 : data.rows));
|
|
370
|
+
}
|
|
371
|
+
function matchesCsvHeader(_x7, _x8) {
|
|
372
|
+
return _matchesCsvHeader.apply(this, arguments);
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Fetch, parse, and process a remote file into Kepler dataset data.
|
|
376
|
+
* Used when reloading an `externally-hosted` dataset from a saved map config
|
|
377
|
+
* and when refreshing a live remote dataset.
|
|
378
|
+
*/
|
|
379
|
+
function _matchesCsvHeader() {
|
|
380
|
+
_matchesCsvHeader = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee4(text, fields) {
|
|
381
|
+
var escape, blankLines, matchesRecord, rows, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch, header, names;
|
|
382
|
+
return _regeneratorRuntime.wrap(function _callee4$(_context4) {
|
|
383
|
+
while (1) switch (_context4.prev = _context4.next) {
|
|
384
|
+
case 0:
|
|
385
|
+
escape = function escape(value) {
|
|
386
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
387
|
+
};
|
|
388
|
+
blankLines = '(?:[ \\t]*[\\r\\n])*'; // Keep exact quote validation: the CSV parser accepts some malformed records.
|
|
389
|
+
matchesRecord = function matchesRecord(header, strict) {
|
|
390
|
+
return [',', '\t', ';', '|'].some(function (delimiter) {
|
|
391
|
+
var record = header.map(function (name) {
|
|
392
|
+
var quoted = escape("\"".concat(name.replace(/"/g, '""'), "\""));
|
|
393
|
+
var needsQuotes = /["\r\n]/.test(name) || (strict ? /[,\t;|]/.test(name) : name.includes(delimiter));
|
|
394
|
+
return needsQuotes ? quoted : "(?:".concat(escape(name), "|").concat(quoted, ")");
|
|
395
|
+
}).join(escape(delimiter));
|
|
396
|
+
return new RegExp("^\\uFEFF?".concat(blankLines).concat(record).concat(blankLines, "$")).test(text);
|
|
397
|
+
});
|
|
398
|
+
}; // Quoted names remain unambiguous even when the loader guesses another delimiter.
|
|
399
|
+
if (!(fields.some(function (name) {
|
|
400
|
+
return name.trim();
|
|
401
|
+
}) && matchesRecord(fields, true))) {
|
|
402
|
+
_context4.next = 5;
|
|
403
|
+
break;
|
|
404
|
+
}
|
|
405
|
+
return _context4.abrupt("return", true);
|
|
406
|
+
case 5:
|
|
407
|
+
if (!(fields.length === 1)) {
|
|
408
|
+
_context4.next = 7;
|
|
409
|
+
break;
|
|
410
|
+
}
|
|
411
|
+
return _context4.abrupt("return", false);
|
|
412
|
+
case 7:
|
|
413
|
+
// Reuse the same streaming parser as import, exposing the header as a row.
|
|
414
|
+
rows = [];
|
|
415
|
+
_iteratorAbruptCompletion = false;
|
|
416
|
+
_didIteratorError = false;
|
|
417
|
+
_context4.prev = 10;
|
|
418
|
+
_iterator = _asyncIterator(KeplerCSVLoader.parseInBatches([new TextEncoder().encode(text)], {
|
|
419
|
+
csv: {
|
|
420
|
+
shape: 'array-row-table',
|
|
421
|
+
// @ts-expect-error loaders.gl accepts false but its type only declares 'auto'.
|
|
422
|
+
header: false
|
|
423
|
+
}
|
|
424
|
+
}));
|
|
425
|
+
case 12:
|
|
426
|
+
_context4.next = 14;
|
|
427
|
+
return _iterator.next();
|
|
428
|
+
case 14:
|
|
429
|
+
if (!(_iteratorAbruptCompletion = !(_step = _context4.sent).done)) {
|
|
430
|
+
_context4.next = 20;
|
|
431
|
+
break;
|
|
432
|
+
}
|
|
433
|
+
batch = _step.value;
|
|
434
|
+
rows.push.apply(rows, _toConsumableArray(batch.data));
|
|
435
|
+
case 17:
|
|
436
|
+
_iteratorAbruptCompletion = false;
|
|
437
|
+
_context4.next = 12;
|
|
438
|
+
break;
|
|
439
|
+
case 20:
|
|
440
|
+
_context4.next = 26;
|
|
441
|
+
break;
|
|
442
|
+
case 22:
|
|
443
|
+
_context4.prev = 22;
|
|
444
|
+
_context4.t0 = _context4["catch"](10);
|
|
445
|
+
_didIteratorError = true;
|
|
446
|
+
_iteratorError = _context4.t0;
|
|
447
|
+
case 26:
|
|
448
|
+
_context4.prev = 26;
|
|
449
|
+
_context4.prev = 27;
|
|
450
|
+
if (!(_iteratorAbruptCompletion && _iterator["return"] != null)) {
|
|
451
|
+
_context4.next = 31;
|
|
452
|
+
break;
|
|
453
|
+
}
|
|
454
|
+
_context4.next = 31;
|
|
455
|
+
return _iterator["return"]();
|
|
456
|
+
case 31:
|
|
457
|
+
_context4.prev = 31;
|
|
458
|
+
if (!_didIteratorError) {
|
|
459
|
+
_context4.next = 34;
|
|
460
|
+
break;
|
|
461
|
+
}
|
|
462
|
+
throw _iteratorError;
|
|
463
|
+
case 34:
|
|
464
|
+
return _context4.finish(31);
|
|
465
|
+
case 35:
|
|
466
|
+
return _context4.finish(26);
|
|
467
|
+
case 36:
|
|
468
|
+
if (!(rows.length !== 1)) {
|
|
469
|
+
_context4.next = 38;
|
|
470
|
+
break;
|
|
471
|
+
}
|
|
472
|
+
return _context4.abrupt("return", false);
|
|
473
|
+
case 38:
|
|
474
|
+
header = rows[0]; // processRowObject uses Object.keys, which sorts integer-named columns.
|
|
475
|
+
names = Object.keys(Object.fromEntries(header.map(function (name) {
|
|
476
|
+
return [name, null];
|
|
477
|
+
})));
|
|
478
|
+
return _context4.abrupt("return", header.length === fields.length && names.length === fields.length && names.every(function (name, i) {
|
|
479
|
+
return name === fields[i];
|
|
480
|
+
}) && matchesRecord(header, false));
|
|
481
|
+
case 41:
|
|
482
|
+
case "end":
|
|
483
|
+
return _context4.stop();
|
|
484
|
+
}
|
|
485
|
+
}, _callee4, null, [[10, 22, 26, 36], [27,, 31, 35]]);
|
|
486
|
+
}));
|
|
487
|
+
return _matchesCsvHeader.apply(this, arguments);
|
|
488
|
+
}
|
|
489
|
+
export function loadExternallyHostedDataset(_x9) {
|
|
490
|
+
return _loadExternallyHostedDataset.apply(this, arguments);
|
|
491
|
+
}
|
|
492
|
+
function _loadExternallyHostedDataset() {
|
|
493
|
+
_loadExternallyHostedDataset = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime.mark(function _callee5(metadata) {
|
|
494
|
+
var _fetched$size, _metadata$csvHeader, _processed;
|
|
495
|
+
var source, format, etag, lastModified, size, bypassCache, onProgress, fetched, batches, content, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, batch, processed, data;
|
|
496
|
+
return _regeneratorRuntime.wrap(function _callee5$(_context5) {
|
|
497
|
+
while (1) switch (_context5.prev = _context5.next) {
|
|
498
|
+
case 0:
|
|
499
|
+
source = metadata.source, format = metadata.format, etag = metadata.etag, lastModified = metadata.lastModified, size = metadata.size, bypassCache = metadata.bypassCache, onProgress = metadata.onProgress;
|
|
500
|
+
_context5.next = 3;
|
|
501
|
+
return fetchRemoteFile(source, {
|
|
502
|
+
format: format,
|
|
503
|
+
etag: etag,
|
|
504
|
+
lastModified: lastModified,
|
|
505
|
+
bypassCache: bypassCache,
|
|
506
|
+
onProgress: onProgress
|
|
507
|
+
});
|
|
508
|
+
case 3:
|
|
509
|
+
fetched = _context5.sent;
|
|
510
|
+
if (!(fetched.notModified || !fetched.file)) {
|
|
511
|
+
_context5.next = 6;
|
|
512
|
+
break;
|
|
513
|
+
}
|
|
514
|
+
return _context5.abrupt("return", {
|
|
515
|
+
data: null,
|
|
516
|
+
notModified: true,
|
|
517
|
+
etag: fetched.etag,
|
|
518
|
+
lastModified: fetched.lastModified,
|
|
519
|
+
size: size
|
|
520
|
+
});
|
|
521
|
+
case 6:
|
|
522
|
+
onProgress === null || onProgress === void 0 || onProgress(_objectSpread(_objectSpread({
|
|
523
|
+
loaded: (_fetched$size = fetched.size) !== null && _fetched$size !== void 0 ? _fetched$size : fetched.file.size
|
|
524
|
+
}, typeof fetched.size === 'number' ? {
|
|
525
|
+
total: fetched.size
|
|
526
|
+
} : {}), {}, {
|
|
527
|
+
percent: 1,
|
|
528
|
+
phase: 'processing'
|
|
529
|
+
}));
|
|
530
|
+
// Let the loading indicator paint a processing state before parse blocks the tab.
|
|
531
|
+
_context5.next = 9;
|
|
532
|
+
return waitForNextPaint();
|
|
533
|
+
case 9:
|
|
534
|
+
_context5.next = 11;
|
|
535
|
+
return readFileInBatches({
|
|
536
|
+
file: fetched.file,
|
|
537
|
+
fileCache: [],
|
|
538
|
+
loaders: [],
|
|
539
|
+
loadOptions: {}
|
|
540
|
+
});
|
|
541
|
+
case 11:
|
|
542
|
+
batches = _context5.sent;
|
|
543
|
+
_iteratorAbruptCompletion2 = false;
|
|
544
|
+
_didIteratorError2 = false;
|
|
545
|
+
_context5.prev = 14;
|
|
546
|
+
_iterator2 = _asyncIterator(batches);
|
|
547
|
+
case 16:
|
|
548
|
+
_context5.next = 18;
|
|
549
|
+
return _iterator2.next();
|
|
550
|
+
case 18:
|
|
551
|
+
if (!(_iteratorAbruptCompletion2 = !(_step2 = _context5.sent).done)) {
|
|
552
|
+
_context5.next = 24;
|
|
553
|
+
break;
|
|
554
|
+
}
|
|
555
|
+
batch = _step2.value;
|
|
556
|
+
content = batch;
|
|
557
|
+
case 21:
|
|
558
|
+
_iteratorAbruptCompletion2 = false;
|
|
559
|
+
_context5.next = 16;
|
|
560
|
+
break;
|
|
561
|
+
case 24:
|
|
562
|
+
_context5.next = 30;
|
|
563
|
+
break;
|
|
564
|
+
case 26:
|
|
565
|
+
_context5.prev = 26;
|
|
566
|
+
_context5.t0 = _context5["catch"](14);
|
|
567
|
+
_didIteratorError2 = true;
|
|
568
|
+
_iteratorError2 = _context5.t0;
|
|
569
|
+
case 30:
|
|
570
|
+
_context5.prev = 30;
|
|
571
|
+
_context5.prev = 31;
|
|
572
|
+
if (!(_iteratorAbruptCompletion2 && _iterator2["return"] != null)) {
|
|
573
|
+
_context5.next = 35;
|
|
574
|
+
break;
|
|
575
|
+
}
|
|
576
|
+
_context5.next = 35;
|
|
577
|
+
return _iterator2["return"]();
|
|
578
|
+
case 35:
|
|
579
|
+
_context5.prev = 35;
|
|
580
|
+
if (!_didIteratorError2) {
|
|
581
|
+
_context5.next = 38;
|
|
582
|
+
break;
|
|
583
|
+
}
|
|
584
|
+
throw _iteratorError2;
|
|
585
|
+
case 38:
|
|
586
|
+
return _context5.finish(35);
|
|
587
|
+
case 39:
|
|
588
|
+
return _context5.finish(30);
|
|
589
|
+
case 40:
|
|
590
|
+
_context5.t1 = (_metadata$csvHeader = metadata.csvHeader) !== null && _metadata$csvHeader !== void 0 && _metadata$csvHeader.length && KeplerCSVLoader.extensions.some(function (extension) {
|
|
591
|
+
return fetched.file.name.toLowerCase().endsWith(".".concat(extension));
|
|
592
|
+
}) && (!content || Array.isArray(content.data) && !content.data.length);
|
|
593
|
+
if (!_context5.t1) {
|
|
594
|
+
_context5.next = 50;
|
|
595
|
+
break;
|
|
596
|
+
}
|
|
597
|
+
_context5.t2 = matchesCsvHeader;
|
|
598
|
+
_context5.next = 45;
|
|
599
|
+
return fetched.file.text();
|
|
600
|
+
case 45:
|
|
601
|
+
_context5.t3 = _context5.sent;
|
|
602
|
+
_context5.t4 = metadata.csvHeader;
|
|
603
|
+
_context5.next = 49;
|
|
604
|
+
return (0, _context5.t2)(_context5.t3, _context5.t4);
|
|
605
|
+
case 49:
|
|
606
|
+
_context5.t1 = _context5.sent;
|
|
607
|
+
case 50:
|
|
608
|
+
if (!_context5.t1) {
|
|
609
|
+
_context5.next = 52;
|
|
610
|
+
break;
|
|
611
|
+
}
|
|
612
|
+
return _context5.abrupt("return", {
|
|
613
|
+
data: {
|
|
614
|
+
fields: [],
|
|
615
|
+
rows: []
|
|
616
|
+
},
|
|
617
|
+
notModified: false,
|
|
618
|
+
etag: fetched.etag,
|
|
619
|
+
lastModified: fetched.lastModified,
|
|
620
|
+
size: fetched.size
|
|
621
|
+
});
|
|
622
|
+
case 52:
|
|
623
|
+
if (content) {
|
|
624
|
+
_context5.next = 54;
|
|
625
|
+
break;
|
|
626
|
+
}
|
|
627
|
+
throw new Error("No data loaded from ".concat(source));
|
|
628
|
+
case 54:
|
|
629
|
+
_context5.next = 56;
|
|
630
|
+
return processFileData({
|
|
631
|
+
content: _objectSpread(_objectSpread({}, content), {}, {
|
|
632
|
+
fileName: content.fileName,
|
|
633
|
+
sourceUrl: source
|
|
634
|
+
}),
|
|
635
|
+
fileCache: []
|
|
636
|
+
});
|
|
637
|
+
case 56:
|
|
638
|
+
processed = _context5.sent;
|
|
639
|
+
data = (_processed = processed[processed.length - 1]) === null || _processed === void 0 ? void 0 : _processed.data;
|
|
640
|
+
if (isTabularProcessorResult(data)) {
|
|
641
|
+
_context5.next = 60;
|
|
642
|
+
break;
|
|
643
|
+
}
|
|
644
|
+
throw new Error("Remote file is not a dataset: ".concat(source));
|
|
645
|
+
case 60:
|
|
646
|
+
return _context5.abrupt("return", {
|
|
647
|
+
data: data,
|
|
648
|
+
notModified: false,
|
|
649
|
+
etag: fetched.etag,
|
|
650
|
+
lastModified: fetched.lastModified,
|
|
651
|
+
size: fetched.size
|
|
652
|
+
});
|
|
653
|
+
case 61:
|
|
654
|
+
case "end":
|
|
655
|
+
return _context5.stop();
|
|
656
|
+
}
|
|
657
|
+
}, _callee5, null, [[14, 26, 30, 40], [31,, 35, 39]]);
|
|
658
|
+
}));
|
|
659
|
+
return _loadExternallyHostedDataset.apply(this, arguments);
|
|
660
|
+
}
|
|
661
|
+
export function isExternallyHostedFile(file) {
|
|
662
|
+
return Boolean(file.keplerSourceUrl);
|
|
663
|
+
}
|
|
664
|
+
//# sourceMappingURL=remote-file.js.map
|