@loaders.gl/core 3.4.0-alpha.1 → 3.4.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +461 -248
- package/dist/es5/core-addons/write-file-browser.js +0 -2
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/iterators/batch-iterators/timed-batch-iterator.js +65 -67
- package/dist/es5/iterators/batch-iterators/timed-batch-iterator.js.map +1 -1
- package/dist/es5/iterators/make-iterator/make-array-buffer-iterator.js +13 -24
- package/dist/es5/iterators/make-iterator/make-array-buffer-iterator.js.map +1 -1
- package/dist/es5/iterators/make-iterator/make-blob-iterator.js +22 -25
- package/dist/es5/iterators/make-iterator/make-blob-iterator.js.map +1 -1
- package/dist/es5/iterators/make-iterator/make-iterator.js.map +1 -1
- package/dist/es5/iterators/make-iterator/make-stream-iterator.js +88 -93
- package/dist/es5/iterators/make-iterator/make-stream-iterator.js.map +1 -1
- package/dist/es5/iterators/make-iterator/make-string-iterator.js +20 -24
- package/dist/es5/iterators/make-iterator/make-string-iterator.js.map +1 -1
- package/dist/es5/iterators/make-stream/make-dom-stream.js +30 -34
- package/dist/es5/iterators/make-stream/make-dom-stream.js.map +1 -1
- package/dist/es5/iterators/make-stream/make-node-stream.js +88 -87
- package/dist/es5/iterators/make-stream/make-node-stream.js.map +1 -1
- package/dist/es5/javascript-utils/is-type.js +0 -3
- package/dist/es5/javascript-utils/is-type.js.map +1 -1
- package/dist/es5/lib/api/encode.js +157 -166
- package/dist/es5/lib/api/encode.js.map +1 -1
- package/dist/es5/lib/api/load-in-batches.js +25 -31
- package/dist/es5/lib/api/load-in-batches.js.map +1 -1
- package/dist/es5/lib/api/load.js +35 -38
- package/dist/es5/lib/api/load.js.map +1 -1
- package/dist/es5/lib/api/loader-options.js.map +1 -1
- package/dist/es5/lib/api/parse-in-batches.js +192 -207
- package/dist/es5/lib/api/parse-in-batches.js.map +1 -1
- package/dist/es5/lib/api/parse-sync.js +0 -7
- package/dist/es5/lib/api/parse-sync.js.map +1 -1
- package/dist/es5/lib/api/parse.js +99 -107
- package/dist/es5/lib/api/parse.js.map +1 -1
- package/dist/es5/lib/api/register-loaders.js +1 -3
- package/dist/es5/lib/api/register-loaders.js.map +1 -1
- package/dist/es5/lib/api/save.js +13 -15
- package/dist/es5/lib/api/save.js.map +1 -1
- package/dist/es5/lib/api/select-loader.js +41 -58
- package/dist/es5/lib/api/select-loader.js.map +1 -1
- package/dist/es5/lib/fetch/fetch-error-message.js +29 -31
- package/dist/es5/lib/fetch/fetch-error-message.js.map +1 -1
- package/dist/es5/lib/fetch/fetch-file.js +23 -25
- package/dist/es5/lib/fetch/fetch-file.js.map +1 -1
- package/dist/es5/lib/fetch/read-array-buffer.js +41 -45
- package/dist/es5/lib/fetch/read-array-buffer.js.map +1 -1
- package/dist/es5/lib/fetch/read-file.js +0 -3
- package/dist/es5/lib/fetch/read-file.js.map +1 -1
- package/dist/es5/lib/fetch/write-file.js +15 -17
- package/dist/es5/lib/fetch/write-file.js.map +1 -1
- package/dist/es5/lib/filesystems/browser-filesystem.js +107 -124
- package/dist/es5/lib/filesystems/browser-filesystem.js.map +1 -1
- package/dist/es5/lib/filesystems/read-array-buffer.js +19 -21
- package/dist/es5/lib/filesystems/read-array-buffer.js.map +1 -1
- package/dist/es5/lib/init.js +1 -3
- package/dist/es5/lib/init.js.map +1 -1
- package/dist/es5/lib/loader-utils/check-errors.js +24 -26
- package/dist/es5/lib/loader-utils/check-errors.js.map +1 -1
- package/dist/es5/lib/loader-utils/get-data.js +130 -145
- package/dist/es5/lib/loader-utils/get-data.js.map +1 -1
- package/dist/es5/lib/loader-utils/get-fetch-function.js +26 -0
- package/dist/es5/lib/loader-utils/get-fetch-function.js.map +1 -0
- package/dist/es5/lib/loader-utils/loader-context.js +2 -5
- package/dist/es5/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/es5/lib/loader-utils/loggers.js +0 -2
- package/dist/es5/lib/loader-utils/loggers.js.map +1 -1
- package/dist/es5/lib/loader-utils/normalize-loader.js +0 -3
- package/dist/es5/lib/loader-utils/normalize-loader.js.map +1 -1
- package/dist/es5/lib/loader-utils/option-defaults.js +0 -2
- package/dist/es5/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/es5/lib/loader-utils/option-utils.js +21 -55
- package/dist/es5/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/es5/lib/progress/fetch-progress.js +92 -98
- package/dist/es5/lib/progress/fetch-progress.js.map +1 -1
- package/dist/es5/lib/utils/log.js +0 -1
- package/dist/es5/lib/utils/log.js.map +1 -1
- package/dist/es5/lib/utils/mime-type-utils.js +0 -3
- package/dist/es5/lib/utils/mime-type-utils.js.map +1 -1
- package/dist/es5/lib/utils/resource-utils.js +0 -4
- package/dist/es5/lib/utils/resource-utils.js.map +1 -1
- package/dist/es5/lib/utils/response-utils.js +118 -131
- package/dist/es5/lib/utils/response-utils.js.map +1 -1
- package/dist/es5/null-loader.js +57 -63
- package/dist/es5/null-loader.js.map +1 -1
- package/dist/es5/workers/null-worker.js.map +1 -1
- package/dist/esm/bundle.js +0 -1
- package/dist/esm/bundle.js.map +1 -1
- package/dist/esm/index.js +0 -12
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/iterators/batch-iterators/timed-batch-iterator.js +0 -1
- package/dist/esm/iterators/batch-iterators/timed-batch-iterator.js.map +1 -1
- package/dist/esm/iterators/make-iterator/make-array-buffer-iterator.js +16 -17
- package/dist/esm/iterators/make-iterator/make-array-buffer-iterator.js.map +1 -1
- package/dist/esm/iterators/make-iterator/make-blob-iterator.js +0 -1
- package/dist/esm/iterators/make-iterator/make-blob-iterator.js.map +1 -1
- package/dist/esm/iterators/make-iterator/make-iterator.js +0 -1
- package/dist/esm/iterators/make-iterator/make-iterator.js.map +1 -1
- package/dist/esm/iterators/make-iterator/make-stream-iterator.js +0 -3
- package/dist/esm/iterators/make-iterator/make-stream-iterator.js.map +1 -1
- package/dist/esm/iterators/make-iterator/make-string-iterator.js +0 -2
- package/dist/esm/iterators/make-iterator/make-string-iterator.js.map +1 -1
- package/dist/esm/iterators/make-stream/make-dom-stream.js +1 -3
- package/dist/esm/iterators/make-stream/make-dom-stream.js.map +1 -1
- package/dist/esm/iterators/make-stream/make-node-stream.js +3 -2
- package/dist/esm/iterators/make-stream/make-node-stream.js.map +1 -1
- package/dist/esm/javascript-utils/is-type.js +0 -4
- package/dist/esm/javascript-utils/is-type.js.map +1 -1
- package/dist/esm/lib/api/encode.js +0 -9
- package/dist/esm/lib/api/encode.js.map +1 -1
- package/dist/esm/lib/api/load-in-batches.js +1 -5
- package/dist/esm/lib/api/load-in-batches.js.map +1 -1
- package/dist/esm/lib/api/load.js +1 -6
- package/dist/esm/lib/api/load.js.map +1 -1
- package/dist/esm/lib/api/parse-in-batches.js +2 -16
- package/dist/esm/lib/api/parse-in-batches.js.map +1 -1
- package/dist/esm/lib/api/parse-sync.js +0 -8
- package/dist/esm/lib/api/parse-sync.js.map +1 -1
- package/dist/esm/lib/api/parse.js +0 -12
- package/dist/esm/lib/api/parse.js.map +1 -1
- package/dist/esm/lib/api/register-loaders.js +0 -3
- package/dist/esm/lib/api/register-loaders.js.map +1 -1
- package/dist/esm/lib/api/save.js.map +1 -1
- package/dist/esm/lib/api/select-loader.js +0 -20
- package/dist/esm/lib/api/select-loader.js.map +1 -1
- package/dist/esm/lib/fetch/fetch-error-message.js.map +1 -1
- package/dist/esm/lib/fetch/fetch-file.js +0 -1
- package/dist/esm/lib/fetch/fetch-file.js.map +1 -1
- package/dist/esm/lib/fetch/read-array-buffer.js +0 -3
- package/dist/esm/lib/fetch/read-array-buffer.js.map +1 -1
- package/dist/esm/lib/fetch/read-file.js +0 -4
- package/dist/esm/lib/fetch/read-file.js.map +1 -1
- package/dist/esm/lib/fetch/write-file.js +0 -1
- package/dist/esm/lib/fetch/write-file.js.map +1 -1
- package/dist/esm/lib/filesystems/browser-filesystem.js +1 -11
- package/dist/esm/lib/filesystems/browser-filesystem.js.map +1 -1
- package/dist/esm/lib/filesystems/read-array-buffer.js +0 -2
- package/dist/esm/lib/filesystems/read-array-buffer.js.map +1 -1
- package/dist/esm/lib/init.js +1 -4
- package/dist/esm/lib/init.js.map +1 -1
- package/dist/esm/lib/loader-utils/check-errors.js +1 -2
- package/dist/esm/lib/loader-utils/check-errors.js.map +1 -1
- package/dist/esm/lib/loader-utils/get-data.js +0 -6
- package/dist/esm/lib/loader-utils/get-data.js.map +1 -1
- package/dist/esm/lib/loader-utils/get-fetch-function.js +18 -0
- package/dist/esm/lib/loader-utils/get-fetch-function.js.map +1 -0
- package/dist/esm/lib/loader-utils/loader-context.js +1 -5
- package/dist/esm/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/esm/lib/loader-utils/loggers.js +0 -3
- package/dist/esm/lib/loader-utils/loggers.js.map +1 -1
- package/dist/esm/lib/loader-utils/normalize-loader.js +0 -3
- package/dist/esm/lib/loader-utils/normalize-loader.js.map +1 -1
- package/dist/esm/lib/loader-utils/option-defaults.js +0 -2
- package/dist/esm/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/esm/lib/loader-utils/option-utils.js +0 -32
- package/dist/esm/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/esm/lib/progress/fetch-progress.js +2 -6
- package/dist/esm/lib/progress/fetch-progress.js.map +1 -1
- package/dist/esm/lib/utils/log.js +0 -1
- package/dist/esm/lib/utils/log.js.map +1 -1
- package/dist/esm/lib/utils/mime-type-utils.js +0 -4
- package/dist/esm/lib/utils/mime-type-utils.js.map +1 -1
- package/dist/esm/lib/utils/resource-utils.js +0 -4
- package/dist/esm/lib/utils/resource-utils.js.map +1 -1
- package/dist/esm/lib/utils/response-utils.js +1 -12
- package/dist/esm/lib/utils/response-utils.js.map +1 -1
- package/dist/esm/null-loader.js +1 -4
- package/dist/esm/null-loader.js.map +1 -1
- package/dist/iterators/make-stream/make-node-stream.d.ts +7 -2
- package/dist/iterators/make-stream/make-node-stream.d.ts.map +1 -1
- package/dist/iterators/make-stream/make-node-stream.js +28 -2
- package/dist/lib/api/encode.d.ts +5 -5
- package/dist/lib/api/encode.d.ts.map +1 -1
- package/dist/lib/api/encode.js +1 -0
- package/dist/lib/api/load-in-batches.js +2 -2
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/load.js +3 -2
- package/dist/lib/fetch/fetch-file.js +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts +9 -0
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -0
- package/dist/lib/loader-utils/get-fetch-function.js +31 -0
- package/dist/lib/loader-utils/loader-context.js +2 -2
- package/dist/lib/loader-utils/option-utils.d.ts +5 -15
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +10 -31
- package/dist/lib/progress/fetch-progress.js +2 -2
- package/dist/null-worker.js +1 -1
- package/package.json +5 -5
- package/src/iterators/make-stream/make-node-stream.ts +8 -2
- package/src/lib/api/encode.ts +7 -6
- package/src/lib/api/load-in-batches.ts +1 -1
- package/src/lib/api/load.ts +3 -2
- package/src/lib/loader-utils/get-fetch-function.ts +38 -0
- package/src/lib/loader-utils/loader-context.ts +1 -1
- package/src/lib/loader-utils/option-utils.ts +17 -44
- package/src/lib/progress/fetch-progress.ts +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"load.js","names":["load","url","loaders","options","context","Array","isArray","isLoaderObject","undefined","
|
|
1
|
+
{"version":3,"file":"load.js","names":["_isType","require","_normalizeLoader","_getFetchFunction","_parse","load","_x","_x2","_x3","_x4","_load","apply","arguments","_asyncToGenerator2","default","_regenerator","mark","_callee","url","loaders","options","context","fetch","data","wrap","_callee$","_context","prev","next","Array","isArray","isLoaderObject","undefined","getFetchFunction","sent","isBlob","parse","abrupt","stop"],"sources":["../../../../src/lib/api/load.ts"],"sourcesContent":["import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {isBlob} from '../../javascript-utils/is-type';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {getFetchFunction} from '../loader-utils/get-fetch-function';\n\nimport {parse} from './parse';\n\n/**\n * Parses `data` using a specified loader\n * Note: Load does duplicate a lot of parse.\n * it can also call fetchFile on string urls, which `parse` won't do.\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\n// implementation signature\nexport async function load(\n url: string | DataType,\n loaders?: Loader | Loader[] | LoaderOptions,\n options?: LoaderOptions,\n context?: LoaderContext\n): Promise<any> {\n // Signature: load(url, options)\n if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n context = undefined; // context not supported in short signature\n options = loaders as LoaderOptions;\n loaders = undefined;\n }\n\n // Select fetch function\n const fetch = getFetchFunction(options);\n\n // at this point, `url` could be already loaded binary data\n let data = url;\n // url is a string, fetch the url\n if (typeof url === 'string') {\n data = await fetch(url);\n // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)\n }\n\n if (isBlob(url)) {\n // The fetch response object will contain blob.name\n // @ts-expect-error TODO - This may not work for overridden fetch functions\n data = await fetch(url);\n }\n\n // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`\n return await parse(data, loaders as Loader[], options);\n}\n"],"mappings":";;;;;;;;;AACA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,gBAAA,GAAAD,OAAA;AACA,IAAAE,iBAAA,GAAAF,OAAA;AAEA,IAAAG,MAAA,GAAAH,OAAA;AAA8B,SAYRI,IAAIA,CAAAC,EAAA,EAAAC,GAAA,EAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,KAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAF,MAAA;EAAAA,KAAA,OAAAG,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAnB,SAAAC,QACLC,GAAsB,EACtBC,OAA2C,EAC3CC,OAAuB,EACvBC,OAAuB;IAAA,IAAAC,KAAA,EAAAC,IAAA;IAAA,OAAAR,YAAA,CAAAD,OAAA,CAAAU,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;QAAA;UAGvB,IAAI,CAACC,KAAK,CAACC,OAAO,CAACX,OAAO,CAAC,IAAI,CAAC,IAAAY,+BAAc,EAACZ,OAAO,CAAC,EAAE;YACvDE,OAAO,GAAGW,SAAS;YACnBZ,OAAO,GAAGD,OAAwB;YAClCA,OAAO,GAAGa,SAAS;UACrB;UAGMV,KAAK,GAAG,IAAAW,kCAAgB,EAACb,OAAO,CAAC;UAGnCG,IAAI,GAAGL,GAAG;UAAA,MAEV,OAAOA,GAAG,KAAK,QAAQ;YAAAQ,QAAA,CAAAE,IAAA;YAAA;UAAA;UAAAF,QAAA,CAAAE,IAAA;UAAA,OACZN,KAAK,CAACJ,GAAG,CAAC;QAAA;UAAvBK,IAAI,GAAAG,QAAA,CAAAQ,IAAA;QAAA;UAAA,KAIF,IAAAC,cAAM,EAACjB,GAAG,CAAC;YAAAQ,QAAA,CAAAE,IAAA;YAAA;UAAA;UAAAF,QAAA,CAAAE,IAAA;UAAA,OAGAN,KAAK,CAACJ,GAAG,CAAC;QAAA;UAAvBK,IAAI,GAAAG,QAAA,CAAAQ,IAAA;QAAA;UAAAR,QAAA,CAAAE,IAAA;UAAA,OAIO,IAAAQ,YAAK,EAACb,IAAI,EAAEJ,OAAO,EAAcC,OAAO,CAAC;QAAA;UAAA,OAAAM,QAAA,CAAAW,MAAA,WAAAX,QAAA,CAAAQ,IAAA;QAAA;QAAA;UAAA,OAAAR,QAAA,CAAAY,IAAA;MAAA;IAAA,GAAArB,OAAA;EAAA,CACvD;EAAA,OAAAP,KAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader-options.js","names":[],"sources":["../../../../src/lib/api/loader-options.ts"],"sourcesContent":["export {setGlobalOptions as setLoaderOptions} from '../loader-utils/option-utils';\nexport {getGlobalLoaderOptions as getLoaderOptions} from '../loader-utils/option-utils';\n"],"mappings":";;;;;;;;;;;;;;;;;AAAA"}
|
|
1
|
+
{"version":3,"file":"loader-options.js","names":["_optionUtils","require"],"sources":["../../../../src/lib/api/loader-options.ts"],"sourcesContent":["export {setGlobalOptions as setLoaderOptions} from '../loader-utils/option-utils';\nexport {getGlobalLoaderOptions as getLoaderOptions} from '../loader-utils/option-utils';\n"],"mappings":";;;;;;;;;;;;;;;;;AAAA,IAAAA,YAAA,GAAAC,OAAA"}
|
|
@@ -31,49 +31,44 @@ function _parseInBatches() {
|
|
|
31
31
|
_parseInBatches = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(data, loaders, options, context) {
|
|
32
32
|
var _getResourceUrlAndTyp, url, loader;
|
|
33
33
|
return _regenerator.default.wrap(function _callee$(_context) {
|
|
34
|
-
while (1) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
return _context.abrupt("return", _context.sent);
|
|
73
|
-
case 17:
|
|
74
|
-
case "end":
|
|
75
|
-
return _context.stop();
|
|
76
|
-
}
|
|
34
|
+
while (1) switch (_context.prev = _context.next) {
|
|
35
|
+
case 0:
|
|
36
|
+
(0, _loaderUtils.assert)(!context || (0, _typeof2.default)(context) === 'object');
|
|
37
|
+
if (!Array.isArray(loaders) && !(0, _normalizeLoader.isLoaderObject)(loaders)) {
|
|
38
|
+
context = undefined;
|
|
39
|
+
options = loaders;
|
|
40
|
+
loaders = undefined;
|
|
41
|
+
}
|
|
42
|
+
_context.next = 4;
|
|
43
|
+
return data;
|
|
44
|
+
case 4:
|
|
45
|
+
data = _context.sent;
|
|
46
|
+
options = options || {};
|
|
47
|
+
_getResourceUrlAndTyp = (0, _resourceUtils.getResourceUrlAndType)(data), url = _getResourceUrlAndTyp.url;
|
|
48
|
+
_context.next = 9;
|
|
49
|
+
return (0, _selectLoader.selectLoader)(data, loaders, options);
|
|
50
|
+
case 9:
|
|
51
|
+
loader = _context.sent;
|
|
52
|
+
if (loader) {
|
|
53
|
+
_context.next = 12;
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
return _context.abrupt("return", null);
|
|
57
|
+
case 12:
|
|
58
|
+
options = (0, _optionUtils.normalizeOptions)(options, loader, loaders, url);
|
|
59
|
+
context = (0, _loaderContext.getLoaderContext)({
|
|
60
|
+
url: url,
|
|
61
|
+
parseInBatches: parseInBatches,
|
|
62
|
+
parse: _parse.parse,
|
|
63
|
+
loaders: loaders
|
|
64
|
+
}, options, context);
|
|
65
|
+
_context.next = 16;
|
|
66
|
+
return parseWithLoaderInBatches(loader, data, options, context);
|
|
67
|
+
case 16:
|
|
68
|
+
return _context.abrupt("return", _context.sent);
|
|
69
|
+
case 17:
|
|
70
|
+
case "end":
|
|
71
|
+
return _context.stop();
|
|
77
72
|
}
|
|
78
73
|
}, _callee);
|
|
79
74
|
}));
|
|
@@ -86,55 +81,51 @@ function _parseWithLoaderInBatches() {
|
|
|
86
81
|
_parseWithLoaderInBatches = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(loader, data, options, context) {
|
|
87
82
|
var outputIterator, metadataBatch, makeMetadataBatchIterator, _makeMetadataBatchIterator;
|
|
88
83
|
return _regenerator.default.wrap(function _callee3$(_context3) {
|
|
89
|
-
while (1) {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
_makeMetadataBatchIterator = function
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
case 9:
|
|
135
|
-
case "end":
|
|
136
|
-
return _context3.stop();
|
|
137
|
-
}
|
|
84
|
+
while (1) switch (_context3.prev = _context3.next) {
|
|
85
|
+
case 0:
|
|
86
|
+
_makeMetadataBatchIterator = function _makeMetadataBatchIte2() {
|
|
87
|
+
_makeMetadataBatchIterator = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee2(iterator) {
|
|
88
|
+
return _regenerator.default.wrap(function _callee2$(_context2) {
|
|
89
|
+
while (1) switch (_context2.prev = _context2.next) {
|
|
90
|
+
case 0:
|
|
91
|
+
_context2.next = 2;
|
|
92
|
+
return metadataBatch;
|
|
93
|
+
case 2:
|
|
94
|
+
return _context2.delegateYield((0, _asyncGeneratorDelegate2.default)(_asyncIterator(iterator), _awaitAsyncGenerator2.default), "t0", 3);
|
|
95
|
+
case 3:
|
|
96
|
+
case "end":
|
|
97
|
+
return _context2.stop();
|
|
98
|
+
}
|
|
99
|
+
}, _callee2);
|
|
100
|
+
}));
|
|
101
|
+
return _makeMetadataBatchIterator.apply(this, arguments);
|
|
102
|
+
};
|
|
103
|
+
makeMetadataBatchIterator = function _makeMetadataBatchIte(_x) {
|
|
104
|
+
return _makeMetadataBatchIterator.apply(this, arguments);
|
|
105
|
+
};
|
|
106
|
+
_context3.next = 4;
|
|
107
|
+
return parseToOutputIterator(loader, data, options, context);
|
|
108
|
+
case 4:
|
|
109
|
+
outputIterator = _context3.sent;
|
|
110
|
+
if (options.metadata) {
|
|
111
|
+
_context3.next = 7;
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
return _context3.abrupt("return", outputIterator);
|
|
115
|
+
case 7:
|
|
116
|
+
metadataBatch = {
|
|
117
|
+
batchType: 'metadata',
|
|
118
|
+
metadata: {
|
|
119
|
+
_loader: loader,
|
|
120
|
+
_context: context
|
|
121
|
+
},
|
|
122
|
+
data: [],
|
|
123
|
+
bytesUsed: 0
|
|
124
|
+
};
|
|
125
|
+
return _context3.abrupt("return", makeMetadataBatchIterator(outputIterator));
|
|
126
|
+
case 9:
|
|
127
|
+
case "end":
|
|
128
|
+
return _context3.stop();
|
|
138
129
|
}
|
|
139
130
|
}, _callee3);
|
|
140
131
|
}));
|
|
@@ -147,66 +138,62 @@ function _parseToOutputIterator() {
|
|
|
147
138
|
_parseToOutputIterator = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee5(loader, data, options, context) {
|
|
148
139
|
var inputIterator, transformedIterator, parseChunkInBatches, _parseChunkInBatches;
|
|
149
140
|
return _regenerator.default.wrap(function _callee5$(_context5) {
|
|
150
|
-
while (1) {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
_parseChunkInBatches = function
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
case 11:
|
|
207
|
-
case "end":
|
|
208
|
-
return _context5.stop();
|
|
209
|
-
}
|
|
141
|
+
while (1) switch (_context5.prev = _context5.next) {
|
|
142
|
+
case 0:
|
|
143
|
+
_parseChunkInBatches = function _parseChunkInBatches3() {
|
|
144
|
+
_parseChunkInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee4() {
|
|
145
|
+
var arrayBuffer, parsedData, batch;
|
|
146
|
+
return _regenerator.default.wrap(function _callee4$(_context4) {
|
|
147
|
+
while (1) switch (_context4.prev = _context4.next) {
|
|
148
|
+
case 0:
|
|
149
|
+
_context4.next = 2;
|
|
150
|
+
return (0, _awaitAsyncGenerator2.default)((0, _loaderUtils.concatenateArrayBuffersAsync)(transformedIterator));
|
|
151
|
+
case 2:
|
|
152
|
+
arrayBuffer = _context4.sent;
|
|
153
|
+
_context4.next = 5;
|
|
154
|
+
return (0, _awaitAsyncGenerator2.default)((0, _parse.parse)(arrayBuffer, loader, _objectSpread(_objectSpread({}, options), {}, {
|
|
155
|
+
mimeType: loader.mimeTypes[0]
|
|
156
|
+
}), context));
|
|
157
|
+
case 5:
|
|
158
|
+
parsedData = _context4.sent;
|
|
159
|
+
batch = {
|
|
160
|
+
mimeType: loader.mimeTypes[0],
|
|
161
|
+
shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',
|
|
162
|
+
batchType: 'data',
|
|
163
|
+
data: parsedData,
|
|
164
|
+
length: Array.isArray(parsedData) ? parsedData.length : 1
|
|
165
|
+
};
|
|
166
|
+
_context4.next = 9;
|
|
167
|
+
return batch;
|
|
168
|
+
case 9:
|
|
169
|
+
case "end":
|
|
170
|
+
return _context4.stop();
|
|
171
|
+
}
|
|
172
|
+
}, _callee4);
|
|
173
|
+
}));
|
|
174
|
+
return _parseChunkInBatches.apply(this, arguments);
|
|
175
|
+
};
|
|
176
|
+
parseChunkInBatches = function _parseChunkInBatches2() {
|
|
177
|
+
return _parseChunkInBatches.apply(this, arguments);
|
|
178
|
+
};
|
|
179
|
+
_context5.next = 4;
|
|
180
|
+
return (0, _getData.getAsyncIterableFromData)(data, options);
|
|
181
|
+
case 4:
|
|
182
|
+
inputIterator = _context5.sent;
|
|
183
|
+
_context5.next = 7;
|
|
184
|
+
return applyInputTransforms(inputIterator, (options === null || options === void 0 ? void 0 : options.transforms) || []);
|
|
185
|
+
case 7:
|
|
186
|
+
transformedIterator = _context5.sent;
|
|
187
|
+
if (!loader.parseInBatches) {
|
|
188
|
+
_context5.next = 10;
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
return _context5.abrupt("return", loader.parseInBatches(transformedIterator, options, context));
|
|
192
|
+
case 10:
|
|
193
|
+
return _context5.abrupt("return", parseChunkInBatches());
|
|
194
|
+
case 11:
|
|
195
|
+
case "end":
|
|
196
|
+
return _context5.stop();
|
|
210
197
|
}
|
|
211
198
|
}, _callee5);
|
|
212
199
|
}));
|
|
@@ -227,63 +214,61 @@ function _applyInputTransforms() {
|
|
|
227
214
|
transformBatches,
|
|
228
215
|
_args6 = arguments;
|
|
229
216
|
return _regenerator.default.wrap(function _callee6$(_context6) {
|
|
230
|
-
while (1) {
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
_context6.next = 14;
|
|
245
|
-
break;
|
|
246
|
-
}
|
|
247
|
-
transformBatches = _step.value;
|
|
248
|
-
iteratorChain = transformBatches(iteratorChain);
|
|
249
|
-
case 11:
|
|
250
|
-
_iteratorAbruptCompletion = false;
|
|
251
|
-
_context6.next = 6;
|
|
252
|
-
break;
|
|
253
|
-
case 14:
|
|
254
|
-
_context6.next = 20;
|
|
217
|
+
while (1) switch (_context6.prev = _context6.next) {
|
|
218
|
+
case 0:
|
|
219
|
+
transforms = _args6.length > 1 && _args6[1] !== undefined ? _args6[1] : [];
|
|
220
|
+
iteratorChain = inputIterator;
|
|
221
|
+
_iteratorAbruptCompletion = false;
|
|
222
|
+
_didIteratorError = false;
|
|
223
|
+
_context6.prev = 4;
|
|
224
|
+
_iterator = _asyncIterator(transforms);
|
|
225
|
+
case 6:
|
|
226
|
+
_context6.next = 8;
|
|
227
|
+
return _iterator.next();
|
|
228
|
+
case 8:
|
|
229
|
+
if (!(_iteratorAbruptCompletion = !(_step = _context6.sent).done)) {
|
|
230
|
+
_context6.next = 14;
|
|
255
231
|
break;
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
232
|
+
}
|
|
233
|
+
transformBatches = _step.value;
|
|
234
|
+
iteratorChain = transformBatches(iteratorChain);
|
|
235
|
+
case 11:
|
|
236
|
+
_iteratorAbruptCompletion = false;
|
|
237
|
+
_context6.next = 6;
|
|
238
|
+
break;
|
|
239
|
+
case 14:
|
|
240
|
+
_context6.next = 20;
|
|
241
|
+
break;
|
|
242
|
+
case 16:
|
|
243
|
+
_context6.prev = 16;
|
|
244
|
+
_context6.t0 = _context6["catch"](4);
|
|
245
|
+
_didIteratorError = true;
|
|
246
|
+
_iteratorError = _context6.t0;
|
|
247
|
+
case 20:
|
|
248
|
+
_context6.prev = 20;
|
|
249
|
+
_context6.prev = 21;
|
|
250
|
+
if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
|
|
268
251
|
_context6.next = 25;
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
_context6.next = 25;
|
|
255
|
+
return _iterator.return();
|
|
256
|
+
case 25:
|
|
257
|
+
_context6.prev = 25;
|
|
258
|
+
if (!_didIteratorError) {
|
|
259
|
+
_context6.next = 28;
|
|
260
|
+
break;
|
|
261
|
+
}
|
|
262
|
+
throw _iteratorError;
|
|
263
|
+
case 28:
|
|
264
|
+
return _context6.finish(25);
|
|
265
|
+
case 29:
|
|
266
|
+
return _context6.finish(20);
|
|
267
|
+
case 30:
|
|
268
|
+
return _context6.abrupt("return", iteratorChain);
|
|
269
|
+
case 31:
|
|
270
|
+
case "end":
|
|
271
|
+
return _context6.stop();
|
|
287
272
|
}
|
|
288
273
|
}, _callee6, null, [[4, 16, 20, 30], [21,, 25, 29]]);
|
|
289
274
|
}));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-in-batches.js","names":["parseInBatches","data","loaders","options","context","assert","Array","isArray","isLoaderObject","undefined","getResourceUrlAndType","url","selectLoader","loader","normalizeOptions","getLoaderContext","parse","parseWithLoaderInBatches","makeMetadataBatchIterator","iterator","metadataBatch","parseToOutputIterator","outputIterator","metadata","batchType","_loader","_context","bytesUsed","parseChunkInBatches","concatenateArrayBuffersAsync","transformedIterator","arrayBuffer","mimeType","mimeTypes","parsedData","batch","shape","length","getAsyncIterableFromData","inputIterator","applyInputTransforms","transforms","iteratorChain","transformBatches"],"sources":["../../../../src/lib/api/parse-in-batches.ts"],"sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {\n BatchableDataType,\n Loader,\n LoaderWithParser,\n LoaderContext,\n LoaderOptions\n} from '@loaders.gl/loader-utils';\nimport {assert, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {normalizeOptions} from '../loader-utils/option-utils';\nimport {getLoaderContext} from '../loader-utils/loader-context';\nimport {getAsyncIterableFromData} from '../loader-utils/get-data';\nimport {getResourceUrlAndType} from '../utils/resource-utils';\nimport {selectLoader} from './select-loader';\n\n// Ensure `parse` is available in context if loader falls back to `parse`\nimport {parse} from './parse';\n\n/**\n * Parses `data` using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport async function parseInBatches(\n data: BatchableDataType,\n loaders?: Loader | Loader[] | LoaderOptions,\n options?: LoaderOptions,\n context?: LoaderContext\n): Promise<AsyncIterable<any>> {\n assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url\n\n // Signature: parseInBatches(data, options, url) - Uses registered loaders\n if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n context = undefined; // context not supported in short signature\n options = loaders as LoaderOptions;\n loaders = undefined;\n }\n\n data = await data; // Resolve any promise\n options = options || {};\n\n // Extract a url for auto detection\n const {url} = getResourceUrlAndType(data);\n\n // Chooses a loader and normalizes it\n // Note - only uses URL and contentType for streams and iterator inputs\n const loader = await selectLoader(data as ArrayBuffer, loaders as Loader[], options);\n // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null\n if (!loader) {\n // @ts-ignore\n return null;\n }\n\n // Normalize options\n // @ts-ignore\n options = normalizeOptions(options, loader, loaders, url);\n // @ts-ignore\n context = getLoaderContext(\n // @ts-ignore\n {url, parseInBatches, parse, loaders: loaders as Loader[]},\n options,\n context\n );\n\n return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);\n}\n\n/**\n * Loader has been selected and context has been prepared, see if we need to emit a metadata batch\n */\nasync function parseWithLoaderInBatches(\n loader: LoaderWithParser,\n data: BatchableDataType,\n options: LoaderOptions,\n context: LoaderContext\n): Promise<AsyncIterable<any>> {\n const outputIterator = await parseToOutputIterator(loader, data, options, context);\n\n // Generate metadata batch if requested\n if (!options.metadata) {\n return outputIterator;\n }\n\n const metadataBatch = {\n batchType: 'metadata',\n metadata: {\n _loader: loader,\n _context: context\n },\n // Populate with some default fields to avoid crashing\n data: [],\n bytesUsed: 0\n };\n\n async function* makeMetadataBatchIterator(iterator) {\n yield metadataBatch;\n yield* iterator;\n }\n\n return makeMetadataBatchIterator(outputIterator);\n}\n\n/**\n * Prep work is done, now it is time to start parsing into an output operator\n * The approach depends on which parse function the loader exposes\n * `parseInBatches` (preferred), `parse` (fallback)\n */\nasync function parseToOutputIterator(\n loader: LoaderWithParser,\n data: BatchableDataType,\n options: LoaderOptions,\n context: LoaderContext\n): Promise<AsyncIterable<any>> {\n // Get an iterator from the input\n const inputIterator = await getAsyncIterableFromData(data, options);\n\n // Apply any iterator transforms (options.transforms)\n const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);\n\n // If loader supports parseInBatches, we are done\n if (loader.parseInBatches) {\n return loader.parseInBatches(transformedIterator, options, context);\n }\n\n // Fallback: load atomically using `parse` concatenating input iterator into single chunk\n async function* parseChunkInBatches() {\n const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);\n // Call `parse` instead of `loader.parse` to ensure we can call workers etc.\n const parsedData = await parse(\n arrayBuffer,\n loader,\n // TODO - Hack: supply loaders MIME type to ensure we match it\n {...options, mimeType: loader.mimeTypes[0]},\n context\n );\n // yield a single batch, the output from loader.parse()\n // TODO - run through batch builder to apply options etc...\n const batch: Batch = {\n mimeType: loader.mimeTypes[0],\n shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',\n batchType: 'data',\n data: parsedData,\n length: Array.isArray(parsedData) ? parsedData.length : 1\n };\n yield batch;\n }\n\n return parseChunkInBatches();\n}\n\ntype TransformBatches = (\n asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n) => AsyncIterable<ArrayBuffer>;\n\n/**\n * Create an iterator chain with any transform iterators (crypto, decompression)\n * @param inputIterator\n * @param options\n */\nasync function applyInputTransforms(\n inputIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n transforms: TransformBatches[] = []\n): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {\n let iteratorChain = inputIterator;\n for await (const transformBatches of transforms) {\n iteratorChain = transformBatches(iteratorChain);\n }\n return iteratorChain;\n}\n"],"mappings":";;;;;;;;;;;;;;AAQA;AACA;AACA;AACA;AACA;AACA;AACA;AAGA;AAA8B;AAAA;AAAA;AAAA;AAAA,SASRA,cAAc;EAAA;AAAA;AAAA;EAAA,4EAA7B,iBACLC,IAAuB,EACvBC,OAA2C,EAC3CC,OAAuB,EACvBC,OAAuB;IAAA;IAAA;MAAA;QAAA;UAAA;YAEvB,IAAAC,mBAAM,EAAC,CAACD,OAAO,IAAI,sBAAOA,OAAO,MAAK,QAAQ,CAAC;;YAG/C,IAAI,CAACE,KAAK,CAACC,OAAO,CAACL,OAAO,CAAC,IAAI,CAAC,IAAAM,+BAAc,EAACN,OAAO,CAAC,EAAE;cACvDE,OAAO,GAAGK,SAAS;cACnBN,OAAO,GAAGD,OAAwB;cAClCA,OAAO,GAAGO,SAAS;YACrB;YAAC;YAAA,OAEYR,IAAI;UAAA;YAAjBA,IAAI;YACJE,OAAO,GAAGA,OAAO,IAAI,CAAC,CAAC;;YAAC,wBAGV,IAAAO,oCAAqB,EAACT,IAAI,CAAC,EAAlCU,GAAG,yBAAHA,GAAG;YAAA;YAAA,OAIW,IAAAC,0BAAY,EAACX,IAAI,EAAiBC,OAAO,EAAcC,OAAO,CAAC;UAAA;YAA9EU,MAAM;YAAA,IAEPA,MAAM;cAAA;cAAA;YAAA;YAAA,iCAEF,IAAI;UAAA;YAKbV,OAAO,GAAG,IAAAW,6BAAgB,EAACX,OAAO,EAAEU,MAAM,EAAEX,OAAO,EAAES,GAAG,CAAC;YAEzDP,OAAO,GAAG,IAAAW,+BAAgB;YAExB;cAACJ,GAAG,EAAHA,GAAG;cAAEX,cAAc,EAAdA,cAAc;cAAEgB,KAAK,EAALA,YAAK;cAAEd,OAAO,EAAEA;YAAmB,CAAC,EAC1DC,OAAO,EACPC,OAAO,CACR;YAAC;YAAA,OAEWa,wBAAwB,CAACJ,MAAM,EAAsBZ,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CAC1F;EAAA;AAAA;AAAA,SAKca,wBAAwB;EAAA;AAAA;AAAA;EAAA,sFAAvC,kBACEJ,MAAwB,EACxBZ,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB;IAAA,mCAoBNc,yBAAyB;IAAA;MAAA;QAAA;UAAA;YAAA;cAAA,yFAAzC,kBAA0CC,QAAQ;gBAAA;kBAAA;oBAAA;sBAAA;wBAAA;wBAChD,OAAMC,aAAa;sBAAA;wBACnB,oFAAOD,QAAQ;sBAAA;sBAAA;wBAAA;oBAAA;kBAAA;gBAAA;cAAA,CAChB;cAAA;YAAA;YAHeD,yBAAyB;cAAA;YAAA;YAAA;YAAA,OAlBZG,qBAAqB,CAACR,MAAM,EAAEZ,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;UAAA;YAA5EkB,cAAc;YAAA,IAGfnB,OAAO,CAACoB,QAAQ;cAAA;cAAA;YAAA;YAAA,kCACZD,cAAc;UAAA;YAGjBF,aAAa,GAAG;cACpBI,SAAS,EAAE,UAAU;cACrBD,QAAQ,EAAE;gBACRE,OAAO,EAAEZ,MAAM;gBACfa,QAAQ,EAAEtB;cACZ,CAAC;cAEDH,IAAI,EAAE,EAAE;cACR0B,SAAS,EAAE;YACb,CAAC;YAAA,kCAOMT,yBAAyB,CAACI,cAAc,CAAC;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACjD;EAAA;AAAA;AAAA,SAOcD,qBAAqB;EAAA;AAAA;AAAA;EAAA,mFAApC,kBACER,MAAwB,EACxBZ,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB;IAAA,wCAcNwB,mBAAmB;IAAA;MAAA;QAAA;UAAA;YAAA;cAAA,mFAAnC;gBAAA;gBAAA;kBAAA;oBAAA;sBAAA;wBAAA;wBAAA,0CAC4B,IAAAC,yCAA4B,EAACC,mBAAmB,CAAC;sBAAA;wBAArEC,WAAW;wBAAA;wBAAA,0CAEQ,IAAAf,YAAK,EAC5Be,WAAW,EACXlB,MAAM,kCAEFV,OAAO;0BAAE6B,QAAQ,EAAEnB,MAAM,CAACoB,SAAS,CAAC,CAAC;wBAAC,IAC1C7B,OAAO,CACR;sBAAA;wBANK8B,UAAU;wBASVC,KAAY,GAAG;0BACnBH,QAAQ,EAAEnB,MAAM,CAACoB,SAAS,CAAC,CAAC,CAAC;0BAC7BG,KAAK,EAAE9B,KAAK,CAACC,OAAO,CAAC2B,UAAU,CAAC,GAAG,WAAW,GAAG,SAAS;0BAC1DV,SAAS,EAAE,MAAM;0BACjBvB,IAAI,EAAEiC,UAAU;0BAChBG,MAAM,EAAE/B,KAAK,CAACC,OAAO,CAAC2B,UAAU,CAAC,GAAGA,UAAU,CAACG,MAAM,GAAG;wBAC1D,CAAC;wBAAA;wBACD,OAAMF,KAAK;sBAAA;sBAAA;wBAAA;oBAAA;kBAAA;gBAAA;cAAA,CACZ;cAAA;YAAA;YApBeP,mBAAmB;cAAA;YAAA;YAAA;YAAA,OAXP,IAAAU,iCAAwB,EAACrC,IAAI,EAAEE,OAAO,CAAC;UAAA;YAA7DoC,aAAa;YAAA;YAAA,OAGeC,oBAAoB,CAACD,aAAa,EAAE,CAAApC,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEsC,UAAU,KAAI,EAAE,CAAC;UAAA;YAA1FX,mBAAmB;YAAA,KAGrBjB,MAAM,CAACb,cAAc;cAAA;cAAA;YAAA;YAAA,kCAChBa,MAAM,CAACb,cAAc,CAAC8B,mBAAmB,EAAE3B,OAAO,EAAEC,OAAO,CAAC;UAAA;YAAA,kCA0B9DwB,mBAAmB,EAAE;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CAC7B;EAAA;AAAA;AAAA,SAWcY,oBAAoB;EAAA;AAAA;AAAA;EAAA,kFAAnC,kBACED,aAAiE;IAAA;MAAA;MAAA;MAAA;MAAA;MAAA;MAAA;MAAA;MAAA;IAAA;MAAA;QAAA;UAAA;YACjEE,UAA8B,8DAAG,EAAE;YAE/BC,aAAa,GAAGH,aAAa;YAAA;YAAA;YAAA;YAAA,2BACIE,UAAU;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAA9BE,gBAAgB;YAC/BD,aAAa,GAAGC,gBAAgB,CAACD,aAAa,CAAC;UAAC;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;YAAA,kCAE3CA,aAAa;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACrB;EAAA;AAAA"}
|
|
1
|
+
{"version":3,"file":"parse-in-batches.js","names":["_loaderUtils","require","_normalizeLoader","_optionUtils","_loaderContext","_getData","_resourceUtils","_selectLoader","_parse","ownKeys","object","enumerableOnly","keys","Object","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","i","arguments","length","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Promise","reject","done","resolve","value","then","n","next","prototype","return","_return","ret","throw","_throw","thr","parseInBatches","_x2","_x3","_x4","_x5","_parseInBatches","_asyncToGenerator2","_regenerator","mark","_callee","data","loaders","options","context","_getResourceUrlAndTyp","url","loader","wrap","_callee$","_context","prev","assert","_typeof2","Array","isArray","isLoaderObject","undefined","sent","getResourceUrlAndType","selectLoader","abrupt","normalizeOptions","getLoaderContext","parse","parseWithLoaderInBatches","stop","_x6","_x7","_x8","_x9","_parseWithLoaderInBatches","_callee3","outputIterator","metadataBatch","makeMetadataBatchIterator","_makeMetadataBatchIterator","_callee3$","_context3","_makeMetadataBatchIte2","_wrapAsyncGenerator2","_callee2","_callee2$","_context2","delegateYield","_asyncGeneratorDelegate2","_awaitAsyncGenerator2","_makeMetadataBatchIte","_x","parseToOutputIterator","metadata","batchType","_loader","bytesUsed","_x10","_x11","_x12","_x13","_parseToOutputIterator","_callee5","inputIterator","transformedIterator","parseChunkInBatches","_parseChunkInBatches","_callee5$","_context5","_parseChunkInBatches3","_callee4","arrayBuffer","parsedData","batch","_callee4$","_context4","concatenateArrayBuffersAsync","mimeType","mimeTypes","shape","_parseChunkInBatches2","getAsyncIterableFromData","applyInputTransforms","transforms","_x14","_applyInputTransforms","_callee6","iteratorChain","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","transformBatches","_args6","_callee6$","_context6","t0","finish"],"sources":["../../../../src/lib/api/parse-in-batches.ts"],"sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {\n BatchableDataType,\n Loader,\n LoaderWithParser,\n LoaderContext,\n LoaderOptions\n} from '@loaders.gl/loader-utils';\nimport {assert, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {normalizeOptions} from '../loader-utils/option-utils';\nimport {getLoaderContext} from '../loader-utils/loader-context';\nimport {getAsyncIterableFromData} from '../loader-utils/get-data';\nimport {getResourceUrlAndType} from '../utils/resource-utils';\nimport {selectLoader} from './select-loader';\n\n// Ensure `parse` is available in context if loader falls back to `parse`\nimport {parse} from './parse';\n\n/**\n * Parses `data` using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport async function parseInBatches(\n data: BatchableDataType,\n loaders?: Loader | Loader[] | LoaderOptions,\n options?: LoaderOptions,\n context?: LoaderContext\n): Promise<AsyncIterable<any>> {\n assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url\n\n // Signature: parseInBatches(data, options, url) - Uses registered loaders\n if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n context = undefined; // context not supported in short signature\n options = loaders as LoaderOptions;\n loaders = undefined;\n }\n\n data = await data; // Resolve any promise\n options = options || {};\n\n // Extract a url for auto detection\n const {url} = getResourceUrlAndType(data);\n\n // Chooses a loader and normalizes it\n // Note - only uses URL and contentType for streams and iterator inputs\n const loader = await selectLoader(data as ArrayBuffer, loaders as Loader[], options);\n // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null\n if (!loader) {\n // @ts-ignore\n return null;\n }\n\n // Normalize options\n // @ts-ignore\n options = normalizeOptions(options, loader, loaders, url);\n // @ts-ignore\n context = getLoaderContext(\n // @ts-ignore\n {url, parseInBatches, parse, loaders: loaders as Loader[]},\n options,\n context\n );\n\n return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);\n}\n\n/**\n * Loader has been selected and context has been prepared, see if we need to emit a metadata batch\n */\nasync function parseWithLoaderInBatches(\n loader: LoaderWithParser,\n data: BatchableDataType,\n options: LoaderOptions,\n context: LoaderContext\n): Promise<AsyncIterable<any>> {\n const outputIterator = await parseToOutputIterator(loader, data, options, context);\n\n // Generate metadata batch if requested\n if (!options.metadata) {\n return outputIterator;\n }\n\n const metadataBatch = {\n batchType: 'metadata',\n metadata: {\n _loader: loader,\n _context: context\n },\n // Populate with some default fields to avoid crashing\n data: [],\n bytesUsed: 0\n };\n\n async function* makeMetadataBatchIterator(iterator) {\n yield metadataBatch;\n yield* iterator;\n }\n\n return makeMetadataBatchIterator(outputIterator);\n}\n\n/**\n * Prep work is done, now it is time to start parsing into an output operator\n * The approach depends on which parse function the loader exposes\n * `parseInBatches` (preferred), `parse` (fallback)\n */\nasync function parseToOutputIterator(\n loader: LoaderWithParser,\n data: BatchableDataType,\n options: LoaderOptions,\n context: LoaderContext\n): Promise<AsyncIterable<any>> {\n // Get an iterator from the input\n const inputIterator = await getAsyncIterableFromData(data, options);\n\n // Apply any iterator transforms (options.transforms)\n const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);\n\n // If loader supports parseInBatches, we are done\n if (loader.parseInBatches) {\n return loader.parseInBatches(transformedIterator, options, context);\n }\n\n // Fallback: load atomically using `parse` concatenating input iterator into single chunk\n async function* parseChunkInBatches() {\n const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);\n // Call `parse` instead of `loader.parse` to ensure we can call workers etc.\n const parsedData = await parse(\n arrayBuffer,\n loader,\n // TODO - Hack: supply loaders MIME type to ensure we match it\n {...options, mimeType: loader.mimeTypes[0]},\n context\n );\n // yield a single batch, the output from loader.parse()\n // TODO - run through batch builder to apply options etc...\n const batch: Batch = {\n mimeType: loader.mimeTypes[0],\n shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',\n batchType: 'data',\n data: parsedData,\n length: Array.isArray(parsedData) ? parsedData.length : 1\n };\n yield batch;\n }\n\n return parseChunkInBatches();\n}\n\ntype TransformBatches = (\n asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n) => AsyncIterable<ArrayBuffer>;\n\n/**\n * Create an iterator chain with any transform iterators (crypto, decompression)\n * @param inputIterator\n * @param options\n */\nasync function applyInputTransforms(\n inputIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n transforms: TransformBatches[] = []\n): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {\n let iteratorChain = inputIterator;\n for await (const transformBatches of transforms) {\n iteratorChain = transformBatches(iteratorChain);\n }\n return iteratorChain;\n}\n"],"mappings":";;;;;;;;;;;;;;AAQA,IAAAA,YAAA,GAAAC,OAAA;AACA,IAAAC,gBAAA,GAAAD,OAAA;AACA,IAAAE,YAAA,GAAAF,OAAA;AACA,IAAAG,cAAA,GAAAH,OAAA;AACA,IAAAI,QAAA,GAAAJ,OAAA;AACA,IAAAK,cAAA,GAAAL,OAAA;AACA,IAAAM,aAAA,GAAAN,OAAA;AAGA,IAAAO,MAAA,GAAAP,OAAA;AAA8B,SAAAQ,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAC,MAAA,CAAAD,IAAA,CAAAF,MAAA,OAAAG,MAAA,CAAAC,qBAAA,QAAAC,OAAA,GAAAF,MAAA,CAAAC,qBAAA,CAAAJ,MAAA,GAAAC,cAAA,KAAAI,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAJ,MAAA,CAAAK,wBAAA,CAAAR,MAAA,EAAAO,GAAA,EAAAE,UAAA,OAAAP,IAAA,CAAAQ,IAAA,CAAAC,KAAA,CAAAT,IAAA,EAAAG,OAAA,YAAAH,IAAA;AAAA,SAAAU,cAAAC,MAAA,aAAAC,CAAA,MAAAA,CAAA,GAAAC,SAAA,CAAAC,MAAA,EAAAF,CAAA,UAAAG,MAAA,WAAAF,SAAA,CAAAD,CAAA,IAAAC,SAAA,CAAAD,CAAA,QAAAA,CAAA,OAAAf,OAAA,CAAAI,MAAA,CAAAc,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAR,MAAA,EAAAM,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAAhB,MAAA,CAAAmB,yBAAA,GAAAnB,MAAA,CAAAoB,gBAAA,CAAAV,MAAA,EAAAV,MAAA,CAAAmB,yBAAA,CAAAL,MAAA,KAAAlB,OAAA,CAAAI,MAAA,CAAAc,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAAhB,MAAA,CAAAqB,cAAA,CAAAX,MAAA,EAAAM,GAAA,EAAAhB,MAAA,CAAAK,wBAAA,CAAAS,MAAA,EAAAE,GAAA,iBAAAN,MAAA;AAAA,SAAAY,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAApC,MAAA,CAAAoC,CAAA,MAAAA,CAAA,SAAAC,OAAA,CAAAC,MAAA,KAAAL,SAAA,CAAAG,CAAA,+BAAAG,IAAA,GAAAH,CAAA,CAAAG,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAJ,CAAA,CAAAK,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAP,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAS,CAAA,GAAAT,CAAA,CAAAU,IAAA,KAAAZ,qBAAA,CAAAa,SAAA,KAAAX,CAAA,QAAAS,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAT,iCAAA,MAAAQ,CAAA,CAAAnC,KAAA,MAAA0B,CAAA,EAAAtB,SAAA,OAAAkC,MAAA,WAAAC,QAAAN,KAAA,QAAAO,GAAA,QAAAd,CAAA,CAAAY,MAAA,oBAAAE,GAAA,GAAAX,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAJ,iCAAA,CAAAa,GAAA,CAAAxC,KAAA,MAAA0B,CAAA,EAAAtB,SAAA,OAAAqC,KAAA,WAAAC,OAAAT,KAAA,QAAAU,GAAA,QAAAjB,CAAA,CAAAY,MAAA,oBAAAK,GAAA,GAAAd,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAN,iCAAA,CAAAgB,GAAA,CAAA3C,KAAA,MAAA0B,CAAA,EAAAtB,SAAA,aAAAoB,qBAAA,CAAAE,CAAA;AAAA,SASRkB,cAAcA,CAAAC,GAAA,EAAAC,GAAA,EAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,eAAA,CAAAjD,KAAA,OAAAI,SAAA;AAAA;AAAA,SAAA6C,gBAAA;EAAAA,eAAA,OAAAC,kBAAA,CAAAxC,OAAA,EAAAyC,YAAA,CAAAzC,OAAA,CAAA0C,IAAA,CAA7B,SAAAC,QACLC,IAAuB,EACvBC,OAA2C,EAC3CC,OAAuB,EACvBC,OAAuB;IAAA,IAAAC,qBAAA,EAAAC,GAAA,EAAAC,MAAA;IAAA,OAAAT,YAAA,CAAAzC,OAAA,CAAAmD,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAA3B,IAAA;QAAA;UAEvB,IAAA6B,mBAAM,EAAC,CAACR,OAAO,IAAI,IAAAS,QAAA,CAAAxD,OAAA,EAAO+C,OAAO,MAAK,QAAQ,CAAC;UAG/C,IAAI,CAACU,KAAK,CAACC,OAAO,CAACb,OAAO,CAAC,IAAI,CAAC,IAAAc,+BAAc,EAACd,OAAO,CAAC,EAAE;YACvDE,OAAO,GAAGa,SAAS;YACnBd,OAAO,GAAGD,OAAwB;YAClCA,OAAO,GAAGe,SAAS;UACrB;UAACP,QAAA,CAAA3B,IAAA;UAAA,OAEYkB,IAAI;QAAA;UAAjBA,IAAI,GAAAS,QAAA,CAAAQ,IAAA;UACJf,OAAO,GAAGA,OAAO,IAAI,CAAC,CAAC;UAACE,qBAAA,GAGV,IAAAc,oCAAqB,EAAClB,IAAI,CAAC,EAAlCK,GAAG,GAAAD,qBAAA,CAAHC,GAAG;UAAAI,QAAA,CAAA3B,IAAA;UAAA,OAIW,IAAAqC,0BAAY,EAACnB,IAAI,EAAiBC,OAAO,EAAcC,OAAO,CAAC;QAAA;UAA9EI,MAAM,GAAAG,QAAA,CAAAQ,IAAA;UAAA,IAEPX,MAAM;YAAAG,QAAA,CAAA3B,IAAA;YAAA;UAAA;UAAA,OAAA2B,QAAA,CAAAW,MAAA,WAEF,IAAI;QAAA;UAKblB,OAAO,GAAG,IAAAmB,6BAAgB,EAACnB,OAAO,EAAEI,MAAM,EAAEL,OAAO,EAAEI,GAAG,CAAC;UAEzDF,OAAO,GAAG,IAAAmB,+BAAgB,EAExB;YAACjB,GAAG,EAAHA,GAAG;YAAEf,cAAc,EAAdA,cAAc;YAAEiC,KAAK,EAALA,YAAK;YAAEtB,OAAO,EAAEA;UAAmB,CAAC,EAC1DC,OAAO,EACPC,OAAO,CACR;UAACM,QAAA,CAAA3B,IAAA;UAAA,OAEW0C,wBAAwB,CAAClB,MAAM,EAAsBN,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;QAAA;UAAA,OAAAM,QAAA,CAAAW,MAAA,WAAAX,QAAA,CAAAQ,IAAA;QAAA;QAAA;UAAA,OAAAR,QAAA,CAAAgB,IAAA;MAAA;IAAA,GAAA1B,OAAA;EAAA,CAC1F;EAAA,OAAAJ,eAAA,CAAAjD,KAAA,OAAAI,SAAA;AAAA;AAAA,SAKc0E,wBAAwBA,CAAAE,GAAA,EAAAC,GAAA,EAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,yBAAA,CAAApF,KAAA,OAAAI,SAAA;AAAA;AAAA,SAAAgF,0BAAA;EAAAA,yBAAA,OAAAlC,kBAAA,CAAAxC,OAAA,EAAAyC,YAAA,CAAAzC,OAAA,CAAA0C,IAAA,CAAvC,SAAAiC,SACEzB,MAAwB,EACxBN,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB;IAAA,IAAA6B,cAAA,EAAAC,aAAA,EAoBNC,yBAAyB,EAAAC,0BAAA;IAAA,OAAAtC,YAAA,CAAAzC,OAAA,CAAAmD,IAAA,UAAA6B,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAA3B,IAAA,GAAA2B,SAAA,CAAAvD,IAAA;QAAA;UAAAqD,0BAAA,YAAAG,uBAAA;YAAAH,0BAAA,OAAAI,oBAAA,CAAAnF,OAAA,EAAAyC,YAAA,CAAAzC,OAAA,CAAA0C,IAAA,CAAzC,SAAA0C,SAA0CxE,QAAQ;cAAA,OAAA6B,YAAA,CAAAzC,OAAA,CAAAmD,IAAA,UAAAkC,UAAAC,SAAA;gBAAA,kBAAAA,SAAA,CAAAhC,IAAA,GAAAgC,SAAA,CAAA5D,IAAA;kBAAA;oBAAA4D,SAAA,CAAA5D,IAAA;oBAChD,OAAMmD,aAAa;kBAAA;oBACnB,OAAAS,SAAA,CAAAC,aAAA,KAAAC,wBAAA,CAAAxF,OAAA,EAAAI,cAAA,CAAOQ,QAAQ,GAAA6E,qBAAA,CAAAzF,OAAA;kBAAA;kBAAA;oBAAA,OAAAsF,SAAA,CAAAjB,IAAA;gBAAA;cAAA,GAAAe,QAAA;YAAA,CAChB;YAAA,OAAAL,0BAAA,CAAAzF,KAAA,OAAAI,SAAA;UAAA;UAHeoF,yBAAyB,YAAAY,sBAAAC,EAAA;YAAA,OAAAZ,0BAAA,CAAAzF,KAAA,OAAAI,SAAA;UAAA;UAAAuF,SAAA,CAAAvD,IAAA;UAAA,OAlBZkE,qBAAqB,CAAC1C,MAAM,EAAEN,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;QAAA;UAA5E6B,cAAc,GAAAK,SAAA,CAAApB,IAAA;UAAA,IAGff,OAAO,CAAC+C,QAAQ;YAAAZ,SAAA,CAAAvD,IAAA;YAAA;UAAA;UAAA,OAAAuD,SAAA,CAAAjB,MAAA,WACZY,cAAc;QAAA;UAGjBC,aAAa,GAAG;YACpBiB,SAAS,EAAE,UAAU;YACrBD,QAAQ,EAAE;cACRE,OAAO,EAAE7C,MAAM;cACfG,QAAQ,EAAEN;YACZ,CAAC;YAEDH,IAAI,EAAE,EAAE;YACRoD,SAAS,EAAE;UACb,CAAC;UAAA,OAAAf,SAAA,CAAAjB,MAAA,WAOMc,yBAAyB,CAACF,cAAc,CAAC;QAAA;QAAA;UAAA,OAAAK,SAAA,CAAAZ,IAAA;MAAA;IAAA,GAAAM,QAAA;EAAA,CACjD;EAAA,OAAAD,yBAAA,CAAApF,KAAA,OAAAI,SAAA;AAAA;AAAA,SAOckG,qBAAqBA,CAAAK,IAAA,EAAAC,IAAA,EAAAC,IAAA,EAAAC,IAAA;EAAA,OAAAC,sBAAA,CAAA/G,KAAA,OAAAI,SAAA;AAAA;AAAA,SAAA2G,uBAAA;EAAAA,sBAAA,OAAA7D,kBAAA,CAAAxC,OAAA,EAAAyC,YAAA,CAAAzC,OAAA,CAAA0C,IAAA,CAApC,SAAA4D,SACEpD,MAAwB,EACxBN,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB;IAAA,IAAAwD,aAAA,EAAAC,mBAAA,EAcNC,mBAAmB,EAAAC,oBAAA;IAAA,OAAAjE,YAAA,CAAAzC,OAAA,CAAAmD,IAAA,UAAAwD,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAtD,IAAA,GAAAsD,SAAA,CAAAlF,IAAA;QAAA;UAAAgF,oBAAA,YAAAG,sBAAA;YAAAH,oBAAA,OAAAvB,oBAAA,CAAAnF,OAAA,EAAAyC,YAAA,CAAAzC,OAAA,CAAA0C,IAAA,CAAnC,SAAAoE,SAAA;cAAA,IAAAC,WAAA,EAAAC,UAAA,EAAAC,KAAA;cAAA,OAAAxE,YAAA,CAAAzC,OAAA,CAAAmD,IAAA,UAAA+D,UAAAC,SAAA;gBAAA,kBAAAA,SAAA,CAAA7D,IAAA,GAAA6D,SAAA,CAAAzF,IAAA;kBAAA;oBAAAyF,SAAA,CAAAzF,IAAA;oBAAA,WAAA+D,qBAAA,CAAAzF,OAAA,EAC4B,IAAAoH,yCAA4B,EAACZ,mBAAmB,CAAC;kBAAA;oBAArEO,WAAW,GAAAI,SAAA,CAAAtD,IAAA;oBAAAsD,SAAA,CAAAzF,IAAA;oBAAA,WAAA+D,qBAAA,CAAAzF,OAAA,EAEQ,IAAAmE,YAAK,EAC5B4C,WAAW,EACX7D,MAAM,EAAA3D,aAAA,CAAAA,aAAA,KAEFuD,OAAO;sBAAEuE,QAAQ,EAAEnE,MAAM,CAACoE,SAAS,CAAC,CAAC;oBAAC,IAC1CvE,OAAO,CACR;kBAAA;oBANKiE,UAAU,GAAAG,SAAA,CAAAtD,IAAA;oBASVoD,KAAY,GAAG;sBACnBI,QAAQ,EAAEnE,MAAM,CAACoE,SAAS,CAAC,CAAC,CAAC;sBAC7BC,KAAK,EAAE9D,KAAK,CAACC,OAAO,CAACsD,UAAU,CAAC,GAAG,WAAW,GAAG,SAAS;sBAC1DlB,SAAS,EAAE,MAAM;sBACjBlD,IAAI,EAAEoE,UAAU;sBAChBrH,MAAM,EAAE8D,KAAK,CAACC,OAAO,CAACsD,UAAU,CAAC,GAAGA,UAAU,CAACrH,MAAM,GAAG;oBAC1D,CAAC;oBAAAwH,SAAA,CAAAzF,IAAA;oBACD,OAAMuF,KAAK;kBAAA;kBAAA;oBAAA,OAAAE,SAAA,CAAA9C,IAAA;gBAAA;cAAA,GAAAyC,QAAA;YAAA,CACZ;YAAA,OAAAJ,oBAAA,CAAApH,KAAA,OAAAI,SAAA;UAAA;UApBe+G,mBAAmB,YAAAe,sBAAA;YAAA,OAAAd,oBAAA,CAAApH,KAAA,OAAAI,SAAA;UAAA;UAAAkH,SAAA,CAAAlF,IAAA;UAAA,OAXP,IAAA+F,iCAAwB,EAAC7E,IAAI,EAAEE,OAAO,CAAC;QAAA;UAA7DyD,aAAa,GAAAK,SAAA,CAAA/C,IAAA;UAAA+C,SAAA,CAAAlF,IAAA;UAAA,OAGegG,oBAAoB,CAACnB,aAAa,EAAE,CAAAzD,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAE6E,UAAU,KAAI,EAAE,CAAC;QAAA;UAA1FnB,mBAAmB,GAAAI,SAAA,CAAA/C,IAAA;UAAA,KAGrBX,MAAM,CAAChB,cAAc;YAAA0E,SAAA,CAAAlF,IAAA;YAAA;UAAA;UAAA,OAAAkF,SAAA,CAAA5C,MAAA,WAChBd,MAAM,CAAChB,cAAc,CAACsE,mBAAmB,EAAE1D,OAAO,EAAEC,OAAO,CAAC;QAAA;UAAA,OAAA6D,SAAA,CAAA5C,MAAA,WA0B9DyC,mBAAmB,EAAE;QAAA;QAAA;UAAA,OAAAG,SAAA,CAAAvC,IAAA;MAAA;IAAA,GAAAiC,QAAA;EAAA,CAC7B;EAAA,OAAAD,sBAAA,CAAA/G,KAAA,OAAAI,SAAA;AAAA;AAAA,SAWcgI,oBAAoBA,CAAAE,IAAA;EAAA,OAAAC,qBAAA,CAAAvI,KAAA,OAAAI,SAAA;AAAA;AAAA,SAAAmI,sBAAA;EAAAA,qBAAA,OAAArF,kBAAA,CAAAxC,OAAA,EAAAyC,YAAA,CAAAzC,OAAA,CAAA0C,IAAA,CAAnC,SAAAoF,SACEvB,aAAiE;IAAA,IAAAoB,UAAA;MAAAI,aAAA;MAAAC,yBAAA;MAAAC,iBAAA;MAAAC,cAAA;MAAAC,SAAA;MAAAC,KAAA;MAAAC,gBAAA;MAAAC,MAAA,GAAA5I,SAAA;IAAA,OAAA+C,YAAA,CAAAzC,OAAA,CAAAmD,IAAA,UAAAoF,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAlF,IAAA,GAAAkF,SAAA,CAAA9G,IAAA;QAAA;UACjEiG,UAA8B,GAAAW,MAAA,CAAA3I,MAAA,QAAA2I,MAAA,QAAA1E,SAAA,GAAA0E,MAAA,MAAG,EAAE;UAE/BP,aAAa,GAAGxB,aAAa;UAAAyB,yBAAA;UAAAC,iBAAA;UAAAO,SAAA,CAAAlF,IAAA;UAAA6E,SAAA,GAAA/H,cAAA,CACIuH,UAAU;QAAA;UAAAa,SAAA,CAAA9G,IAAA;UAAA,OAAAyG,SAAA,CAAAzG,IAAA;QAAA;UAAA,MAAAsG,yBAAA,KAAAI,KAAA,GAAAI,SAAA,CAAA3E,IAAA,EAAAxC,IAAA;YAAAmH,SAAA,CAAA9G,IAAA;YAAA;UAAA;UAA9B2G,gBAAgB,GAAAD,KAAA,CAAA7G,KAAA;UAC/BwG,aAAa,GAAGM,gBAAgB,CAACN,aAAa,CAAC;QAAC;UAAAC,yBAAA;UAAAQ,SAAA,CAAA9G,IAAA;UAAA;QAAA;UAAA8G,SAAA,CAAA9G,IAAA;UAAA;QAAA;UAAA8G,SAAA,CAAAlF,IAAA;UAAAkF,SAAA,CAAAC,EAAA,GAAAD,SAAA;UAAAP,iBAAA;UAAAC,cAAA,GAAAM,SAAA,CAAAC,EAAA;QAAA;UAAAD,SAAA,CAAAlF,IAAA;UAAAkF,SAAA,CAAAlF,IAAA;UAAA,MAAA0E,yBAAA,IAAAG,SAAA,CAAAvG,MAAA;YAAA4G,SAAA,CAAA9G,IAAA;YAAA;UAAA;UAAA8G,SAAA,CAAA9G,IAAA;UAAA,OAAAyG,SAAA,CAAAvG,MAAA;QAAA;UAAA4G,SAAA,CAAAlF,IAAA;UAAA,KAAA2E,iBAAA;YAAAO,SAAA,CAAA9G,IAAA;YAAA;UAAA;UAAA,MAAAwG,cAAA;QAAA;UAAA,OAAAM,SAAA,CAAAE,MAAA;QAAA;UAAA,OAAAF,SAAA,CAAAE,MAAA;QAAA;UAAA,OAAAF,SAAA,CAAAxE,MAAA,WAE3C+D,aAAa;QAAA;QAAA;UAAA,OAAAS,SAAA,CAAAnE,IAAA;MAAA;IAAA,GAAAyD,QAAA;EAAA,CACrB;EAAA,OAAAD,qBAAA,CAAAvI,KAAA,OAAAI,SAAA;AAAA"}
|
|
@@ -15,23 +15,19 @@ var _loaderContext = require("../loader-utils/loader-context");
|
|
|
15
15
|
var _resourceUtils = require("../utils/resource-utils");
|
|
16
16
|
function parseSync(data, loaders, options, context) {
|
|
17
17
|
(0, _loaderUtils.assert)(!context || (0, _typeof2.default)(context) === 'object');
|
|
18
|
-
|
|
19
18
|
if (!Array.isArray(loaders) && !(0, _normalizeLoader.isLoaderObject)(loaders)) {
|
|
20
19
|
context = undefined;
|
|
21
20
|
options = loaders;
|
|
22
21
|
loaders = undefined;
|
|
23
22
|
}
|
|
24
23
|
options = options || {};
|
|
25
|
-
|
|
26
24
|
var typedLoaders = loaders;
|
|
27
25
|
var candidateLoaders = (0, _loaderContext.getLoadersFromContext)(typedLoaders, context);
|
|
28
26
|
var loader = (0, _selectLoader.selectLoaderSync)(data, candidateLoaders, options);
|
|
29
27
|
if (!loader) {
|
|
30
28
|
return null;
|
|
31
29
|
}
|
|
32
|
-
|
|
33
30
|
options = (0, _optionUtils.normalizeOptions)(options, loader, candidateLoaders);
|
|
34
|
-
|
|
35
31
|
var _getResourceUrlAndTyp = (0, _resourceUtils.getResourceUrlAndType)(data),
|
|
36
32
|
url = _getResourceUrlAndTyp.url;
|
|
37
33
|
var parse = function parse() {
|
|
@@ -45,17 +41,14 @@ function parseSync(data, loaders, options, context) {
|
|
|
45
41
|
}, options);
|
|
46
42
|
return parseWithLoaderSync(loader, data, options, context);
|
|
47
43
|
}
|
|
48
|
-
|
|
49
44
|
function parseWithLoaderSync(loader, data, options, context) {
|
|
50
45
|
data = (0, _getData.getArrayBufferOrStringFromDataSync)(data, loader, options);
|
|
51
46
|
if (loader.parseTextSync && typeof data === 'string') {
|
|
52
47
|
return loader.parseTextSync(data, options);
|
|
53
48
|
}
|
|
54
|
-
|
|
55
49
|
if (loader.parseSync && data instanceof ArrayBuffer) {
|
|
56
50
|
return loader.parseSync(data, options, context);
|
|
57
51
|
}
|
|
58
|
-
|
|
59
52
|
throw new Error("".concat(loader.name, " loader: 'parseSync' not supported by this loader, use 'parse' instead. ").concat(context.url || ''));
|
|
60
53
|
}
|
|
61
54
|
//# sourceMappingURL=parse-sync.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-sync.js","names":["parseSync","data","loaders","options","context","assert","Array","isArray","isLoaderObject","undefined","typedLoaders","candidateLoaders","getLoadersFromContext","loader","selectLoaderSync","normalizeOptions","getResourceUrlAndType","url","parse","Error","getLoaderContext","parseWithLoaderSync","getArrayBufferOrStringFromDataSync","parseTextSync","ArrayBuffer","name"],"sources":["../../../../src/lib/api/parse-sync.ts"],"sourcesContent":["import type {\n SyncDataType,\n Loader,\n LoaderWithParser,\n LoaderContext,\n LoaderOptions\n} from '@loaders.gl/loader-utils';\nimport {assert} from '@loaders.gl/loader-utils';\nimport {selectLoaderSync} from './select-loader';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {normalizeOptions} from '../loader-utils/option-utils';\nimport {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';\nimport {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';\nimport {getResourceUrlAndType} from '../utils/resource-utils';\n\n/**\n * Parses `data` synchronously using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport function parseSync(\n data: SyncDataType,\n loaders?: Loader | Loader[] | LoaderOptions,\n options?: LoaderOptions,\n context?: LoaderContext\n): any {\n assert(!context || typeof context === 'object'); // parseSync no longer accepts final url\n\n // Signature: parseSync(data, options)\n // Uses registered loaders\n if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n context = undefined; // context not supported in short signature\n options = loaders as LoaderOptions;\n loaders = undefined;\n }\n\n options = options || {};\n\n // Chooses a loader (and normalizes it)\n // Also use any loaders in the context, new loaders take priority\n const typedLoaders = loaders as Loader | Loader[] | undefined;\n const candidateLoaders = getLoadersFromContext(typedLoaders, context);\n const loader = selectLoaderSync(data, candidateLoaders, options);\n // Note: if nothrow option was set, it is possible that no loader was found, if so just return null\n if (!loader) {\n return null;\n }\n\n // Normalize options\n options = normalizeOptions(options, loader, candidateLoaders);\n\n // Extract a url for auto detection\n const {url} = getResourceUrlAndType(data);\n\n const parse = () => {\n throw new Error('parseSync called parse');\n };\n context = getLoaderContext({url, parseSync, parse, loaders: loaders as Loader[]}, options);\n\n return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);\n}\n\n// TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator\nfunction parseWithLoaderSync(\n loader: LoaderWithParser,\n data: SyncDataType,\n options: LoaderOptions,\n context: LoaderContext\n) {\n data = getArrayBufferOrStringFromDataSync(data, loader, options);\n\n if (loader.parseTextSync && typeof data === 'string') {\n return loader.parseTextSync(data, options); // , context, loader);\n }\n\n if (loader.parseSync && data instanceof ArrayBuffer) {\n return loader.parseSync(data, options, context); // , loader);\n }\n\n // TBD - If synchronous parser not available, return null\n throw new Error(\n `${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${\n context.url || ''\n }`\n );\n}\n"],"mappings":";;;;;;;;AAOA;AACA;AACA;AACA;AACA;AACA;AACA;AASO,SAASA,
|
|
1
|
+
{"version":3,"file":"parse-sync.js","names":["_loaderUtils","require","_selectLoader","_normalizeLoader","_optionUtils","_getData","_loaderContext","_resourceUtils","parseSync","data","loaders","options","context","assert","_typeof2","default","Array","isArray","isLoaderObject","undefined","typedLoaders","candidateLoaders","getLoadersFromContext","loader","selectLoaderSync","normalizeOptions","_getResourceUrlAndTyp","getResourceUrlAndType","url","parse","Error","getLoaderContext","parseWithLoaderSync","getArrayBufferOrStringFromDataSync","parseTextSync","ArrayBuffer","concat","name"],"sources":["../../../../src/lib/api/parse-sync.ts"],"sourcesContent":["import type {\n SyncDataType,\n Loader,\n LoaderWithParser,\n LoaderContext,\n LoaderOptions\n} from '@loaders.gl/loader-utils';\nimport {assert} from '@loaders.gl/loader-utils';\nimport {selectLoaderSync} from './select-loader';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {normalizeOptions} from '../loader-utils/option-utils';\nimport {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';\nimport {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';\nimport {getResourceUrlAndType} from '../utils/resource-utils';\n\n/**\n * Parses `data` synchronously using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport function parseSync(\n data: SyncDataType,\n loaders?: Loader | Loader[] | LoaderOptions,\n options?: LoaderOptions,\n context?: LoaderContext\n): any {\n assert(!context || typeof context === 'object'); // parseSync no longer accepts final url\n\n // Signature: parseSync(data, options)\n // Uses registered loaders\n if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n context = undefined; // context not supported in short signature\n options = loaders as LoaderOptions;\n loaders = undefined;\n }\n\n options = options || {};\n\n // Chooses a loader (and normalizes it)\n // Also use any loaders in the context, new loaders take priority\n const typedLoaders = loaders as Loader | Loader[] | undefined;\n const candidateLoaders = getLoadersFromContext(typedLoaders, context);\n const loader = selectLoaderSync(data, candidateLoaders, options);\n // Note: if nothrow option was set, it is possible that no loader was found, if so just return null\n if (!loader) {\n return null;\n }\n\n // Normalize options\n options = normalizeOptions(options, loader, candidateLoaders);\n\n // Extract a url for auto detection\n const {url} = getResourceUrlAndType(data);\n\n const parse = () => {\n throw new Error('parseSync called parse');\n };\n context = getLoaderContext({url, parseSync, parse, loaders: loaders as Loader[]}, options);\n\n return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);\n}\n\n// TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator\nfunction parseWithLoaderSync(\n loader: LoaderWithParser,\n data: SyncDataType,\n options: LoaderOptions,\n context: LoaderContext\n) {\n data = getArrayBufferOrStringFromDataSync(data, loader, options);\n\n if (loader.parseTextSync && typeof data === 'string') {\n return loader.parseTextSync(data, options); // , context, loader);\n }\n\n if (loader.parseSync && data instanceof ArrayBuffer) {\n return loader.parseSync(data, options, context); // , loader);\n }\n\n // TBD - If synchronous parser not available, return null\n throw new Error(\n `${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${\n context.url || ''\n }`\n );\n}\n"],"mappings":";;;;;;;;AAOA,IAAAA,YAAA,GAAAC,OAAA;AACA,IAAAC,aAAA,GAAAD,OAAA;AACA,IAAAE,gBAAA,GAAAF,OAAA;AACA,IAAAG,YAAA,GAAAH,OAAA;AACA,IAAAI,QAAA,GAAAJ,OAAA;AACA,IAAAK,cAAA,GAAAL,OAAA;AACA,IAAAM,cAAA,GAAAN,OAAA;AASO,SAASO,SAASA,CACvBC,IAAkB,EAClBC,OAA2C,EAC3CC,OAAuB,EACvBC,OAAuB,EAClB;EACL,IAAAC,mBAAM,EAAC,CAACD,OAAO,IAAI,IAAAE,QAAA,CAAAC,OAAA,EAAOH,OAAO,MAAK,QAAQ,CAAC;EAI/C,IAAI,CAACI,KAAK,CAACC,OAAO,CAACP,OAAO,CAAC,IAAI,CAAC,IAAAQ,+BAAc,EAACR,OAAO,CAAC,EAAE;IACvDE,OAAO,GAAGO,SAAS;IACnBR,OAAO,GAAGD,OAAwB;IAClCA,OAAO,GAAGS,SAAS;EACrB;EAEAR,OAAO,GAAGA,OAAO,IAAI,CAAC,CAAC;EAIvB,IAAMS,YAAY,GAAGV,OAAwC;EAC7D,IAAMW,gBAAgB,GAAG,IAAAC,oCAAqB,EAACF,YAAY,EAAER,OAAO,CAAC;EACrE,IAAMW,MAAM,GAAG,IAAAC,8BAAgB,EAACf,IAAI,EAAEY,gBAAgB,EAAEV,OAAO,CAAC;EAEhE,IAAI,CAACY,MAAM,EAAE;IACX,OAAO,IAAI;EACb;EAGAZ,OAAO,GAAG,IAAAc,6BAAgB,EAACd,OAAO,EAAEY,MAAM,EAAEF,gBAAgB,CAAC;EAG7D,IAAAK,qBAAA,GAAc,IAAAC,oCAAqB,EAAClB,IAAI,CAAC;IAAlCmB,GAAG,GAAAF,qBAAA,CAAHE,GAAG;EAEV,IAAMC,KAAK,GAAG,SAARA,KAAKA,CAAA,EAAS;IAClB,MAAM,IAAIC,KAAK,CAAC,wBAAwB,CAAC;EAC3C,CAAC;EACDlB,OAAO,GAAG,IAAAmB,+BAAgB,EAAC;IAACH,GAAG,EAAHA,GAAG;IAAEpB,SAAS,EAATA,SAAS;IAAEqB,KAAK,EAALA,KAAK;IAAEnB,OAAO,EAAEA;EAAmB,CAAC,EAAEC,OAAO,CAAC;EAE1F,OAAOqB,mBAAmB,CAACT,MAAM,EAAsBd,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;AAChF;AAGA,SAASoB,mBAAmBA,CAC1BT,MAAwB,EACxBd,IAAkB,EAClBE,OAAsB,EACtBC,OAAsB,EACtB;EACAH,IAAI,GAAG,IAAAwB,2CAAkC,EAACxB,IAAI,EAAEc,MAAM,EAAEZ,OAAO,CAAC;EAEhE,IAAIY,MAAM,CAACW,aAAa,IAAI,OAAOzB,IAAI,KAAK,QAAQ,EAAE;IACpD,OAAOc,MAAM,CAACW,aAAa,CAACzB,IAAI,EAAEE,OAAO,CAAC;EAC5C;EAEA,IAAIY,MAAM,CAACf,SAAS,IAAIC,IAAI,YAAY0B,WAAW,EAAE;IACnD,OAAOZ,MAAM,CAACf,SAAS,CAACC,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;EACjD;EAGA,MAAM,IAAIkB,KAAK,IAAAM,MAAA,CACVb,MAAM,CAACc,IAAI,8EAAAD,MAAA,CACZxB,OAAO,CAACgB,GAAG,IAAI,EAAE,EAEpB;AACH"}
|