@loaders.gl/core 4.2.0-alpha.4 → 4.2.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core-addons/write-file-browser.js +62 -1
- package/dist/dist.dev.js +366 -301
- package/dist/dist.min.js +15 -0
- package/dist/index.cjs +60 -71
- package/dist/index.cjs.map +7 -0
- package/dist/index.d.ts +20 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.js +18 -13
- package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js +21 -15
- package/dist/iterators/make-iterator/make-blob-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-blob-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-blob-iterator.js +18 -10
- package/dist/iterators/make-iterator/make-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-iterator.js +29 -18
- package/dist/iterators/make-iterator/make-stream-iterator.js +86 -23
- package/dist/iterators/make-iterator/make-string-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-string-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-string-iterator.js +20 -10
- package/dist/iterators/make-stream/make-stream.js +47 -29
- package/dist/javascript-utils/is-type.js +25 -19
- package/dist/lib/api/encode-table.js +40 -35
- package/dist/lib/api/encode.js +112 -73
- package/dist/lib/api/load-in-batches.js +35 -21
- package/dist/lib/api/load.js +35 -20
- package/dist/lib/api/loader-options.d.ts +2 -2
- package/dist/lib/api/loader-options.d.ts.map +1 -1
- package/dist/lib/api/loader-options.js +3 -1
- package/dist/lib/api/parse-in-batches.js +106 -69
- package/dist/lib/api/parse-sync.js +42 -33
- package/dist/lib/api/parse.js +73 -61
- package/dist/lib/api/register-loaders.js +23 -14
- package/dist/lib/api/select-loader.js +216 -163
- package/dist/lib/common.js +3 -1
- package/dist/lib/fetch/fetch-error-message.js +17 -12
- package/dist/lib/fetch/fetch-file.js +26 -15
- package/dist/lib/fetch/read-array-buffer.js +30 -15
- package/dist/lib/filesystems/browser-filesystem.js +127 -69
- package/dist/lib/filesystems/read-array-buffer.js +14 -6
- package/dist/lib/init.js +13 -7
- package/dist/lib/loader-utils/check-errors.js +37 -16
- package/dist/lib/loader-utils/get-data.js +110 -88
- package/dist/lib/loader-utils/get-fetch-function.js +24 -13
- package/dist/lib/loader-utils/loader-context.js +50 -31
- package/dist/lib/loader-utils/loggers.js +35 -44
- package/dist/lib/loader-utils/normalize-loader.js +45 -32
- package/dist/lib/loader-utils/option-defaults.js +38 -34
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +133 -80
- package/dist/lib/progress/fetch-progress.js +54 -47
- package/dist/lib/utils/log.js +4 -4
- package/dist/lib/utils/mime-type-utils.js +34 -11
- package/dist/lib/utils/resource-utils.js +77 -45
- package/dist/lib/utils/response-utils.js +97 -74
- package/dist/lib/utils/url-utils.js +6 -4
- package/dist/null-loader.js +44 -30
- package/dist/null-worker-node.js +4 -2
- package/dist/null-worker.js +4 -2
- package/dist/workers/null-worker-node.js +3 -1
- package/dist/workers/null-worker.js +3 -1
- package/package.json +12 -11
- package/src/lib/api/parse-in-batches.ts +6 -5
- package/src/lib/loader-utils/option-utils.ts +3 -1
- package/src/lib/utils/resource-utils.ts +1 -1
- package/dist/core-addons/README.md +0 -1
- package/dist/core-addons/write-file-browser.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-blob-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-stream-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-string-iterator.js.map +0 -1
- package/dist/iterators/make-stream/make-stream.js.map +0 -1
- package/dist/javascript-utils/is-type.js.map +0 -1
- package/dist/lib/api/encode-table.js.map +0 -1
- package/dist/lib/api/encode.js.map +0 -1
- package/dist/lib/api/load-in-batches.js.map +0 -1
- package/dist/lib/api/load.js.map +0 -1
- package/dist/lib/api/loader-options.js.map +0 -1
- package/dist/lib/api/parse-in-batches.js.map +0 -1
- package/dist/lib/api/parse-sync.js.map +0 -1
- package/dist/lib/api/parse.js.map +0 -1
- package/dist/lib/api/register-loaders.js.map +0 -1
- package/dist/lib/api/select-loader.js.map +0 -1
- package/dist/lib/common.js.map +0 -1
- package/dist/lib/fetch/fetch-error-message.js.map +0 -1
- package/dist/lib/fetch/fetch-file.js.map +0 -1
- package/dist/lib/fetch/read-array-buffer.js.map +0 -1
- package/dist/lib/filesystems/browser-filesystem.js.map +0 -1
- package/dist/lib/filesystems/read-array-buffer.js.map +0 -1
- package/dist/lib/init.js.map +0 -1
- package/dist/lib/loader-utils/check-errors.js.map +0 -1
- package/dist/lib/loader-utils/get-data.js.map +0 -1
- package/dist/lib/loader-utils/get-fetch-function.js.map +0 -1
- package/dist/lib/loader-utils/loader-context.js.map +0 -1
- package/dist/lib/loader-utils/loggers.js.map +0 -1
- package/dist/lib/loader-utils/normalize-loader.js.map +0 -1
- package/dist/lib/loader-utils/option-defaults.js.map +0 -1
- package/dist/lib/loader-utils/option-utils.js.map +0 -1
- package/dist/lib/progress/fetch-progress.js.map +0 -1
- package/dist/lib/utils/log.js.map +0 -1
- package/dist/lib/utils/mime-type-utils.js.map +0 -1
- package/dist/lib/utils/resource-utils.js.map +0 -1
- package/dist/lib/utils/response-utils.js.map +0 -1
- package/dist/lib/utils/url-utils.js.map +0 -1
- package/dist/null-loader.js.map +0 -1
- package/dist/workers/null-worker-node.js.map +0 -1
- package/dist/workers/null-worker.js.map +0 -1
|
@@ -1,28 +1,42 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { isLoaderObject } from "../loader-utils/normalize-loader.js";
|
|
2
5
|
import { getFetchFunction } from "../loader-utils/get-fetch-function.js";
|
|
3
6
|
import { parseInBatches } from "./parse-in-batches.js";
|
|
4
7
|
export function loadInBatches(files, loaders, options, context) {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
8
|
+
let loadersArray;
|
|
9
|
+
// Signature: load(url, options)
|
|
10
|
+
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
11
|
+
context = undefined; // context not supported in short signature
|
|
12
|
+
options = loaders;
|
|
13
|
+
loadersArray = undefined;
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
loadersArray = loaders;
|
|
17
|
+
}
|
|
18
|
+
// Select fetch function
|
|
19
|
+
const fetch = getFetchFunction(options || {});
|
|
20
|
+
// Single url/file
|
|
21
|
+
if (!Array.isArray(files)) {
|
|
22
|
+
return loadOneFileInBatches(files, loadersArray, options || {}, fetch);
|
|
23
|
+
}
|
|
24
|
+
// Multiple URLs / files
|
|
25
|
+
const promises = files.map((file) => loadOneFileInBatches(file, loadersArray, options || {}, fetch));
|
|
26
|
+
// No point in waiting here for all responses before starting to stream individual streams?
|
|
27
|
+
return promises;
|
|
19
28
|
}
|
|
20
29
|
async function loadOneFileInBatches(file, loaders, options, fetch) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
30
|
+
if (typeof file === 'string') {
|
|
31
|
+
const url = file;
|
|
32
|
+
const response = await fetch(url);
|
|
33
|
+
// pick right overload
|
|
34
|
+
return Array.isArray(loaders)
|
|
35
|
+
? await parseInBatches(response, loaders, options)
|
|
36
|
+
: await parseInBatches(response, loaders, options);
|
|
37
|
+
}
|
|
38
|
+
// pick right overload
|
|
39
|
+
return Array.isArray(loaders)
|
|
40
|
+
? await parseInBatches(file, loaders, options)
|
|
41
|
+
: await parseInBatches(file, loaders, options);
|
|
27
42
|
}
|
|
28
|
-
//# sourceMappingURL=load-in-batches.js.map
|
package/dist/lib/api/load.js
CHANGED
|
@@ -1,26 +1,41 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { isBlob } from "../../javascript-utils/is-type.js";
|
|
2
5
|
import { isLoaderObject } from "../loader-utils/normalize-loader.js";
|
|
3
6
|
import { getFetchFunction } from "../loader-utils/get-fetch-function.js";
|
|
4
7
|
import { parse } from "./parse.js";
|
|
8
|
+
// implementation signature
|
|
5
9
|
export async function load(url, loaders, options, context) {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
10
|
+
let resolvedLoaders;
|
|
11
|
+
let resolvedOptions;
|
|
12
|
+
// Signature: load(url, options)
|
|
13
|
+
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
14
|
+
resolvedLoaders = [];
|
|
15
|
+
resolvedOptions = loaders;
|
|
16
|
+
context = undefined; // context not supported in short signature
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
resolvedLoaders = loaders;
|
|
20
|
+
resolvedOptions = options;
|
|
21
|
+
}
|
|
22
|
+
// Select fetch function
|
|
23
|
+
const fetch = getFetchFunction(resolvedOptions);
|
|
24
|
+
// at this point, `url` could be already loaded binary data
|
|
25
|
+
let data = url;
|
|
26
|
+
// url is a string, fetch the url
|
|
27
|
+
if (typeof url === 'string') {
|
|
28
|
+
data = await fetch(url);
|
|
29
|
+
// URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
|
|
30
|
+
}
|
|
31
|
+
if (isBlob(url)) {
|
|
32
|
+
// The fetch response object will contain blob.name
|
|
33
|
+
// @ts-expect-error TODO - This may not work for overridden fetch functions
|
|
34
|
+
data = await fetch(url);
|
|
35
|
+
}
|
|
36
|
+
// Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
|
|
37
|
+
// return await parse(data, loaders as Loader[], options);
|
|
38
|
+
return Array.isArray(resolvedLoaders)
|
|
39
|
+
? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
|
|
40
|
+
: await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
|
|
25
41
|
}
|
|
26
|
-
//# sourceMappingURL=load.js.map
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { setGlobalOptions as setLoaderOptions } from
|
|
2
|
-
export { getGlobalLoaderOptions as getLoaderOptions } from
|
|
1
|
+
export { setGlobalOptions as setLoaderOptions } from "../loader-utils/option-utils.js";
|
|
2
|
+
export { getGlobalLoaderOptions as getLoaderOptions } from "../loader-utils/option-utils.js";
|
|
3
3
|
//# sourceMappingURL=loader-options.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader-options.d.ts","sourceRoot":"","sources":["../../../src/lib/api/loader-options.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,gBAAgB,IAAI,gBAAgB,EAAC,
|
|
1
|
+
{"version":3,"file":"loader-options.d.ts","sourceRoot":"","sources":["../../../src/lib/api/loader-options.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,gBAAgB,IAAI,gBAAgB,EAAC,wCAAqC;AAClF,OAAO,EAAC,sBAAsB,IAAI,gBAAgB,EAAC,wCAAqC"}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
export { setGlobalOptions as setLoaderOptions } from "../loader-utils/option-utils.js";
|
|
2
5
|
export { getGlobalLoaderOptions as getLoaderOptions } from "../loader-utils/option-utils.js";
|
|
3
|
-
//# sourceMappingURL=loader-options.js.map
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { isTable, makeBatchFromTable } from '@loaders.gl/schema';
|
|
2
5
|
import { concatenateArrayBuffersAsync } from '@loaders.gl/loader-utils';
|
|
3
6
|
import { isLoaderObject } from "../loader-utils/normalize-loader.js";
|
|
@@ -6,84 +9,118 @@ import { getLoaderContext } from "../loader-utils/loader-context.js";
|
|
|
6
9
|
import { getAsyncIterableFromData } from "../loader-utils/get-data.js";
|
|
7
10
|
import { getResourceUrl } from "../utils/resource-utils.js";
|
|
8
11
|
import { selectLoader } from "./select-loader.js";
|
|
12
|
+
// Ensure `parse` is available in context if loader falls back to `parse`
|
|
9
13
|
import { parse } from "./parse.js";
|
|
14
|
+
/**
|
|
15
|
+
* Parses `data` using a specified loader
|
|
16
|
+
* @param data
|
|
17
|
+
* @param loaders
|
|
18
|
+
* @param options
|
|
19
|
+
* @param context
|
|
20
|
+
*/
|
|
10
21
|
export async function parseInBatches(data, loaders, options, context) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
22
|
+
const loaderArray = Array.isArray(loaders) ? loaders : undefined;
|
|
23
|
+
// Signature: parseInBatches(data, options, url) - Uses registered loaders
|
|
24
|
+
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
25
|
+
context = undefined; // context not supported in short signature
|
|
26
|
+
options = loaders;
|
|
27
|
+
loaders = undefined;
|
|
28
|
+
}
|
|
29
|
+
data = await data; // Resolve any promise
|
|
30
|
+
options = options || {};
|
|
31
|
+
// Extract a url for auto detection
|
|
32
|
+
const url = getResourceUrl(data);
|
|
33
|
+
// Chooses a loader and normalizes it
|
|
34
|
+
// Note - only uses URL and contentType for streams and iterator inputs
|
|
35
|
+
const loader = await selectLoader(data, loaders, options);
|
|
36
|
+
// Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
|
|
37
|
+
if (!loader) {
|
|
38
|
+
return [];
|
|
39
|
+
}
|
|
40
|
+
// Normalize options
|
|
41
|
+
options = normalizeOptions(options, loader, loaderArray, url);
|
|
42
|
+
context = getLoaderContext({ url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray }, options, context || null);
|
|
43
|
+
return await parseWithLoaderInBatches(loader, data, options, context);
|
|
32
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Loader has been selected and context has been prepared, see if we need to emit a metadata batch
|
|
47
|
+
*/
|
|
33
48
|
async function parseWithLoaderInBatches(loader, data, options, context) {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
49
|
+
const outputIterator = await parseToOutputIterator(loader, data, options, context);
|
|
50
|
+
// Generate metadata batch if requested
|
|
51
|
+
if (!options.metadata) {
|
|
52
|
+
return outputIterator;
|
|
53
|
+
}
|
|
54
|
+
const metadataBatch = {
|
|
55
|
+
shape: 'metadata',
|
|
56
|
+
batchType: 'metadata',
|
|
57
|
+
metadata: {
|
|
58
|
+
_loader: loader,
|
|
59
|
+
_context: context
|
|
60
|
+
},
|
|
61
|
+
// Populate with some default fields to avoid crashing
|
|
62
|
+
data: [],
|
|
63
|
+
bytesUsed: 0
|
|
64
|
+
};
|
|
65
|
+
async function* makeMetadataBatchIterator(iterator) {
|
|
66
|
+
yield metadataBatch;
|
|
67
|
+
yield* iterator;
|
|
68
|
+
}
|
|
69
|
+
return makeMetadataBatchIterator(outputIterator);
|
|
53
70
|
}
|
|
71
|
+
/**
|
|
72
|
+
* Prep work is done, now it is time to start parsing into an output operator
|
|
73
|
+
* The approach depends on which parse function the loader exposes
|
|
74
|
+
* `parseInBatches` (preferred), `parse` (fallback)
|
|
75
|
+
*/
|
|
54
76
|
async function parseToOutputIterator(loader, data, options, context) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
77
|
+
// Get an iterator from the input
|
|
78
|
+
const inputIterator = await getAsyncIterableFromData(data, options);
|
|
79
|
+
// Apply any iterator transforms (options.transforms)
|
|
80
|
+
const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);
|
|
81
|
+
// If loader supports parseInBatches, we are done
|
|
82
|
+
if (loader.parseInBatches) {
|
|
83
|
+
return loader.parseInBatches(transformedIterator, options, context);
|
|
84
|
+
}
|
|
85
|
+
return parseChunkInBatches(transformedIterator, loader, options, context);
|
|
61
86
|
}
|
|
87
|
+
// Fallback: load atomically using `parse` concatenating input iterator into single chunk
|
|
62
88
|
async function* parseChunkInBatches(transformedIterator, loader, options, context) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
89
|
+
const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
|
|
90
|
+
// Call `parse` instead of `loader.parse` to ensure we can call workers etc.
|
|
91
|
+
const parsedData = await parse(arrayBuffer, loader,
|
|
92
|
+
// TODO - Hack: supply loaders MIME type to ensure we match it
|
|
93
|
+
{ ...options, mimeType: loader.mimeTypes[0] }, context);
|
|
94
|
+
// yield a single batch, the output from loader.parse() repackaged as a batch
|
|
95
|
+
const batch = convertDataToBatch(parsedData, loader);
|
|
96
|
+
yield batch;
|
|
70
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* Convert parsed data into a single batch
|
|
100
|
+
* @todo run through batch builder to apply options etc...
|
|
101
|
+
*/
|
|
71
102
|
function convertDataToBatch(parsedData, loader) {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
103
|
+
// prettier-ignore
|
|
104
|
+
const batch = isTable(parsedData)
|
|
105
|
+
? makeBatchFromTable(parsedData)
|
|
106
|
+
: {
|
|
107
|
+
shape: 'unknown',
|
|
108
|
+
batchType: 'data',
|
|
109
|
+
data: parsedData,
|
|
110
|
+
length: Array.isArray(parsedData) ? parsedData.length : 1
|
|
111
|
+
};
|
|
112
|
+
batch.mimeType = loader.mimeTypes[0];
|
|
113
|
+
return batch;
|
|
80
114
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
115
|
+
/**
|
|
116
|
+
* Create an iterator chain with any transform iterators (crypto, decompression)
|
|
117
|
+
* @param inputIterator
|
|
118
|
+
* @param options
|
|
119
|
+
*/
|
|
120
|
+
async function applyInputTransforms(inputIterator, transforms = []) {
|
|
121
|
+
let iteratorChain = inputIterator;
|
|
122
|
+
for await (const transformBatches of transforms) {
|
|
123
|
+
iteratorChain = transformBatches(iteratorChain);
|
|
124
|
+
}
|
|
125
|
+
return iteratorChain;
|
|
88
126
|
}
|
|
89
|
-
//# sourceMappingURL=parse-in-batches.js.map
|
|
@@ -1,43 +1,52 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { selectLoaderSync } from "./select-loader.js";
|
|
2
5
|
import { isLoaderObject } from "../loader-utils/normalize-loader.js";
|
|
3
6
|
import { normalizeOptions } from "../loader-utils/option-utils.js";
|
|
4
7
|
import { getArrayBufferOrStringFromDataSync } from "../loader-utils/get-data.js";
|
|
5
8
|
import { getLoaderContext, getLoadersFromContext } from "../loader-utils/loader-context.js";
|
|
6
9
|
import { getResourceUrl } from "../utils/resource-utils.js";
|
|
10
|
+
/**
|
|
11
|
+
* Parses `data` synchronously using a specified loader
|
|
12
|
+
*/
|
|
7
13
|
export function parseSync(data, loaders, options, context) {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
14
|
+
// Signature: parseSync(data, options)
|
|
15
|
+
// Uses registered loaders
|
|
16
|
+
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
17
|
+
context = undefined; // context not supported in short signature
|
|
18
|
+
options = loaders;
|
|
19
|
+
loaders = undefined;
|
|
20
|
+
}
|
|
21
|
+
options = options || {};
|
|
22
|
+
// Chooses a loader (and normalizes it)
|
|
23
|
+
// Also use any loaders in the context, new loaders take priority
|
|
24
|
+
const typedLoaders = loaders;
|
|
25
|
+
const candidateLoaders = getLoadersFromContext(typedLoaders, context);
|
|
26
|
+
const loader = selectLoaderSync(data, candidateLoaders, options);
|
|
27
|
+
// Note: if nothrow option was set, it is possible that no loader was found, if so just return null
|
|
28
|
+
if (!loader) {
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
// Normalize options
|
|
32
|
+
options = normalizeOptions(options, loader, candidateLoaders);
|
|
33
|
+
// Extract a url for auto detection
|
|
34
|
+
const url = getResourceUrl(data);
|
|
35
|
+
const parse = () => {
|
|
36
|
+
throw new Error('parseSync called parse (which is async');
|
|
37
|
+
};
|
|
38
|
+
context = getLoaderContext({ url, _parseSync: parse, _parse: parse, loaders: loaders }, options, context || null);
|
|
39
|
+
return parseWithLoaderSync(loader, data, options, context);
|
|
32
40
|
}
|
|
41
|
+
// TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
|
|
33
42
|
function parseWithLoaderSync(loader, data, options, context) {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
43
|
+
data = getArrayBufferOrStringFromDataSync(data, loader, options);
|
|
44
|
+
if (loader.parseTextSync && typeof data === 'string') {
|
|
45
|
+
return loader.parseTextSync(data, options); // , context, loader);
|
|
46
|
+
}
|
|
47
|
+
if (loader.parseSync && data instanceof ArrayBuffer) {
|
|
48
|
+
return loader.parseSync(data, options, context); // , loader);
|
|
49
|
+
}
|
|
50
|
+
// TBD - If synchronous parser not available, return null
|
|
51
|
+
throw new Error(`${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${context.url || ''}`);
|
|
42
52
|
}
|
|
43
|
-
//# sourceMappingURL=parse-sync.js.map
|
package/dist/lib/api/parse.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { parseWithWorker, canParseWithWorker } from '@loaders.gl/loader-utils';
|
|
2
5
|
import { assert, validateWorkerVersion } from '@loaders.gl/worker-utils';
|
|
3
6
|
import { isLoaderObject } from "../loader-utils/normalize-loader.js";
|
|
@@ -8,68 +11,77 @@ import { getArrayBufferOrStringFromData } from "../loader-utils/get-data.js";
|
|
|
8
11
|
import { getLoaderContext, getLoadersFromContext } from "../loader-utils/loader-context.js";
|
|
9
12
|
import { getResourceUrl } from "../utils/resource-utils.js";
|
|
10
13
|
import { selectLoader } from "./select-loader.js";
|
|
14
|
+
/**
|
|
15
|
+
* Parses `data` using a specified loader
|
|
16
|
+
* @param data
|
|
17
|
+
* @param loaders
|
|
18
|
+
* @param options
|
|
19
|
+
* @param context
|
|
20
|
+
*/
|
|
21
|
+
// implementation signature
|
|
11
22
|
export async function parse(data, loaders, options, context) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
23
|
+
// Signature: parse(data, options, context | url)
|
|
24
|
+
// Uses registered loaders
|
|
25
|
+
if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
26
|
+
context = undefined; // context not supported in short signature
|
|
27
|
+
options = loaders;
|
|
28
|
+
loaders = undefined;
|
|
29
|
+
}
|
|
30
|
+
data = await data; // Resolve any promise
|
|
31
|
+
options = options || {}; // Could be invalid...
|
|
32
|
+
// Extract a url for auto detection
|
|
33
|
+
const url = getResourceUrl(data);
|
|
34
|
+
// Chooses a loader (and normalizes it)
|
|
35
|
+
// Also use any loaders in the context, new loaders take priority
|
|
36
|
+
const typedLoaders = loaders;
|
|
37
|
+
const candidateLoaders = getLoadersFromContext(typedLoaders, context);
|
|
38
|
+
// todo hacky type cast
|
|
39
|
+
const loader = await selectLoader(data, candidateLoaders, options);
|
|
40
|
+
// Note: if no loader was found, if so just return null
|
|
41
|
+
if (!loader) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
// Normalize options
|
|
45
|
+
// @ts-expect-error
|
|
46
|
+
options = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
|
|
47
|
+
// Get a context (if already present, will be unchanged)
|
|
48
|
+
context = getLoaderContext(
|
|
49
|
+
// @ts-expect-error
|
|
50
|
+
{ url, _parse: parse, loaders: candidateLoaders }, options, context || null);
|
|
51
|
+
return await parseWithLoader(loader, data, options, context);
|
|
33
52
|
}
|
|
53
|
+
// TODO: support progress and abort
|
|
54
|
+
// TODO - should accept loader.parseAsyncIterator and concatenate.
|
|
34
55
|
async function parseWithLoader(loader, data, options, context) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
if (loaderWithParser.parseText && typeof data === 'string') {
|
|
67
|
-
return await loaderWithParser.parseText(data, options, context);
|
|
68
|
-
}
|
|
69
|
-
if (loaderWithParser.parse) {
|
|
70
|
-
return await loaderWithParser.parse(data, options, context);
|
|
71
|
-
}
|
|
72
|
-
assert(!loaderWithParser.parseSync);
|
|
73
|
-
throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
|
|
56
|
+
validateWorkerVersion(loader);
|
|
57
|
+
options = mergeLoaderOptions(loader.options, options);
|
|
58
|
+
if (isResponse(data)) {
|
|
59
|
+
// Serialize to support passing the response to web worker
|
|
60
|
+
const response = data;
|
|
61
|
+
const { ok, redirected, status, statusText, type, url } = response;
|
|
62
|
+
const headers = Object.fromEntries(response.headers.entries());
|
|
63
|
+
// @ts-expect-error TODO - fix this
|
|
64
|
+
context.response = { headers, ok, redirected, status, statusText, type, url };
|
|
65
|
+
}
|
|
66
|
+
data = await getArrayBufferOrStringFromData(data, loader, options);
|
|
67
|
+
const loaderWithParser = loader;
|
|
68
|
+
// First check for synchronous text parser, wrap results in promises
|
|
69
|
+
if (loaderWithParser.parseTextSync && typeof data === 'string') {
|
|
70
|
+
return loaderWithParser.parseTextSync(data, options, context);
|
|
71
|
+
}
|
|
72
|
+
// If we have a workerUrl and the loader can parse the given options efficiently in a worker
|
|
73
|
+
if (canParseWithWorker(loader, options)) {
|
|
74
|
+
return await parseWithWorker(loader, data, options, context, parse);
|
|
75
|
+
}
|
|
76
|
+
// Check for asynchronous parser
|
|
77
|
+
if (loaderWithParser.parseText && typeof data === 'string') {
|
|
78
|
+
return await loaderWithParser.parseText(data, options, context);
|
|
79
|
+
}
|
|
80
|
+
if (loaderWithParser.parse) {
|
|
81
|
+
return await loaderWithParser.parse(data, options, context);
|
|
82
|
+
}
|
|
83
|
+
// This should not happen, all sync loaders should also offer `parse` function
|
|
84
|
+
assert(!loaderWithParser.parseSync);
|
|
85
|
+
// TBD - If asynchronous parser not available, return null
|
|
86
|
+
throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
|
|
74
87
|
}
|
|
75
|
-
//# sourceMappingURL=parse.js.map
|
|
@@ -1,25 +1,34 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { normalizeLoader } from "../loader-utils/normalize-loader.js";
|
|
2
5
|
import { getGlobalLoaderState } from "../loader-utils/option-utils.js";
|
|
6
|
+
/**
|
|
7
|
+
* Store global registered loaders on the global object to increase chances of cross loaders-version interoperability
|
|
8
|
+
* This use case is not reliable but can help when testing new versions of loaders.gl with existing frameworks
|
|
9
|
+
*/
|
|
3
10
|
const getGlobalLoaderRegistry = () => {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
11
|
+
const state = getGlobalLoaderState();
|
|
12
|
+
state.loaderRegistry = state.loaderRegistry || [];
|
|
13
|
+
return state.loaderRegistry;
|
|
7
14
|
};
|
|
15
|
+
/** Register a list of global loaders */
|
|
8
16
|
export function registerLoaders(loaders) {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
17
|
+
const loaderRegistry = getGlobalLoaderRegistry();
|
|
18
|
+
loaders = Array.isArray(loaders) ? loaders : [loaders];
|
|
19
|
+
for (const loader of loaders) {
|
|
20
|
+
const normalizedLoader = normalizeLoader(loader);
|
|
21
|
+
if (!loaderRegistry.find((registeredLoader) => normalizedLoader === registeredLoader)) {
|
|
22
|
+
// add to the beginning of the loaderRegistry, so the last registeredLoader get picked
|
|
23
|
+
loaderRegistry.unshift(normalizedLoader);
|
|
24
|
+
}
|
|
15
25
|
}
|
|
16
|
-
}
|
|
17
26
|
}
|
|
18
27
|
export function getRegisteredLoaders() {
|
|
19
|
-
|
|
28
|
+
return getGlobalLoaderRegistry();
|
|
20
29
|
}
|
|
30
|
+
/** @deprecated For testing only */
|
|
21
31
|
export function _unregisterLoaders() {
|
|
22
|
-
|
|
23
|
-
|
|
32
|
+
const state = getGlobalLoaderState();
|
|
33
|
+
state.loaderRegistry = [];
|
|
24
34
|
}
|
|
25
|
-
//# sourceMappingURL=register-loaders.js.map
|