@loaders.gl/core 4.4.0-alpha.2 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/core-addons/write-file-browser.js +1 -0
- package/dist/core-addons/write-file-browser.js.map +1 -0
- package/dist/dist.dev.js +641 -286
- package/dist/dist.min.js +3 -3
- package/dist/index.cjs +411 -262
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -0
- package/dist/iterators/batch-iterators/timed-batch-iterator.js +1 -0
- package/dist/iterators/batch-iterators/timed-batch-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js +1 -0
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-blob-iterator.js +1 -0
- package/dist/iterators/make-iterator/make-blob-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-iterator.js +7 -3
- package/dist/iterators/make-iterator/make-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-stream-iterator.js +1 -0
- package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-string-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-string-iterator.js +3 -1
- package/dist/iterators/make-iterator/make-string-iterator.js.map +1 -0
- package/dist/iterators/make-stream/make-stream.js +1 -0
- package/dist/iterators/make-stream/make-stream.js.map +1 -0
- package/dist/lib/api/create-data-source.js +1 -0
- package/dist/lib/api/create-data-source.js.map +1 -0
- package/dist/lib/api/encode-table.d.ts.map +1 -1
- package/dist/lib/api/encode-table.js +3 -2
- package/dist/lib/api/encode-table.js.map +1 -0
- package/dist/lib/api/encode.d.ts.map +1 -1
- package/dist/lib/api/encode.js +3 -2
- package/dist/lib/api/encode.js.map +1 -0
- package/dist/lib/api/load-in-batches.js +1 -0
- package/dist/lib/api/load-in-batches.js.map +1 -0
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/load.js +15 -1
- package/dist/lib/api/load.js.map +1 -0
- package/dist/lib/api/loader-options.js +1 -0
- package/dist/lib/api/loader-options.js.map +1 -0
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +7 -6
- package/dist/lib/api/parse-in-batches.js.map +1 -0
- package/dist/lib/api/parse-sync.d.ts.map +1 -1
- package/dist/lib/api/parse-sync.js +4 -3
- package/dist/lib/api/parse-sync.js.map +1 -0
- package/dist/lib/api/parse.d.ts.map +1 -1
- package/dist/lib/api/parse.js +8 -9
- package/dist/lib/api/parse.js.map +1 -0
- package/dist/lib/api/register-loaders.js +1 -0
- package/dist/lib/api/register-loaders.js.map +1 -0
- package/dist/lib/api/select-loader.d.ts +3 -3
- package/dist/lib/api/select-loader.d.ts.map +1 -1
- package/dist/lib/api/select-loader.js +36 -15
- package/dist/lib/api/select-loader.js.map +1 -0
- package/dist/lib/api/select-source.js +1 -0
- package/dist/lib/api/select-source.js.map +1 -0
- package/dist/lib/common.js +1 -0
- package/dist/lib/common.js.map +1 -0
- package/dist/lib/fetch/fetch-error-message.js +1 -0
- package/dist/lib/fetch/fetch-error-message.js.map +1 -0
- package/dist/lib/fetch/fetch-error.js +1 -0
- package/dist/lib/fetch/fetch-error.js.map +1 -0
- package/dist/lib/fetch/fetch-file.js +1 -0
- package/dist/lib/fetch/fetch-file.js.map +1 -0
- package/dist/lib/fetch/read-array-buffer.js +1 -0
- package/dist/lib/fetch/read-array-buffer.js.map +1 -0
- package/dist/lib/filesystems/browser-filesystem.js +1 -0
- package/dist/lib/filesystems/browser-filesystem.js.map +1 -0
- package/dist/lib/filesystems/read-array-buffer.js +1 -0
- package/dist/lib/filesystems/read-array-buffer.js.map +1 -0
- package/dist/lib/init.js +2 -1
- package/dist/lib/init.js.map +1 -0
- package/dist/lib/loader-utils/check-errors.js +1 -0
- package/dist/lib/loader-utils/check-errors.js.map +1 -0
- package/dist/lib/loader-utils/get-data.d.ts +18 -1
- package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-data.js +47 -38
- package/dist/lib/loader-utils/get-data.js.map +1 -0
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +7 -5
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -0
- package/dist/lib/loader-utils/loader-context.d.ts +2 -2
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +1 -0
- package/dist/lib/loader-utils/loader-context.js.map +1 -0
- package/dist/lib/loader-utils/loggers.js +1 -0
- package/dist/lib/loader-utils/loggers.js.map +1 -0
- package/dist/lib/loader-utils/normalize-loader.js +1 -0
- package/dist/lib/loader-utils/normalize-loader.js.map +1 -0
- package/dist/lib/loader-utils/option-defaults.d.ts +22 -0
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +63 -33
- package/dist/lib/loader-utils/option-defaults.js.map +1 -0
- package/dist/lib/loader-utils/option-utils.d.ts +10 -4
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +123 -19
- package/dist/lib/loader-utils/option-utils.js.map +1 -0
- package/dist/lib/progress/fetch-progress.js +1 -0
- package/dist/lib/progress/fetch-progress.js.map +1 -0
- package/dist/lib/utils/mime-type-utils.js +1 -0
- package/dist/lib/utils/mime-type-utils.js.map +1 -0
- package/dist/lib/utils/resource-utils.d.ts.map +1 -1
- package/dist/lib/utils/resource-utils.js +8 -10
- package/dist/lib/utils/resource-utils.js.map +1 -0
- package/dist/lib/utils/response-utils.js +2 -1
- package/dist/lib/utils/response-utils.js.map +1 -0
- package/dist/lib/utils/url-utils.js +1 -0
- package/dist/lib/utils/url-utils.js.map +1 -0
- package/dist/null-loader.d.ts +3 -3
- package/dist/null-loader.d.ts.map +1 -1
- package/dist/null-loader.js +2 -1
- package/dist/null-loader.js.map +1 -0
- package/dist/null-worker-node.js +5 -2
- package/dist/null-worker.js +5 -2
- package/dist/workers/null-worker-node.js +1 -0
- package/dist/workers/null-worker-node.js.map +1 -0
- package/dist/workers/null-worker.js +1 -0
- package/dist/workers/null-worker.js.map +1 -0
- package/package.json +13 -10
- package/src/index.ts +1 -1
- package/src/iterators/make-iterator/make-iterator.ts +8 -5
- package/src/iterators/make-iterator/make-string-iterator.ts +2 -1
- package/src/lib/api/encode-table.ts +2 -1
- package/src/lib/api/encode.ts +8 -2
- package/src/lib/api/load.ts +15 -1
- package/src/lib/api/parse-in-batches.ts +26 -19
- package/src/lib/api/parse-sync.ts +6 -5
- package/src/lib/api/parse.ts +15 -11
- package/src/lib/api/select-loader.ts +68 -24
- package/src/lib/loader-utils/get-data.ts +62 -48
- package/src/lib/loader-utils/get-fetch-function.ts +6 -5
- package/src/lib/loader-utils/loader-context.ts +2 -2
- package/src/lib/loader-utils/option-defaults.ts +68 -35
- package/src/lib/loader-utils/option-utils.ts +146 -29
- package/src/lib/utils/resource-utils.ts +9 -12
- package/src/lib/utils/response-utils.ts +2 -2
- package/src/null-loader.ts +6 -3
- package/dist/javascript-utils/is-type.d.ts +0 -21
- package/dist/javascript-utils/is-type.d.ts.map +0 -1
- package/dist/javascript-utils/is-type.js +0 -25
- package/src/javascript-utils/is-type.ts +0 -51
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/core",
|
|
3
|
-
"version": "4.4.0
|
|
3
|
+
"version": "4.4.0",
|
|
4
4
|
"description": "The core API for working with loaders.gl loaders and writers",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -32,13 +32,16 @@
|
|
|
32
32
|
"types": "./dist/index.d.ts",
|
|
33
33
|
"import": "./dist/index.js",
|
|
34
34
|
"require": "./dist/index.cjs"
|
|
35
|
+
},
|
|
36
|
+
"./null-worker.js": {
|
|
37
|
+
"import": "./dist/null-worker.js"
|
|
38
|
+
},
|
|
39
|
+
"./null-worker-node.js": {
|
|
40
|
+
"import": "./dist/null-worker-node.js"
|
|
35
41
|
}
|
|
36
42
|
},
|
|
37
43
|
"sideEffects": false,
|
|
38
44
|
"browser": {
|
|
39
|
-
"./dist/iterators/make-stream/make-node-stream.js": "./dist/iterators/make-stream/make-dom-stream.js",
|
|
40
|
-
"./src/iterators/make-stream/make-node-stream.js": "./src/iterators/make-stream/make-dom-stream.js",
|
|
41
|
-
"./src/iterators/make-stream/make-node-stream.ts": "./src/iterators/make-stream/make-dom-stream.ts",
|
|
42
45
|
"fs": false,
|
|
43
46
|
"stream": false
|
|
44
47
|
},
|
|
@@ -55,11 +58,11 @@
|
|
|
55
58
|
"build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
|
|
56
59
|
},
|
|
57
60
|
"dependencies": {
|
|
58
|
-
"@loaders.gl/loader-utils": "4.4.0
|
|
59
|
-
"@loaders.gl/schema": "4.4.0
|
|
60
|
-
"@loaders.gl/schema-utils": "4.4.0
|
|
61
|
-
"@loaders.gl/worker-utils": "4.4.0
|
|
62
|
-
"@probe.gl/log": "^4.
|
|
61
|
+
"@loaders.gl/loader-utils": "4.4.0",
|
|
62
|
+
"@loaders.gl/schema": "4.4.0",
|
|
63
|
+
"@loaders.gl/schema-utils": "4.4.0",
|
|
64
|
+
"@loaders.gl/worker-utils": "4.4.0",
|
|
65
|
+
"@probe.gl/log": "^4.1.1"
|
|
63
66
|
},
|
|
64
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "b8a23bc05946bde7089936ff3ea1651ee9a45536"
|
|
65
68
|
}
|
package/src/index.ts
CHANGED
|
@@ -9,7 +9,7 @@ import {makeArrayBufferIterator} from './make-array-buffer-iterator';
|
|
|
9
9
|
import {makeBlobIterator} from './make-blob-iterator';
|
|
10
10
|
import type {StreamIteratorOptions} from './make-stream-iterator';
|
|
11
11
|
import {makeStreamIterator} from './make-stream-iterator';
|
|
12
|
-
import {isBlob, isReadableStream, isResponse} from '
|
|
12
|
+
import {isBlob, isReadableStream, isResponse} from '@loaders.gl/loader-utils';
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* @param [options.chunkSize]
|
|
@@ -38,14 +38,17 @@ export function makeIterator(
|
|
|
38
38
|
return makeArrayBufferIterator(data, options);
|
|
39
39
|
}
|
|
40
40
|
if (isBlob(data)) {
|
|
41
|
-
return makeBlobIterator(data
|
|
41
|
+
return makeBlobIterator(data, options);
|
|
42
42
|
}
|
|
43
43
|
if (isReadableStream(data)) {
|
|
44
|
-
return makeStreamIterator(data
|
|
44
|
+
return makeStreamIterator(data, options);
|
|
45
45
|
}
|
|
46
46
|
if (isResponse(data)) {
|
|
47
|
-
const
|
|
48
|
-
|
|
47
|
+
const responseBody = data.body;
|
|
48
|
+
if (!responseBody) {
|
|
49
|
+
throw new Error('Readable stream not available on Response');
|
|
50
|
+
}
|
|
51
|
+
return makeStreamIterator(responseBody as ReadableStream, options);
|
|
49
52
|
}
|
|
50
53
|
throw new Error('makeIterator');
|
|
51
54
|
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
+
import {ensureArrayBuffer} from '@loaders.gl/loader-utils';
|
|
5
6
|
import type {IteratorOptions} from './make-iterator';
|
|
6
7
|
|
|
7
8
|
const DEFAULT_CHUNK_SIZE = 256 * 1024;
|
|
@@ -27,6 +28,6 @@ export function* makeStringIterator(
|
|
|
27
28
|
offset += chunkLength;
|
|
28
29
|
|
|
29
30
|
// yield an ArrayBuffer chunk
|
|
30
|
-
yield textEncoder.encode(chunk);
|
|
31
|
+
yield ensureArrayBuffer(textEncoder.encode(chunk));
|
|
31
32
|
}
|
|
32
33
|
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
/* global TextEncoder, TextDecoder */
|
|
7
7
|
import {
|
|
8
8
|
concatenateArrayBuffers,
|
|
9
|
+
ensureArrayBuffer,
|
|
9
10
|
WriterOptionsType,
|
|
10
11
|
WriterWithEncoder
|
|
11
12
|
} from '@loaders.gl/loader-utils';
|
|
@@ -22,7 +23,7 @@ export async function encodeTable<WriterT extends WriterWithEncoder = WriterWith
|
|
|
22
23
|
|
|
23
24
|
if (writer.encodeText) {
|
|
24
25
|
const text = await writer.encodeText(data, options);
|
|
25
|
-
return new TextEncoder().encode(text);
|
|
26
|
+
return ensureArrayBuffer(new TextEncoder().encode(text));
|
|
26
27
|
}
|
|
27
28
|
|
|
28
29
|
if (writer.encodeInBatches) {
|
package/src/lib/api/encode.ts
CHANGED
|
@@ -9,7 +9,13 @@ import type {
|
|
|
9
9
|
WriterDataType,
|
|
10
10
|
WriterBatchType
|
|
11
11
|
} from '@loaders.gl/loader-utils';
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
canEncodeWithWorker,
|
|
14
|
+
NodeFile,
|
|
15
|
+
resolvePath,
|
|
16
|
+
isBrowser,
|
|
17
|
+
ensureArrayBuffer
|
|
18
|
+
} from '@loaders.gl/loader-utils';
|
|
13
19
|
import {processOnWorker} from '@loaders.gl/worker-utils';
|
|
14
20
|
import {fetchFile} from '../fetch/fetch-file';
|
|
15
21
|
import {getLoaderOptions} from './loader-options';
|
|
@@ -52,7 +58,7 @@ export function encodeSync<WriterT extends WriterWithEncoder>(
|
|
|
52
58
|
return writer.encodeSync(data, options);
|
|
53
59
|
}
|
|
54
60
|
if (writer.encodeTextSync) {
|
|
55
|
-
return new TextEncoder().encode(writer.encodeTextSync(data, options));
|
|
61
|
+
return ensureArrayBuffer(new TextEncoder().encode(writer.encodeTextSync(data, options)));
|
|
56
62
|
}
|
|
57
63
|
throw new Error(`Writer ${writer.name} could not synchronously encode data`);
|
|
58
64
|
}
|
package/src/lib/api/load.ts
CHANGED
|
@@ -12,9 +12,10 @@ import type {
|
|
|
12
12
|
LoaderArrayOptionsType,
|
|
13
13
|
LoaderArrayReturnType
|
|
14
14
|
} from '@loaders.gl/loader-utils';
|
|
15
|
-
import {isBlob} from '
|
|
15
|
+
import {isBlob} from '@loaders.gl/loader-utils';
|
|
16
16
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
17
17
|
import {getFetchFunction} from '../loader-utils/get-fetch-function';
|
|
18
|
+
import {normalizeLoaderOptions} from '../loader-utils/option-utils';
|
|
18
19
|
|
|
19
20
|
import {parse} from './parse';
|
|
20
21
|
|
|
@@ -97,6 +98,19 @@ export async function load(
|
|
|
97
98
|
data = await fetch(url);
|
|
98
99
|
}
|
|
99
100
|
|
|
101
|
+
if (typeof url === 'string') {
|
|
102
|
+
const normalizedOptions = normalizeLoaderOptions(resolvedOptions || {});
|
|
103
|
+
if (!normalizedOptions.core?.baseUrl) {
|
|
104
|
+
resolvedOptions = {
|
|
105
|
+
...resolvedOptions,
|
|
106
|
+
core: {
|
|
107
|
+
...resolvedOptions?.core,
|
|
108
|
+
baseUrl: url
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
100
114
|
// Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
|
|
101
115
|
// return await parse(data, loaders as Loader[], options);
|
|
102
116
|
return Array.isArray(resolvedLoaders)
|
|
@@ -7,13 +7,15 @@ import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
|
|
|
7
7
|
import type {
|
|
8
8
|
Loader,
|
|
9
9
|
LoaderWithParser,
|
|
10
|
+
StrictLoaderOptions,
|
|
10
11
|
LoaderOptions,
|
|
11
12
|
LoaderContext,
|
|
12
13
|
BatchableDataType,
|
|
13
14
|
LoaderOptionsType,
|
|
14
15
|
LoaderBatchType,
|
|
15
16
|
LoaderArrayOptionsType,
|
|
16
|
-
LoaderArrayBatchType
|
|
17
|
+
LoaderArrayBatchType,
|
|
18
|
+
TransformBatches
|
|
17
19
|
} from '@loaders.gl/loader-utils';
|
|
18
20
|
import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
|
|
19
21
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
@@ -80,7 +82,7 @@ export async function parseInBatches(
|
|
|
80
82
|
// Signature: parseInBatches(data, options, url) - Uses registered loaders
|
|
81
83
|
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
82
84
|
context = undefined; // context not supported in short signature
|
|
83
|
-
options = loaders as LoaderOptions;
|
|
85
|
+
options = loaders as unknown as LoaderOptions;
|
|
84
86
|
loaders = undefined;
|
|
85
87
|
}
|
|
86
88
|
|
|
@@ -92,21 +94,21 @@ export async function parseInBatches(
|
|
|
92
94
|
|
|
93
95
|
// Chooses a loader and normalizes it
|
|
94
96
|
// Note - only uses URL and contentType for streams and iterator inputs
|
|
95
|
-
const loader = await selectLoader(data
|
|
97
|
+
const loader = await selectLoader(data, loaders, options);
|
|
96
98
|
// Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
|
|
97
99
|
if (!loader) {
|
|
98
100
|
return [];
|
|
99
101
|
}
|
|
100
102
|
|
|
101
103
|
// Normalize options
|
|
102
|
-
|
|
104
|
+
const strictOptions = normalizeOptions(options, loader, loaderArray, url);
|
|
103
105
|
context = getLoaderContext(
|
|
104
106
|
{url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
|
|
105
|
-
|
|
107
|
+
strictOptions,
|
|
106
108
|
context || null
|
|
107
109
|
);
|
|
108
110
|
|
|
109
|
-
return await parseWithLoaderInBatches(loader as LoaderWithParser, data,
|
|
111
|
+
return await parseWithLoaderInBatches(loader as LoaderWithParser, data, strictOptions, context);
|
|
110
112
|
}
|
|
111
113
|
|
|
112
114
|
/**
|
|
@@ -115,13 +117,13 @@ export async function parseInBatches(
|
|
|
115
117
|
async function parseWithLoaderInBatches(
|
|
116
118
|
loader: LoaderWithParser,
|
|
117
119
|
data: BatchableDataType,
|
|
118
|
-
options:
|
|
120
|
+
options: StrictLoaderOptions,
|
|
119
121
|
context: LoaderContext
|
|
120
122
|
): Promise<AsyncIterable<unknown>> {
|
|
121
123
|
const outputIterator = await parseToOutputIterator(loader, data, options, context);
|
|
122
124
|
|
|
123
125
|
// Generate metadata batch if requested
|
|
124
|
-
if (!options
|
|
126
|
+
if (!options?.core?.metadata) {
|
|
125
127
|
return outputIterator;
|
|
126
128
|
}
|
|
127
129
|
|
|
@@ -155,14 +157,17 @@ async function parseWithLoaderInBatches(
|
|
|
155
157
|
async function parseToOutputIterator(
|
|
156
158
|
loader: LoaderWithParser,
|
|
157
159
|
data: BatchableDataType,
|
|
158
|
-
options:
|
|
160
|
+
options: StrictLoaderOptions,
|
|
159
161
|
context: LoaderContext
|
|
160
162
|
): Promise<AsyncIterable<unknown>> {
|
|
161
163
|
// Get an iterator from the input
|
|
162
164
|
const inputIterator = await getAsyncIterableFromData(data, options);
|
|
163
165
|
|
|
164
166
|
// Apply any iterator transforms (options.transforms)
|
|
165
|
-
const transformedIterator = await applyInputTransforms(
|
|
167
|
+
const transformedIterator = await applyInputTransforms(
|
|
168
|
+
inputIterator,
|
|
169
|
+
options?.core?.transforms || []
|
|
170
|
+
);
|
|
166
171
|
|
|
167
172
|
// If loader supports parseInBatches, we are done
|
|
168
173
|
if (loader.parseInBatches) {
|
|
@@ -174,9 +179,11 @@ async function parseToOutputIterator(
|
|
|
174
179
|
|
|
175
180
|
// Fallback: load atomically using `parse` concatenating input iterator into single chunk
|
|
176
181
|
async function* parseChunkInBatches(
|
|
177
|
-
transformedIterator:
|
|
182
|
+
transformedIterator:
|
|
183
|
+
| Iterable<ArrayBufferLike | ArrayBufferView>
|
|
184
|
+
| AsyncIterable<ArrayBufferLike | ArrayBufferView>,
|
|
178
185
|
loader: Loader,
|
|
179
|
-
options:
|
|
186
|
+
options: StrictLoaderOptions,
|
|
180
187
|
context: LoaderContext
|
|
181
188
|
): AsyncIterable<Batch> {
|
|
182
189
|
const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
|
|
@@ -185,7 +192,7 @@ async function* parseChunkInBatches(
|
|
|
185
192
|
arrayBuffer,
|
|
186
193
|
loader,
|
|
187
194
|
// TODO - Hack: supply loaders MIME type to ensure we match it
|
|
188
|
-
{...options, mimeType: loader.mimeTypes[0]},
|
|
195
|
+
{...options, core: {...options?.core, mimeType: loader.mimeTypes[0]}},
|
|
189
196
|
context
|
|
190
197
|
);
|
|
191
198
|
|
|
@@ -215,19 +222,19 @@ function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
|
|
|
215
222
|
return batch;
|
|
216
223
|
}
|
|
217
224
|
|
|
218
|
-
type TransformBatches = (
|
|
219
|
-
asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
|
|
220
|
-
) => AsyncIterable<ArrayBuffer>;
|
|
221
|
-
|
|
222
225
|
/**
|
|
223
226
|
* Create an iterator chain with any transform iterators (crypto, decompression)
|
|
224
227
|
* @param inputIterator
|
|
225
228
|
* @param options
|
|
226
229
|
*/
|
|
227
230
|
async function applyInputTransforms(
|
|
228
|
-
inputIterator:
|
|
231
|
+
inputIterator:
|
|
232
|
+
| AsyncIterable<ArrayBufferLike | ArrayBufferView>
|
|
233
|
+
| Iterable<ArrayBufferLike | ArrayBufferView>,
|
|
229
234
|
transforms: TransformBatches[] = []
|
|
230
|
-
): Promise<
|
|
235
|
+
): Promise<
|
|
236
|
+
AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>
|
|
237
|
+
> {
|
|
231
238
|
let iteratorChain = inputIterator;
|
|
232
239
|
for await (const transformBatches of transforms) {
|
|
233
240
|
iteratorChain = transformBatches(iteratorChain);
|
|
@@ -11,7 +11,8 @@ import type {
|
|
|
11
11
|
LoaderOptionsType,
|
|
12
12
|
LoaderReturnType,
|
|
13
13
|
LoaderArrayOptionsType,
|
|
14
|
-
LoaderArrayReturnType
|
|
14
|
+
LoaderArrayReturnType,
|
|
15
|
+
StrictLoaderOptions
|
|
15
16
|
} from '@loaders.gl/loader-utils';
|
|
16
17
|
import {selectLoaderSync} from './select-loader';
|
|
17
18
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
@@ -84,7 +85,7 @@ export function parseSync(
|
|
|
84
85
|
}
|
|
85
86
|
|
|
86
87
|
// Normalize options
|
|
87
|
-
|
|
88
|
+
const strictOptions = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
|
|
88
89
|
|
|
89
90
|
// Extract a url for auto detection
|
|
90
91
|
const url = getResourceUrl(data);
|
|
@@ -94,18 +95,18 @@ export function parseSync(
|
|
|
94
95
|
};
|
|
95
96
|
context = getLoaderContext(
|
|
96
97
|
{url, _parseSync: parse, _parse: parse, loaders: loaders as Loader[]},
|
|
97
|
-
|
|
98
|
+
strictOptions,
|
|
98
99
|
context || null
|
|
99
100
|
);
|
|
100
101
|
|
|
101
|
-
return parseWithLoaderSync(loader as LoaderWithParser, data,
|
|
102
|
+
return parseWithLoaderSync(loader as LoaderWithParser, data, strictOptions, context);
|
|
102
103
|
}
|
|
103
104
|
|
|
104
105
|
// TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
|
|
105
106
|
function parseWithLoaderSync(
|
|
106
107
|
loader: LoaderWithParser,
|
|
107
108
|
data: SyncDataType,
|
|
108
|
-
options:
|
|
109
|
+
options: StrictLoaderOptions,
|
|
109
110
|
context: LoaderContext
|
|
110
111
|
) {
|
|
111
112
|
data = getArrayBufferOrStringFromDataSync(data, loader, options);
|
package/src/lib/api/parse.ts
CHANGED
|
@@ -11,12 +11,17 @@ import type {
|
|
|
11
11
|
LoaderOptionsType,
|
|
12
12
|
LoaderReturnType,
|
|
13
13
|
LoaderArrayOptionsType,
|
|
14
|
-
LoaderArrayReturnType
|
|
14
|
+
LoaderArrayReturnType,
|
|
15
|
+
StrictLoaderOptions
|
|
16
|
+
} from '@loaders.gl/loader-utils';
|
|
17
|
+
import {
|
|
18
|
+
parseWithWorker,
|
|
19
|
+
canParseWithWorker,
|
|
20
|
+
mergeOptions,
|
|
21
|
+
isResponse
|
|
15
22
|
} from '@loaders.gl/loader-utils';
|
|
16
|
-
import {parseWithWorker, canParseWithWorker, mergeOptions} from '@loaders.gl/loader-utils';
|
|
17
23
|
import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
|
|
18
24
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
19
|
-
import {isResponse} from '../../javascript-utils/is-type';
|
|
20
25
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
21
26
|
import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
|
|
22
27
|
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
|
|
@@ -100,18 +105,18 @@ export async function parse(
|
|
|
100
105
|
}
|
|
101
106
|
|
|
102
107
|
// Normalize options
|
|
103
|
-
// @ts-expect-error
|
|
104
|
-
|
|
108
|
+
// @ts-expect-error candidateLoaders
|
|
109
|
+
const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
|
|
105
110
|
|
|
106
111
|
// Get a context (if already present, will be unchanged)
|
|
107
112
|
context = getLoaderContext(
|
|
108
113
|
// @ts-expect-error
|
|
109
114
|
{url, _parse: parse, loaders: candidateLoaders},
|
|
110
|
-
|
|
115
|
+
strictOptions,
|
|
111
116
|
context || null
|
|
112
117
|
);
|
|
113
118
|
|
|
114
|
-
return await parseWithLoader(loader, data,
|
|
119
|
+
return await parseWithLoader(loader, data, strictOptions, context);
|
|
115
120
|
}
|
|
116
121
|
|
|
117
122
|
// TODO: support progress and abort
|
|
@@ -119,7 +124,7 @@ export async function parse(
|
|
|
119
124
|
async function parseWithLoader(
|
|
120
125
|
loader: Loader,
|
|
121
126
|
data,
|
|
122
|
-
options:
|
|
127
|
+
options: StrictLoaderOptions,
|
|
123
128
|
context: LoaderContext
|
|
124
129
|
): Promise<unknown> {
|
|
125
130
|
validateWorkerVersion(loader);
|
|
@@ -128,9 +133,8 @@ async function parseWithLoader(
|
|
|
128
133
|
|
|
129
134
|
if (isResponse(data)) {
|
|
130
135
|
// Serialize to support passing the response to web worker
|
|
131
|
-
const
|
|
132
|
-
const
|
|
133
|
-
const headers = Object.fromEntries(response.headers.entries());
|
|
136
|
+
const {ok, redirected, status, statusText, type, url} = data;
|
|
137
|
+
const headers = Object.fromEntries(data.headers.entries());
|
|
134
138
|
// @ts-expect-error TODO - fix this
|
|
135
139
|
context.response = {headers, ok, redirected, status, statusText, type, url};
|
|
136
140
|
}
|
|
@@ -2,14 +2,21 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import type {LoaderContext, LoaderOptions, Loader} from '@loaders.gl/loader-utils';
|
|
6
|
-
import {
|
|
5
|
+
import type {LoaderContext, LoaderOptions, Loader, DataType} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {
|
|
7
|
+
compareArrayBuffers,
|
|
8
|
+
path,
|
|
9
|
+
log,
|
|
10
|
+
isBlob,
|
|
11
|
+
ensureArrayBuffer,
|
|
12
|
+
isArrayBufferLike
|
|
13
|
+
} from '@loaders.gl/loader-utils';
|
|
7
14
|
import {TypedArray} from '@loaders.gl/schema';
|
|
8
15
|
import {normalizeLoader} from '../loader-utils/normalize-loader';
|
|
16
|
+
import {normalizeLoaderOptions} from '../loader-utils/option-utils';
|
|
9
17
|
import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
|
|
10
18
|
import {compareMIMETypes} from '../utils/mime-type-utils';
|
|
11
19
|
import {getRegisteredLoaders} from './register-loaders';
|
|
12
|
-
import {isBlob} from '../../javascript-utils/is-type';
|
|
13
20
|
import {stripQueryString} from '../utils/url-utils';
|
|
14
21
|
|
|
15
22
|
const EXT_PATTERN = /\.([^.]+)$/;
|
|
@@ -28,7 +35,7 @@ const EXT_PATTERN = /\.([^.]+)$/;
|
|
|
28
35
|
* @param context used internally, applications should not provide this parameter
|
|
29
36
|
*/
|
|
30
37
|
export async function selectLoader(
|
|
31
|
-
data:
|
|
38
|
+
data: DataType,
|
|
32
39
|
loaders: Loader[] | Loader = [],
|
|
33
40
|
options?: LoaderOptions,
|
|
34
41
|
context?: LoaderContext
|
|
@@ -37,8 +44,29 @@ export async function selectLoader(
|
|
|
37
44
|
return null;
|
|
38
45
|
}
|
|
39
46
|
|
|
47
|
+
const normalizedOptions = normalizeLoaderOptions(options || {});
|
|
48
|
+
normalizedOptions.core ||= {};
|
|
49
|
+
|
|
50
|
+
if (data instanceof Response && mayContainText(data)) {
|
|
51
|
+
const text = await data.clone().text();
|
|
52
|
+
const textLoader = selectLoaderSync(
|
|
53
|
+
text,
|
|
54
|
+
loaders,
|
|
55
|
+
{...normalizedOptions, core: {...normalizedOptions.core, nothrow: true}},
|
|
56
|
+
context
|
|
57
|
+
);
|
|
58
|
+
if (textLoader) {
|
|
59
|
+
return textLoader;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
40
63
|
// First make a sync attempt, disabling exceptions
|
|
41
|
-
let loader = selectLoaderSync(
|
|
64
|
+
let loader = selectLoaderSync(
|
|
65
|
+
data,
|
|
66
|
+
loaders,
|
|
67
|
+
{...normalizedOptions, core: {...normalizedOptions.core, nothrow: true}},
|
|
68
|
+
context
|
|
69
|
+
);
|
|
42
70
|
if (loader) {
|
|
43
71
|
return loader;
|
|
44
72
|
}
|
|
@@ -46,18 +74,31 @@ export async function selectLoader(
|
|
|
46
74
|
// For Blobs and Files, try to asynchronously read a small initial slice and test again with that
|
|
47
75
|
// to see if we can detect by initial content
|
|
48
76
|
if (isBlob(data)) {
|
|
49
|
-
data = await
|
|
50
|
-
loader = selectLoaderSync(data, loaders,
|
|
77
|
+
data = await data.slice(0, 10).arrayBuffer();
|
|
78
|
+
loader = selectLoaderSync(data, loaders, normalizedOptions, context);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (!loader && data instanceof Response && mayContainText(data)) {
|
|
82
|
+
const text = await data.clone().text();
|
|
83
|
+
loader = selectLoaderSync(text, loaders, normalizedOptions, context);
|
|
51
84
|
}
|
|
52
85
|
|
|
53
86
|
// no loader available
|
|
54
|
-
if (!loader && !
|
|
87
|
+
if (!loader && !normalizedOptions.core.nothrow) {
|
|
55
88
|
throw new Error(getNoValidLoaderMessage(data));
|
|
56
89
|
}
|
|
57
90
|
|
|
58
91
|
return loader;
|
|
59
92
|
}
|
|
60
93
|
|
|
94
|
+
function mayContainText(response: Response): boolean {
|
|
95
|
+
const mimeType = getResourceMIMEType(response);
|
|
96
|
+
return Boolean(
|
|
97
|
+
mimeType &&
|
|
98
|
+
(mimeType.startsWith('text/') || mimeType === 'application/json' || mimeType.endsWith('+json'))
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
|
|
61
102
|
/**
|
|
62
103
|
* Find a loader that matches file extension and/or initial file content
|
|
63
104
|
* Search the loaders array argument for a loader that matches url extension or initial data
|
|
@@ -68,7 +109,7 @@ export async function selectLoader(
|
|
|
68
109
|
* @param context used internally, applications should not provide this parameter
|
|
69
110
|
*/
|
|
70
111
|
export function selectLoaderSync(
|
|
71
|
-
data:
|
|
112
|
+
data: DataType,
|
|
72
113
|
loaders: Loader[] | Loader = [],
|
|
73
114
|
options?: LoaderOptions,
|
|
74
115
|
context?: LoaderContext
|
|
@@ -77,6 +118,9 @@ export function selectLoaderSync(
|
|
|
77
118
|
return null;
|
|
78
119
|
}
|
|
79
120
|
|
|
121
|
+
const normalizedOptions = normalizeLoaderOptions(options || {});
|
|
122
|
+
normalizedOptions.core ||= {};
|
|
123
|
+
|
|
80
124
|
// eslint-disable-next-line complexity
|
|
81
125
|
// if only a single loader was provided (not as array), force its use
|
|
82
126
|
// TODO - Should this behavior be kept and documented?
|
|
@@ -92,17 +136,17 @@ export function selectLoaderSync(
|
|
|
92
136
|
candidateLoaders = candidateLoaders.concat(loaders);
|
|
93
137
|
}
|
|
94
138
|
// Then fall back to registered loaders
|
|
95
|
-
if (!
|
|
139
|
+
if (!normalizedOptions.core.ignoreRegisteredLoaders) {
|
|
96
140
|
candidateLoaders.push(...getRegisteredLoaders());
|
|
97
141
|
}
|
|
98
142
|
|
|
99
143
|
// TODO - remove support for legacy loaders
|
|
100
144
|
normalizeLoaders(candidateLoaders);
|
|
101
145
|
|
|
102
|
-
const loader = selectLoaderInternal(data, candidateLoaders,
|
|
146
|
+
const loader = selectLoaderInternal(data, candidateLoaders, normalizedOptions, context);
|
|
103
147
|
|
|
104
148
|
// no loader available
|
|
105
|
-
if (!loader && !
|
|
149
|
+
if (!loader && !normalizedOptions.core.nothrow) {
|
|
106
150
|
throw new Error(getNoValidLoaderMessage(data));
|
|
107
151
|
}
|
|
108
152
|
|
|
@@ -112,7 +156,7 @@ export function selectLoaderSync(
|
|
|
112
156
|
/** Implements loaders selection logic */
|
|
113
157
|
// eslint-disable-next-line complexity
|
|
114
158
|
function selectLoaderInternal(
|
|
115
|
-
data:
|
|
159
|
+
data: DataType,
|
|
116
160
|
loaders: Loader[],
|
|
117
161
|
options?: LoaderOptions,
|
|
118
162
|
context?: LoaderContext
|
|
@@ -126,9 +170,9 @@ function selectLoaderInternal(
|
|
|
126
170
|
let reason: string = '';
|
|
127
171
|
|
|
128
172
|
// if options.mimeType is supplied, it takes precedence
|
|
129
|
-
if (options?.mimeType) {
|
|
130
|
-
loader = findLoaderByMIMEType(loaders, options?.mimeType);
|
|
131
|
-
reason = `match forced by supplied MIME type ${options?.mimeType}`;
|
|
173
|
+
if (options?.core?.mimeType) {
|
|
174
|
+
loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
|
|
175
|
+
reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
|
|
132
176
|
}
|
|
133
177
|
|
|
134
178
|
// Look up loader by url
|
|
@@ -146,8 +190,8 @@ function selectLoaderInternal(
|
|
|
146
190
|
reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
|
|
147
191
|
|
|
148
192
|
// Look up loader by fallback mime type
|
|
149
|
-
if (options?.fallbackMimeType) {
|
|
150
|
-
loader = loader || findLoaderByMIMEType(loaders, options?.fallbackMimeType);
|
|
193
|
+
if (options?.core?.fallbackMimeType) {
|
|
194
|
+
loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
|
|
151
195
|
reason = reason || (loader ? `matched fallback MIME type ${type}` : '');
|
|
152
196
|
}
|
|
153
197
|
|
|
@@ -171,7 +215,7 @@ function validHTTPResponse(data: unknown): boolean {
|
|
|
171
215
|
}
|
|
172
216
|
|
|
173
217
|
/** Generate a helpful message to help explain why loader selection failed. */
|
|
174
|
-
function getNoValidLoaderMessage(data:
|
|
218
|
+
function getNoValidLoaderMessage(data: DataType): string {
|
|
175
219
|
const url = getResourceUrl(data);
|
|
176
220
|
const type = getResourceMIMEType(data);
|
|
177
221
|
|
|
@@ -264,23 +308,23 @@ function testDataAgainstText(data: string, loader: Loader): boolean {
|
|
|
264
308
|
return tests.some((test) => data.startsWith(test as string));
|
|
265
309
|
}
|
|
266
310
|
|
|
267
|
-
function testDataAgainstBinary(data:
|
|
311
|
+
function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
|
|
268
312
|
const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
|
|
269
313
|
return tests.some((test) => testBinary(data, byteOffset, loader, test));
|
|
270
314
|
}
|
|
271
315
|
|
|
272
316
|
function testBinary(
|
|
273
|
-
data:
|
|
317
|
+
data: ArrayBufferLike,
|
|
274
318
|
byteOffset: number,
|
|
275
319
|
loader: Loader,
|
|
276
320
|
test?: ArrayBuffer | string | ((b: ArrayBuffer) => boolean)
|
|
277
321
|
): boolean {
|
|
278
|
-
if (test
|
|
322
|
+
if (isArrayBufferLike(test)) {
|
|
279
323
|
return compareArrayBuffers(test, data, test.byteLength);
|
|
280
324
|
}
|
|
281
325
|
switch (typeof test) {
|
|
282
326
|
case 'function':
|
|
283
|
-
return test(data);
|
|
327
|
+
return test(ensureArrayBuffer(data));
|
|
284
328
|
|
|
285
329
|
case 'string':
|
|
286
330
|
// Magic bytes check: If `test` is a string, check if binary data starts with that strings
|
|
@@ -305,7 +349,7 @@ function getFirstCharacters(data: string | ArrayBuffer | TypedArray, length: num
|
|
|
305
349
|
return '';
|
|
306
350
|
}
|
|
307
351
|
|
|
308
|
-
function getMagicString(arrayBuffer:
|
|
352
|
+
function getMagicString(arrayBuffer: ArrayBufferLike, byteOffset: number, length: number): string {
|
|
309
353
|
if (arrayBuffer.byteLength < byteOffset + length) {
|
|
310
354
|
return '';
|
|
311
355
|
}
|