@loaders.gl/core 4.4.0-alpha.1 → 4.4.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/core-addons/write-file-browser.js +1 -0
- package/dist/core-addons/write-file-browser.js.map +1 -0
- package/dist/dist.dev.js +368 -146
- package/dist/dist.min.js +3 -3
- package/dist/index.cjs +339 -227
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -0
- package/dist/iterators/batch-iterators/timed-batch-iterator.js +1 -0
- package/dist/iterators/batch-iterators/timed-batch-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js +1 -0
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-blob-iterator.js +1 -0
- package/dist/iterators/make-iterator/make-blob-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-iterator.js +7 -3
- package/dist/iterators/make-iterator/make-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-stream-iterator.js +1 -0
- package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -0
- package/dist/iterators/make-iterator/make-string-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-string-iterator.js +3 -1
- package/dist/iterators/make-iterator/make-string-iterator.js.map +1 -0
- package/dist/iterators/make-stream/make-stream.js +1 -0
- package/dist/iterators/make-stream/make-stream.js.map +1 -0
- package/dist/lib/api/create-data-source.js +1 -0
- package/dist/lib/api/create-data-source.js.map +1 -0
- package/dist/lib/api/encode-table.d.ts.map +1 -1
- package/dist/lib/api/encode-table.js +3 -2
- package/dist/lib/api/encode-table.js.map +1 -0
- package/dist/lib/api/encode.d.ts.map +1 -1
- package/dist/lib/api/encode.js +3 -2
- package/dist/lib/api/encode.js.map +1 -0
- package/dist/lib/api/load-in-batches.js +1 -0
- package/dist/lib/api/load-in-batches.js.map +1 -0
- package/dist/lib/api/load.js +2 -1
- package/dist/lib/api/load.js.map +1 -0
- package/dist/lib/api/loader-options.js +1 -0
- package/dist/lib/api/loader-options.js.map +1 -0
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +7 -6
- package/dist/lib/api/parse-in-batches.js.map +1 -0
- package/dist/lib/api/parse-sync.d.ts.map +1 -1
- package/dist/lib/api/parse-sync.js +4 -3
- package/dist/lib/api/parse-sync.js.map +1 -0
- package/dist/lib/api/parse.d.ts.map +1 -1
- package/dist/lib/api/parse.js +8 -9
- package/dist/lib/api/parse.js.map +1 -0
- package/dist/lib/api/register-loaders.js +1 -0
- package/dist/lib/api/register-loaders.js.map +1 -0
- package/dist/lib/api/select-loader.d.ts +3 -3
- package/dist/lib/api/select-loader.d.ts.map +1 -1
- package/dist/lib/api/select-loader.js +20 -15
- package/dist/lib/api/select-loader.js.map +1 -0
- package/dist/lib/api/select-source.js +1 -0
- package/dist/lib/api/select-source.js.map +1 -0
- package/dist/lib/common.js +1 -0
- package/dist/lib/common.js.map +1 -0
- package/dist/lib/fetch/fetch-error-message.js +1 -0
- package/dist/lib/fetch/fetch-error-message.js.map +1 -0
- package/dist/lib/fetch/fetch-error.js +1 -0
- package/dist/lib/fetch/fetch-error.js.map +1 -0
- package/dist/lib/fetch/fetch-file.js +1 -0
- package/dist/lib/fetch/fetch-file.js.map +1 -0
- package/dist/lib/fetch/read-array-buffer.js +1 -0
- package/dist/lib/fetch/read-array-buffer.js.map +1 -0
- package/dist/lib/filesystems/browser-filesystem.js +1 -0
- package/dist/lib/filesystems/browser-filesystem.js.map +1 -0
- package/dist/lib/filesystems/read-array-buffer.js +1 -0
- package/dist/lib/filesystems/read-array-buffer.js.map +1 -0
- package/dist/lib/init.js +2 -1
- package/dist/lib/init.js.map +1 -0
- package/dist/lib/loader-utils/check-errors.js +1 -0
- package/dist/lib/loader-utils/check-errors.js.map +1 -0
- package/dist/lib/loader-utils/get-data.d.ts +18 -1
- package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-data.js +47 -38
- package/dist/lib/loader-utils/get-data.js.map +1 -0
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +7 -5
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -0
- package/dist/lib/loader-utils/loader-context.d.ts +2 -2
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +1 -0
- package/dist/lib/loader-utils/loader-context.js.map +1 -0
- package/dist/lib/loader-utils/loggers.js +1 -0
- package/dist/lib/loader-utils/loggers.js.map +1 -0
- package/dist/lib/loader-utils/normalize-loader.js +1 -0
- package/dist/lib/loader-utils/normalize-loader.js.map +1 -0
- package/dist/lib/loader-utils/option-defaults.d.ts +22 -0
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +50 -20
- package/dist/lib/loader-utils/option-defaults.js.map +1 -0
- package/dist/lib/loader-utils/option-utils.d.ts +9 -3
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +108 -13
- package/dist/lib/loader-utils/option-utils.js.map +1 -0
- package/dist/lib/progress/fetch-progress.js +1 -0
- package/dist/lib/progress/fetch-progress.js.map +1 -0
- package/dist/lib/utils/mime-type-utils.js +1 -0
- package/dist/lib/utils/mime-type-utils.js.map +1 -0
- package/dist/lib/utils/resource-utils.d.ts.map +1 -1
- package/dist/lib/utils/resource-utils.js +8 -10
- package/dist/lib/utils/resource-utils.js.map +1 -0
- package/dist/lib/utils/response-utils.js +2 -1
- package/dist/lib/utils/response-utils.js.map +1 -0
- package/dist/lib/utils/url-utils.js +1 -0
- package/dist/lib/utils/url-utils.js.map +1 -0
- package/dist/null-loader.d.ts +3 -3
- package/dist/null-loader.d.ts.map +1 -1
- package/dist/null-loader.js +2 -1
- package/dist/null-loader.js.map +1 -0
- package/dist/null-worker-node.js +5 -2
- package/dist/null-worker.js +5 -2
- package/dist/workers/null-worker-node.js +1 -0
- package/dist/workers/null-worker-node.js.map +1 -0
- package/dist/workers/null-worker.js +1 -0
- package/dist/workers/null-worker.js.map +1 -0
- package/package.json +12 -6
- package/src/index.ts +1 -1
- package/src/iterators/make-iterator/make-iterator.ts +8 -5
- package/src/iterators/make-iterator/make-string-iterator.ts +2 -1
- package/src/lib/api/encode-table.ts +2 -1
- package/src/lib/api/encode.ts +8 -2
- package/src/lib/api/load.ts +1 -1
- package/src/lib/api/parse-in-batches.ts +26 -19
- package/src/lib/api/parse-sync.ts +6 -5
- package/src/lib/api/parse.ts +15 -11
- package/src/lib/api/select-loader.ts +42 -24
- package/src/lib/loader-utils/get-data.ts +62 -48
- package/src/lib/loader-utils/get-fetch-function.ts +6 -5
- package/src/lib/loader-utils/loader-context.ts +2 -2
- package/src/lib/loader-utils/option-defaults.ts +54 -22
- package/src/lib/loader-utils/option-utils.ts +125 -19
- package/src/lib/utils/resource-utils.ts +9 -12
- package/src/lib/utils/response-utils.ts +2 -2
- package/src/null-loader.ts +6 -3
- package/dist/javascript-utils/is-type.d.ts +0 -21
- package/dist/javascript-utils/is-type.d.ts.map +0 -1
- package/dist/javascript-utils/is-type.js +0 -25
- package/src/javascript-utils/is-type.ts +0 -51
|
@@ -9,21 +9,30 @@ import type {
|
|
|
9
9
|
Loader,
|
|
10
10
|
LoaderOptions
|
|
11
11
|
} from '@loaders.gl/loader-utils';
|
|
12
|
-
import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
|
|
13
12
|
import {
|
|
13
|
+
concatenateArrayBuffersAsync,
|
|
14
|
+
isPromise,
|
|
14
15
|
isResponse,
|
|
15
16
|
isReadableStream,
|
|
16
17
|
isAsyncIterable,
|
|
17
18
|
isIterable,
|
|
18
19
|
isIterator,
|
|
19
20
|
isBlob,
|
|
20
|
-
isBuffer
|
|
21
|
-
|
|
21
|
+
isBuffer,
|
|
22
|
+
isArrayBufferLike,
|
|
23
|
+
toArrayBuffer,
|
|
24
|
+
toArrayBufferView
|
|
25
|
+
} from '@loaders.gl/loader-utils';
|
|
22
26
|
import {makeIterator} from '../../iterators/make-iterator/make-iterator';
|
|
23
27
|
import {checkResponse, makeResponse} from '../utils/response-utils';
|
|
24
28
|
|
|
25
29
|
const ERR_DATA = 'Cannot convert supplied data type';
|
|
26
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Returns an {@link ArrayBuffer} or string from the provided data synchronously.
|
|
33
|
+
* Supports `ArrayBuffer`, `ArrayBufferView`, and `ArrayBufferLike` (e.g. `SharedArrayBuffer`)
|
|
34
|
+
* while preserving typed array view offsets.
|
|
35
|
+
*/
|
|
27
36
|
// eslint-disable-next-line complexity
|
|
28
37
|
export function getArrayBufferOrStringFromDataSync(
|
|
29
38
|
data: SyncDataType,
|
|
@@ -35,52 +44,32 @@ export function getArrayBufferOrStringFromDataSync(
|
|
|
35
44
|
}
|
|
36
45
|
|
|
37
46
|
if (isBuffer(data)) {
|
|
38
|
-
// @ts-ignore
|
|
39
47
|
data = data.buffer;
|
|
40
48
|
}
|
|
41
49
|
|
|
42
|
-
if (data
|
|
43
|
-
const
|
|
44
|
-
if (loader.text && !loader.binary) {
|
|
45
|
-
const textDecoder = new TextDecoder('utf8');
|
|
46
|
-
return textDecoder.decode(arrayBuffer);
|
|
47
|
-
}
|
|
48
|
-
return arrayBuffer;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
// We may need to handle offsets
|
|
52
|
-
if (ArrayBuffer.isView(data)) {
|
|
53
|
-
// TextDecoder is invoked on typed arrays and will handle offsets
|
|
50
|
+
if (isArrayBufferLike(data)) {
|
|
51
|
+
const bufferSource = toArrayBufferView(data);
|
|
54
52
|
if (loader.text && !loader.binary) {
|
|
55
53
|
const textDecoder = new TextDecoder('utf8');
|
|
56
|
-
return textDecoder.decode(
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
let arrayBuffer = data.buffer;
|
|
60
|
-
|
|
61
|
-
// Since we are returning the underlying arrayBuffer, we must create a new copy
|
|
62
|
-
// if this typed array / Buffer is a partial view into the ArryayBuffer
|
|
63
|
-
// TODO - this is a potentially unnecessary copy
|
|
64
|
-
const byteLength = data.byteLength || data.length;
|
|
65
|
-
if (data.byteOffset !== 0 || byteLength !== arrayBuffer.byteLength) {
|
|
66
|
-
// console.warn(`loaders.gl copying arraybuffer of length ${byteLength}`);
|
|
67
|
-
arrayBuffer = arrayBuffer.slice(data.byteOffset, data.byteOffset + byteLength);
|
|
54
|
+
return textDecoder.decode(bufferSource);
|
|
68
55
|
}
|
|
69
|
-
return
|
|
56
|
+
return toArrayBuffer(bufferSource);
|
|
70
57
|
}
|
|
71
58
|
|
|
72
59
|
throw new Error(ERR_DATA);
|
|
73
60
|
}
|
|
74
61
|
|
|
75
|
-
|
|
62
|
+
/**
|
|
63
|
+
* Resolves the provided data into an {@link ArrayBuffer} or string asynchronously.
|
|
64
|
+
* Accepts the full {@link DataType} surface including responses and async iterables.
|
|
65
|
+
*/
|
|
76
66
|
export async function getArrayBufferOrStringFromData(
|
|
77
67
|
data: DataType,
|
|
78
68
|
loader: Loader,
|
|
79
69
|
options: LoaderOptions
|
|
80
70
|
): Promise<ArrayBuffer | string> {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
return getArrayBufferOrStringFromDataSync(data as string | ArrayBuffer, loader, options);
|
|
71
|
+
if (typeof data === 'string' || isArrayBufferLike(data)) {
|
|
72
|
+
return getArrayBufferOrStringFromDataSync(data as SyncDataType, loader, options);
|
|
84
73
|
}
|
|
85
74
|
|
|
86
75
|
// Blobs and files are FileReader compatible
|
|
@@ -89,9 +78,8 @@ export async function getArrayBufferOrStringFromData(
|
|
|
89
78
|
}
|
|
90
79
|
|
|
91
80
|
if (isResponse(data)) {
|
|
92
|
-
|
|
93
|
-
await
|
|
94
|
-
return loader.binary ? await response.arrayBuffer() : await response.text();
|
|
81
|
+
await checkResponse(data);
|
|
82
|
+
return loader.binary ? await data.arrayBuffer() : await data.text();
|
|
95
83
|
}
|
|
96
84
|
|
|
97
85
|
if (isReadableStream(data)) {
|
|
@@ -101,29 +89,41 @@ export async function getArrayBufferOrStringFromData(
|
|
|
101
89
|
|
|
102
90
|
if (isIterable(data) || isAsyncIterable(data)) {
|
|
103
91
|
// Assume arrayBuffer iterator - attempt to concatenate
|
|
104
|
-
return concatenateArrayBuffersAsync(data as AsyncIterable<
|
|
92
|
+
return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);
|
|
105
93
|
}
|
|
106
94
|
|
|
107
95
|
throw new Error(ERR_DATA);
|
|
108
96
|
}
|
|
109
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Normalizes batchable inputs into async iterables for batch parsing flows.
|
|
100
|
+
* Supports synchronous iterables, async iterables, fetch responses, readable streams, and
|
|
101
|
+
* single binary chunks (including typed array views and `ArrayBufferLike` values).
|
|
102
|
+
*/
|
|
110
103
|
export async function getAsyncIterableFromData(
|
|
111
104
|
data: BatchableDataType,
|
|
112
105
|
options: LoaderOptions
|
|
113
|
-
): Promise<
|
|
106
|
+
): Promise<
|
|
107
|
+
AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>
|
|
108
|
+
> {
|
|
109
|
+
if (isPromise(data)) {
|
|
110
|
+
data = await data;
|
|
111
|
+
}
|
|
112
|
+
|
|
114
113
|
if (isIterator(data)) {
|
|
115
114
|
return data as AsyncIterable<ArrayBuffer>;
|
|
116
115
|
}
|
|
117
116
|
|
|
118
117
|
if (isResponse(data)) {
|
|
119
|
-
const response = data as Response;
|
|
120
118
|
// Note Since this function is not async, we currently can't load error message, just status
|
|
121
|
-
await checkResponse(
|
|
119
|
+
await checkResponse(data);
|
|
122
120
|
// TODO - bug in polyfill, body can be a Promise under Node.js
|
|
123
121
|
// eslint-disable-next-line @typescript-eslint/await-thenable
|
|
124
|
-
const body = await
|
|
125
|
-
|
|
126
|
-
|
|
122
|
+
const body = await data.body;
|
|
123
|
+
if (!body) {
|
|
124
|
+
throw new Error(ERR_DATA);
|
|
125
|
+
}
|
|
126
|
+
return makeIterator(body, options as any);
|
|
127
127
|
}
|
|
128
128
|
|
|
129
129
|
if (isBlob(data) || isReadableStream(data)) {
|
|
@@ -131,38 +131,52 @@ export async function getAsyncIterableFromData(
|
|
|
131
131
|
}
|
|
132
132
|
|
|
133
133
|
if (isAsyncIterable(data)) {
|
|
134
|
-
return data as AsyncIterable<
|
|
134
|
+
return data as AsyncIterable<ArrayBufferLike | ArrayBufferView>;
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
+
if (isIterable(data)) {
|
|
138
|
+
return data as Iterable<ArrayBufferLike | ArrayBufferView>;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// @ts-expect-error TODO - fix type mess
|
|
137
142
|
return getIterableFromData(data);
|
|
138
143
|
}
|
|
139
144
|
|
|
145
|
+
/**
|
|
146
|
+
* Returns a readable stream for streaming loader inputs when available.
|
|
147
|
+
*/
|
|
140
148
|
export async function getReadableStream(data: BatchableDataType): Promise<ReadableStream> {
|
|
141
149
|
if (isReadableStream(data)) {
|
|
142
150
|
return data as ReadableStream;
|
|
143
151
|
}
|
|
144
152
|
if (isResponse(data)) {
|
|
145
153
|
// @ts-ignore
|
|
154
|
+
if (!data.body) {
|
|
155
|
+
throw new Error(ERR_DATA);
|
|
156
|
+
}
|
|
146
157
|
return data.body;
|
|
147
158
|
}
|
|
148
159
|
const response = await makeResponse(data);
|
|
149
160
|
// @ts-ignore
|
|
161
|
+
if (!response.body) {
|
|
162
|
+
throw new Error(ERR_DATA);
|
|
163
|
+
}
|
|
150
164
|
return response.body;
|
|
151
165
|
}
|
|
152
166
|
|
|
153
167
|
// HELPERS
|
|
154
168
|
|
|
155
|
-
function getIterableFromData(data) {
|
|
169
|
+
function getIterableFromData(data: string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView) {
|
|
156
170
|
// generate an iterator that emits a single chunk
|
|
157
171
|
if (ArrayBuffer.isView(data)) {
|
|
158
172
|
return (function* oneChunk() {
|
|
159
|
-
yield data
|
|
173
|
+
yield toArrayBuffer(data);
|
|
160
174
|
})();
|
|
161
175
|
}
|
|
162
176
|
|
|
163
|
-
if (data
|
|
177
|
+
if (isArrayBufferLike(data)) {
|
|
164
178
|
return (function* oneChunk() {
|
|
165
|
-
yield data;
|
|
179
|
+
yield toArrayBuffer(data);
|
|
166
180
|
})();
|
|
167
181
|
}
|
|
168
182
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import type {LoaderContext, LoaderOptions, FetchLike} from '@loaders.gl/loader-utils';
|
|
6
|
-
import {isObject} from '
|
|
6
|
+
import {isObject} from '@loaders.gl/loader-utils';
|
|
7
7
|
import {fetchFile} from '../fetch/fetch-file';
|
|
8
8
|
import {getGlobalLoaderOptions} from './option-utils';
|
|
9
9
|
|
|
@@ -19,15 +19,16 @@ export function getFetchFunction(
|
|
|
19
19
|
const globalOptions = getGlobalLoaderOptions();
|
|
20
20
|
|
|
21
21
|
const loaderOptions = options || globalOptions;
|
|
22
|
+
const fetchOption = loaderOptions.fetch ?? loaderOptions.core?.fetch;
|
|
22
23
|
|
|
23
24
|
// options.fetch can be a function
|
|
24
|
-
if (typeof
|
|
25
|
-
return
|
|
25
|
+
if (typeof fetchOption === 'function') {
|
|
26
|
+
return fetchOption;
|
|
26
27
|
}
|
|
27
28
|
|
|
28
29
|
// options.fetch can be an options object
|
|
29
|
-
if (isObject(
|
|
30
|
-
return (url) => fetchFile(url,
|
|
30
|
+
if (isObject(fetchOption)) {
|
|
31
|
+
return (url) => fetchFile(url, fetchOption as RequestInit);
|
|
31
32
|
}
|
|
32
33
|
|
|
33
34
|
// else refer to context (from parent loader) if available
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import type {Loader,
|
|
5
|
+
import type {Loader, LoaderContext, StrictLoaderOptions} from '@loaders.gl/loader-utils';
|
|
6
6
|
import {getFetchFunction} from './get-fetch-function';
|
|
7
7
|
import {extractQueryString, stripQueryString} from '../utils/url-utils';
|
|
8
8
|
import {path} from '@loaders.gl/loader-utils';
|
|
@@ -21,7 +21,7 @@ type LoaderContextProps = Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderCont
|
|
|
21
21
|
*/
|
|
22
22
|
export function getLoaderContext(
|
|
23
23
|
context: LoaderContextProps,
|
|
24
|
-
options:
|
|
24
|
+
options: StrictLoaderOptions,
|
|
25
25
|
parentContext: LoaderContext | null
|
|
26
26
|
): LoaderContext {
|
|
27
27
|
// For recursive calls, we already have a context
|
|
@@ -7,34 +7,66 @@ import {isBrowser} from '@loaders.gl/loader-utils';
|
|
|
7
7
|
import {ConsoleLog} from './loggers';
|
|
8
8
|
|
|
9
9
|
export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
10
|
+
core: {
|
|
11
|
+
baseUri: undefined,
|
|
12
|
+
// baseUri
|
|
13
|
+
fetch: null,
|
|
14
|
+
mimeType: undefined,
|
|
15
|
+
fallbackMimeType: undefined,
|
|
16
|
+
ignoreRegisteredLoaders: undefined,
|
|
17
|
+
nothrow: false,
|
|
18
|
+
log: new ConsoleLog(), // A probe.gl compatible (`log.log()()` syntax) that just logs to console
|
|
19
|
+
useLocalLibraries: false,
|
|
20
|
+
|
|
21
|
+
CDN: 'https://unpkg.com/@loaders.gl',
|
|
22
|
+
worker: true, // By default, use worker if provided by loader.
|
|
23
|
+
maxConcurrency: 3, // How many worker instances should be created for each loader.
|
|
24
|
+
maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
|
|
25
|
+
reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
|
|
26
|
+
_nodeWorkers: false, // By default do not support node workers
|
|
27
|
+
_workerType: '', // 'test' to use locally generated workers
|
|
28
|
+
|
|
29
|
+
limit: 0,
|
|
30
|
+
_limitMB: 0,
|
|
31
|
+
batchSize: 'auto',
|
|
32
|
+
batchDebounceMs: 0,
|
|
33
|
+
metadata: false, // TODO - currently only implemented for parseInBatches, adds initial metadata batch,
|
|
34
|
+
transforms: []
|
|
35
|
+
}
|
|
31
36
|
};
|
|
32
37
|
|
|
33
38
|
export const REMOVED_LOADER_OPTIONS = {
|
|
39
|
+
// baseUri
|
|
40
|
+
baseUri: 'core.baseUri',
|
|
41
|
+
fetch: 'core.fetch',
|
|
42
|
+
mimeType: 'core.mimeType',
|
|
43
|
+
fallbackMimeType: 'core.fallbackMimeType',
|
|
44
|
+
ignoreRegisteredLoaders: 'core.ignoreRegisteredLoaders',
|
|
45
|
+
nothrow: 'core.nothrow',
|
|
46
|
+
log: 'core.log',
|
|
47
|
+
useLocalLibraries: 'core.useLocalLibraries',
|
|
48
|
+
|
|
49
|
+
CDN: 'core.CDN',
|
|
50
|
+
worker: 'core.worker',
|
|
51
|
+
maxConcurrency: 'core.maxConcurrency',
|
|
52
|
+
maxMobileConcurrency: 'core.maxMobileConcurrency',
|
|
53
|
+
reuseWorkers: 'core.reuseWorkers',
|
|
54
|
+
_nodeWorkers: 'core.nodeWorkers',
|
|
55
|
+
_workerType: 'core._workerType',
|
|
56
|
+
_worker: 'core._workerType',
|
|
57
|
+
|
|
58
|
+
limit: 'core.limit',
|
|
59
|
+
_limitMB: 'core._limitMB',
|
|
60
|
+
batchSize: 'core.batchSize',
|
|
61
|
+
batchDebounceMs: 'core.batchDebounceMs',
|
|
62
|
+
metadata: 'core.metadata',
|
|
63
|
+
transforms: 'core.transforms',
|
|
64
|
+
|
|
65
|
+
// Older deprecations
|
|
34
66
|
throws: 'nothrow',
|
|
35
67
|
dataType: '(no longer used)',
|
|
36
68
|
uri: 'baseUri',
|
|
37
|
-
// Warn if fetch options are used on
|
|
69
|
+
// Warn if fetch options are used on toplevel
|
|
38
70
|
method: 'fetch.method',
|
|
39
71
|
headers: 'fetch.headers',
|
|
40
72
|
body: 'fetch.body',
|
|
@@ -2,11 +2,41 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
5
|
+
import {
|
|
6
|
+
Loader,
|
|
7
|
+
LoaderOptions,
|
|
8
|
+
registerJSModules,
|
|
9
|
+
isPureObject,
|
|
10
|
+
isObject,
|
|
11
|
+
StrictLoaderOptions
|
|
12
|
+
} from '@loaders.gl/loader-utils';
|
|
7
13
|
import {probeLog, NullLog} from './loggers';
|
|
8
14
|
import {DEFAULT_LOADER_OPTIONS, REMOVED_LOADER_OPTIONS} from './option-defaults';
|
|
9
15
|
|
|
16
|
+
const CORE_LOADER_OPTION_KEYS = [
|
|
17
|
+
'baseUri',
|
|
18
|
+
'fetch',
|
|
19
|
+
'mimeType',
|
|
20
|
+
'fallbackMimeType',
|
|
21
|
+
'ignoreRegisteredLoaders',
|
|
22
|
+
'nothrow',
|
|
23
|
+
'log',
|
|
24
|
+
'useLocalLibraries',
|
|
25
|
+
'CDN',
|
|
26
|
+
'worker',
|
|
27
|
+
'maxConcurrency',
|
|
28
|
+
'maxMobileConcurrency',
|
|
29
|
+
'reuseWorkers',
|
|
30
|
+
'_nodeWorkers',
|
|
31
|
+
'_workerType',
|
|
32
|
+
'limit',
|
|
33
|
+
'_limitMB',
|
|
34
|
+
'batchSize',
|
|
35
|
+
'batchDebounceMs',
|
|
36
|
+
'metadata',
|
|
37
|
+
'transforms'
|
|
38
|
+
] as const;
|
|
39
|
+
|
|
10
40
|
/**
|
|
11
41
|
* Global state for loaders.gl. Stored on `globalThis.loaders._state`
|
|
12
42
|
*/
|
|
@@ -37,11 +67,14 @@ export function getGlobalLoaderState(): GlobalLoaderState {
|
|
|
37
67
|
* NOTE: This use case is not reliable but can help when testing new versions of loaders.gl with existing frameworks
|
|
38
68
|
* @returns global loader options merged with default loader options
|
|
39
69
|
*/
|
|
40
|
-
export function getGlobalLoaderOptions():
|
|
70
|
+
export function getGlobalLoaderOptions(): StrictLoaderOptions {
|
|
41
71
|
const state = getGlobalLoaderState();
|
|
42
72
|
// Ensure all default loader options from this library are mentioned
|
|
43
|
-
state.globalOptions = state.globalOptions || {
|
|
44
|
-
|
|
73
|
+
state.globalOptions = state.globalOptions || {
|
|
74
|
+
...DEFAULT_LOADER_OPTIONS,
|
|
75
|
+
core: {...DEFAULT_LOADER_OPTIONS.core}
|
|
76
|
+
};
|
|
77
|
+
return normalizeLoaderOptions(state.globalOptions);
|
|
45
78
|
}
|
|
46
79
|
|
|
47
80
|
/**
|
|
@@ -69,12 +102,31 @@ export function normalizeOptions(
|
|
|
69
102
|
loader: Loader,
|
|
70
103
|
loaders?: Loader[],
|
|
71
104
|
url?: string
|
|
72
|
-
):
|
|
105
|
+
): StrictLoaderOptions {
|
|
73
106
|
loaders = loaders || [];
|
|
74
107
|
loaders = Array.isArray(loaders) ? loaders : [loaders];
|
|
75
108
|
|
|
76
109
|
validateOptions(options, loaders);
|
|
77
|
-
return normalizeOptionsInternal(loader, options, url);
|
|
110
|
+
return normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Returns a copy of the provided options with deprecated top-level core fields moved into `core`
|
|
115
|
+
* and removed from the top level. This keeps global options from leaking deprecated aliases into
|
|
116
|
+
* loader-specific option maps during normalization.
|
|
117
|
+
*/
|
|
118
|
+
export function normalizeLoaderOptions(options: LoaderOptions): StrictLoaderOptions {
|
|
119
|
+
const normalized = cloneLoaderOptions(options);
|
|
120
|
+
moveDeprecatedTopLevelOptionsToCore(normalized);
|
|
121
|
+
for (const key of CORE_LOADER_OPTION_KEYS) {
|
|
122
|
+
if (normalized.core && normalized.core[key] !== undefined) {
|
|
123
|
+
delete (normalized as Record<string, unknown>)[key];
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (normalized.core && normalized.core._workerType !== undefined) {
|
|
127
|
+
delete (normalized as any)._worker;
|
|
128
|
+
}
|
|
129
|
+
return normalized as StrictLoaderOptions;
|
|
78
130
|
}
|
|
79
131
|
|
|
80
132
|
// VALIDATE OPTIONS
|
|
@@ -89,10 +141,8 @@ function validateOptions(options: LoaderOptions, loaders: Loader[]): void {
|
|
|
89
141
|
validateOptionsObject(options, null, DEFAULT_LOADER_OPTIONS, REMOVED_LOADER_OPTIONS, loaders);
|
|
90
142
|
for (const loader of loaders) {
|
|
91
143
|
// Get the scoped, loader specific options from the user supplied options
|
|
92
|
-
const idOptions: Record<string, unknown> =
|
|
93
|
-
string,
|
|
94
|
-
unknown
|
|
95
|
-
>;
|
|
144
|
+
const idOptions: Record<string, unknown> =
|
|
145
|
+
((options && options[loader.id]) as Record<string, unknown>) || {};
|
|
96
146
|
|
|
97
147
|
// Get scoped, loader specific default and deprecated options from the selected loader
|
|
98
148
|
const loaderOptions = (loader.options && loader.options[loader.id]) || {};
|
|
@@ -165,16 +215,23 @@ function normalizeOptionsInternal(
|
|
|
165
215
|
const loaderDefaultOptions = loader.options || {};
|
|
166
216
|
|
|
167
217
|
const mergedOptions = {...loaderDefaultOptions};
|
|
168
|
-
|
|
169
|
-
|
|
218
|
+
if (loaderDefaultOptions.core) {
|
|
219
|
+
mergedOptions.core = {...loaderDefaultOptions.core};
|
|
220
|
+
}
|
|
221
|
+
moveDeprecatedTopLevelOptionsToCore(mergedOptions);
|
|
170
222
|
|
|
171
223
|
// LOGGING: options.log can be set to `null` to defeat logging
|
|
172
|
-
if (mergedOptions.log === null) {
|
|
173
|
-
mergedOptions.
|
|
224
|
+
if (mergedOptions.core?.log === null) {
|
|
225
|
+
mergedOptions.core = {...mergedOptions.core, log: new NullLog()};
|
|
174
226
|
}
|
|
175
227
|
|
|
176
|
-
mergeNestedFields(mergedOptions, getGlobalLoaderOptions());
|
|
177
|
-
|
|
228
|
+
mergeNestedFields(mergedOptions, normalizeLoaderOptions(getGlobalLoaderOptions()));
|
|
229
|
+
|
|
230
|
+
const userOptions = normalizeLoaderOptions(options);
|
|
231
|
+
mergeNestedFields(mergedOptions, userOptions);
|
|
232
|
+
|
|
233
|
+
addUrlOptions(mergedOptions, url);
|
|
234
|
+
addDeprecatedTopLevelOptions(mergedOptions);
|
|
178
235
|
|
|
179
236
|
return mergedOptions;
|
|
180
237
|
}
|
|
@@ -208,7 +265,56 @@ function mergeNestedFields(mergedOptions: LoaderOptions, options: LoaderOptions)
|
|
|
208
265
|
* TODO - should these be injected on context instead of options?
|
|
209
266
|
*/
|
|
210
267
|
function addUrlOptions(options: LoaderOptions, url?: string): void {
|
|
211
|
-
if (url
|
|
212
|
-
|
|
268
|
+
if (!url) {
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
const hasTopLevelBaseUri = options.baseUri !== undefined;
|
|
272
|
+
const hasCoreBaseUri = options.core?.baseUri !== undefined;
|
|
273
|
+
if (!hasTopLevelBaseUri && !hasCoreBaseUri) {
|
|
274
|
+
options.core ||= {};
|
|
275
|
+
options.core.baseUri = url;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function cloneLoaderOptions(options: LoaderOptions): LoaderOptions {
|
|
280
|
+
const clonedOptions = {...options};
|
|
281
|
+
if (options.core) {
|
|
282
|
+
clonedOptions.core = {...options.core};
|
|
283
|
+
}
|
|
284
|
+
return clonedOptions;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function moveDeprecatedTopLevelOptionsToCore(options: LoaderOptions): void {
|
|
288
|
+
for (const key of CORE_LOADER_OPTION_KEYS) {
|
|
289
|
+
if ((options as Record<string, unknown>)[key] !== undefined) {
|
|
290
|
+
const coreOptions = (options.core = options.core || {});
|
|
291
|
+
const coreRecord = coreOptions as Record<string, unknown>;
|
|
292
|
+
// Treat deprecated top-level core options as aliases to `options.core`, but never override an explicitly
|
|
293
|
+
// provided `options.core` value.
|
|
294
|
+
if (coreRecord[key] === undefined) {
|
|
295
|
+
coreRecord[key] = (options as Record<string, unknown>)[key];
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Support the older internal `_worker` alias (used by some tests and integrations) for `_workerType`.
|
|
301
|
+
const workerTypeAlias = (options as any)._worker;
|
|
302
|
+
if (workerTypeAlias !== undefined) {
|
|
303
|
+
options.core ||= {};
|
|
304
|
+
if (options.core._workerType === undefined) {
|
|
305
|
+
options.core._workerType = workerTypeAlias;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function addDeprecatedTopLevelOptions(options: LoaderOptions): void {
|
|
311
|
+
const coreOptions = options.core as Record<string, unknown> | undefined;
|
|
312
|
+
if (!coreOptions) {
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
for (const key of CORE_LOADER_OPTION_KEYS) {
|
|
316
|
+
if (coreOptions[key] !== undefined) {
|
|
317
|
+
(options as Record<string, unknown>)[key] = coreOptions[key];
|
|
318
|
+
}
|
|
213
319
|
}
|
|
214
320
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {isResponse, isBlob} from '
|
|
5
|
+
import {isResponse, isBlob} from '@loaders.gl/loader-utils';
|
|
6
6
|
import {parseMIMEType, parseMIMETypeFromURL} from './mime-type-utils';
|
|
7
7
|
import {stripQueryString} from './url-utils';
|
|
8
8
|
|
|
@@ -22,16 +22,15 @@ export type Resource = Response | Blob | string;
|
|
|
22
22
|
export function getResourceUrl(resource: unknown): string {
|
|
23
23
|
// If resource is a `Response`, it contains the information directly as a field
|
|
24
24
|
if (isResponse(resource)) {
|
|
25
|
-
|
|
26
|
-
return response.url;
|
|
25
|
+
return resource.url;
|
|
27
26
|
}
|
|
28
27
|
|
|
29
28
|
// If the resource is a Blob or a File (subclass of Blob)
|
|
30
29
|
if (isBlob(resource)) {
|
|
31
|
-
const blob = resource as Blob;
|
|
32
30
|
// File objects have a "name" property. Blob objects don't have any
|
|
33
31
|
// url (name) information
|
|
34
|
-
|
|
32
|
+
const fileName = 'name' in resource ? (resource as File).name : '';
|
|
33
|
+
return fileName || '';
|
|
35
34
|
}
|
|
36
35
|
|
|
37
36
|
if (typeof resource === 'string') {
|
|
@@ -52,16 +51,14 @@ export function getResourceUrl(resource: unknown): string {
|
|
|
52
51
|
export function getResourceMIMEType(resource: unknown): string {
|
|
53
52
|
// If resource is a response, it contains the information directly
|
|
54
53
|
if (isResponse(resource)) {
|
|
55
|
-
const
|
|
56
|
-
const
|
|
57
|
-
const noQueryUrl = stripQueryString(response.url);
|
|
54
|
+
const contentTypeHeader = resource.headers.get('content-type') || '';
|
|
55
|
+
const noQueryUrl = stripQueryString(resource.url);
|
|
58
56
|
return parseMIMEType(contentTypeHeader) || parseMIMETypeFromURL(noQueryUrl);
|
|
59
57
|
}
|
|
60
58
|
|
|
61
59
|
// If the resource is a Blob or a File (subclass of Blob)
|
|
62
60
|
if (isBlob(resource)) {
|
|
63
|
-
|
|
64
|
-
return blob.type || '';
|
|
61
|
+
return resource.type || '';
|
|
65
62
|
}
|
|
66
63
|
|
|
67
64
|
if (typeof resource === 'string') {
|
|
@@ -81,11 +78,11 @@ export function getResourceMIMEType(resource: unknown): string {
|
|
|
81
78
|
*/
|
|
82
79
|
export function getResourceContentLength(resource: unknown): number {
|
|
83
80
|
if (isResponse(resource)) {
|
|
84
|
-
const response = resource
|
|
81
|
+
const response = resource;
|
|
85
82
|
return response.headers['content-length'] || -1;
|
|
86
83
|
}
|
|
87
84
|
if (isBlob(resource)) {
|
|
88
|
-
const blob = resource
|
|
85
|
+
const blob = resource;
|
|
89
86
|
return blob.size;
|
|
90
87
|
}
|
|
91
88
|
if (typeof resource === 'string') {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {isResponse} from '
|
|
5
|
+
import {isResponse} from '@loaders.gl/loader-utils';
|
|
6
6
|
import {FetchError} from '../fetch/fetch-error';
|
|
7
7
|
import {getResourceContentLength, getResourceUrl, getResourceMIMEType} from './resource-utils';
|
|
8
8
|
import {shortenUrlForDisplay} from './url-utils';
|
|
@@ -15,7 +15,7 @@ import {shortenUrlForDisplay} from './url-utils';
|
|
|
15
15
|
*/
|
|
16
16
|
export async function makeResponse(resource: unknown): Promise<Response> {
|
|
17
17
|
if (isResponse(resource)) {
|
|
18
|
-
return resource
|
|
18
|
+
return resource;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
// Add content-length header if possible
|
package/src/null-loader.ts
CHANGED
|
@@ -45,8 +45,11 @@ export const NullLoader = {
|
|
|
45
45
|
version: VERSION,
|
|
46
46
|
mimeTypes: ['application/x.empty'],
|
|
47
47
|
extensions: ['null'],
|
|
48
|
-
parse: async (
|
|
49
|
-
|
|
48
|
+
parse: async (
|
|
49
|
+
arrayBuffer: ArrayBufferLike | ArrayBufferView,
|
|
50
|
+
options?: NullLoaderOptions,
|
|
51
|
+
context?: LoaderContext
|
|
52
|
+
) => parseSync(arrayBuffer, options || {}, context),
|
|
50
53
|
parseSync,
|
|
51
54
|
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
52
55
|
for await (const batch of asyncIterator) {
|
|
@@ -64,7 +67,7 @@ export const NullLoader = {
|
|
|
64
67
|
* web worker. The `context` parameter is stripped using JSON.stringify & parse.
|
|
65
68
|
*/
|
|
66
69
|
function parseSync(
|
|
67
|
-
arrayBuffer:
|
|
70
|
+
arrayBuffer: ArrayBufferLike | ArrayBufferView,
|
|
68
71
|
options?: NullLoaderOptions,
|
|
69
72
|
context?: LoaderContext
|
|
70
73
|
): null {
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import type { Readable } from 'stream';
|
|
2
|
-
/** A DOM or Node readable stream */
|
|
3
|
-
export type ReadableStreamType = ReadableStream | Readable;
|
|
4
|
-
export declare const isObject: (x: unknown) => boolean;
|
|
5
|
-
export declare const isPureObject: (x: any) => boolean;
|
|
6
|
-
export declare const isPromise: (x: any) => boolean;
|
|
7
|
-
export declare const isIterable: (x: any) => boolean;
|
|
8
|
-
export declare const isAsyncIterable: (x: any) => boolean;
|
|
9
|
-
export declare const isIterator: (x: any) => boolean;
|
|
10
|
-
export declare const isResponse: (x: any) => boolean;
|
|
11
|
-
export declare const isFile: (x: unknown) => boolean;
|
|
12
|
-
export declare const isBlob: (x: unknown) => boolean;
|
|
13
|
-
/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
|
|
14
|
-
export declare const isBuffer: (x: any) => boolean;
|
|
15
|
-
export declare const isWritableDOMStream: (x: any) => boolean;
|
|
16
|
-
export declare const isReadableDOMStream: (x: any) => boolean;
|
|
17
|
-
export declare const isWritableNodeStream: (x: any) => boolean;
|
|
18
|
-
export declare const isReadableNodeStream: (x: any) => boolean;
|
|
19
|
-
export declare const isReadableStream: (x: unknown) => boolean;
|
|
20
|
-
export declare const isWritableStream: (x: unknown) => boolean;
|
|
21
|
-
//# sourceMappingURL=is-type.d.ts.map
|