@loaders.gl/core 5.0.0-alpha.13 → 5.0.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.dev.js +190 -63
- package/dist/dist.min.js +3 -3
- package/dist/index.cjs +205 -72
- package/dist/index.cjs.map +2 -2
- package/dist/lib/api/parse-file.js +2 -0
- package/dist/lib/api/parse-file.js.map +1 -1
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +5 -4
- package/dist/lib/api/parse-in-batches.js.map +1 -1
- package/dist/lib/api/parse.js +4 -2
- package/dist/lib/api/parse.js.map +1 -1
- package/dist/lib/init.js +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +3 -1
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +4 -0
- package/dist/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +1 -0
- package/dist/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +1 -1
- package/dist/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/loader-utils/index.cjs +231 -92
- package/dist/loader-utils/index.cjs.map +2 -2
- package/dist/loader-utils/index.d.ts +1 -1
- package/dist/loader-utils/index.d.ts.map +1 -1
- package/dist/loader-utils/index.js +1 -1
- package/dist/loader-utils/index.js.map +1 -1
- package/dist/loader-utils/json-loader.d.ts +1 -1
- package/dist/loader-utils/json-loader.js +2 -2
- package/dist/loader-utils/lib/log-utils/log.js +1 -1
- package/dist/loader-utils/lib/request-utils/request-credentials.d.ts +3 -1
- package/dist/loader-utils/lib/request-utils/request-credentials.d.ts.map +1 -1
- package/dist/loader-utils/lib/request-utils/request-credentials.js +10 -16
- package/dist/loader-utils/lib/request-utils/request-credentials.js.map +1 -1
- package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.d.ts +3 -0
- package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.d.ts.map +1 -1
- package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.js +13 -12
- package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.js.map +1 -1
- package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.d.ts +6 -0
- package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.d.ts.map +1 -1
- package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.js +38 -7
- package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.js.map +1 -1
- package/dist/loader-utils/loader-types.d.ts +12 -4
- package/dist/loader-utils/loader-types.d.ts.map +1 -1
- package/dist/loader-utils/loader-types.js.map +1 -1
- package/dist/null-loader.js +1 -1
- package/dist/null-worker-node.js +60 -15
- package/dist/null-worker.js +59 -15
- package/dist/schema-utils/lib/table/tables/convert-table.d.ts +1 -1
- package/dist/schema-utils/lib/table/tables/convert-table.js +1 -1
- package/dist/worker-utils/index.cjs +170 -49
- package/dist/worker-utils/index.cjs.map +2 -2
- package/dist/worker-utils/index.d.ts +2 -2
- package/dist/worker-utils/index.d.ts.map +1 -1
- package/dist/worker-utils/index.js +1 -1
- package/dist/worker-utils/index.js.map +1 -1
- package/dist/worker-utils/lib/env-utils/version.js +2 -2
- package/dist/worker-utils/lib/worker-api/create-worker.d.ts.map +1 -1
- package/dist/worker-utils/lib/worker-api/create-worker.js +40 -2
- package/dist/worker-utils/lib/worker-api/create-worker.js.map +1 -1
- package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts +2 -0
- package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts.map +1 -1
- package/dist/worker-utils/lib/worker-api/get-worker-url.js +7 -11
- package/dist/worker-utils/lib/worker-api/get-worker-url.js.map +1 -1
- package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts +6 -0
- package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts.map +1 -1
- package/dist/worker-utils/lib/worker-api/process-on-worker.js +64 -27
- package/dist/worker-utils/lib/worker-api/process-on-worker.js.map +1 -1
- package/dist/worker-utils/lib/worker-farm/worker-job.d.ts.map +1 -1
- package/dist/worker-utils/lib/worker-farm/worker-job.js +3 -2
- package/dist/worker-utils/lib/worker-farm/worker-job.js.map +1 -1
- package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts +1 -1
- package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts.map +1 -1
- package/dist/worker-utils/lib/worker-farm/worker-pool.js +65 -12
- package/dist/worker-utils/lib/worker-farm/worker-pool.js.map +1 -1
- package/dist/worker-utils/null-worker-node.js +36 -2
- package/dist/worker-utils/null-worker-node.js.map +2 -2
- package/dist/worker-utils/null-worker.js +36 -2
- package/dist/worker-utils/null-worker.js.map +2 -2
- package/dist/worker-utils/types.d.ts +25 -4
- package/dist/worker-utils/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/lib/api/parse-file.ts +2 -0
- package/src/lib/api/parse-in-batches.ts +5 -5
- package/src/lib/api/parse.ts +4 -2
- package/src/lib/loader-utils/get-fetch-function.ts +4 -2
- package/src/lib/loader-utils/loader-context.ts +4 -0
- package/src/lib/loader-utils/option-defaults.ts +1 -0
- package/src/lib/loader-utils/option-utils.ts +1 -1
- package/src/loader-utils/index.ts +1 -0
- package/src/loader-utils/json-loader.ts +1 -1
- package/src/loader-utils/lib/request-utils/request-credentials.ts +10 -12
- package/src/loader-utils/lib/worker-loader-utils/create-loader-worker.ts +38 -15
- package/src/loader-utils/lib/worker-loader-utils/parse-with-worker.ts +52 -7
- package/src/loader-utils/loader-types.ts +16 -4
- package/src/schema-utils/lib/table/tables/convert-table.ts +1 -1
- package/src/worker-utils/index.ts +3 -1
- package/src/worker-utils/lib/worker-api/create-worker.ts +41 -2
- package/src/worker-utils/lib/worker-api/get-worker-url.ts +13 -14
- package/src/worker-utils/lib/worker-api/process-on-worker.ts +128 -73
- package/src/worker-utils/lib/worker-farm/worker-job.ts +4 -2
- package/src/worker-utils/lib/worker-farm/worker-pool.ts +65 -12
- package/src/worker-utils/types.ts +26 -3
package/src/lib/api/parse.ts
CHANGED
|
@@ -97,8 +97,10 @@ export async function parse(
|
|
|
97
97
|
loaders = undefined;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
(options?.signal || context?.signal)?.throwIfAborted();
|
|
100
101
|
data = await data; // Resolve any promise
|
|
101
|
-
options = options || ({} as LoaderOptions);
|
|
102
|
+
options = options || ({} as LoaderOptions);
|
|
103
|
+
(options.signal || context?.signal)?.throwIfAborted(); // Could be invalid...
|
|
102
104
|
|
|
103
105
|
// Extract a url for auto detection
|
|
104
106
|
const url = getResourceUrl(data);
|
|
@@ -144,7 +146,6 @@ export async function parse(
|
|
|
144
146
|
return await parseWithLoader(loader, data, strictOptions, context);
|
|
145
147
|
}
|
|
146
148
|
|
|
147
|
-
// TODO: support progress and abort
|
|
148
149
|
// TODO - should accept loader.parseAsyncIterator and concatenate.
|
|
149
150
|
async function parseWithLoader(
|
|
150
151
|
loader: Loader,
|
|
@@ -152,6 +153,7 @@ async function parseWithLoader(
|
|
|
152
153
|
options: StrictLoaderOptions,
|
|
153
154
|
context: LoaderContext
|
|
154
155
|
): Promise<unknown> {
|
|
156
|
+
(options.signal || context.signal)?.throwIfAborted();
|
|
155
157
|
validateWorkerVersion(loader);
|
|
156
158
|
|
|
157
159
|
options = mergeOptions(loader.options, options);
|
|
@@ -25,7 +25,7 @@ export function getFetchFunction(
|
|
|
25
25
|
|
|
26
26
|
// options.fetch can be a function
|
|
27
27
|
if (typeof fetchOption === 'function') {
|
|
28
|
-
fetchFunction = fetchOption;
|
|
28
|
+
fetchFunction = fetchOption as FetchLike;
|
|
29
29
|
} else if (isObject(fetchOption)) {
|
|
30
30
|
fetchFunction = context?.fetch || fetchFile;
|
|
31
31
|
} else if (context?.fetch) {
|
|
@@ -34,10 +34,12 @@ export function getFetchFunction(
|
|
|
34
34
|
fetchFunction = fetchFile;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
const signal = loaderOptions.signal || context?.signal;
|
|
38
|
+
const requestOptions = isObject(fetchOption) ? (fetchOption as RequestInit) : undefined;
|
|
37
39
|
return createAuthenticatedFetch({
|
|
38
40
|
fetch: fetchFunction,
|
|
39
41
|
credentials: loaderOptions.core?.credentials || [],
|
|
40
42
|
authentications: loaderOptions.core?.authentications,
|
|
41
|
-
fetchOptions:
|
|
43
|
+
fetchOptions: signal ? {...requestOptions, signal} : requestOptions
|
|
42
44
|
});
|
|
43
45
|
}
|
|
@@ -31,6 +31,8 @@ export function getLoaderContext(
|
|
|
31
31
|
if (parentContext) {
|
|
32
32
|
return {
|
|
33
33
|
...parentContext,
|
|
34
|
+
signal: options.signal || parentContext.signal,
|
|
35
|
+
reportProgress: options.onProgress || parentContext.reportProgress,
|
|
34
36
|
coreApi: parentContext.coreApi || context.coreApi || coreApi,
|
|
35
37
|
_parse: parentContext._parse || context._parse,
|
|
36
38
|
fetch: getFetchFunction(options, parentContext)
|
|
@@ -39,6 +41,8 @@ export function getLoaderContext(
|
|
|
39
41
|
|
|
40
42
|
const newContext: LoaderContext = {
|
|
41
43
|
...context,
|
|
44
|
+
signal: options.signal || context.signal,
|
|
45
|
+
reportProgress: options.onProgress || context.reportProgress,
|
|
42
46
|
fetch: getFetchFunction(options, context),
|
|
43
47
|
coreApi: context.coreApi || coreApi
|
|
44
48
|
};
|
|
@@ -24,6 +24,7 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
|
|
|
24
24
|
CDN: 'https://unpkg.com/@loaders.gl',
|
|
25
25
|
worker: true, // By default, use worker if provided by loader.
|
|
26
26
|
workerThreshold: 0.1,
|
|
27
|
+
workerInBatches: undefined,
|
|
27
28
|
maxConcurrency: 3, // How many worker instances should be created for each loader.
|
|
28
29
|
maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
|
|
29
30
|
reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
|
|
@@ -219,7 +219,7 @@ function validateOptionsObject(
|
|
|
219
219
|
for (const key in options) {
|
|
220
220
|
// If top level option value is an object it could options for a loader, so ignore
|
|
221
221
|
const isSubOptions = !id && isObject(options[key]);
|
|
222
|
-
const isBaseUriOption =
|
|
222
|
+
const isBaseUriOption = !id && ['baseUri', 'signal', 'onProgress'].includes(key);
|
|
223
223
|
const isWorkerUrlOption = key === 'workerUrl' && id;
|
|
224
224
|
// <loader>.workerUrl requires special handling as it is now auto-generated and no longer specified as a default option.
|
|
225
225
|
if (!(key in defaultOptions) && !isBaseUriOption && !isWorkerUrlOption) {
|
|
@@ -112,6 +112,7 @@ export {
|
|
|
112
112
|
parseWithWorker,
|
|
113
113
|
parseWithWorkerInBatches,
|
|
114
114
|
canParseWithWorker,
|
|
115
|
+
canParseWithWorkerInBatches,
|
|
115
116
|
shouldParseWithWorker
|
|
116
117
|
} from './lib/worker-loader-utils/parse-with-worker';
|
|
117
118
|
export {canEncodeWithWorker} from './lib/worker-loader-utils/encode-with-worker';
|
|
@@ -13,7 +13,7 @@ export type JSONLoaderOptions = LoaderOptions;
|
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* A JSON Micro loader (minimal bundle size)
|
|
16
|
-
* Alternative to `@loaders.gl/json`
|
|
16
|
+
* Alternative to `@loaders.gl/tables/json`
|
|
17
17
|
*/
|
|
18
18
|
export const JSONLoader = {
|
|
19
19
|
dataType: null as unknown as Table,
|
|
@@ -211,7 +211,9 @@ export class BearerTokenAuthentication extends TokenAuthentication {
|
|
|
211
211
|
}
|
|
212
212
|
}
|
|
213
213
|
|
|
214
|
-
/**
|
|
214
|
+
/**
|
|
215
|
+
* Wraps fetch with exact-origin credentials and one refresh replay, preserving abort reasons.
|
|
216
|
+
*/
|
|
215
217
|
export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): FetchLike {
|
|
216
218
|
const suppliedFetch = options.fetch || ((url, requestOptions) => fetch(url, requestOptions));
|
|
217
219
|
const existingMetadata = (
|
|
@@ -253,7 +255,7 @@ export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): Fe
|
|
|
253
255
|
|
|
254
256
|
const authenticatedFetch: FetchLike = async (url, requestOptions = {}) => {
|
|
255
257
|
requestOptions = mergeFetchOptions(fetchOptions, requestOptions);
|
|
256
|
-
|
|
258
|
+
requestOptions.signal?.throwIfAborted();
|
|
257
259
|
// Request defaults alone must not normalize URLs or defer the transport invocation.
|
|
258
260
|
if (!credentials.length) return baseFetch(url, requestOptions);
|
|
259
261
|
const resolvedURL = parseHTTPURL(url);
|
|
@@ -283,7 +285,7 @@ export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): Fe
|
|
|
283
285
|
credential.refreshStatusCodes?.includes(response.status)
|
|
284
286
|
);
|
|
285
287
|
if (!refreshableCredentials.length && !shouldRetrySignature) return response;
|
|
286
|
-
|
|
288
|
+
requestOptions.signal?.throwIfAborted();
|
|
287
289
|
|
|
288
290
|
const refreshedTokens = new Map<RequestCredential, string>();
|
|
289
291
|
await Promise.all(
|
|
@@ -295,7 +297,7 @@ export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): Fe
|
|
|
295
297
|
if (!refreshedTokens.size && !shouldRetrySignature) return response;
|
|
296
298
|
|
|
297
299
|
await response.body?.cancel().catch(() => {});
|
|
298
|
-
|
|
300
|
+
requestOptions.signal?.throwIfAborted();
|
|
299
301
|
const replayAuthorization = resolvedURL
|
|
300
302
|
? await authorizeRequest(
|
|
301
303
|
resolvedURL,
|
|
@@ -340,9 +342,10 @@ export function getAuthenticatedFetch(options: LoaderOptions = {}): FetchLike {
|
|
|
340
342
|
fetchFunction = (url, requestOptions) => fetch(url, requestOptions);
|
|
341
343
|
}
|
|
342
344
|
|
|
345
|
+
const fetchOptions = typeof fetchOption === 'object' && fetchOption ? fetchOption : undefined;
|
|
343
346
|
return createAuthenticatedFetch({
|
|
344
347
|
fetch: fetchFunction,
|
|
345
|
-
fetchOptions:
|
|
348
|
+
fetchOptions: options.signal ? {...fetchOptions, signal: options.signal} : fetchOptions,
|
|
346
349
|
credentials: options.core?.credentials || [],
|
|
347
350
|
authentications: options.core?.authentications
|
|
348
351
|
});
|
|
@@ -395,7 +398,7 @@ async function authenticateRequest(
|
|
|
395
398
|
for (const credential of credentials) {
|
|
396
399
|
const origin = getRequestOrigin(request.url);
|
|
397
400
|
if (!origin || !normalizeRequestOrigins(credential.origins).includes(origin)) continue;
|
|
398
|
-
|
|
401
|
+
signal?.throwIfAborted();
|
|
399
402
|
const authenticatedRequest = await credential.authenticate({
|
|
400
403
|
url: request.url,
|
|
401
404
|
options: {...request.options, headers: new Headers(request.options.headers)},
|
|
@@ -418,7 +421,7 @@ async function authenticateRequest(
|
|
|
418
421
|
};
|
|
419
422
|
appliedCredentials.push(credential);
|
|
420
423
|
}
|
|
421
|
-
|
|
424
|
+
signal?.throwIfAborted();
|
|
422
425
|
return {...request, credentials: appliedCredentials};
|
|
423
426
|
}
|
|
424
427
|
|
|
@@ -569,8 +572,3 @@ function isReplayableRequest(options: RequestInit): boolean {
|
|
|
569
572
|
options.body instanceof ReadableStream
|
|
570
573
|
);
|
|
571
574
|
}
|
|
572
|
-
|
|
573
|
-
/** Creates the standard DOM exception used for an aborted fetch. */
|
|
574
|
-
function createAbortError(): DOMException {
|
|
575
|
-
return new DOMException('The operation was aborted.', 'AbortError');
|
|
576
|
-
}
|
|
@@ -4,7 +4,14 @@
|
|
|
4
4
|
|
|
5
5
|
/* eslint-disable no-restricted-globals */
|
|
6
6
|
import type {CoreAPI} from '../sources/data-source';
|
|
7
|
-
import type {
|
|
7
|
+
import type {
|
|
8
|
+
LoaderWithParser,
|
|
9
|
+
LoaderOptions,
|
|
10
|
+
StrictLoaderOptions,
|
|
11
|
+
LoaderContext,
|
|
12
|
+
Loader
|
|
13
|
+
} from '../../loader-types';
|
|
14
|
+
import type {WorkerContext} from '@loaders.gl/core/worker-utils';
|
|
8
15
|
import {createWorker} from '@loaders.gl/core/worker-utils';
|
|
9
16
|
// import {validateLoaderVersion} from './validate-loader-version';
|
|
10
17
|
|
|
@@ -43,6 +50,7 @@ export async function processLoaderWorkerData(
|
|
|
43
50
|
input: ArrayBuffer,
|
|
44
51
|
options: {[key: string]: any} = {},
|
|
45
52
|
workerContext?: {
|
|
53
|
+
reportProgress?: WorkerContext['reportProgress'];
|
|
46
54
|
process?: (data: any, options?: LoaderOptions, context?: Record<string, any>) => any;
|
|
47
55
|
},
|
|
48
56
|
loaderContext: Record<string, any> = {}
|
|
@@ -53,6 +61,7 @@ export async function processLoaderWorkerData(
|
|
|
53
61
|
options,
|
|
54
62
|
context: {
|
|
55
63
|
...loaderContext,
|
|
64
|
+
reportProgress: workerContext?.reportProgress,
|
|
56
65
|
coreApi: createWorkerCoreApi(),
|
|
57
66
|
_parse: createParseOnMainThread(workerContext?.process)
|
|
58
67
|
} as LoaderContext
|
|
@@ -71,6 +80,7 @@ export async function* processLoaderWorkerBatches(
|
|
|
71
80
|
| Iterable<ArrayBufferLike | ArrayBufferView>,
|
|
72
81
|
options: {[key: string]: any} = {},
|
|
73
82
|
workerContext?: {
|
|
83
|
+
reportProgress?: WorkerContext['reportProgress'];
|
|
74
84
|
process?: (data: any, options?: LoaderOptions, context?: Record<string, any>) => any;
|
|
75
85
|
},
|
|
76
86
|
loaderContext: Record<string, any> = {}
|
|
@@ -79,8 +89,10 @@ export async function* processLoaderWorkerBatches(
|
|
|
79
89
|
throw new Error(`${loader.id} loader does not support batched parsing`);
|
|
80
90
|
}
|
|
81
91
|
|
|
82
|
-
const
|
|
92
|
+
const parserOptions = getWorkerLoaderOptions(loader, options);
|
|
93
|
+
const resultIterator = loader.parseInBatches(inputIterator, parserOptions, {
|
|
83
94
|
...loaderContext,
|
|
95
|
+
reportProgress: workerContext?.reportProgress,
|
|
84
96
|
coreApi: createWorkerCoreApi(),
|
|
85
97
|
_parse: createParseOnMainThread(workerContext?.process)
|
|
86
98
|
} as LoaderContext);
|
|
@@ -181,8 +193,17 @@ function getSerializableLoaderContext(context?: LoaderContext) {
|
|
|
181
193
|
if (!context) {
|
|
182
194
|
return undefined;
|
|
183
195
|
}
|
|
184
|
-
const {
|
|
185
|
-
|
|
196
|
+
const {
|
|
197
|
+
fetch,
|
|
198
|
+
loaders,
|
|
199
|
+
coreApi,
|
|
200
|
+
signal,
|
|
201
|
+
reportProgress,
|
|
202
|
+
_parse,
|
|
203
|
+
_parseSync,
|
|
204
|
+
_parseInBatches,
|
|
205
|
+
...serializableContext
|
|
206
|
+
} = context;
|
|
186
207
|
return JSON.parse(JSON.stringify(serializableContext));
|
|
187
208
|
}
|
|
188
209
|
|
|
@@ -217,17 +238,19 @@ async function parseData({
|
|
|
217
238
|
// Preserve worker-supplied module overrides (for example a caller-provided
|
|
218
239
|
// Draco URL) while still applying the loader's defaults. Functions cannot be
|
|
219
240
|
// structured-cloned, but URL strings and other serializable options can.
|
|
220
|
-
options =
|
|
221
|
-
...options,
|
|
222
|
-
modules: {
|
|
223
|
-
...((loader && loader.options && loader.options.modules) || {}),
|
|
224
|
-
...(options.modules || {})
|
|
225
|
-
},
|
|
226
|
-
core: {
|
|
227
|
-
...options.core,
|
|
228
|
-
worker: false
|
|
229
|
-
}
|
|
230
|
-
};
|
|
241
|
+
options = getWorkerLoaderOptions(loader, options);
|
|
231
242
|
|
|
232
243
|
return await parser(data, {...options}, context, loader);
|
|
233
244
|
}
|
|
245
|
+
|
|
246
|
+
/** Applies loader module defaults and prevents recursive worker dispatch inside a worker parser. */
|
|
247
|
+
function getWorkerLoaderOptions(
|
|
248
|
+
loader: LoaderWithParser,
|
|
249
|
+
options: LoaderOptions
|
|
250
|
+
): StrictLoaderOptions {
|
|
251
|
+
return {
|
|
252
|
+
...options,
|
|
253
|
+
modules: {...loader.options?.modules, ...options.modules},
|
|
254
|
+
core: {...options.core, worker: false}
|
|
255
|
+
} as StrictLoaderOptions;
|
|
256
|
+
}
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
6
|
canProcessOnWorker,
|
|
7
|
+
canProcessOnWorkerInBatches,
|
|
7
8
|
isBrowser,
|
|
8
9
|
processOnWorker,
|
|
9
10
|
processOnWorkerInBatches
|
|
@@ -65,6 +66,27 @@ export function canParseWithWorker(loader: Loader, options?: StrictLoaderOptions
|
|
|
65
66
|
return Boolean(canProcessOnWorker(loader, workerOptions));
|
|
66
67
|
}
|
|
67
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Checks loader batch-parser transport capability and the selected worker's batch protocol.
|
|
71
|
+
* @param loader Loader declaring compatible batch parsing and transport.
|
|
72
|
+
* @param options Loader and worker selection options.
|
|
73
|
+
*/
|
|
74
|
+
export function canParseWithWorkerInBatches(
|
|
75
|
+
loader: Loader,
|
|
76
|
+
options?: StrictLoaderOptions
|
|
77
|
+
): boolean {
|
|
78
|
+
// Encryption key callbacks must stay on the caller; JSON worker options omit functions.
|
|
79
|
+
const parquetOptions = options?.parquet as {keyRetriever?: unknown} | undefined;
|
|
80
|
+
if (loader.id === 'parquet' && typeof parquetOptions?.keyRetriever === 'function') {
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
return Boolean(
|
|
84
|
+
loader.workerInBatches &&
|
|
85
|
+
canParseWithWorker(loader, options) &&
|
|
86
|
+
canProcessOnWorkerInBatches(loader, getWorkerOptions(options))
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
68
90
|
/**
|
|
69
91
|
* Determines whether an atomic parse should use a worker, including optional loader work estimates.
|
|
70
92
|
* @param loader Loader metadata or parser-bearing loader.
|
|
@@ -118,11 +140,15 @@ export async function parseWithWorker(
|
|
|
118
140
|
parseOnMainThread?: ParseOnMainThread,
|
|
119
141
|
signal?: AbortSignal
|
|
120
142
|
) {
|
|
121
|
-
const workerSignal = signal || getWorkerAbortSignal(options);
|
|
143
|
+
const workerSignal = signal || getWorkerAbortSignal(options) || context?.signal;
|
|
122
144
|
const result = await processOnWorker(
|
|
123
145
|
loader,
|
|
124
146
|
data,
|
|
125
|
-
{
|
|
147
|
+
{
|
|
148
|
+
...getWorkerOptions(options, loader),
|
|
149
|
+
signal: workerSignal,
|
|
150
|
+
onProgress: options?.onProgress || context?.reportProgress
|
|
151
|
+
},
|
|
126
152
|
{
|
|
127
153
|
process: async (input, processOptions, _workerContext, parseContext) => {
|
|
128
154
|
if (!parseOnMainThread) {
|
|
@@ -168,11 +194,15 @@ export async function* parseWithWorkerInBatches(
|
|
|
168
194
|
parseOnMainThread?: ParseOnMainThread,
|
|
169
195
|
signal?: AbortSignal
|
|
170
196
|
): AsyncIterable<unknown> {
|
|
171
|
-
const workerSignal = signal || getWorkerAbortSignal(options);
|
|
197
|
+
const workerSignal = signal || getWorkerAbortSignal(options) || context?.signal;
|
|
172
198
|
const outputIterator = processOnWorkerInBatches(
|
|
173
199
|
loader,
|
|
174
200
|
inputIterator,
|
|
175
|
-
{
|
|
201
|
+
{
|
|
202
|
+
...getWorkerOptions(options, loader),
|
|
203
|
+
signal: workerSignal,
|
|
204
|
+
onProgress: options?.onProgress || context?.reportProgress
|
|
205
|
+
},
|
|
176
206
|
{
|
|
177
207
|
process: async (input, processOptions, _workerContext, parseContext) => {
|
|
178
208
|
if (!parseOnMainThread) {
|
|
@@ -204,6 +234,7 @@ export async function* parseWithWorkerInBatches(
|
|
|
204
234
|
|
|
205
235
|
/** Returns the loader-specific abort signal used to cancel a worker job. */
|
|
206
236
|
function getWorkerAbortSignal(options?: StrictLoaderOptions): AbortSignal | undefined {
|
|
237
|
+
if (options?.signal) return options.signal;
|
|
207
238
|
const parquetSignal = (options as {parquet?: {signal?: unknown}} | undefined)?.parquet?.signal;
|
|
208
239
|
if (typeof AbortSignal !== 'undefined' && parquetSignal instanceof AbortSignal) {
|
|
209
240
|
return parquetSignal;
|
|
@@ -278,10 +309,15 @@ function resolveWorkerLoaders(
|
|
|
278
309
|
function getWorkerOptions(options: StrictLoaderOptions = {}, loader?: Loader) {
|
|
279
310
|
const optionsToSerialize = {
|
|
280
311
|
...options,
|
|
312
|
+
signal: undefined,
|
|
313
|
+
onProgress: undefined,
|
|
281
314
|
...(options.core ? {core: {...options.core, authentications: undefined}} : {})
|
|
282
315
|
};
|
|
283
316
|
const serializedOptions = JSON.parse(
|
|
284
|
-
JSON.stringify(optionsToSerialize, (_key, value) =>
|
|
317
|
+
JSON.stringify(optionsToSerialize, (_key, value) => {
|
|
318
|
+
if (typeof AbortSignal !== 'undefined' && value instanceof AbortSignal) return undefined;
|
|
319
|
+
return typeof value === 'bigint' ? null : value;
|
|
320
|
+
})
|
|
285
321
|
);
|
|
286
322
|
restoreBigIntOptions(optionsToSerialize, serializedOptions);
|
|
287
323
|
const workerOptions: Record<string, any> = {
|
|
@@ -317,8 +353,17 @@ function getSerializableLoaderContext(context?: LoaderContext) {
|
|
|
317
353
|
if (!context) {
|
|
318
354
|
return {};
|
|
319
355
|
}
|
|
320
|
-
const {
|
|
321
|
-
|
|
356
|
+
const {
|
|
357
|
+
fetch,
|
|
358
|
+
loaders,
|
|
359
|
+
coreApi,
|
|
360
|
+
signal,
|
|
361
|
+
reportProgress,
|
|
362
|
+
_parse,
|
|
363
|
+
_parseSync,
|
|
364
|
+
_parseInBatches,
|
|
365
|
+
...serializableContext
|
|
366
|
+
} = context;
|
|
322
367
|
return JSON.parse(JSON.stringify(serializableContext));
|
|
323
368
|
}
|
|
324
369
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import type {Format} from './format-types';
|
|
6
|
-
import type {LoadWorker} from '@loaders.gl/core/worker-utils';
|
|
6
|
+
import type {LoadWorker, WorkerOptions, WorkerProgress} from '@loaders.gl/core/worker-utils';
|
|
7
7
|
import {DataType, FetchLike, TransformBatches} from './types';
|
|
8
8
|
import {ReadableFile} from './lib/files/file';
|
|
9
9
|
import type {CoreAPI} from './lib/sources/data-source';
|
|
@@ -18,7 +18,7 @@ export type SearchParams = Record<string, string | number | boolean>;
|
|
|
18
18
|
/**
|
|
19
19
|
* Core Loader Options
|
|
20
20
|
*/
|
|
21
|
-
export type StrictLoaderOptions = {
|
|
21
|
+
export type StrictLoaderOptions = Pick<WorkerOptions, 'signal' | 'onProgress'> & {
|
|
22
22
|
/** Experimental common scan request. Format loaders implement the fields they support. */
|
|
23
23
|
_scan?: ExperimentalScanOptions;
|
|
24
24
|
/** Query parameters appended to the initial URL request. */
|
|
@@ -77,6 +77,8 @@ export type StrictLoaderOptions = {
|
|
|
77
77
|
CDN?: string | null;
|
|
78
78
|
/** Set to `false` to disable workers, or `'auto'` to use loader work estimates. */
|
|
79
79
|
worker?: boolean | 'auto';
|
|
80
|
+
/** Declares batch support for a custom worker URL, or disables batch offloading. The loader must also advertise workerInBatches. */
|
|
81
|
+
workerInBatches?: boolean;
|
|
80
82
|
/** Work-score threshold used by `worker: 'auto'`. */
|
|
81
83
|
workerThreshold?: number;
|
|
82
84
|
/** Number of concurrent workers (per loader) on desktop browser */
|
|
@@ -97,13 +99,17 @@ export type StrictLoaderOptions = {
|
|
|
97
99
|
modules?: Record<string, unknown>;
|
|
98
100
|
|
|
99
101
|
// Accept other keys (loader options objects, e.g. `options.csv`, `options.json` ...)
|
|
100
|
-
[loaderId: string]:
|
|
102
|
+
[loaderId: string]:
|
|
103
|
+
| Record<string, unknown>
|
|
104
|
+
| AbortSignal
|
|
105
|
+
| ((progress: WorkerProgress) => void)
|
|
106
|
+
| undefined;
|
|
101
107
|
};
|
|
102
108
|
|
|
103
109
|
/**
|
|
104
110
|
* Core Loader Options
|
|
105
111
|
*/
|
|
106
|
-
export type LoaderOptions = {
|
|
112
|
+
export type LoaderOptions = Pick<WorkerOptions, 'signal' | 'onProgress'> & {
|
|
107
113
|
/** Experimental common scan request. Format loaders implement the fields they support. */
|
|
108
114
|
_scan?: ExperimentalScanOptions;
|
|
109
115
|
/** Query parameters appended to the initial URL request. */
|
|
@@ -193,6 +199,8 @@ export type Loader<DataT = any, BatchT = any, LoaderOptionsT = StrictLoaderOptio
|
|
|
193
199
|
version: string;
|
|
194
200
|
/** A boolean, or a URL */
|
|
195
201
|
worker?: string | boolean;
|
|
202
|
+
/** Whether the worker and its batch transport support this loader's batch parser. */
|
|
203
|
+
workerInBatches?: boolean;
|
|
196
204
|
/** Browser worker filename when it differs from the loader id. */
|
|
197
205
|
workerFile?: string;
|
|
198
206
|
/** Optional Node.js worker filename, such as a CommonJS `.cjs` bundle. */
|
|
@@ -332,6 +340,10 @@ export type LoaderWithParser<
|
|
|
332
340
|
* threads back to main thread.
|
|
333
341
|
*/
|
|
334
342
|
export type LoaderContext = {
|
|
343
|
+
/** Cancels nested asynchronous operations where supported. */
|
|
344
|
+
signal?: AbortSignal;
|
|
345
|
+
/** Reports progress within a loader-defined phase. */
|
|
346
|
+
reportProgress?: (progress: WorkerProgress) => void;
|
|
335
347
|
/** Loader list provided to top-level loader call plus any sub loaders */
|
|
336
348
|
loaders?: Loader[] | null;
|
|
337
349
|
/** If URL is available. */
|
|
@@ -112,7 +112,7 @@ export function convertToObjectRowTable(table: Table): ObjectRowTable {
|
|
|
112
112
|
|
|
113
113
|
/**
|
|
114
114
|
* Convert a table to apache arrow format
|
|
115
|
-
* @note this depends on the `@loaders.gl/arrow module being imported
|
|
115
|
+
* @note this depends on the `@loaders.gl/tables/arrow module being imported
|
|
116
116
|
*/
|
|
117
117
|
export function convertToArrowTable(table: Table): ArrowTable {
|
|
118
118
|
const schema = table.schema || deduceTableSchema(table);
|
|
@@ -10,6 +10,7 @@ export type {
|
|
|
10
10
|
LoadWorker,
|
|
11
11
|
WorkerObject,
|
|
12
12
|
WorkerOptions,
|
|
13
|
+
WorkerProgress,
|
|
13
14
|
WorkerContext,
|
|
14
15
|
WorkerJobContext,
|
|
15
16
|
Process,
|
|
@@ -41,7 +42,8 @@ export {
|
|
|
41
42
|
processOnWorker,
|
|
42
43
|
processOnWorkerInBatches,
|
|
43
44
|
preloadWorker,
|
|
44
|
-
canProcessOnWorker
|
|
45
|
+
canProcessOnWorker,
|
|
46
|
+
canProcessOnWorkerInBatches
|
|
45
47
|
} from './lib/worker-api/process-on-worker';
|
|
46
48
|
export {createWorker} from './lib/worker-api/create-worker';
|
|
47
49
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import type {
|
|
6
|
+
WorkerProgress,
|
|
6
7
|
WorkerMessageType,
|
|
7
8
|
WorkerMessagePayload,
|
|
8
9
|
WorkerContext,
|
|
@@ -42,6 +43,9 @@ export async function createWorker(
|
|
|
42
43
|
|
|
43
44
|
// eslint-disable-next-line complexity
|
|
44
45
|
WorkerBody.onmessage = async (type: WorkerMessageType, payload: WorkerMessagePayload) => {
|
|
46
|
+
const reporter =
|
|
47
|
+
type === 'process' || type === 'process-in-batches' ? createProgressReporter() : undefined;
|
|
48
|
+
const jobContext = {...context, reportProgress: reporter?.report};
|
|
45
49
|
try {
|
|
46
50
|
switch (type) {
|
|
47
51
|
case 'preload':
|
|
@@ -55,9 +59,10 @@ export async function createWorker(
|
|
|
55
59
|
const result = await process(
|
|
56
60
|
payload.input,
|
|
57
61
|
payload.options || {},
|
|
58
|
-
|
|
62
|
+
jobContext,
|
|
59
63
|
payload.context || {}
|
|
60
64
|
);
|
|
65
|
+
reporter?.finish();
|
|
61
66
|
WorkerBody.postMessage('done', {result});
|
|
62
67
|
break;
|
|
63
68
|
|
|
@@ -73,13 +78,14 @@ export async function createWorker(
|
|
|
73
78
|
const resultIterator = processInBatches(
|
|
74
79
|
createDemandDrivenIterator(inputBatches),
|
|
75
80
|
payload.options || {},
|
|
76
|
-
|
|
81
|
+
jobContext,
|
|
77
82
|
payload.context || {}
|
|
78
83
|
);
|
|
79
84
|
for await (const batch of resultIterator) {
|
|
80
85
|
await WorkerBody.postMessage('output-batch', {result: batch});
|
|
81
86
|
await outputAcknowledgements.next();
|
|
82
87
|
}
|
|
88
|
+
reporter?.finish();
|
|
83
89
|
await WorkerBody.postMessage('done', {});
|
|
84
90
|
} finally {
|
|
85
91
|
activeInputBatches = null;
|
|
@@ -111,6 +117,7 @@ export async function createWorker(
|
|
|
111
117
|
default:
|
|
112
118
|
}
|
|
113
119
|
} catch (error) {
|
|
120
|
+
reporter?.finish();
|
|
114
121
|
const message = error instanceof Error ? error.message : '';
|
|
115
122
|
WorkerBody.postMessage('error', {error: message});
|
|
116
123
|
}
|
|
@@ -164,3 +171,35 @@ function processOnMainThread(arrayBuffer: ArrayBuffer, options = {}, jobContext
|
|
|
164
171
|
WorkerBody.postMessage('process', payload);
|
|
165
172
|
});
|
|
166
173
|
}
|
|
174
|
+
|
|
175
|
+
/** Coalesces same-phase reports per event-loop turn and flushes before settlement. */
|
|
176
|
+
function createProgressReporter() {
|
|
177
|
+
let pending: WorkerProgress | undefined;
|
|
178
|
+
let finished = false;
|
|
179
|
+
let scheduled = false;
|
|
180
|
+
/** Posts the latest pending measurement. */
|
|
181
|
+
const flush = (): void => {
|
|
182
|
+
scheduled = false;
|
|
183
|
+
if (pending) {
|
|
184
|
+
WorkerBody.postMessage('progress', {progress: pending});
|
|
185
|
+
pending = undefined;
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
return {
|
|
189
|
+
/** Records progress while retaining phase transitions. */
|
|
190
|
+
report(progress: WorkerProgress): void {
|
|
191
|
+
if (finished) return;
|
|
192
|
+
if (pending && pending.phase !== progress.phase) flush();
|
|
193
|
+
pending = {...progress};
|
|
194
|
+
if (!scheduled) {
|
|
195
|
+
scheduled = true;
|
|
196
|
+
setTimeout(flush, 0);
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
/** Flushes final progress and suppresses reports from stale callbacks. */
|
|
200
|
+
finish(): void {
|
|
201
|
+
flush();
|
|
202
|
+
finished = true;
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
}
|
|
@@ -32,23 +32,10 @@ export function getCustomWorkerURL(
|
|
|
32
32
|
worker: WorkerObject,
|
|
33
33
|
options: WorkerOptions = {}
|
|
34
34
|
): string | null {
|
|
35
|
-
const workerOptions = options[worker.id] || {};
|
|
36
|
-
|
|
37
35
|
const workerFile = isBrowser
|
|
38
36
|
? worker.workerFile || `${worker.id}-worker.js`
|
|
39
37
|
: worker.workerNode || `${worker.id}-worker-node.js`;
|
|
40
|
-
|
|
41
|
-
let url = workerOptions.workerUrl;
|
|
42
|
-
|
|
43
|
-
// HACK: Allow for non-nested workerUrl for the CompressionWorker.
|
|
44
|
-
// For the compression worker, workerOptions is currently not nested correctly. For most loaders,
|
|
45
|
-
// you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker
|
|
46
|
-
// puts options at the top level, not within a `compression` key (its `id`). For this reason, the
|
|
47
|
-
// above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not
|
|
48
|
-
// break backwards compatibility, we allow the CompressionWorker to have options at the top level.
|
|
49
|
-
if (!url && worker.id === 'compression') {
|
|
50
|
-
url = options.workerUrl;
|
|
51
|
-
}
|
|
38
|
+
let url = getConfiguredWorkerURL(worker, options);
|
|
52
39
|
|
|
53
40
|
// If URL is test, generate local loaders.gl url
|
|
54
41
|
// @ts-ignore _workerType
|
|
@@ -71,6 +58,18 @@ export function getCustomWorkerURL(
|
|
|
71
58
|
return url || null;
|
|
72
59
|
}
|
|
73
60
|
|
|
61
|
+
/** Returns an explicit worker URL override, excluding generated local test URLs. */
|
|
62
|
+
export function getConfiguredWorkerURL(
|
|
63
|
+
worker: WorkerObject,
|
|
64
|
+
options: WorkerOptions = {}
|
|
65
|
+
): string | null {
|
|
66
|
+
const workerOptions = options[worker.id] || {};
|
|
67
|
+
// Compression also supports the historical non-nested URL override.
|
|
68
|
+
return (
|
|
69
|
+
workerOptions.workerUrl || (worker.id === 'compression' ? options.workerUrl : null) || null
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
74
73
|
/**
|
|
75
74
|
* Generates a worker URL using overrides first and the published CDN artifact as a fallback.
|
|
76
75
|
* @param worker Worker descriptor to resolve.
|