@loaders.gl/core 5.0.0-alpha.13 → 5.0.0-alpha.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/dist.dev.js +190 -63
  2. package/dist/dist.min.js +3 -3
  3. package/dist/index.cjs +205 -72
  4. package/dist/index.cjs.map +2 -2
  5. package/dist/lib/api/parse-file.js +2 -0
  6. package/dist/lib/api/parse-file.js.map +1 -1
  7. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  8. package/dist/lib/api/parse-in-batches.js +5 -4
  9. package/dist/lib/api/parse-in-batches.js.map +1 -1
  10. package/dist/lib/api/parse.js +4 -2
  11. package/dist/lib/api/parse.js.map +1 -1
  12. package/dist/lib/init.js +1 -1
  13. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  14. package/dist/lib/loader-utils/get-fetch-function.js +3 -1
  15. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  16. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  17. package/dist/lib/loader-utils/loader-context.js +4 -0
  18. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  19. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  20. package/dist/lib/loader-utils/option-defaults.js +1 -0
  21. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  22. package/dist/lib/loader-utils/option-utils.js +1 -1
  23. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  24. package/dist/loader-utils/index.cjs +231 -92
  25. package/dist/loader-utils/index.cjs.map +2 -2
  26. package/dist/loader-utils/index.d.ts +1 -1
  27. package/dist/loader-utils/index.d.ts.map +1 -1
  28. package/dist/loader-utils/index.js +1 -1
  29. package/dist/loader-utils/index.js.map +1 -1
  30. package/dist/loader-utils/json-loader.d.ts +1 -1
  31. package/dist/loader-utils/json-loader.js +2 -2
  32. package/dist/loader-utils/lib/log-utils/log.js +1 -1
  33. package/dist/loader-utils/lib/request-utils/request-credentials.d.ts +3 -1
  34. package/dist/loader-utils/lib/request-utils/request-credentials.d.ts.map +1 -1
  35. package/dist/loader-utils/lib/request-utils/request-credentials.js +10 -16
  36. package/dist/loader-utils/lib/request-utils/request-credentials.js.map +1 -1
  37. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.d.ts +3 -0
  38. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.d.ts.map +1 -1
  39. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.js +13 -12
  40. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.js.map +1 -1
  41. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.d.ts +6 -0
  42. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.d.ts.map +1 -1
  43. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.js +38 -7
  44. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.js.map +1 -1
  45. package/dist/loader-utils/loader-types.d.ts +12 -4
  46. package/dist/loader-utils/loader-types.d.ts.map +1 -1
  47. package/dist/loader-utils/loader-types.js.map +1 -1
  48. package/dist/null-loader.js +1 -1
  49. package/dist/null-worker-node.js +60 -15
  50. package/dist/null-worker.js +59 -15
  51. package/dist/schema-utils/lib/table/tables/convert-table.d.ts +1 -1
  52. package/dist/schema-utils/lib/table/tables/convert-table.js +1 -1
  53. package/dist/worker-utils/index.cjs +170 -49
  54. package/dist/worker-utils/index.cjs.map +2 -2
  55. package/dist/worker-utils/index.d.ts +2 -2
  56. package/dist/worker-utils/index.d.ts.map +1 -1
  57. package/dist/worker-utils/index.js +1 -1
  58. package/dist/worker-utils/index.js.map +1 -1
  59. package/dist/worker-utils/lib/env-utils/version.js +2 -2
  60. package/dist/worker-utils/lib/worker-api/create-worker.d.ts.map +1 -1
  61. package/dist/worker-utils/lib/worker-api/create-worker.js +40 -2
  62. package/dist/worker-utils/lib/worker-api/create-worker.js.map +1 -1
  63. package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts +2 -0
  64. package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts.map +1 -1
  65. package/dist/worker-utils/lib/worker-api/get-worker-url.js +7 -11
  66. package/dist/worker-utils/lib/worker-api/get-worker-url.js.map +1 -1
  67. package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts +6 -0
  68. package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts.map +1 -1
  69. package/dist/worker-utils/lib/worker-api/process-on-worker.js +64 -27
  70. package/dist/worker-utils/lib/worker-api/process-on-worker.js.map +1 -1
  71. package/dist/worker-utils/lib/worker-farm/worker-job.d.ts.map +1 -1
  72. package/dist/worker-utils/lib/worker-farm/worker-job.js +3 -2
  73. package/dist/worker-utils/lib/worker-farm/worker-job.js.map +1 -1
  74. package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts +1 -1
  75. package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts.map +1 -1
  76. package/dist/worker-utils/lib/worker-farm/worker-pool.js +65 -12
  77. package/dist/worker-utils/lib/worker-farm/worker-pool.js.map +1 -1
  78. package/dist/worker-utils/null-worker-node.js +36 -2
  79. package/dist/worker-utils/null-worker-node.js.map +2 -2
  80. package/dist/worker-utils/null-worker.js +36 -2
  81. package/dist/worker-utils/null-worker.js.map +2 -2
  82. package/dist/worker-utils/types.d.ts +25 -4
  83. package/dist/worker-utils/types.d.ts.map +1 -1
  84. package/package.json +4 -4
  85. package/src/lib/api/parse-file.ts +2 -0
  86. package/src/lib/api/parse-in-batches.ts +5 -5
  87. package/src/lib/api/parse.ts +4 -2
  88. package/src/lib/loader-utils/get-fetch-function.ts +4 -2
  89. package/src/lib/loader-utils/loader-context.ts +4 -0
  90. package/src/lib/loader-utils/option-defaults.ts +1 -0
  91. package/src/lib/loader-utils/option-utils.ts +1 -1
  92. package/src/loader-utils/index.ts +1 -0
  93. package/src/loader-utils/json-loader.ts +1 -1
  94. package/src/loader-utils/lib/request-utils/request-credentials.ts +10 -12
  95. package/src/loader-utils/lib/worker-loader-utils/create-loader-worker.ts +38 -15
  96. package/src/loader-utils/lib/worker-loader-utils/parse-with-worker.ts +52 -7
  97. package/src/loader-utils/loader-types.ts +16 -4
  98. package/src/schema-utils/lib/table/tables/convert-table.ts +1 -1
  99. package/src/worker-utils/index.ts +3 -1
  100. package/src/worker-utils/lib/worker-api/create-worker.ts +41 -2
  101. package/src/worker-utils/lib/worker-api/get-worker-url.ts +13 -14
  102. package/src/worker-utils/lib/worker-api/process-on-worker.ts +128 -73
  103. package/src/worker-utils/lib/worker-farm/worker-job.ts +4 -2
  104. package/src/worker-utils/lib/worker-farm/worker-pool.ts +65 -12
  105. package/src/worker-utils/types.ts +26 -3
@@ -97,8 +97,10 @@ export async function parse(
97
97
  loaders = undefined;
98
98
  }
99
99
 
100
+ (options?.signal || context?.signal)?.throwIfAborted();
100
101
  data = await data; // Resolve any promise
101
- options = options || ({} as LoaderOptions); // Could be invalid...
102
+ options = options || ({} as LoaderOptions);
103
+ (options.signal || context?.signal)?.throwIfAborted(); // Could be invalid...
102
104
 
103
105
  // Extract a url for auto detection
104
106
  const url = getResourceUrl(data);
@@ -144,7 +146,6 @@ export async function parse(
144
146
  return await parseWithLoader(loader, data, strictOptions, context);
145
147
  }
146
148
 
147
- // TODO: support progress and abort
148
149
  // TODO - should accept loader.parseAsyncIterator and concatenate.
149
150
  async function parseWithLoader(
150
151
  loader: Loader,
@@ -152,6 +153,7 @@ async function parseWithLoader(
152
153
  options: StrictLoaderOptions,
153
154
  context: LoaderContext
154
155
  ): Promise<unknown> {
156
+ (options.signal || context.signal)?.throwIfAborted();
155
157
  validateWorkerVersion(loader);
156
158
 
157
159
  options = mergeOptions(loader.options, options);
@@ -25,7 +25,7 @@ export function getFetchFunction(
25
25
 
26
26
  // options.fetch can be a function
27
27
  if (typeof fetchOption === 'function') {
28
- fetchFunction = fetchOption;
28
+ fetchFunction = fetchOption as FetchLike;
29
29
  } else if (isObject(fetchOption)) {
30
30
  fetchFunction = context?.fetch || fetchFile;
31
31
  } else if (context?.fetch) {
@@ -34,10 +34,12 @@ export function getFetchFunction(
34
34
  fetchFunction = fetchFile;
35
35
  }
36
36
 
37
+ const signal = loaderOptions.signal || context?.signal;
38
+ const requestOptions = isObject(fetchOption) ? (fetchOption as RequestInit) : undefined;
37
39
  return createAuthenticatedFetch({
38
40
  fetch: fetchFunction,
39
41
  credentials: loaderOptions.core?.credentials || [],
40
42
  authentications: loaderOptions.core?.authentications,
41
- fetchOptions: isObject(fetchOption) ? (fetchOption as RequestInit) : undefined
43
+ fetchOptions: signal ? {...requestOptions, signal} : requestOptions
42
44
  });
43
45
  }
@@ -31,6 +31,8 @@ export function getLoaderContext(
31
31
  if (parentContext) {
32
32
  return {
33
33
  ...parentContext,
34
+ signal: options.signal || parentContext.signal,
35
+ reportProgress: options.onProgress || parentContext.reportProgress,
34
36
  coreApi: parentContext.coreApi || context.coreApi || coreApi,
35
37
  _parse: parentContext._parse || context._parse,
36
38
  fetch: getFetchFunction(options, parentContext)
@@ -39,6 +41,8 @@ export function getLoaderContext(
39
41
 
40
42
  const newContext: LoaderContext = {
41
43
  ...context,
44
+ signal: options.signal || context.signal,
45
+ reportProgress: options.onProgress || context.reportProgress,
42
46
  fetch: getFetchFunction(options, context),
43
47
  coreApi: context.coreApi || coreApi
44
48
  };
@@ -24,6 +24,7 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
24
24
  CDN: 'https://unpkg.com/@loaders.gl',
25
25
  worker: true, // By default, use worker if provided by loader.
26
26
  workerThreshold: 0.1,
27
+ workerInBatches: undefined,
27
28
  maxConcurrency: 3, // How many worker instances should be created for each loader.
28
29
  maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
29
30
  reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
@@ -219,7 +219,7 @@ function validateOptionsObject(
219
219
  for (const key in options) {
220
220
  // If top level option value is an object it could options for a loader, so ignore
221
221
  const isSubOptions = !id && isObject(options[key]);
222
- const isBaseUriOption = key === 'baseUri' && !id;
222
+ const isBaseUriOption = !id && ['baseUri', 'signal', 'onProgress'].includes(key);
223
223
  const isWorkerUrlOption = key === 'workerUrl' && id;
224
224
  // <loader>.workerUrl requires special handling as it is now auto-generated and no longer specified as a default option.
225
225
  if (!(key in defaultOptions) && !isBaseUriOption && !isWorkerUrlOption) {
@@ -112,6 +112,7 @@ export {
112
112
  parseWithWorker,
113
113
  parseWithWorkerInBatches,
114
114
  canParseWithWorker,
115
+ canParseWithWorkerInBatches,
115
116
  shouldParseWithWorker
116
117
  } from './lib/worker-loader-utils/parse-with-worker';
117
118
  export {canEncodeWithWorker} from './lib/worker-loader-utils/encode-with-worker';
@@ -13,7 +13,7 @@ export type JSONLoaderOptions = LoaderOptions;
13
13
 
14
14
  /**
15
15
  * A JSON Micro loader (minimal bundle size)
16
- * Alternative to `@loaders.gl/json`
16
+ * Alternative to `@loaders.gl/tables/json`
17
17
  */
18
18
  export const JSONLoader = {
19
19
  dataType: null as unknown as Table,
@@ -211,7 +211,9 @@ export class BearerTokenAuthentication extends TokenAuthentication {
211
211
  }
212
212
  }
213
213
 
214
- /** Wraps a fetch function with exact-origin credential application and one refresh replay. */
214
+ /**
215
+ * Wraps fetch with exact-origin credentials and one refresh replay, preserving abort reasons.
216
+ */
215
217
  export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): FetchLike {
216
218
  const suppliedFetch = options.fetch || ((url, requestOptions) => fetch(url, requestOptions));
217
219
  const existingMetadata = (
@@ -253,7 +255,7 @@ export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): Fe
253
255
 
254
256
  const authenticatedFetch: FetchLike = async (url, requestOptions = {}) => {
255
257
  requestOptions = mergeFetchOptions(fetchOptions, requestOptions);
256
- if (requestOptions.signal?.aborted) throw createAbortError();
258
+ requestOptions.signal?.throwIfAborted();
257
259
  // Request defaults alone must not normalize URLs or defer the transport invocation.
258
260
  if (!credentials.length) return baseFetch(url, requestOptions);
259
261
  const resolvedURL = parseHTTPURL(url);
@@ -283,7 +285,7 @@ export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): Fe
283
285
  credential.refreshStatusCodes?.includes(response.status)
284
286
  );
285
287
  if (!refreshableCredentials.length && !shouldRetrySignature) return response;
286
- if (requestOptions.signal?.aborted) throw createAbortError();
288
+ requestOptions.signal?.throwIfAborted();
287
289
 
288
290
  const refreshedTokens = new Map<RequestCredential, string>();
289
291
  await Promise.all(
@@ -295,7 +297,7 @@ export function createAuthenticatedFetch(options: AuthenticatedFetchOptions): Fe
295
297
  if (!refreshedTokens.size && !shouldRetrySignature) return response;
296
298
 
297
299
  await response.body?.cancel().catch(() => {});
298
- if (requestOptions.signal?.aborted) throw createAbortError();
300
+ requestOptions.signal?.throwIfAborted();
299
301
  const replayAuthorization = resolvedURL
300
302
  ? await authorizeRequest(
301
303
  resolvedURL,
@@ -340,9 +342,10 @@ export function getAuthenticatedFetch(options: LoaderOptions = {}): FetchLike {
340
342
  fetchFunction = (url, requestOptions) => fetch(url, requestOptions);
341
343
  }
342
344
 
345
+ const fetchOptions = typeof fetchOption === 'object' && fetchOption ? fetchOption : undefined;
343
346
  return createAuthenticatedFetch({
344
347
  fetch: fetchFunction,
345
- fetchOptions: typeof fetchOption === 'object' && fetchOption ? fetchOption : undefined,
348
+ fetchOptions: options.signal ? {...fetchOptions, signal: options.signal} : fetchOptions,
346
349
  credentials: options.core?.credentials || [],
347
350
  authentications: options.core?.authentications
348
351
  });
@@ -395,7 +398,7 @@ async function authenticateRequest(
395
398
  for (const credential of credentials) {
396
399
  const origin = getRequestOrigin(request.url);
397
400
  if (!origin || !normalizeRequestOrigins(credential.origins).includes(origin)) continue;
398
- if (signal?.aborted) throw createAbortError();
401
+ signal?.throwIfAborted();
399
402
  const authenticatedRequest = await credential.authenticate({
400
403
  url: request.url,
401
404
  options: {...request.options, headers: new Headers(request.options.headers)},
@@ -418,7 +421,7 @@ async function authenticateRequest(
418
421
  };
419
422
  appliedCredentials.push(credential);
420
423
  }
421
- if (signal?.aborted) throw createAbortError();
424
+ signal?.throwIfAborted();
422
425
  return {...request, credentials: appliedCredentials};
423
426
  }
424
427
 
@@ -569,8 +572,3 @@ function isReplayableRequest(options: RequestInit): boolean {
569
572
  options.body instanceof ReadableStream
570
573
  );
571
574
  }
572
-
573
- /** Creates the standard DOM exception used for an aborted fetch. */
574
- function createAbortError(): DOMException {
575
- return new DOMException('The operation was aborted.', 'AbortError');
576
- }
@@ -4,7 +4,14 @@
4
4
 
5
5
  /* eslint-disable no-restricted-globals */
6
6
  import type {CoreAPI} from '../sources/data-source';
7
- import type {LoaderWithParser, LoaderOptions, LoaderContext, Loader} from '../../loader-types';
7
+ import type {
8
+ LoaderWithParser,
9
+ LoaderOptions,
10
+ StrictLoaderOptions,
11
+ LoaderContext,
12
+ Loader
13
+ } from '../../loader-types';
14
+ import type {WorkerContext} from '@loaders.gl/core/worker-utils';
8
15
  import {createWorker} from '@loaders.gl/core/worker-utils';
9
16
  // import {validateLoaderVersion} from './validate-loader-version';
10
17
 
@@ -43,6 +50,7 @@ export async function processLoaderWorkerData(
43
50
  input: ArrayBuffer,
44
51
  options: {[key: string]: any} = {},
45
52
  workerContext?: {
53
+ reportProgress?: WorkerContext['reportProgress'];
46
54
  process?: (data: any, options?: LoaderOptions, context?: Record<string, any>) => any;
47
55
  },
48
56
  loaderContext: Record<string, any> = {}
@@ -53,6 +61,7 @@ export async function processLoaderWorkerData(
53
61
  options,
54
62
  context: {
55
63
  ...loaderContext,
64
+ reportProgress: workerContext?.reportProgress,
56
65
  coreApi: createWorkerCoreApi(),
57
66
  _parse: createParseOnMainThread(workerContext?.process)
58
67
  } as LoaderContext
@@ -71,6 +80,7 @@ export async function* processLoaderWorkerBatches(
71
80
  | Iterable<ArrayBufferLike | ArrayBufferView>,
72
81
  options: {[key: string]: any} = {},
73
82
  workerContext?: {
83
+ reportProgress?: WorkerContext['reportProgress'];
74
84
  process?: (data: any, options?: LoaderOptions, context?: Record<string, any>) => any;
75
85
  },
76
86
  loaderContext: Record<string, any> = {}
@@ -79,8 +89,10 @@ export async function* processLoaderWorkerBatches(
79
89
  throw new Error(`${loader.id} loader does not support batched parsing`);
80
90
  }
81
91
 
82
- const resultIterator = loader.parseInBatches(inputIterator, options, {
92
+ const parserOptions = getWorkerLoaderOptions(loader, options);
93
+ const resultIterator = loader.parseInBatches(inputIterator, parserOptions, {
83
94
  ...loaderContext,
95
+ reportProgress: workerContext?.reportProgress,
84
96
  coreApi: createWorkerCoreApi(),
85
97
  _parse: createParseOnMainThread(workerContext?.process)
86
98
  } as LoaderContext);
@@ -181,8 +193,17 @@ function getSerializableLoaderContext(context?: LoaderContext) {
181
193
  if (!context) {
182
194
  return undefined;
183
195
  }
184
- const {fetch, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext} =
185
- context;
196
+ const {
197
+ fetch,
198
+ loaders,
199
+ coreApi,
200
+ signal,
201
+ reportProgress,
202
+ _parse,
203
+ _parseSync,
204
+ _parseInBatches,
205
+ ...serializableContext
206
+ } = context;
186
207
  return JSON.parse(JSON.stringify(serializableContext));
187
208
  }
188
209
 
@@ -217,17 +238,19 @@ async function parseData({
217
238
  // Preserve worker-supplied module overrides (for example a caller-provided
218
239
  // Draco URL) while still applying the loader's defaults. Functions cannot be
219
240
  // structured-cloned, but URL strings and other serializable options can.
220
- options = {
221
- ...options,
222
- modules: {
223
- ...((loader && loader.options && loader.options.modules) || {}),
224
- ...(options.modules || {})
225
- },
226
- core: {
227
- ...options.core,
228
- worker: false
229
- }
230
- };
241
+ options = getWorkerLoaderOptions(loader, options);
231
242
 
232
243
  return await parser(data, {...options}, context, loader);
233
244
  }
245
+
246
+ /** Applies loader module defaults and prevents recursive worker dispatch inside a worker parser. */
247
+ function getWorkerLoaderOptions(
248
+ loader: LoaderWithParser,
249
+ options: LoaderOptions
250
+ ): StrictLoaderOptions {
251
+ return {
252
+ ...options,
253
+ modules: {...loader.options?.modules, ...options.modules},
254
+ core: {...options.core, worker: false}
255
+ } as StrictLoaderOptions;
256
+ }
@@ -4,6 +4,7 @@
4
4
 
5
5
  import {
6
6
  canProcessOnWorker,
7
+ canProcessOnWorkerInBatches,
7
8
  isBrowser,
8
9
  processOnWorker,
9
10
  processOnWorkerInBatches
@@ -65,6 +66,27 @@ export function canParseWithWorker(loader: Loader, options?: StrictLoaderOptions
65
66
  return Boolean(canProcessOnWorker(loader, workerOptions));
66
67
  }
67
68
 
69
+ /**
70
+ * Checks loader batch-parser transport capability and the selected worker's batch protocol.
71
+ * @param loader Loader declaring compatible batch parsing and transport.
72
+ * @param options Loader and worker selection options.
73
+ */
74
+ export function canParseWithWorkerInBatches(
75
+ loader: Loader,
76
+ options?: StrictLoaderOptions
77
+ ): boolean {
78
+ // Encryption key callbacks must stay on the caller; JSON worker options omit functions.
79
+ const parquetOptions = options?.parquet as {keyRetriever?: unknown} | undefined;
80
+ if (loader.id === 'parquet' && typeof parquetOptions?.keyRetriever === 'function') {
81
+ return false;
82
+ }
83
+ return Boolean(
84
+ loader.workerInBatches &&
85
+ canParseWithWorker(loader, options) &&
86
+ canProcessOnWorkerInBatches(loader, getWorkerOptions(options))
87
+ );
88
+ }
89
+
68
90
  /**
69
91
  * Determines whether an atomic parse should use a worker, including optional loader work estimates.
70
92
  * @param loader Loader metadata or parser-bearing loader.
@@ -118,11 +140,15 @@ export async function parseWithWorker(
118
140
  parseOnMainThread?: ParseOnMainThread,
119
141
  signal?: AbortSignal
120
142
  ) {
121
- const workerSignal = signal || getWorkerAbortSignal(options);
143
+ const workerSignal = signal || getWorkerAbortSignal(options) || context?.signal;
122
144
  const result = await processOnWorker(
123
145
  loader,
124
146
  data,
125
- {...getWorkerOptions(options, loader), signal: workerSignal},
147
+ {
148
+ ...getWorkerOptions(options, loader),
149
+ signal: workerSignal,
150
+ onProgress: options?.onProgress || context?.reportProgress
151
+ },
126
152
  {
127
153
  process: async (input, processOptions, _workerContext, parseContext) => {
128
154
  if (!parseOnMainThread) {
@@ -168,11 +194,15 @@ export async function* parseWithWorkerInBatches(
168
194
  parseOnMainThread?: ParseOnMainThread,
169
195
  signal?: AbortSignal
170
196
  ): AsyncIterable<unknown> {
171
- const workerSignal = signal || getWorkerAbortSignal(options);
197
+ const workerSignal = signal || getWorkerAbortSignal(options) || context?.signal;
172
198
  const outputIterator = processOnWorkerInBatches(
173
199
  loader,
174
200
  inputIterator,
175
- {...getWorkerOptions(options), signal: workerSignal},
201
+ {
202
+ ...getWorkerOptions(options, loader),
203
+ signal: workerSignal,
204
+ onProgress: options?.onProgress || context?.reportProgress
205
+ },
176
206
  {
177
207
  process: async (input, processOptions, _workerContext, parseContext) => {
178
208
  if (!parseOnMainThread) {
@@ -204,6 +234,7 @@ export async function* parseWithWorkerInBatches(
204
234
 
205
235
  /** Returns the loader-specific abort signal used to cancel a worker job. */
206
236
  function getWorkerAbortSignal(options?: StrictLoaderOptions): AbortSignal | undefined {
237
+ if (options?.signal) return options.signal;
207
238
  const parquetSignal = (options as {parquet?: {signal?: unknown}} | undefined)?.parquet?.signal;
208
239
  if (typeof AbortSignal !== 'undefined' && parquetSignal instanceof AbortSignal) {
209
240
  return parquetSignal;
@@ -278,10 +309,15 @@ function resolveWorkerLoaders(
278
309
  function getWorkerOptions(options: StrictLoaderOptions = {}, loader?: Loader) {
279
310
  const optionsToSerialize = {
280
311
  ...options,
312
+ signal: undefined,
313
+ onProgress: undefined,
281
314
  ...(options.core ? {core: {...options.core, authentications: undefined}} : {})
282
315
  };
283
316
  const serializedOptions = JSON.parse(
284
- JSON.stringify(optionsToSerialize, (_key, value) => (typeof value === 'bigint' ? null : value))
317
+ JSON.stringify(optionsToSerialize, (_key, value) => {
318
+ if (typeof AbortSignal !== 'undefined' && value instanceof AbortSignal) return undefined;
319
+ return typeof value === 'bigint' ? null : value;
320
+ })
285
321
  );
286
322
  restoreBigIntOptions(optionsToSerialize, serializedOptions);
287
323
  const workerOptions: Record<string, any> = {
@@ -317,8 +353,17 @@ function getSerializableLoaderContext(context?: LoaderContext) {
317
353
  if (!context) {
318
354
  return {};
319
355
  }
320
- const {fetch, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext} =
321
- context;
356
+ const {
357
+ fetch,
358
+ loaders,
359
+ coreApi,
360
+ signal,
361
+ reportProgress,
362
+ _parse,
363
+ _parseSync,
364
+ _parseInBatches,
365
+ ...serializableContext
366
+ } = context;
322
367
  return JSON.parse(JSON.stringify(serializableContext));
323
368
  }
324
369
 
@@ -3,7 +3,7 @@
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {Format} from './format-types';
6
- import type {LoadWorker} from '@loaders.gl/core/worker-utils';
6
+ import type {LoadWorker, WorkerOptions, WorkerProgress} from '@loaders.gl/core/worker-utils';
7
7
  import {DataType, FetchLike, TransformBatches} from './types';
8
8
  import {ReadableFile} from './lib/files/file';
9
9
  import type {CoreAPI} from './lib/sources/data-source';
@@ -18,7 +18,7 @@ export type SearchParams = Record<string, string | number | boolean>;
18
18
  /**
19
19
  * Core Loader Options
20
20
  */
21
- export type StrictLoaderOptions = {
21
+ export type StrictLoaderOptions = Pick<WorkerOptions, 'signal' | 'onProgress'> & {
22
22
  /** Experimental common scan request. Format loaders implement the fields they support. */
23
23
  _scan?: ExperimentalScanOptions;
24
24
  /** Query parameters appended to the initial URL request. */
@@ -77,6 +77,8 @@ export type StrictLoaderOptions = {
77
77
  CDN?: string | null;
78
78
  /** Set to `false` to disable workers, or `'auto'` to use loader work estimates. */
79
79
  worker?: boolean | 'auto';
80
+ /** Declares batch support for a custom worker URL, or disables batch offloading. The loader must also advertise workerInBatches. */
81
+ workerInBatches?: boolean;
80
82
  /** Work-score threshold used by `worker: 'auto'`. */
81
83
  workerThreshold?: number;
82
84
  /** Number of concurrent workers (per loader) on desktop browser */
@@ -97,13 +99,17 @@ export type StrictLoaderOptions = {
97
99
  modules?: Record<string, unknown>;
98
100
 
99
101
  // Accept other keys (loader options objects, e.g. `options.csv`, `options.json` ...)
100
- [loaderId: string]: Record<string, unknown> | undefined;
102
+ [loaderId: string]:
103
+ | Record<string, unknown>
104
+ | AbortSignal
105
+ | ((progress: WorkerProgress) => void)
106
+ | undefined;
101
107
  };
102
108
 
103
109
  /**
104
110
  * Core Loader Options
105
111
  */
106
- export type LoaderOptions = {
112
+ export type LoaderOptions = Pick<WorkerOptions, 'signal' | 'onProgress'> & {
107
113
  /** Experimental common scan request. Format loaders implement the fields they support. */
108
114
  _scan?: ExperimentalScanOptions;
109
115
  /** Query parameters appended to the initial URL request. */
@@ -193,6 +199,8 @@ export type Loader<DataT = any, BatchT = any, LoaderOptionsT = StrictLoaderOptio
193
199
  version: string;
194
200
  /** A boolean, or a URL */
195
201
  worker?: string | boolean;
202
+ /** Whether the worker and its batch transport support this loader's batch parser. */
203
+ workerInBatches?: boolean;
196
204
  /** Browser worker filename when it differs from the loader id. */
197
205
  workerFile?: string;
198
206
  /** Optional Node.js worker filename, such as a CommonJS `.cjs` bundle. */
@@ -332,6 +340,10 @@ export type LoaderWithParser<
332
340
  * threads back to main thread.
333
341
  */
334
342
  export type LoaderContext = {
343
+ /** Cancels nested asynchronous operations where supported. */
344
+ signal?: AbortSignal;
345
+ /** Reports progress within a loader-defined phase. */
346
+ reportProgress?: (progress: WorkerProgress) => void;
335
347
  /** Loader list provided to top-level loader call plus any sub loaders */
336
348
  loaders?: Loader[] | null;
337
349
  /** If URL is available. */
@@ -112,7 +112,7 @@ export function convertToObjectRowTable(table: Table): ObjectRowTable {
112
112
 
113
113
  /**
114
114
  * Convert a table to apache arrow format
115
- * @note this depends on the `@loaders.gl/arrow module being imported
115
+ * @note this depends on the `@loaders.gl/tables/arrow module being imported
116
116
  */
117
117
  export function convertToArrowTable(table: Table): ArrowTable {
118
118
  const schema = table.schema || deduceTableSchema(table);
@@ -10,6 +10,7 @@ export type {
10
10
  LoadWorker,
11
11
  WorkerObject,
12
12
  WorkerOptions,
13
+ WorkerProgress,
13
14
  WorkerContext,
14
15
  WorkerJobContext,
15
16
  Process,
@@ -41,7 +42,8 @@ export {
41
42
  processOnWorker,
42
43
  processOnWorkerInBatches,
43
44
  preloadWorker,
44
- canProcessOnWorker
45
+ canProcessOnWorker,
46
+ canProcessOnWorkerInBatches
45
47
  } from './lib/worker-api/process-on-worker';
46
48
  export {createWorker} from './lib/worker-api/create-worker';
47
49
 
@@ -3,6 +3,7 @@
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {
6
+ WorkerProgress,
6
7
  WorkerMessageType,
7
8
  WorkerMessagePayload,
8
9
  WorkerContext,
@@ -42,6 +43,9 @@ export async function createWorker(
42
43
 
43
44
  // eslint-disable-next-line complexity
44
45
  WorkerBody.onmessage = async (type: WorkerMessageType, payload: WorkerMessagePayload) => {
46
+ const reporter =
47
+ type === 'process' || type === 'process-in-batches' ? createProgressReporter() : undefined;
48
+ const jobContext = {...context, reportProgress: reporter?.report};
45
49
  try {
46
50
  switch (type) {
47
51
  case 'preload':
@@ -55,9 +59,10 @@ export async function createWorker(
55
59
  const result = await process(
56
60
  payload.input,
57
61
  payload.options || {},
58
- context,
62
+ jobContext,
59
63
  payload.context || {}
60
64
  );
65
+ reporter?.finish();
61
66
  WorkerBody.postMessage('done', {result});
62
67
  break;
63
68
 
@@ -73,13 +78,14 @@ export async function createWorker(
73
78
  const resultIterator = processInBatches(
74
79
  createDemandDrivenIterator(inputBatches),
75
80
  payload.options || {},
76
- context,
81
+ jobContext,
77
82
  payload.context || {}
78
83
  );
79
84
  for await (const batch of resultIterator) {
80
85
  await WorkerBody.postMessage('output-batch', {result: batch});
81
86
  await outputAcknowledgements.next();
82
87
  }
88
+ reporter?.finish();
83
89
  await WorkerBody.postMessage('done', {});
84
90
  } finally {
85
91
  activeInputBatches = null;
@@ -111,6 +117,7 @@ export async function createWorker(
111
117
  default:
112
118
  }
113
119
  } catch (error) {
120
+ reporter?.finish();
114
121
  const message = error instanceof Error ? error.message : '';
115
122
  WorkerBody.postMessage('error', {error: message});
116
123
  }
@@ -164,3 +171,35 @@ function processOnMainThread(arrayBuffer: ArrayBuffer, options = {}, jobContext
164
171
  WorkerBody.postMessage('process', payload);
165
172
  });
166
173
  }
174
+
175
+ /** Coalesces same-phase reports per event-loop turn and flushes before settlement. */
176
+ function createProgressReporter() {
177
+ let pending: WorkerProgress | undefined;
178
+ let finished = false;
179
+ let scheduled = false;
180
+ /** Posts the latest pending measurement. */
181
+ const flush = (): void => {
182
+ scheduled = false;
183
+ if (pending) {
184
+ WorkerBody.postMessage('progress', {progress: pending});
185
+ pending = undefined;
186
+ }
187
+ };
188
+ return {
189
+ /** Records progress while retaining phase transitions. */
190
+ report(progress: WorkerProgress): void {
191
+ if (finished) return;
192
+ if (pending && pending.phase !== progress.phase) flush();
193
+ pending = {...progress};
194
+ if (!scheduled) {
195
+ scheduled = true;
196
+ setTimeout(flush, 0);
197
+ }
198
+ },
199
+ /** Flushes final progress and suppresses reports from stale callbacks. */
200
+ finish(): void {
201
+ flush();
202
+ finished = true;
203
+ }
204
+ };
205
+ }
@@ -32,23 +32,10 @@ export function getCustomWorkerURL(
32
32
  worker: WorkerObject,
33
33
  options: WorkerOptions = {}
34
34
  ): string | null {
35
- const workerOptions = options[worker.id] || {};
36
-
37
35
  const workerFile = isBrowser
38
36
  ? worker.workerFile || `${worker.id}-worker.js`
39
37
  : worker.workerNode || `${worker.id}-worker-node.js`;
40
-
41
- let url = workerOptions.workerUrl;
42
-
43
- // HACK: Allow for non-nested workerUrl for the CompressionWorker.
44
- // For the compression worker, workerOptions is currently not nested correctly. For most loaders,
45
- // you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker
46
- // puts options at the top level, not within a `compression` key (its `id`). For this reason, the
47
- // above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not
48
- // break backwards compatibility, we allow the CompressionWorker to have options at the top level.
49
- if (!url && worker.id === 'compression') {
50
- url = options.workerUrl;
51
- }
38
+ let url = getConfiguredWorkerURL(worker, options);
52
39
 
53
40
  // If URL is test, generate local loaders.gl url
54
41
  // @ts-ignore _workerType
@@ -71,6 +58,18 @@ export function getCustomWorkerURL(
71
58
  return url || null;
72
59
  }
73
60
 
61
+ /** Returns an explicit worker URL override, excluding generated local test URLs. */
62
+ export function getConfiguredWorkerURL(
63
+ worker: WorkerObject,
64
+ options: WorkerOptions = {}
65
+ ): string | null {
66
+ const workerOptions = options[worker.id] || {};
67
+ // Compression also supports the historical non-nested URL override.
68
+ return (
69
+ workerOptions.workerUrl || (worker.id === 'compression' ? options.workerUrl : null) || null
70
+ );
71
+ }
72
+
74
73
  /**
75
74
  * Generates a worker URL using overrides first and the published CDN artifact as a fallback.
76
75
  * @param worker Worker descriptor to resolve.