@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/dist/worker-utils/lib/worker-api/create-worker.d.ts

@@ -0,0 +1,9 @@
+import type { WorkerJobContext, Process, ProcessInBatches } from "../../types.js";
+export type ProcessOnMainThread = (data: any, options?: {
+    [key: string]: any;
+}, jobContext?: WorkerJobContext) => any;
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ */
+export declare function createWorker(process: Process, processInBatches?: ProcessInBatches): Promise<void>;
+//# sourceMappingURL=create-worker.d.ts.map

package/dist/worker-utils/lib/worker-api/create-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-worker.d.ts","sourceRoot":"","sources":["../../../../src/worker-utils/lib/worker-api/create-worker.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAIV,gBAAgB,EAChB,OAAO,EACP,gBAAgB,EACjB,uBAAoB;AAUrB,MAAM,MAAM,mBAAmB,GAAG,CAChC,IAAI,EAAE,GAAG,EACT,OAAO,CAAC,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,EAC9B,UAAU,CAAC,EAAE,gBAAgB,KAC1B,GAAG,CAAC;AAET;;GAEG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,OAAO,EAChB,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,CAAC,IAAI,CAAC,CAoFf"}

package/dist/worker-utils/lib/worker-api/create-worker.js

@@ -0,0 +1,123 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import AsyncQueue from "../async-queue/async-queue.js";
+import WorkerBody from "../worker-farm/worker-body.js";
+// import {validateWorkerVersion} from './validate-worker-version';
+/** Counter for jobs */
+let requestId = 0;
+let activeInputBatches = null;
+let activeOutputAcknowledgements = null;
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ */
+export async function createWorker(process, processInBatches) {
+    if (!(await WorkerBody.inWorkerThread())) {
+        return;
+    }
+    const context = {
+        process: processOnMainThread
+    };
+    // eslint-disable-next-line complexity
+    WorkerBody.onmessage = async (type, payload) => {
+        try {
+            switch (type) {
+                case 'preload':
+                    WorkerBody.postMessage('done', {});
+                    break;
+                case 'process':
+                    if (!process) {
+                        throw new Error('Worker does not support atomic processing');
+                    }
+                    const result = await process(payload.input, payload.options || {}, context, payload.context || {});
+                    WorkerBody.postMessage('done', { result });
+                    break;
+                case 'process-in-batches':
+                    if (!processInBatches) {
+                        throw new Error('Worker does not support batched processing');
+                    }
+                    const inputBatches = new AsyncQueue();
+                    const outputAcknowledgements = new AsyncQueue();
+                    activeInputBatches = inputBatches;
+                    activeOutputAcknowledgements = outputAcknowledgements;
+                    try {
+                        const resultIterator = processInBatches(createDemandDrivenIterator(inputBatches), payload.options || {}, context, payload.context || {});
+                        for await (const batch of resultIterator) {
+                            await WorkerBody.postMessage('output-batch', { result: batch });
+                            await outputAcknowledgements.next();
+                        }
+                        await WorkerBody.postMessage('done', {});
+                    }
+                    finally {
+                        activeInputBatches = null;
+                        activeOutputAcknowledgements = null;
+                    }
+                    break;
+                case 'input-batch':
+                    if (!activeInputBatches) {
+                        throw new Error('Worker has no active batched processing session');
+                    }
+                    activeInputBatches.push(payload.input);
+                    break;
+                case 'input-done':
+                    if (!activeInputBatches) {
+                        throw new Error('Worker has no active batched processing session');
+                    }
+                    activeInputBatches.close();
+                    break;
+                case 'output-ack':
+                    if (!activeOutputAcknowledgements) {
+                        throw new Error('Worker has no active batched processing session');
+                    }
+                    activeOutputAcknowledgements.push(undefined);
+                    break;
+                default:
+            }
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : '';
+            WorkerBody.postMessage('error', { error: message });
+        }
+    };
+}
+/** Requests exactly one input batch whenever the worker-side processor advances its iterator. */
+async function* createDemandDrivenIterator(inputBatches) {
+    while (true) {
+        await WorkerBody.postMessage('input-request', {});
+        const nextBatch = await inputBatches.next();
+        if (nextBatch.done) {
+            return;
+        }
+        yield nextBatch.value;
+    }
+}
+function processOnMainThread(arrayBuffer, options = {}, jobContext = {}) {
+    return new Promise((resolve, reject) => {
+        const id = requestId++;
+        /**
+         */
+        const onMessage = (type, payload) => {
+            if (payload.id !== id) {
+                // not ours
+                return;
+            }
+            switch (type) {
+                case 'done':
+                    WorkerBody.removeEventListener(onMessage);
+                    resolve(payload.result);
+                    break;
+                case 'error':
+                    WorkerBody.removeEventListener(onMessage);
+                    reject(payload.error);
+                    break;
+                default:
+                // ignore
+            }
+        };
+        WorkerBody.addEventListener(onMessage);
+        // Ask the main thread to decode data
+        const payload = { id, input: arrayBuffer, options, context: jobContext };
+        WorkerBody.postMessage('process', payload);
+    });
+}
+//# sourceMappingURL=create-worker.js.map

package/dist/worker-utils/lib/worker-api/create-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-worker.js","sourceRoot":"","sources":["../../../../src/worker-utils/lib/worker-api/create-worker.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAUpC,OAAO,UAAU,sCAAmC;AACpD,OAAO,UAAU,sCAAmC;AACpD,mEAAmE;AAEnE,uBAAuB;AACvB,IAAI,SAAS,GAAG,CAAC,CAAC;AAClB,IAAI,kBAAkB,GAA2B,IAAI,CAAC;AACtD,IAAI,4BAA4B,GAA4B,IAAI,CAAC;AAQjE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAgB,EAChB,gBAAmC;IAEnC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,cAAc,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAkB;QAC7B,OAAO,EAAE,mBAAmB;KAC7B,CAAC;IAEF,sCAAsC;IACtC,UAAU,CAAC,SAAS,GAAG,KAAK,EAAE,IAAuB,EAAE,OAA6B,EAAE,EAAE;QACtF,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,SAAS;oBACZ,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;oBACnC,MAAM;gBAER,KAAK,SAAS;oBACZ,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;oBAC/D,CAAC;oBACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAC1B,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,OAAO,IAAI,EAAE,EACrB,OAAO,EACP,OAAO,CAAC,OAAO,IAAI,EAAE,CACtB,CAAC;oBACF,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC;oBACzC,MAAM;gBAER,KAAK,oBAAoB;oBACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACtB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;oBAChE,CAAC;oBACD,MAAM,YAAY,GAAG,IAAI,UAAU,EAAO,CAAC;oBAC3C,MAAM,sBAAsB,GAAG,IAAI,UAAU,EAAQ,CAAC;oBACtD,kBAAkB,GAAG,YAAY,CAAC;oBAClC,4BAA4B,GAAG,sBAAsB,CAAC;oBACtD,IAAI,CAAC;wBACH,MAAM,cAAc,GAAG,gBAAgB,CACrC,0BAA0B,CAAC,YAAY,CAAC,EACxC,OAAO,CAAC,OAAO,IAAI,EAAE,EACrB,OAAO,EACP,OAAO,CAAC,OAAO,IAAI,EAAE,CACtB,CAAC;wBACF,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;4BACzC,MAAM,UAAU,CAAC,WAAW,CAAC,cAAc,EAAE,EAAC,MAAM,EAAE,KAAK,EAAC,CAAC,CAAC;4BAC9D,MAAM,sBAAsB,CAAC,IAAI,EAAE,CAAC;wBACtC,CAAC;wBACD,MAAM,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;oBAC3C,CAAC;4BAAS,CAAC;wBACT,kBAAkB,GAAG,IAAI,CAAC;wBAC1B,4BAA4B,GAAG,IAAI,CAAC;oBACtC,CAAC;oBACD,MAAM;gBAER,KAAK,aAAa;oBAChB,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;oBACrE,CAAC;oBACD,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACvC,MAAM;gBAER,KAAK,YAAY;oBACf,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;oBACrE,CAAC;oBACD,kBAAkB,CAAC,KAAK,EAAE,CAAC;oBAC3B,MAAM;gBAER,KAAK,YAAY;oBACf,IAAI,CAAC,4BAA4B,EAAE,CAAC;wBAClC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;oBACrE,CAAC;oBACD,4BAA4B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC7C,MAAM;gBAER,QAAQ;YACV,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,UAAU,CAAC,WAAW,CAAC,OAAO,EAAE,EAAC,KAAK,EAAE,OAAO,EAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,iGAAiG;AACjG,KAAK,SAAS,CAAC,CAAC,0BAA0B,CAAC,YAA6B;IACtE,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,UAAU,CAAC,WAAW,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,SAAS,CAAC,KAAK,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAwB,EAAE,OAAO,GAAG,EAAE,EAAE,UAAU,GAAG,EAAE;IAClF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;QAEvB;WACG;QACH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,OAA6B,EAAE,EAAE;YAChE,IAAI,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;gBACtB,WAAW;gBACX,OAAO;YACT,CAAC;YAED,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,MAAM;oBACT,UAAU,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;oBAC1C,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBACxB,MAAM;gBAER,KAAK,OAAO;oBACV,UAAU,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;oBAC1C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACtB,MAAM;gBAER,QAAQ;gBACR,SAAS;YACX,CAAC;QACH,CAAC,CAAC;QAEF,UAAU,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAEvC,qCAAqC;QACrC,MAAM,OAAO,GAAG,EAAC,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAC,CAAC;QACvE,UAAU,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts

@@ -0,0 +1,27 @@
+import type { WorkerObject, WorkerOptions } from "../../types.js";
+/**
+ * Gets worker object's name (for debugging in Chrome thread inspector window)
+ */
+export declare function getWorkerName(worker: WorkerObject): string;
+/**
+ * Returns an explicitly configured or test worker URL.
+ * @param worker Worker descriptor to resolve.
+ * @param options Worker options that can override the descriptor.
+ * @returns A worker URL, or `null` when URL-based fallback resolution is still required.
+ */
+export declare function getCustomWorkerURL(worker: WorkerObject, options?: WorkerOptions): string | null;
+/**
+ * Generates a worker URL using overrides first and the published CDN artifact as a fallback.
+ * @param worker Worker descriptor to resolve.
+ * @param options Worker options that can override the descriptor.
+ * @returns A loadable worker URL.
+ */
+export declare function getWorkerURL(worker: WorkerObject, options?: WorkerOptions): string;
+/**
+ * Returns the generated URL for a published pre-built worker.
+ * @param worker Worker descriptor to resolve.
+ * @param warn Whether to warn when an uninjected development version uses the npm tag.
+ * @returns The CDN worker URL.
+ */
+export declare function getDefaultWorkerURL(worker: WorkerObject, warn?: boolean): string;
+//# sourceMappingURL=get-worker-url.d.ts.map

package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-worker-url.d.ts","sourceRoot":"","sources":["../../../../src/worker-utils/lib/worker-api/get-worker-url.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAC,YAAY,EAAE,aAAa,EAAC,uBAAoB;AAQ7D;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAG1D;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,aAAkB,GAC1B,MAAM,GAAG,IAAI,CAsCf;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,GAAE,aAAkB,GAAG,MAAM,CAStF;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,GAAE,OAAe,GAAG,MAAM,CAcvF"}

package/dist/worker-utils/lib/worker-api/get-worker-url.js

@@ -0,0 +1,103 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { assert } from "../env-utils/assert.js";
+import { isBrowser } from "../env-utils/globals.js";
+import { VERSION } from "../env-utils/version.js";
+import { NPM_TAG } from "../npm-tag.js";
+const warnedWorkerVersionFallbacks = new Set();
+/**
+ * Gets worker object's name (for debugging in Chrome thread inspector window)
+ */
+export function getWorkerName(worker) {
+    const warning = worker.version !== VERSION ? ` (worker-utils@${VERSION})` : '';
+    return `${worker.name}@${worker.version}${warning}`;
+}
+/**
+ * Returns an explicitly configured or test worker URL.
+ * @param worker Worker descriptor to resolve.
+ * @param options Worker options that can override the descriptor.
+ * @returns A worker URL, or `null` when URL-based fallback resolution is still required.
+ */
+export function getCustomWorkerURL(worker, options = {}) {
+    const workerOptions = options[worker.id] || {};
+    const workerFile = isBrowser
+        ? worker.workerFile || `${worker.id}-worker.js`
+        : worker.workerNode || `${worker.id}-worker-node.js`;
+    let url = workerOptions.workerUrl;
+    // HACK: Allow for non-nested workerUrl for the CompressionWorker.
+    // For the compression worker, workerOptions is currently not nested correctly. For most loaders,
+    // you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker
+    // puts options at the top level, not within a `compression` key (its `id`). For this reason, the
+    // above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not
+    // break backwards compatibility, we allow the CompressionWorker to have options at the top level.
+    if (!url && worker.id === 'compression') {
+        url = options.workerUrl;
+    }
+    // If URL is test, generate local loaders.gl url
+    // @ts-ignore _workerType
+    const workerType = options._workerType || options?.core?._workerType;
+    if (!url && workerType === 'test') {
+        if (isBrowser) {
+            url = `modules/${worker.module}/dist/${workerFile}`;
+        }
+        else {
+            // In the test environment the ts-node loader requires TypeScript code
+            const workerSourceDirectory = worker.module === 'core' && worker.workerNode?.startsWith('worker-utils/')
+                ? 'worker-utils/workers'
+                : 'workers';
+            const workerSourceNode = worker.workerSourceNode || `${workerSourceDirectory}/${worker.id}-worker-node.ts`;
+            url = `modules/${worker.module}/src/${workerSourceNode}`;
+        }
+    }
+    return url || null;
+}
+/**
+ * Generates a worker URL using overrides first and the published CDN artifact as a fallback.
+ * @param worker Worker descriptor to resolve.
+ * @param options Worker options that can override the descriptor.
+ * @returns A loadable worker URL.
+ */
+export function getWorkerURL(worker, options = {}) {
+    const customWorkerUrl = getCustomWorkerURL(worker, options);
+    const descriptorWorkerUrl = typeof worker.worker === 'string' ? worker.worker : null;
+    const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
+    assert(url);
+    // Allow user to override location
+    return url;
+}
+/**
+ * Returns the generated URL for a published pre-built worker.
+ * @param worker Worker descriptor to resolve.
+ * @param warn Whether to warn when an uninjected development version uses the npm tag.
+ * @returns The CDN worker URL.
+ */
+export function getDefaultWorkerURL(worker, warn = false) {
+    const workerFile = isBrowser
+        ? worker.workerFile || `${worker.id}-worker.js`
+        : worker.workerNode || `${worker.id}-worker-node.js`;
+    let version = worker.version;
+    if (version === 'latest') {
+        version = NPM_TAG;
+    }
+    const versionTag = version ? `@${version}` : '';
+    const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+    if (warn) {
+        warnIfUsingNpmTagFallback(worker, url);
+    }
+    return url;
+}
+/** Warn once when a worker falls back to the npm tag because __VERSION__ was not injected. */
+function warnIfUsingNpmTagFallback(worker, url) {
+    if (worker.version !== 'latest') {
+        return;
+    }
+    const workerId = `${worker.module}:${worker.id}`;
+    if (warnedWorkerVersionFallbacks.has(workerId)) {
+        return;
+    }
+    warnedWorkerVersionFallbacks.add(workerId);
+    // eslint-disable-next-line no-console
+    console.warn(`loaders.gl: ${worker.name} loader worker version is "latest" because __VERSION__ was not injected. Fetching ${url} from CDN.`);
+}
+//# sourceMappingURL=get-worker-url.js.map

package/dist/worker-utils/lib/worker-api/get-worker-url.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-worker-url.js","sourceRoot":"","sources":["../../../../src/worker-utils/lib/worker-api/get-worker-url.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAOpC,OAAO,EAAC,MAAM,EAAC,+BAA4B;AAC3C,OAAO,EAAC,SAAS,EAAC,gCAA6B;AAC/C,OAAO,EAAC,OAAO,EAAC,gCAA6B;AAC7C,OAAO,EAAC,OAAO,EAAC,sBAAmB;AAEnC,MAAM,4BAA4B,GAAG,IAAI,GAAG,EAAU,CAAC;AAEvD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAAoB;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,kBAAkB,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/E,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,GAAG,OAAO,EAAE,CAAC;AACtD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAoB,EACpB,UAAyB,EAAE;IAE3B,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;IAE/C,MAAM,UAAU,GAAG,SAAS;QAC1B,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,MAAM,CAAC,EAAE,YAAY;QAC/C,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAEvD,IAAI,GAAG,GAAG,aAAa,CAAC,SAAS,CAAC;IAElC,kEAAkE;IAClE,iGAAiG;IACjG,kGAAkG;IAClG,iGAAiG;IACjG,kGAAkG;IAClG,kGAAkG;IAClG,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,KAAK,aAAa,EAAE,CAAC;QACxC,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,gDAAgD;IAChD,yBAAyB;IACzB,MAAM,UAAU,GAAI,OAAe,CAAC,WAAW,IAAK,OAAe,EAAE,IAAI,EAAE,WAAW,CAAC;IACvF,IAAI,CAAC,GAAG,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAClC,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,GAAG,WAAW,MAAM,CAAC,MAAM,SAAS,UAAU,EAAE,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,sEAAsE;YACtE,MAAM,qBAAqB,GACzB,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,eAAe,CAAC;gBACxE,CAAC,CAAC,sBAAsB;gBACxB,CAAC,CAAC,SAAS,CAAC;YAChB,MAAM,gBAAgB,GACpB,MAAM,CAAC,gBAAgB,IAAI,GAAG,qBAAqB,IAAI,MAAM,CAAC,EAAE,iBAAiB,CAAC;YACpF,GAAG,GAAG,WAAW,MAAM,CAAC,MAAM,QAAQ,gBAAgB,EAAE,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,OAAO,GAAG,IAAI,IAAI,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,MAAoB,EAAE,UAAyB,EAAE;IAC5E,MAAM,eAAe,GAAG,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5D,MAAM,mBAAmB,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACrF,MAAM,GAAG,GAAG,eAAe,IAAI,mBAAmB,IAAI,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAExF,MAAM,CAAC,GAAG,CAAC,CAAC;IAEZ,kCAAkC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB,EAAE,OAAgB,KAAK;IAC7E,MAAM,UAAU,GAAG,SAAS;QAC1B,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,MAAM,CAAC,EAAE,YAAY;QAC/C,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,MAAM,CAAC,EAAE,iBAAiB,CAAC;IACvD,IAAI,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC7B,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,GAAG,OAAO,CAAC;IACpB,CAAC;IACD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAChD,MAAM,GAAG,GAAG,iCAAiC,MAAM,CAAC,MAAM,GAAG,UAAU,SAAS,UAAU,EAAE,CAAC;IAC7F,IAAI,IAAI,EAAE,CAAC;QACT,yBAAyB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8FAA8F;AAC9F,SAAS,yBAAyB,CAAC,MAAoB,EAAE,GAAW;IAClE,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;IACjD,IAAI,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,OAAO;IACT,CAAC;IAED,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3C,sCAAsC;IACtC,OAAO,CAAC,IAAI,CACV,eAAe,MAAM,CAAC,IAAI,qFAAqF,GAAG,YAAY,CAC/H,CAAC;AACJ,CAAC"}

package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts

@@ -0,0 +1,44 @@
+import type { WorkerObject, WorkerOptions, WorkerContext, WorkerJobContext } from "../../types.js";
+/** Options for worker processing */
+export type ProcessOnWorkerOptions = WorkerOptions & {
+    /** Cancels the job by terminating its worker. */
+    signal?: AbortSignal;
+    /** Diagnostic name shown for the worker job. */
+    jobName?: string;
+    [key: string]: any;
+};
+/** Options for preloading workers. */
+export type PreloadWorkerOptions = {
+    /** Number of workers to warm in the worker pool. */
+    count?: number;
+};
+/**
+ * Determines if we can parse with worker
+ * @param loader
+ * @param data
+ * @param options
+ */
+export declare function canProcessOnWorker(worker: WorkerObject, options?: WorkerOptions): boolean;
+/**
+ * This function expects that the worker thread sends certain messages,
+ * Creating such a worker can be automated if the worker is wrapper by a call to
+ * createWorker in @loaders.gl/core/worker-utils.
+ */
+export declare function processOnWorker(worker: WorkerObject, data: any, options?: ProcessOnWorkerOptions, context?: WorkerContext, jobContext?: WorkerJobContext): Promise<any>;
+/**
+ * Processes an input iterator on one leased worker and streams output batches back.
+ *
+ * The worker remains assigned to this job for the complete iterator lifetime, allowing
+ * `processInBatches` implementations to retain parser or encoder state between input batches.
+ * Input is demand-driven: the main thread advances the source only when that worker requests the
+ * next batch.
+ */
+export declare function processOnWorkerInBatches<InputBatch = any, OutputBatch = any>(worker: WorkerObject, input: AsyncIterable<InputBatch> | Iterable<InputBatch>, options?: ProcessOnWorkerOptions, context?: WorkerContext, jobContext?: WorkerJobContext): AsyncIterable<OutputBatch>;
+/**
+ * Warm-start one or more workers in the same pool used by processOnWorker.
+ * @param worker Worker object to preload.
+ * @param options Worker options used to resolve the worker pool.
+ * @param preloadOptions Preload options.
+ */
+export declare function preloadWorker(worker: WorkerObject, options?: ProcessOnWorkerOptions, preloadOptions?: PreloadWorkerOptions): Promise<void>;
+//# sourceMappingURL=process-on-worker.d.ts.map

package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-on-worker.d.ts","sourceRoot":"","sources":["../../../../src/worker-utils/lib/worker-api/process-on-worker.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,YAAY,EACZ,aAAa,EACb,aAAa,EACb,gBAAgB,EAGjB,uBAAoB;AAcrB,oCAAoC;AACpC,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG;IACnD,iDAAiD;IACjD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB,CAAC;AAEF,sCAAsC;AACtC,MAAM,MAAM,oBAAoB,GAAG;IACjC,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,aAAa,WAS/E;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,GAAG,EACT,OAAO,GAAE,sBAA2B,EACpC,OAAO,GAAE,aAAkB,EAC3B,UAAU,GAAE,gBAAqB,GAChC,OAAO,CAAC,GAAG,CAAC,CAsCd;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,GAAG,GAAG,EAAE,WAAW,GAAG,GAAG,EAC1E,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,EACvD,OAAO,GAAE,sBAA2B,EACpC,OAAO,GAAE,aAAkB,EAC3B,UAAU,GAAE,gBAAqB,GAChC,aAAa,CAAC,WAAW,CAAC,CAE5B;AAsKD;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,sBAA2B,EACpC,cAAc,GAAE,oBAAyB,GACxC,OAAO,CAAC,IAAI,CAAC,CAaf"}

package/dist/worker-utils/lib/worker-api/process-on-worker.js

@@ -0,0 +1,312 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import AsyncQueue from "../async-queue/async-queue.js";
+import WorkerFarm from "../worker-farm/worker-farm.js";
+import { getCustomWorkerURL, getDefaultWorkerURL, getWorkerURL, getWorkerName } from "./get-worker-url.js";
+import { getTransferListForWriter } from "../worker-utils/get-transfer-list.js";
+import { isBrowser } from "../env-utils/globals.js";
+/**
+ * Determines if we can parse with worker
+ * @param loader
+ * @param data
+ * @param options
+ */
+export function canProcessOnWorker(worker, options) {
+    if (!WorkerFarm.isSupported()) {
+        return false;
+    }
+    const workerOptions = options?.[worker.id];
+    return Boolean((worker.worker || worker.loadWorker || workerOptions?.workerUrl) && options?.worker);
+}
+/**
+ * This function expects that the worker thread sends certain messages,
+ * Creating such a worker can be automated if the worker is wrapper by a call to
+ * createWorker in @loaders.gl/core/worker-utils.
+ */
+export async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
+    throwIfAborted(options.signal);
+    const name = getWorkerName(worker);
+    const workerFarm = WorkerFarm.getWorkerFarm(options);
+    const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
+    const jobName = options.jobName || worker.name;
+    const job = await workerPool.startJob(jobName, 
+    // eslint-disable-next-line
+    onMessage.bind(null, context));
+    const abortJob = () => job.abort(getAbortReason(options.signal));
+    options.signal?.addEventListener('abort', abortJob, { once: true });
+    if (options.signal?.aborted) {
+        abortJob();
+    }
+    try {
+        // AbortSignal cannot be structured-cloned. It controls the job on this thread only.
+        const { signal: _signal, ...workerOptions } = options;
+        const transferableOptions = getTransferListForWriter(workerOptions);
+        const transferableContext = getTransferListForWriter(jobContext);
+        if (job.isRunning) {
+            job.postMessage('process', {
+                input: data,
+                options: transferableOptions,
+                context: transferableContext
+            });
+        }
+        const result = await job.result;
+        return result.result;
+    }
+    finally {
+        options.signal?.removeEventListener('abort', abortJob);
+    }
+}
+/**
+ * Processes an input iterator on one leased worker and streams output batches back.
+ *
+ * The worker remains assigned to this job for the complete iterator lifetime, allowing
+ * `processInBatches` implementations to retain parser or encoder state between input batches.
+ * Input is demand-driven: the main thread advances the source only when that worker requests the
+ * next batch.
+ */
+export function processOnWorkerInBatches(worker, input, options = {}, context = {}, jobContext = {}) {
+    return processOnWorkerInBatchesIterator(worker, input, options, context, jobContext);
+}
+/** Implements stateful worker batch processing as an async generator. */
+async function* processOnWorkerInBatchesIterator(worker, input, options, context, jobContext) {
+    throwIfAborted(options.signal);
+    const name = getWorkerName(worker);
+    const workerFarm = WorkerFarm.getWorkerFarm(options);
+    const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
+    const outputBatches = new AsyncQueue();
+    const inputIterator = getAsyncIterator(input);
+    let outputFinished = false;
+    let inputFinished = false;
+    let inputRequest = Promise.resolve();
+    const batchJobFailed = new Error('Worker batch job failed');
+    const finishOutput = () => {
+        if (!outputFinished) {
+            outputFinished = true;
+            outputBatches.close();
+        }
+    };
+    const failOutput = (_error) => {
+        if (!outputFinished) {
+            outputFinished = true;
+            outputBatches.enqueue(batchJobFailed);
+            outputBatches.close();
+        }
+    };
+    const job = await workerPool.startJob(options.jobName || worker.name, (activeJob, type, payload) => {
+        switch (type) {
+            case 'input-request':
+                inputRequest = inputRequest
+                    .then(async () => {
+                    const nextBatch = await inputIterator.next();
+                    if (!activeJob.isRunning) {
+                        return;
+                    }
+                    if (nextBatch.done) {
+                        inputFinished = true;
+                        activeJob.postMessage('input-done', {});
+                    }
+                    else {
+                        activeJob.postMessage('input-batch', { input: nextBatch.value });
+                    }
+                })
+                    .catch(error => activeJob.abort(error));
+                break;
+            case 'output-batch':
+                outputBatches.push(payload.result);
+                break;
+            case 'done':
+                activeJob.done(payload);
+                finishOutput();
+                break;
+            case 'error': {
+                const error = new Error(payload.error || 'Worker batch processing failed');
+                activeJob.error(error);
+                failOutput(error);
+                break;
+            }
+            case 'process':
+                void onMessage(context, activeJob, type, payload);
+                break;
+            default: {
+                const error = new Error(`Unexpected worker batch message: ${type}`);
+                activeJob.error(error);
+                failOutput(error);
+                break;
+            }
+        }
+    });
+    const abortJob = () => job.abort(getAbortReason(options.signal));
+    options.signal?.addEventListener('abort', abortJob, { once: true });
+    if (options.signal?.aborted) {
+        abortJob();
+    }
+    void job.result.catch(failOutput);
+    try {
+        const { signal: _signal, ...workerOptions } = options;
+        if (job.isRunning) {
+            job.postMessage('process-in-batches', {
+                options: getTransferListForWriter(workerOptions),
+                context: getTransferListForWriter(jobContext)
+            });
+        }
+        try {
+            for await (const outputBatch of outputBatches) {
+                yield outputBatch;
+                if (job.isRunning) {
+                    job.postMessage('output-ack', {});
+                }
+            }
+        }
+        catch (error) {
+            if (error === batchJobFailed) {
+                await job.result;
+            }
+            throw error;
+        }
+        await job.result;
+    }
+    finally {
+        options.signal?.removeEventListener('abort', abortJob);
+        if (job.isRunning) {
+            job.abort(createAbortError('Worker batch iterator was closed'));
+        }
+        if (!inputFinished) {
+            closeInputIterator(inputIterator);
+        }
+    }
+}
+/** Throws a cross-runtime abort error when a signal is already aborted. */
+function throwIfAborted(signal) {
+    if (!signal?.aborted) {
+        return;
+    }
+    throw getAbortReason(signal);
+}
+/** Returns a caller-provided abort reason or a cross-runtime `AbortError`. */
+function getAbortReason(signal) {
+    return signal?.reason ?? createAbortError('Worker job was aborted');
+}
+/** Creates an abort error without requiring the DOMException global. */
+function createAbortError(message) {
+    const error = new Error(message);
+    error.name = 'AbortError';
+    return error;
+}
+/** Returns an asynchronous iterator for either accepted source kind. */
+function getAsyncIterator(input) {
+    if (Symbol.asyncIterator in input) {
+        return input[Symbol.asyncIterator]();
+    }
+    return (async function* iterateSynchronously() {
+        yield* input;
+    })();
+}
+/** Closes an input iterator without allowing a blocked return to delay cancellation. */
+function closeInputIterator(inputIterator) {
+    try {
+        void Promise.resolve(inputIterator.return?.()).catch(() => undefined);
+    }
+    catch {
+        // The worker job has already been aborted; iterator cleanup is best effort.
+    }
+}
+/**
+ * Warm-start one or more workers in the same pool used by processOnWorker.
+ * @param worker Worker object to preload.
+ * @param options Worker options used to resolve the worker pool.
+ * @param preloadOptions Preload options.
+ */
+export async function preloadWorker(worker, options = {}, preloadOptions = {}) {
+    const name = getWorkerName(worker);
+    const workerFarm = WorkerFarm.getWorkerFarm(options);
+    const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
+    const count = preloadOptions.count ?? options.maxConcurrency ?? options.core?.maxConcurrency ?? 1;
+    const preloadJobs = Array.from({ length: count }, async () => {
+        const job = await workerPool.startJob(`${worker.name} preload`, onPreloadMessage);
+        job.postMessage('preload', {});
+        return await job.result;
+    });
+    await Promise.all(preloadJobs);
+}
+/**
+ * Resolves the worker source while preserving URL overrides and classic-worker fallbacks.
+ * @param worker Worker descriptor to resolve.
+ * @param options Options for the active worker operation.
+ * @param name Versioned worker name.
+ * @returns A target suitable for worker-pool lookup.
+ */
+function getWorkerPoolTarget(worker, options, name) {
+    if (options.source) {
+        return { name, source: options.source };
+    }
+    const customWorkerUrl = getCustomWorkerURL(worker, options);
+    if (customWorkerUrl) {
+        return { name, url: customWorkerUrl };
+    }
+    if (isBrowser && worker.loadWorker) {
+        return {
+            name,
+            loadWorker: worker.loadWorker,
+            getUrl: () => getWorkerURL(worker, options),
+            urlKey: getDefaultWorkerURL(worker)
+        };
+    }
+    return { name, url: getWorkerURL(worker, options) };
+}
+/**
+ * Job completes when we receive the result
+ * @param job
+ * @param message
+ */
+async function onMessage(context, job, type, payload) {
+    switch (type) {
+        case 'done':
+            // Worker is done
+            job.done(payload);
+            break;
+        case 'error':
+            // Worker encountered an error
+            job.error(new Error(payload.error));
+            break;
+        case 'process':
+            // Worker is asking for us (main thread) to process something
+            const { id, input, options } = payload;
+            try {
+                if (!context.process) {
+                    job.postMessage('error', { id, error: 'Worker not set up to process on main thread' });
+                    return;
+                }
+                const result = await context.process(input, options, undefined, payload.context || {});
+                job.postMessage('done', { id, result });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : 'unknown error';
+                job.postMessage('error', { id, error: message });
+            }
+            break;
+        default:
+            // eslint-disable-next-line
+            console.warn(`process-on-worker: unknown message ${type}`);
+    }
+}
+/**
+ * Completes a preload job when the worker acknowledges the preload message.
+ * @param job Worker job.
+ * @param type Worker message type.
+ * @param payload Worker message payload.
+ */
+function onPreloadMessage(job, type, payload) {
+    switch (type) {
+        case 'done':
+            job.done(payload);
+            break;
+        case 'error':
+            job.error(new Error(payload.error));
+            break;
+        default:
+            // eslint-disable-next-line
+            console.warn(`process-on-worker: unknown preload message ${type}`);
+    }
+}
+//# sourceMappingURL=process-on-worker.js.map