@loaders.gl/worker-utils 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/lib/worker-farm/worker-pool.js

@@ -0,0 +1,151 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const globals_1 = require("../env-utils/globals");
+const worker_thread_1 = __importDefault(require("./worker-thread"));
+const worker_job_1 = __importDefault(require("./worker-job"));
+/**
+ * Process multiple data messages with small pool of identical workers
+ */
+class WorkerPool {
+    /**
+     * @param processor - worker function
+     * @param maxConcurrency - max count of workers
+     */
+    constructor(props) {
+        this.name = 'unnamed';
+        this.maxConcurrency = 1;
+        this.maxMobileConcurrency = 1;
+        this.onDebug = () => { };
+        this.reuseWorkers = true;
+        this.props = {};
+        this.jobQueue = [];
+        this.idleQueue = [];
+        this.count = 0;
+        this.isDestroyed = false;
+        this.source = props.source;
+        this.url = props.url;
+        this.setProps(props);
+    }
+    /**
+     * Terminates all workers in the pool
+     * @note Can free up significant memory
+     */
+    destroy() {
+        // Destroy idle workers, active Workers will be destroyed on completion
+        this.idleQueue.forEach((worker) => worker.destroy());
+        this.isDestroyed = true;
+    }
+    setProps(props) {
+        this.props = { ...this.props, ...props };
+        if (props.name !== undefined) {
+            this.name = props.name;
+        }
+        if (props.maxConcurrency !== undefined) {
+            this.maxConcurrency = props.maxConcurrency;
+        }
+        if (props.maxMobileConcurrency !== undefined) {
+            this.maxMobileConcurrency = props.maxMobileConcurrency;
+        }
+        if (props.reuseWorkers !== undefined) {
+            this.reuseWorkers = props.reuseWorkers;
+        }
+        if (props.onDebug !== undefined) {
+            this.onDebug = props.onDebug;
+        }
+    }
+    async startJob(name, onMessage = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
+        // Promise resolves when thread starts working on this job
+        const startPromise = new Promise((onStart) => {
+            // Promise resolves when thread completes or fails working on this job
+            this.jobQueue.push({ name, onMessage, onError, onStart });
+            return this;
+        });
+        this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
+        return await startPromise;
+    }
+    // PRIVATE
+    /**
+     * Starts first queued job if worker is available or can be created
+     * Called when job is started and whenever a worker returns to the idleQueue
+     */
+    async _startQueuedJob() {
+        if (!this.jobQueue.length) {
+            return;
+        }
+        const workerThread = this._getAvailableWorker();
+        if (!workerThread) {
+            return;
+        }
+        // We have a worker, dequeue and start the job
+        const queuedJob = this.jobQueue.shift();
+        if (queuedJob) {
+            // Emit a debug event
+            // @ts-ignore
+            this.onDebug({
+                message: 'Starting job',
+                name: queuedJob.name,
+                workerThread,
+                backlog: this.jobQueue.length
+            });
+            // Create a worker job to let the app access thread and manage job completion
+            const job = new worker_job_1.default(queuedJob.name, workerThread);
+            // Set the worker thread's message handlers
+            workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
+            workerThread.onError = (error) => queuedJob.onError(job, error);
+            // Resolve the start promise so that the app can start sending messages to worker
+            queuedJob.onStart(job);
+            // Wait for the app to signal that the job is complete, then return worker to queue
+            try {
+                await job.result;
+            }
+            finally {
+                this.returnWorkerToQueue(workerThread);
+            }
+        }
+    }
+    /**
+     * Returns a worker to the idle queue
+     * Destroys the worker if
+     *  - pool is destroyed
+     *  - if this pool doesn't reuse workers
+     *  - if maxConcurrency has been lowered
+     * @param worker
+     */
+    returnWorkerToQueue(worker) {
+        const shouldDestroyWorker = this.isDestroyed || !this.reuseWorkers || this.count > this._getMaxConcurrency();
+        if (shouldDestroyWorker) {
+            worker.destroy();
+            this.count--;
+        }
+        else {
+            this.idleQueue.push(worker);
+        }
+        if (!this.isDestroyed) {
+            this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
+        }
+    }
+    /**
+     * Returns idle worker or creates new worker if maxConcurrency has not been reached
+     */
+    _getAvailableWorker() {
+        // If a worker has completed and returned to the queue, it can be used
+        if (this.idleQueue.length > 0) {
+            return this.idleQueue.shift() || null;
+        }
+        // Create fresh worker if we haven't yet created the max amount of worker threads for this worker source
+        if (this.count < this._getMaxConcurrency()) {
+            this.count++;
+            const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
+            return new worker_thread_1.default({ name, source: this.source, url: this.url });
+        }
+        // No worker available, have to wait
+        return null;
+    }
+    _getMaxConcurrency() {
+        return globals_1.isMobile ? this.maxMobileConcurrency : this.maxConcurrency;
+    }
+}
+exports.default = WorkerPool;

package/dist/lib/worker-farm/worker-thread.d.ts

@@ -0,0 +1,42 @@
+export declare type WorkerThreadProps = {
+    name: string;
+    source?: string;
+    url?: string;
+};
+/**
+ * Represents one worker thread
+ */
+export default class WorkerThread {
+    readonly name: string;
+    readonly source: string | undefined;
+    readonly url: string | undefined;
+    terminated: boolean;
+    worker: Worker;
+    onMessage: (message: any) => void;
+    onError: (error: Error) => void;
+    private _loadableURL;
+    static isSupported(): boolean;
+    constructor(props: WorkerThreadProps);
+    /**
+     * Terminate this worker thread
+     * @note Can free up significant memory
+     */
+    destroy(): void;
+    get isRunning(): boolean;
+    /**
+     * Send a message to this worker thread
+     * @param data any data structure, ideally consisting mostly of transferrable objects
+     * @param transferList If not supplied, calculated automatically by traversing data
+     */
+    postMessage(data: any, transferList?: any[]): void;
+    /**
+     * Generate a standard Error from an ErrorEvent
+     * @param {ErrorEvent} event
+     */
+    _getErrorFromErrorEvent(event: ErrorEvent): Error;
+    /**
+     * Creates a worker thread on the browser
+     */
+    _createBrowserWorker(): Worker;
+}
+//# sourceMappingURL=worker-thread.d.ts.map

package/dist/lib/worker-farm/worker-thread.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-thread.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-thread.ts"],"names":[],"mappings":"AAMA,oBAAY,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,YAAY;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,UAAU,EAAE,OAAO,CAAS;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,CAAC;IAClC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAEhC,OAAO,CAAC,YAAY,CAAc;IAElC,MAAM,CAAC,WAAW,IAAI,OAAO;gBAIjB,KAAK,EAAE,iBAAiB;IAYpC;;;OAGG;IACH,OAAO,IAAI,IAAI;IAQf,IAAI,SAAS,YAEZ;IAED;;;;OAIG;IACH,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,YAAY,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI;IAQlD;;;OAGG;IACH,uBAAuB,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK;IAiBjD;;OAEG;IACH,oBAAoB;CAqBrB"}

package/dist/lib/worker-farm/worker-thread.js

@@ -0,0 +1,95 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const assert_1 = require("../env-utils/assert");
+const get_loadable_worker_url_1 = require("../worker-utils/get-loadable-worker-url");
+const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
+const NOOP = () => { };
+/**
+ * Represents one worker thread
+ */
+class WorkerThread {
+    constructor(props) {
+        this.terminated = false;
+        this._loadableURL = '';
+        const { name, source, url } = props;
+        (0, assert_1.assert)(source || url); // Either source or url must be defined
+        this.name = name;
+        this.source = source;
+        this.url = url;
+        this.onMessage = NOOP;
+        this.onError = (error) => console.log(error); // eslint-disable-line
+        this.worker = this._createBrowserWorker();
+    }
+    static isSupported() {
+        return typeof Worker !== 'undefined';
+    }
+    /**
+     * Terminate this worker thread
+     * @note Can free up significant memory
+     */
+    destroy() {
+        this.onMessage = NOOP;
+        this.onError = NOOP;
+        // @ts-ignore
+        this.worker.terminate();
+        this.terminated = true;
+    }
+    get isRunning() {
+        return Boolean(this.onMessage);
+    }
+    /**
+     * Send a message to this worker thread
+     * @param data any data structure, ideally consisting mostly of transferrable objects
+     * @param transferList If not supplied, calculated automatically by traversing data
+     */
+    postMessage(data, transferList) {
+        transferList = transferList || (0, get_transfer_list_1.getTransferList)(data);
+        // @ts-ignore
+        this.worker.postMessage(data, transferList);
+    }
+    // PRIVATE
+    /**
+     * Generate a standard Error from an ErrorEvent
+     * @param {ErrorEvent} event
+     */
+    _getErrorFromErrorEvent(event) {
+        // Note Error object does not have the expected fields if loading failed completely
+        // https://developer.mozilla.org/en-US/docs/Web/API/Worker#Event_handlers
+        // https://developer.mozilla.org/en-US/docs/Web/API/ErrorEvent
+        let message = 'Failed to load ';
+        message += `worker ${this.name} from ${this.url}. `;
+        if (event.message) {
+            message += `${event.message} in `;
+        }
+        // const hasFilename = event.filename && !event.filename.startsWith('blob:');
+        // message += hasFilename ? event.filename : this.source.slice(0, 100);
+        if (event.lineno) {
+            message += `:${event.lineno}:${event.colno}`;
+        }
+        return new Error(message);
+    }
+    /**
+     * Creates a worker thread on the browser
+     */
+    _createBrowserWorker() {
+        this._loadableURL = (0, get_loadable_worker_url_1.getLoadableWorkerURL)({ source: this.source, url: this.url });
+        const worker = new Worker(this._loadableURL, { name: this.name });
+        worker.onmessage = (event) => {
+            if (!event.data) {
+                this.onError(new Error('No data received'));
+            }
+            else {
+                this.onMessage(event.data);
+            }
+        };
+        // This callback represents an uncaught exception in the worker thread
+        worker.onerror = (error) => {
+            this.onError(this._getErrorFromErrorEvent(error));
+            this.terminated = true;
+        };
+        // TODO - not clear when this would be called, for now just log in case it happens
+        worker.onmessageerror = (event) => console.error(event); // eslint-disable-line
+        return worker;
+    }
+}
+exports.default = WorkerThread;

package/dist/lib/worker-utils/get-loadable-worker-url.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Creates a loadable URL from worker source or URL
+ * that can be used to create `Worker` instances.
+ * Due to CORS issues it may be necessary to wrap a URL in a small importScripts
+ * @param props
+ * @param props.source Worker source
+ * @param props.url Worker URL
+ * @returns loadable url
+ */
+export declare function getLoadableWorkerURL(props: {
+    source?: string;
+    url?: string;
+}): any;
+//# sourceMappingURL=get-loadable-worker-url.d.ts.map

package/dist/lib/worker-utils/get-loadable-worker-url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-loadable-worker-url.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-utils/get-loadable-worker-url.ts"],"names":[],"mappings":"AAIA;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAC,OAmB1E"}

package/dist/lib/worker-utils/get-loadable-worker-url.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getLoadableWorkerURL = void 0;
+const assert_1 = require("../env-utils/assert");
+const workerURLCache = new Map();
+/**
+ * Creates a loadable URL from worker source or URL
+ * that can be used to create `Worker` instances.
+ * Due to CORS issues it may be necessary to wrap a URL in a small importScripts
+ * @param props
+ * @param props.source Worker source
+ * @param props.url Worker URL
+ * @returns loadable url
+ */
+function getLoadableWorkerURL(props) {
+    (0, assert_1.assert)((props.source && !props.url) || (!props.source && props.url)); // Either source or url must be defined
+    let workerURL = workerURLCache.get(props.source || props.url);
+    if (!workerURL) {
+        // Differentiate worker urls from worker source code
+        if (props.url) {
+            workerURL = getLoadableWorkerURLFromURL(props.url);
+            workerURLCache.set(props.url, workerURL);
+        }
+        if (props.source) {
+            workerURL = getLoadableWorkerURLFromSource(props.source);
+            workerURLCache.set(props.source, workerURL);
+        }
+    }
+    (0, assert_1.assert)(workerURL);
+    return workerURL;
+}
+exports.getLoadableWorkerURL = getLoadableWorkerURL;
+/**
+ * Build a loadable worker URL from worker URL
+ * @param url
+ * @returns loadable URL
+ */
+function getLoadableWorkerURLFromURL(url) {
+    // A local script url, we can use it to initialize a Worker directly
+    if (!url.startsWith('http')) {
+        return url;
+    }
+    // A remote script, we need to use `importScripts` to load from different origin
+    const workerSource = buildScriptSource(url);
+    return getLoadableWorkerURLFromSource(workerSource);
+}
+/**
+ * Build a loadable worker URL from worker source
+ * @param workerSource
+ * @returns loadable url
+ */
+function getLoadableWorkerURLFromSource(workerSource) {
+    // NOTE: webworkify was previously used
+    // const blob = webworkify(workerSource, {bare: true});
+    const blob = new Blob([workerSource], { type: 'application/javascript' });
+    return URL.createObjectURL(blob);
+}
+/**
+ * Per spec, worker cannot be initialized with a script from a different origin
+ * However a local worker script can still import scripts from other origins,
+ * so we simply build a wrapper script.
+ *
+ * @param workerUrl
+ * @returns source
+ */
+function buildScriptSource(workerUrl) {
+    return `\
+try {
+  importScripts('${workerUrl}');
+} catch (error) {
+  console.error(error);
+  throw error;
+}`;
+}

package/dist/lib/worker-utils/get-transfer-list.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Returns an array of Transferrable objects that can be used with postMessage
+ * https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage
+ * @param object data to be sent via postMessage
+ * @param recursive - not for application use
+ * @param transfers - not for application use
+ * @returns a transfer list that can be passed to postMessage
+ */
+export declare function getTransferList(object: any, recursive?: boolean, transfers?: Set<any>): Transferable[];
+//# sourceMappingURL=get-transfer-list.d.ts.map

package/dist/lib/worker-utils/get-transfer-list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-transfer-list.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-utils/get-transfer-list.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,GAAG,EACX,SAAS,GAAE,OAAc,EACzB,SAAS,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GACnB,YAAY,EAAE,CAwBhB"}

package/dist/lib/worker-utils/get-transfer-list.js

@@ -0,0 +1,61 @@
+"use strict";
+// NOTE - there is a copy of this function is both in core and loader-utils
+// core does not need all the utils in loader-utils, just this one.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTransferList = void 0;
+/**
+ * Returns an array of Transferrable objects that can be used with postMessage
+ * https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage
+ * @param object data to be sent via postMessage
+ * @param recursive - not for application use
+ * @param transfers - not for application use
+ * @returns a transfer list that can be passed to postMessage
+ */
+function getTransferList(object, recursive = true, transfers) {
+    // Make sure that items in the transfer list is unique
+    const transfersSet = transfers || new Set();
+    if (!object) {
+        // ignore
+    }
+    else if (isTransferable(object)) {
+        transfersSet.add(object);
+    }
+    else if (isTransferable(object.buffer)) {
+        // Typed array
+        transfersSet.add(object.buffer);
+    }
+    else if (ArrayBuffer.isView(object)) {
+        // object is a TypeArray viewing into a SharedArrayBuffer (not transferable)
+        // Do not iterate through the content in this case
+    }
+    else if (recursive && typeof object === 'object') {
+        for (const key in object) {
+            // Avoid perf hit - only go one level deep
+            getTransferList(object[key], recursive, transfersSet);
+        }
+    }
+    // If transfers is defined, is internal recursive call
+    // Otherwise it's called by the user
+    return transfers === undefined ? Array.from(transfersSet) : [];
+}
+exports.getTransferList = getTransferList;
+// https://developer.mozilla.org/en-US/docs/Web/API/Transferable
+function isTransferable(object) {
+    if (!object) {
+        return false;
+    }
+    if (object instanceof ArrayBuffer) {
+        return true;
+    }
+    if (typeof MessagePort !== 'undefined' && object instanceof MessagePort) {
+        return true;
+    }
+    if (typeof ImageBitmap !== 'undefined' && object instanceof ImageBitmap) {
+        return true;
+    }
+    // @ts-ignore
+    if (typeof OffscreenCanvas !== 'undefined' && object instanceof OffscreenCanvas) {
+        return true;
+    }
+    return false;
+}

package/dist/lib/worker-utils/remove-nontransferable-options.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Safely stringify JSON (drop non serializable values like functions and regexps)
+ * @param value
+ */
+export declare function removeNontransferableOptions(object: object): object;
+//# sourceMappingURL=remove-nontransferable-options.d.ts.map

package/dist/lib/worker-utils/remove-nontransferable-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remove-nontransferable-options.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-utils/remove-nontransferable-options.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAKnE"}

package/dist/lib/worker-utils/remove-nontransferable-options.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeNontransferableOptions = void 0;
+/**
+ * Safely stringify JSON (drop non serializable values like functions and regexps)
+ * @param value
+ */
+function removeNontransferableOptions(object) {
+    // options.log object contains functions which cannot be transferred
+    // TODO - decide how to handle logging on workers
+    // TODO - warn if options stringification is long
+    return JSON.parse(stringifyJSON(object));
+}
+exports.removeNontransferableOptions = removeNontransferableOptions;
+function stringifyJSON(v) {
+    const cache = new Set();
+    return JSON.stringify(v, (key, value) => {
+        if (typeof value === 'object' && value !== null) {
+            if (cache.has(value)) {
+                // Circular reference found
+                try {
+                    // If this value does not reference a parent it can be deduped
+                    return JSON.parse(JSON.stringify(value));
+                }
+                catch (err) {
+                    // discard key if value cannot be deduped
+                    return undefined;
+                }
+            }
+            // Store value in our set
+            cache.add(value);
+        }
+        return value;
+    });
+}