@loaders.gl/worker-utils 4.0.0-alpha.9 → 4.0.0-beta.2

package/dist/lib/worker-api/create-worker.js

@@ -1,91 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createWorker = void 0;
-const async_queue_1 = __importDefault(require("../async-queue/async-queue"));
-const worker_body_1 = __importDefault(require("../worker-farm/worker-body"));
-// import {validateWorkerVersion} from './validate-worker-version';
-/** Counter for jobs */
-let requestId = 0;
-let inputBatches;
-let options;
-/**
- * Set up a WebWorkerGlobalScope to talk with the main thread
- */
-function createWorker(process, processInBatches) {
-    if (!worker_body_1.default.inWorkerThread()) {
-        return;
-    }
-    const context = {
-        process: processOnMainThread
-    };
-    // eslint-disable-next-line complexity
-    worker_body_1.default.onmessage = async (type, payload) => {
-        try {
-            switch (type) {
-                case 'process':
-                    if (!process) {
-                        throw new Error('Worker does not support atomic processing');
-                    }
-                    const result = await process(payload.input, payload.options || {}, context);
-                    worker_body_1.default.postMessage('done', { result });
-                    break;
-                case 'process-in-batches':
-                    if (!processInBatches) {
-                        throw new Error('Worker does not support batched processing');
-                    }
-                    inputBatches = new async_queue_1.default();
-                    options = payload.options || {};
-                    const resultIterator = processInBatches(inputBatches, options, context);
-                    for await (const batch of resultIterator) {
-                        worker_body_1.default.postMessage('output-batch', { result: batch });
-                    }
-                    worker_body_1.default.postMessage('done', {});
-                    break;
-                case 'input-batch':
-                    inputBatches.push(payload.input);
-                    break;
-                case 'input-done':
-                    inputBatches.close();
-                    break;
-                default:
-            }
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : '';
-            worker_body_1.default.postMessage('error', { error: message });
-        }
-    };
-}
-exports.createWorker = createWorker;
-function processOnMainThread(arrayBuffer, options = {}) {
-    return new Promise((resolve, reject) => {
-        const id = requestId++;
-        /**
-         */
-        const onMessage = (type, payload) => {
-            if (payload.id !== id) {
-                // not ours
-                return;
-            }
-            switch (type) {
-                case 'done':
-                    worker_body_1.default.removeEventListener(onMessage);
-                    resolve(payload.result);
-                    break;
-                case 'error':
-                    worker_body_1.default.removeEventListener(onMessage);
-                    reject(payload.error);
-                    break;
-                default:
-                // ignore
-            }
-        };
-        worker_body_1.default.addEventListener(onMessage);
-        // Ask the main thread to decode data
-        const payload = { id, input: arrayBuffer, options };
-        worker_body_1.default.postMessage('process', payload);
-    });
-}

package/dist/lib/worker-api/get-worker-url.js

@@ -1,59 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getWorkerURL = exports.getWorkerName = void 0;
-const assert_1 = require("../env-utils/assert");
-const globals_1 = require("../env-utils/globals");
-const version_1 = require("../env-utils/version");
-const NPM_TAG = 'beta'; // 'beta', or 'latest' on release-branch
-const VERSION = typeof version_1.VERSION !== 'undefined' ? version_1.VERSION : NPM_TAG;
-/**
- * Gets worker object's name (for debugging in Chrome thread inspector window)
- */
-function getWorkerName(worker) {
-    const warning = worker.version !== VERSION ? ` (worker-utils@${VERSION})` : '';
-    return `${worker.name}@${worker.version}${warning}`;
-}
-exports.getWorkerName = getWorkerName;
-/**
- * Generate a worker URL based on worker object and options
- * @returns A URL to one of the following:
- * - a published worker on unpkg CDN
- * - a local test worker
- * - a URL provided by the user in options
- */
-function getWorkerURL(worker, options = {}) {
-    const workerOptions = options[worker.id] || {};
-    const workerFile = globals_1.isBrowser ? `${worker.id}-worker.js` : `${worker.id}-worker-node.js`;
-    let url = workerOptions.workerUrl;
-    // HACK: Allow for non-nested workerUrl for the CompressionWorker.
-    // For the compression worker, workerOptions is currently not nested correctly. For most loaders,
-    // you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker
-    // puts options at the top level, not within a `compression` key (its `id`). For this reason, the
-    // above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not
-    // break backwards compatibility, we allow the CompressionWorker to have options at the top level.
-    if (!url && worker.id === 'compression') {
-        url = options.workerUrl;
-    }
-    // If URL is test, generate local loaders.gl url
-    // @ts-ignore _workerType
-    if (options._workerType === 'test') {
-        url = `modules/${worker.module}/dist/${workerFile}`;
-    }
-    // If url override is not provided, generate a URL to published version on npm CDN unpkg.com
-    if (!url) {
-        // GENERATE
-        let version = worker.version;
-        // On master we need to load npm alpha releases published with the `beta` tag
-        if (version === 'latest') {
-            // throw new Error('latest worker version specified');
-            version = NPM_TAG;
-        }
-        const versionTag = version ? `@${version}` : '';
-        url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
-    }
-    (0, assert_1.assert)(url);
-    // Allow user to override location
-    return url;
-}
-exports.getWorkerURL = getWorkerURL;

package/dist/lib/worker-api/process-on-worker.js

@@ -1,83 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.processOnWorker = exports.canProcessOnWorker = void 0;
-const worker_farm_1 = __importDefault(require("../worker-farm/worker-farm"));
-const get_worker_url_1 = require("./get-worker-url");
-const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
-/**
- * Determines if we can parse with worker
- * @param loader
- * @param data
- * @param options
- */
-function canProcessOnWorker(worker, options) {
-    if (!worker_farm_1.default.isSupported()) {
-        return false;
-    }
-    return worker.worker && options?.worker;
-}
-exports.canProcessOnWorker = canProcessOnWorker;
-/**
- * This function expects that the worker thread sends certain messages,
- * Creating such a worker can be automated if the worker is wrapper by a call to
- * createWorker in @loaders.gl/worker-utils.
- */
-async function processOnWorker(worker, data, options = {}, context = {}) {
-    const name = (0, get_worker_url_1.getWorkerName)(worker);
-    const workerFarm = worker_farm_1.default.getWorkerFarm(options);
-    const { source } = options;
-    const workerPoolProps = { name, source };
-    if (!source) {
-        workerPoolProps.url = (0, get_worker_url_1.getWorkerURL)(worker, options);
-    }
-    const workerPool = workerFarm.getWorkerPool(workerPoolProps);
-    const jobName = options.jobName || worker.name;
-    const job = await workerPool.startJob(jobName, 
-    // eslint-disable-next-line
-    onMessage.bind(null, context));
-    // Kick off the processing in the worker
-    const transferableOptions = (0, get_transfer_list_1.getTransferListForWriter)(options);
-    job.postMessage('process', { input: data, options: transferableOptions });
-    const result = await job.result;
-    return result.result;
-}
-exports.processOnWorker = processOnWorker;
-/**
- * Job completes when we receive the result
- * @param job
- * @param message
- */
-async function onMessage(context, job, type, payload) {
-    switch (type) {
-        case 'done':
-            // Worker is done
-            job.done(payload);
-            break;
-        case 'error':
-            // Worker encountered an error
-            job.error(new Error(payload.error));
-            break;
-        case 'process':
-            // Worker is asking for us (main thread) to process something
-            const { id, input, options } = payload;
-            try {
-                if (!context.process) {
-                    job.postMessage('error', { id, error: 'Worker not set up to process on main thread' });
-                    return;
-                }
-                const result = await context.process(input, options);
-                job.postMessage('done', { id, result });
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : 'unknown error';
-                job.postMessage('error', { id, error: message });
-            }
-            break;
-        default:
-            // eslint-disable-next-line
-            console.warn(`process-on-worker: unknown message ${type}`);
-    }
-}

package/dist/lib/worker-api/validate-worker-version.js

@@ -1,35 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateWorkerVersion = void 0;
-const assert_1 = require("../env-utils/assert");
-const version_1 = require("../env-utils/version");
-/**
- * Check if worker is compatible with this library version
- * @param worker
- * @param libVersion
- * @returns `true` if the two versions are compatible
- */
-function validateWorkerVersion(worker, coreVersion = version_1.VERSION) {
-    (0, assert_1.assert)(worker, 'no worker provided');
-    const workerVersion = worker.version;
-    if (!coreVersion || !workerVersion) {
-        return false;
-    }
-    // TODO enable when fix the __version__ injection
-    // const coreVersions = parseVersion(coreVersion);
-    // const workerVersions = parseVersion(workerVersion);
-    // assert(
-    //   coreVersion.major === workerVersion.major && coreVersion.minor <= workerVersion.minor,
-    //   `worker: ${worker.name} is not compatible. ${coreVersion.major}.${
-    //     coreVersion.minor
-    //   }+ is required.`
-    // );
-    return true;
-}
-exports.validateWorkerVersion = validateWorkerVersion;
-// @ts-ignore
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-function parseVersion(version) {
-    const parts = version.split('.').map(Number);
-    return { major: parts[0], minor: parts[1] };
-}

package/dist/lib/worker-farm/worker-body.js

@@ -1,111 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
-/** Vile hack to defeat over-zealous bundlers from stripping out the require */
-function getParentPort() {
-    // const isNode = globalThis.process;
-    let parentPort;
-    try {
-        // prettier-ignore
-        eval('globalThis.parentPort = require(\'worker_threads\').parentPort'); // eslint-disable-line no-eval
-        parentPort = globalThis.parentPort;
-        // eslint-disable-next-line no-empty
-    }
-    catch { }
-    return parentPort;
-}
-const onMessageWrapperMap = new Map();
-/**
- * Type safe wrapper for worker code
- */
-class WorkerBody {
-    /** Check that we are actually in a worker thread */
-    static inWorkerThread() {
-        return typeof self !== 'undefined' || Boolean(getParentPort());
-    }
-    /*
-     * (type: WorkerMessageType, payload: WorkerMessagePayload) => any
-     */
-    static set onmessage(onMessage) {
-        function handleMessage(message) {
-            // Confusingly the message itself also has a 'type' field which is always set to 'message'
-            const parentPort = getParentPort();
-            const { type, payload } = parentPort ? message : message.data;
-            // if (!isKnownMessage(message)) {
-            //   return;
-            // }
-            onMessage(type, payload);
-        }
-        const parentPort = getParentPort();
-        if (parentPort) {
-            parentPort.on('message', handleMessage);
-            // if (message == 'exit') { parentPort.unref(); }
-            // eslint-disable-next-line
-            parentPort.on('exit', () => console.debug('Node worker closing'));
-        }
-        else {
-            // eslint-disable-next-line no-restricted-globals
-            globalThis.onmessage = handleMessage;
-        }
-    }
-    static addEventListener(onMessage) {
-        let onMessageWrapper = onMessageWrapperMap.get(onMessage);
-        if (!onMessageWrapper) {
-            onMessageWrapper = (message) => {
-                if (!isKnownMessage(message)) {
-                    return;
-                }
-                // Confusingly in the browser, the message itself also has a 'type' field which is always set to 'message'
-                const parentPort = getParentPort();
-                const { type, payload } = parentPort ? message : message.data;
-                onMessage(type, payload);
-            };
-        }
-        const parentPort = getParentPort();
-        if (parentPort) {
-            console.error('not implemented'); // eslint-disable-line
-        }
-        else {
-            globalThis.addEventListener('message', onMessageWrapper);
-        }
-    }
-    static removeEventListener(onMessage) {
-        const onMessageWrapper = onMessageWrapperMap.get(onMessage);
-        onMessageWrapperMap.delete(onMessage);
-        const parentPort = getParentPort();
-        if (parentPort) {
-            console.error('not implemented'); // eslint-disable-line
-        }
-        else {
-            globalThis.removeEventListener('message', onMessageWrapper);
-        }
-    }
-    /**
-     * Send a message from a worker to creating thread (main thread)
-     * @param type
-     * @param payload
-     */
-    static postMessage(type, payload) {
-        const data = { source: 'loaders.gl', type, payload };
-        // console.log('posting message', data);
-        const transferList = (0, get_transfer_list_1.getTransferList)(payload);
-        const parentPort = getParentPort();
-        if (parentPort) {
-            parentPort.postMessage(data, transferList);
-            // console.log('posted message', data);
-        }
-        else {
-            // @ts-ignore
-            globalThis.postMessage(data, transferList);
-        }
-    }
-}
-exports.default = WorkerBody;
-// Filter out noise messages sent to workers
-function isKnownMessage(message) {
-    const { type, data } = message;
-    return (type === 'message' &&
-        data &&
-        typeof data.source === 'string' &&
-        data.source.startsWith('loaders.gl'));
-}

package/dist/lib/worker-farm/worker-farm.js

@@ -1,89 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const worker_pool_1 = __importDefault(require("./worker-pool"));
-const worker_thread_1 = __importDefault(require("./worker-thread"));
-const DEFAULT_PROPS = {
-    maxConcurrency: 3,
-    maxMobileConcurrency: 1,
-    reuseWorkers: true,
-    onDebug: () => { }
-};
-/**
- * Process multiple jobs with a "farm" of different workers in worker pools.
- */
-class WorkerFarm {
-    /** Checks if workers are supported on this platform */
-    static isSupported() {
-        return worker_thread_1.default.isSupported();
-    }
-    /** Get the singleton instance of the global worker farm */
-    static getWorkerFarm(props = {}) {
-        WorkerFarm._workerFarm = WorkerFarm._workerFarm || new WorkerFarm({});
-        WorkerFarm._workerFarm.setProps(props);
-        return WorkerFarm._workerFarm;
-    }
-    /** get global instance with WorkerFarm.getWorkerFarm() */
-    constructor(props) {
-        this.workerPools = new Map();
-        this.props = { ...DEFAULT_PROPS };
-        this.setProps(props);
-        /** @type Map<string, WorkerPool>} */
-        this.workerPools = new Map();
-    }
-    /**
-     * Terminate all workers in the farm
-     * @note Can free up significant memory
-     */
-    destroy() {
-        for (const workerPool of this.workerPools.values()) {
-            workerPool.destroy();
-        }
-        this.workerPools = new Map();
-    }
-    /**
-     * Set props used when initializing worker pools
-     * @param props
-     */
-    setProps(props) {
-        this.props = { ...this.props, ...props };
-        // Update worker pool props
-        for (const workerPool of this.workerPools.values()) {
-            workerPool.setProps(this._getWorkerPoolProps());
-        }
-    }
-    /**
-     * Returns a worker pool for the specified worker
-     * @param options - only used first time for a specific worker name
-     * @param options.name - the name of the worker - used to identify worker pool
-     * @param options.url -
-     * @param options.source -
-     * @example
-     *   const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
-     */
-    getWorkerPool(options) {
-        const { name, source, url } = options;
-        let workerPool = this.workerPools.get(name);
-        if (!workerPool) {
-            workerPool = new worker_pool_1.default({
-                name,
-                source,
-                url
-            });
-            workerPool.setProps(this._getWorkerPoolProps());
-            this.workerPools.set(name, workerPool);
-        }
-        return workerPool;
-    }
-    _getWorkerPoolProps() {
-        return {
-            maxConcurrency: this.props.maxConcurrency,
-            maxMobileConcurrency: this.props.maxMobileConcurrency,
-            reuseWorkers: this.props.reuseWorkers,
-            onDebug: this.props.onDebug
-        };
-    }
-}
-exports.default = WorkerFarm;

package/dist/lib/worker-farm/worker-job.js

@@ -1,47 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const assert_1 = require("../env-utils/assert");
-/**
- * Represents one Job handled by a WorkerPool or WorkerFarm
- */
-class WorkerJob {
-    constructor(jobName, workerThread) {
-        this.isRunning = true;
-        this._resolve = () => { };
-        this._reject = () => { };
-        this.name = jobName;
-        this.workerThread = workerThread;
-        this.result = new Promise((resolve, reject) => {
-            this._resolve = resolve;
-            this._reject = reject;
-        });
-    }
-    /**
-     * Send a message to the job's worker thread
-     * @param data any data structure, ideally consisting mostly of transferrable objects
-     */
-    postMessage(type, payload) {
-        this.workerThread.postMessage({
-            source: 'loaders.gl',
-            type,
-            payload
-        });
-    }
-    /**
-     * Call to resolve the `result` Promise with the supplied value
-     */
-    done(value) {
-        (0, assert_1.assert)(this.isRunning);
-        this.isRunning = false;
-        this._resolve(value);
-    }
-    /**
-     * Call to reject the `result` Promise with the supplied error
-     */
-    error(error) {
-        (0, assert_1.assert)(this.isRunning);
-        this.isRunning = false;
-        this._reject(error);
-    }
-}
-exports.default = WorkerJob;

package/dist/lib/worker-farm/worker-pool.js

@@ -1,164 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const globals_1 = require("../env-utils/globals");
-const worker_thread_1 = __importDefault(require("./worker-thread"));
-const worker_job_1 = __importDefault(require("./worker-job"));
-/**
- * Process multiple data messages with small pool of identical workers
- */
-class WorkerPool {
-    /** Checks if workers are supported on this platform */
-    static isSupported() {
-        return worker_thread_1.default.isSupported();
-    }
-    /**
-     * @param processor - worker function
-     * @param maxConcurrency - max count of workers
-     */
-    constructor(props) {
-        this.name = 'unnamed';
-        this.maxConcurrency = 1;
-        this.maxMobileConcurrency = 1;
-        this.onDebug = () => { };
-        this.reuseWorkers = true;
-        this.props = {};
-        this.jobQueue = [];
-        this.idleQueue = [];
-        this.count = 0;
-        this.isDestroyed = false;
-        this.source = props.source;
-        this.url = props.url;
-        this.setProps(props);
-    }
-    /**
-     * Terminates all workers in the pool
-     * @note Can free up significant memory
-     */
-    destroy() {
-        // Destroy idle workers, active Workers will be destroyed on completion
-        this.idleQueue.forEach((worker) => worker.destroy());
-        this.isDestroyed = true;
-    }
-    setProps(props) {
-        this.props = { ...this.props, ...props };
-        if (props.name !== undefined) {
-            this.name = props.name;
-        }
-        if (props.maxConcurrency !== undefined) {
-            this.maxConcurrency = props.maxConcurrency;
-        }
-        if (props.maxMobileConcurrency !== undefined) {
-            this.maxMobileConcurrency = props.maxMobileConcurrency;
-        }
-        if (props.reuseWorkers !== undefined) {
-            this.reuseWorkers = props.reuseWorkers;
-        }
-        if (props.onDebug !== undefined) {
-            this.onDebug = props.onDebug;
-        }
-    }
-    async startJob(name, onMessage = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
-        // Promise resolves when thread starts working on this job
-        const startPromise = new Promise((onStart) => {
-            // Promise resolves when thread completes or fails working on this job
-            this.jobQueue.push({ name, onMessage, onError, onStart });
-            return this;
-        });
-        this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
-        return await startPromise;
-    }
-    // PRIVATE
-    /**
-     * Starts first queued job if worker is available or can be created
-     * Called when job is started and whenever a worker returns to the idleQueue
-     */
-    async _startQueuedJob() {
-        if (!this.jobQueue.length) {
-            return;
-        }
-        const workerThread = this._getAvailableWorker();
-        if (!workerThread) {
-            return;
-        }
-        // We have a worker, dequeue and start the job
-        const queuedJob = this.jobQueue.shift();
-        if (queuedJob) {
-            // Emit a debug event
-            // @ts-ignore
-            this.onDebug({
-                message: 'Starting job',
-                name: queuedJob.name,
-                workerThread,
-                backlog: this.jobQueue.length
-            });
-            // Create a worker job to let the app access thread and manage job completion
-            const job = new worker_job_1.default(queuedJob.name, workerThread);
-            // Set the worker thread's message handlers
-            workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
-            workerThread.onError = (error) => queuedJob.onError(job, error);
-            // Resolve the start promise so that the app can start sending messages to worker
-            queuedJob.onStart(job);
-            // Wait for the app to signal that the job is complete, then return worker to queue
-            try {
-                await job.result;
-            }
-            finally {
-                this.returnWorkerToQueue(workerThread);
-            }
-        }
-    }
-    /**
-     * Returns a worker to the idle queue
-     * Destroys the worker if
-     *  - pool is destroyed
-     *  - if this pool doesn't reuse workers
-     *  - if maxConcurrency has been lowered
-     * @param worker
-     */
-    returnWorkerToQueue(worker) {
-        const shouldDestroyWorker = 
-        // Workers on Node.js prevent the process from exiting.
-        // Until we figure out how to close them before exit, we always destroy them
-        !globals_1.isBrowser ||
-            // If the pool is destroyed, there is no reason to keep the worker around
-            this.isDestroyed ||
-            // If the app has disabled worker reuse, any completed workers should be destroyed
-            !this.reuseWorkers ||
-            // If concurrency has been lowered, this worker might be surplus to requirements
-            this.count > this._getMaxConcurrency();
-        if (shouldDestroyWorker) {
-            worker.destroy();
-            this.count--;
-        }
-        else {
-            this.idleQueue.push(worker);
-        }
-        if (!this.isDestroyed) {
-            this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
-        }
-    }
-    /**
-     * Returns idle worker or creates new worker if maxConcurrency has not been reached
-     */
-    _getAvailableWorker() {
-        // If a worker has completed and returned to the queue, it can be used
-        if (this.idleQueue.length > 0) {
-            return this.idleQueue.shift() || null;
-        }
-        // Create fresh worker if we haven't yet created the max amount of worker threads for this worker source
-        if (this.count < this._getMaxConcurrency()) {
-            this.count++;
-            const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
-            return new worker_thread_1.default({ name, source: this.source, url: this.url });
-        }
-        // No worker available, have to wait
-        return null;
-    }
-    _getMaxConcurrency() {
-        return globals_1.isMobile ? this.maxMobileConcurrency : this.maxConcurrency;
-    }
-}
-exports.default = WorkerPool;