@loaders.gl/worker-utils 3.4.11 → 3.4.12

package/dist/lib/process-utils/child-process-proxy.js

@@ -1,136 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-/* eslint-disable no-console */
-// Avoid using named imports for Node builtins to help with "empty" resolution
-// for bundlers targeting browser environments. Access imports & types
-// through the `ChildProcess` object (e.g. `ChildProcess.spawn`, `ChildProcess.ChildProcess`).
-const ChildProcess = __importStar(require("child_process"));
-const process_utils_1 = require("./process-utils");
-const DEFAULT_PROPS = {
-    command: '',
-    arguments: [],
-    port: 5000,
-    autoPort: true,
-    wait: 2000,
-    onSuccess: (processProxy) => {
-        console.log(`Started ${processProxy.props.command}`);
-    }
-};
-/**
- * Manager for a Node.js child process
- * Prepares arguments, starts, stops and tracks output
- */
-class ChildProcessProxy {
-    // constructor(props?: {id?: string});
-    constructor({ id = 'browser-driver' } = {}) {
-        this.props = { ...DEFAULT_PROPS };
-        this.childProcess = null;
-        this.port = 0;
-        this.id = id;
-    }
-    /** Starts a child process with the provided props */
-    async start(props) {
-        props = { ...DEFAULT_PROPS, ...props };
-        this.props = props;
-        const args = [...props.arguments];
-        // If portArg is set, we can look up an available port
-        this.port = Number(props.port);
-        if (props.portArg) {
-            if (props.autoPort) {
-                this.port = await (0, process_utils_1.getAvailablePort)(props.port);
-            }
-            args.push(props.portArg, String(this.port));
-        }
-        return await new Promise((resolve, reject) => {
-            try {
-                this._setTimeout(() => {
-                    if (props.onSuccess) {
-                        props.onSuccess(this);
-                    }
-                    resolve({});
-                });
-                console.log(`Spawning ${props.command} ${props.arguments.join(' ')}`);
-                const childProcess = ChildProcess.spawn(props.command, args, props.spawn);
-                this.childProcess = childProcess;
-                childProcess.stdout.on('data', (data) => {
-                    console.log(data.toString());
-                });
-                childProcess.stderr.on('data', (data) => {
-                    console.log(`Child process wrote to stderr: "${data}".`);
-                    if (!props.ignoreStderr) {
-                        this._clearTimeout();
-                        reject(new Error(data));
-                    }
-                });
-                childProcess.on('error', (error) => {
-                    console.log(`Child process errored with ${error}`);
-                    this._clearTimeout();
-                    reject(error);
-                });
-                childProcess.on('close', (code) => {
-                    console.log(`Child process exited with ${code}`);
-                    this.childProcess = null;
-                    this._clearTimeout();
-                    resolve({});
-                });
-            }
-            catch (error) {
-                reject(error);
-            }
-        });
-    }
-    /** Stops a running child process */
-    async stop() {
-        if (this.childProcess) {
-            this.childProcess.kill();
-            this.childProcess = null;
-        }
-    }
-    /** Exits this process */
-    async exit(statusCode = 0) {
-        try {
-            await this.stop();
-            // eslint-disable-next-line no-process-exit
-            process.exit(statusCode);
-        }
-        catch (error) {
-            console.error(error.message || error);
-            // eslint-disable-next-line no-process-exit
-            process.exit(1);
-        }
-    }
-    _setTimeout(callback) {
-        if (Number(this.props.wait) > 0) {
-            this.successTimer = setTimeout(callback, this.props.wait);
-        }
-    }
-    _clearTimeout() {
-        if (this.successTimer) {
-            clearTimeout(this.successTimer);
-        }
-    }
-}
-exports.default = ChildProcessProxy;

package/dist/lib/process-utils/process-utils.js

@@ -1,35 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAvailablePort = void 0;
-const child_process_1 = __importDefault(require("child_process"));
-// Get an available port
-// Works on Unix systems
-function getAvailablePort(defaultPort = 3000) {
-    return new Promise((resolve) => {
-        // Get a list of all ports in use
-        child_process_1.default.exec('lsof -i -P -n | grep LISTEN', (error, stdout) => {
-            if (error) {
-                // likely no permission, e.g. CI
-                resolve(defaultPort);
-                return;
-            }
-            const portsInUse = [];
-            const regex = /:(\d+) \(LISTEN\)/;
-            stdout.split('\n').forEach((line) => {
-                const match = regex.exec(line);
-                if (match) {
-                    portsInUse.push(Number(match[1]));
-                }
-            });
-            let port = defaultPort;
-            while (portsInUse.includes(port)) {
-                port++;
-            }
-            resolve(port);
-        });
-    });
-}
-exports.getAvailablePort = getAvailablePort;

package/dist/lib/worker-api/create-worker.js

@@ -1,91 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createWorker = void 0;
-const async_queue_1 = __importDefault(require("../async-queue/async-queue"));
-const worker_body_1 = __importDefault(require("../worker-farm/worker-body"));
-// import {validateWorkerVersion} from './validate-worker-version';
-/** Counter for jobs */
-let requestId = 0;
-let inputBatches;
-let options;
-/**
- * Set up a WebWorkerGlobalScope to talk with the main thread
- */
-function createWorker(process, processInBatches) {
-    if (!worker_body_1.default.inWorkerThread()) {
-        return;
-    }
-    const context = {
-        process: processOnMainThread
-    };
-    // eslint-disable-next-line complexity
-    worker_body_1.default.onmessage = async (type, payload) => {
-        try {
-            switch (type) {
-                case 'process':
-                    if (!process) {
-                        throw new Error('Worker does not support atomic processing');
-                    }
-                    const result = await process(payload.input, payload.options || {}, context);
-                    worker_body_1.default.postMessage('done', { result });
-                    break;
-                case 'process-in-batches':
-                    if (!processInBatches) {
-                        throw new Error('Worker does not support batched processing');
-                    }
-                    inputBatches = new async_queue_1.default();
-                    options = payload.options || {};
-                    const resultIterator = processInBatches(inputBatches, options, context);
-                    for await (const batch of resultIterator) {
-                        worker_body_1.default.postMessage('output-batch', { result: batch });
-                    }
-                    worker_body_1.default.postMessage('done', {});
-                    break;
-                case 'input-batch':
-                    inputBatches.push(payload.input);
-                    break;
-                case 'input-done':
-                    inputBatches.close();
-                    break;
-                default:
-            }
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : '';
-            worker_body_1.default.postMessage('error', { error: message });
-        }
-    };
-}
-exports.createWorker = createWorker;
-function processOnMainThread(arrayBuffer, options = {}) {
-    return new Promise((resolve, reject) => {
-        const id = requestId++;
-        /**
-         */
-        const onMessage = (type, payload) => {
-            if (payload.id !== id) {
-                // not ours
-                return;
-            }
-            switch (type) {
-                case 'done':
-                    worker_body_1.default.removeEventListener(onMessage);
-                    resolve(payload.result);
-                    break;
-                case 'error':
-                    worker_body_1.default.removeEventListener(onMessage);
-                    reject(payload.error);
-                    break;
-                default:
-                // ignore
-            }
-        };
-        worker_body_1.default.addEventListener(onMessage);
-        // Ask the main thread to decode data
-        const payload = { id, input: arrayBuffer, options };
-        worker_body_1.default.postMessage('process', payload);
-    });
-}

package/dist/lib/worker-api/get-worker-url.js

@@ -1,58 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getWorkerURL = exports.getWorkerName = void 0;
-const assert_1 = require("../env-utils/assert");
-const version_1 = require("../env-utils/version");
-const NPM_TAG = 'latest'; // 'beta', or 'latest' on release-branch
-const VERSION = typeof version_1.VERSION !== 'undefined' ? version_1.VERSION : NPM_TAG;
-/**
- * Gets worker object's name (for debugging in Chrome thread inspector window)
- */
-function getWorkerName(worker) {
-    const warning = worker.version !== VERSION ? ` (worker-utils@${VERSION})` : '';
-    return `${worker.name}@${worker.version}${warning}`;
-}
-exports.getWorkerName = getWorkerName;
-/**
- * Generate a worker URL based on worker object and options
- * @returns A URL to one of the following:
- * - a published worker on unpkg CDN
- * - a local test worker
- * - a URL provided by the user in options
- */
-function getWorkerURL(worker, options = {}) {
-    const workerOptions = options[worker.id] || {};
-    const workerFile = `${worker.id}-worker.js`;
-    let url = workerOptions.workerUrl;
-    // HACK: Allow for non-nested workerUrl for the CompressionWorker.
-    // For the compression worker, workerOptions is currently not nested correctly. For most loaders,
-    // you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker
-    // puts options at the top level, not within a `compression` key (its `id`). For this reason, the
-    // above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not
-    // break backwards compatibility, we allow the CompressionWorker to have options at the top level.
-    if (!url && worker.id === 'compression') {
-        url = options.workerUrl;
-    }
-    // If URL is test, generate local loaders.gl url
-    // @ts-ignore _workerType
-    if (options._workerType === 'test') {
-        url = `modules/${worker.module}/dist/${workerFile}`;
-    }
-    // If url override is not provided, generate a URL to published version on npm CDN unpkg.com
-    if (!url) {
-        // GENERATE
-        let version = worker.version;
-        // On master we need to load npm alpha releases published with the `beta` tag
-        if (version === 'latest') {
-            // throw new Error('latest worker version specified');
-            version = NPM_TAG;
-        }
-        const versionTag = version ? `@${version}` : '';
-        url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
-    }
-    (0, assert_1.assert)(url);
-    // Allow user to override location
-    return url;
-}
-exports.getWorkerURL = getWorkerURL;

package/dist/lib/worker-api/process-on-worker.js

@@ -1,83 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.processOnWorker = exports.canProcessOnWorker = void 0;
-const worker_farm_1 = __importDefault(require("../worker-farm/worker-farm"));
-const get_worker_url_1 = require("./get-worker-url");
-const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
-/**
- * Determines if we can parse with worker
- * @param loader
- * @param data
- * @param options
- */
-function canProcessOnWorker(worker, options) {
-    if (!worker_farm_1.default.isSupported()) {
-        return false;
-    }
-    return worker.worker && options?.worker;
-}
-exports.canProcessOnWorker = canProcessOnWorker;
-/**
- * This function expects that the worker thread sends certain messages,
- * Creating such a worker can be automated if the worker is wrapper by a call to
- * createWorker in @loaders.gl/worker-utils.
- */
-async function processOnWorker(worker, data, options = {}, context = {}) {
-    const name = (0, get_worker_url_1.getWorkerName)(worker);
-    const workerFarm = worker_farm_1.default.getWorkerFarm(options);
-    const { source } = options;
-    const workerPoolProps = { name, source };
-    if (!source) {
-        workerPoolProps.url = (0, get_worker_url_1.getWorkerURL)(worker, options);
-    }
-    const workerPool = workerFarm.getWorkerPool(workerPoolProps);
-    const jobName = options.jobName || worker.name;
-    const job = await workerPool.startJob(jobName, 
-    // eslint-disable-next-line
-    onMessage.bind(null, context));
-    // Kick off the processing in the worker
-    const transferableOptions = (0, get_transfer_list_1.getTransferListForWriter)(options);
-    job.postMessage('process', { input: data, options: transferableOptions });
-    const result = await job.result;
-    return result.result;
-}
-exports.processOnWorker = processOnWorker;
-/**
- * Job completes when we receive the result
- * @param job
- * @param message
- */
-async function onMessage(context, job, type, payload) {
-    switch (type) {
-        case 'done':
-            // Worker is done
-            job.done(payload);
-            break;
-        case 'error':
-            // Worker encountered an error
-            job.error(new Error(payload.error));
-            break;
-        case 'process':
-            // Worker is asking for us (main thread) to process something
-            const { id, input, options } = payload;
-            try {
-                if (!context.process) {
-                    job.postMessage('error', { id, error: 'Worker not set up to process on main thread' });
-                    return;
-                }
-                const result = await context.process(input, options);
-                job.postMessage('done', { id, result });
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : 'unknown error';
-                job.postMessage('error', { id, error: message });
-            }
-            break;
-        default:
-            // eslint-disable-next-line
-            console.warn(`process-on-worker: unknown message ${type}`);
-    }
-}

package/dist/lib/worker-api/validate-worker-version.js

@@ -1,35 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateWorkerVersion = void 0;
-const assert_1 = require("../env-utils/assert");
-const version_1 = require("../env-utils/version");
-/**
- * Check if worker is compatible with this library version
- * @param worker
- * @param libVersion
- * @returns `true` if the two versions are compatible
- */
-function validateWorkerVersion(worker, coreVersion = version_1.VERSION) {
-    (0, assert_1.assert)(worker, 'no worker provided');
-    const workerVersion = worker.version;
-    if (!coreVersion || !workerVersion) {
-        return false;
-    }
-    // TODO enable when fix the __version__ injection
-    // const coreVersions = parseVersion(coreVersion);
-    // const workerVersions = parseVersion(workerVersion);
-    // assert(
-    //   coreVersion.major === workerVersion.major && coreVersion.minor <= workerVersion.minor,
-    //   `worker: ${worker.name} is not compatible. ${coreVersion.major}.${
-    //     coreVersion.minor
-    //   }+ is required.`
-    // );
-    return true;
-}
-exports.validateWorkerVersion = validateWorkerVersion;
-// @ts-ignore
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-function parseVersion(version) {
-    const parts = version.split('.').map(Number);
-    return { major: parts[0], minor: parts[1] };
-}

package/dist/lib/worker-farm/worker-body.js

@@ -1,111 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
-/** Vile hack to defeat over-zealous bundlers from stripping out the require */
-function getParentPort() {
-    // const isNode = globalThis.process;
-    let parentPort;
-    try {
-        // prettier-ignore
-        eval('globalThis.parentPort = require(\'worker_threads\').parentPort'); // eslint-disable-line no-eval
-        parentPort = globalThis.parentPort;
-        // eslint-disable-next-line no-empty
-    }
-    catch { }
-    return parentPort;
-}
-const onMessageWrapperMap = new Map();
-/**
- * Type safe wrapper for worker code
- */
-class WorkerBody {
-    /** Check that we are actually in a worker thread */
-    static inWorkerThread() {
-        return typeof self !== 'undefined' || Boolean(getParentPort());
-    }
-    /*
-     * (type: WorkerMessageType, payload: WorkerMessagePayload) => any
-     */
-    static set onmessage(onMessage) {
-        function handleMessage(message) {
-            // Confusingly the message itself also has a 'type' field which is always set to 'message'
-            const parentPort = getParentPort();
-            const { type, payload } = parentPort ? message : message.data;
-            // if (!isKnownMessage(message)) {
-            //   return;
-            // }
-            onMessage(type, payload);
-        }
-        const parentPort = getParentPort();
-        if (parentPort) {
-            parentPort.on('message', handleMessage);
-            // if (message == 'exit') { parentPort.unref(); }
-            // eslint-disable-next-line
-            parentPort.on('exit', () => console.debug('Node worker closing'));
-        }
-        else {
-            // eslint-disable-next-line no-restricted-globals
-            globalThis.onmessage = handleMessage;
-        }
-    }
-    static addEventListener(onMessage) {
-        let onMessageWrapper = onMessageWrapperMap.get(onMessage);
-        if (!onMessageWrapper) {
-            onMessageWrapper = (message) => {
-                if (!isKnownMessage(message)) {
-                    return;
-                }
-                // Confusingly in the browser, the message itself also has a 'type' field which is always set to 'message'
-                const parentPort = getParentPort();
-                const { type, payload } = parentPort ? message : message.data;
-                onMessage(type, payload);
-            };
-        }
-        const parentPort = getParentPort();
-        if (parentPort) {
-            console.error('not implemented'); // eslint-disable-line
-        }
-        else {
-            globalThis.addEventListener('message', onMessageWrapper);
-        }
-    }
-    static removeEventListener(onMessage) {
-        const onMessageWrapper = onMessageWrapperMap.get(onMessage);
-        onMessageWrapperMap.delete(onMessage);
-        const parentPort = getParentPort();
-        if (parentPort) {
-            console.error('not implemented'); // eslint-disable-line
-        }
-        else {
-            globalThis.removeEventListener('message', onMessageWrapper);
-        }
-    }
-    /**
-     * Send a message from a worker to creating thread (main thread)
-     * @param type
-     * @param payload
-     */
-    static postMessage(type, payload) {
-        const data = { source: 'loaders.gl', type, payload };
-        // console.log('posting message', data);
-        const transferList = (0, get_transfer_list_1.getTransferList)(payload);
-        const parentPort = getParentPort();
-        if (parentPort) {
-            parentPort.postMessage(data, transferList);
-            // console.log('posted message', data);
-        }
-        else {
-            // @ts-ignore
-            globalThis.postMessage(data, transferList);
-        }
-    }
-}
-exports.default = WorkerBody;
-// Filter out noise messages sent to workers
-function isKnownMessage(message) {
-    const { type, data } = message;
-    return (type === 'message' &&
-        data &&
-        typeof data.source === 'string' &&
-        data.source.startsWith('loaders.gl'));
-}

package/dist/lib/worker-farm/worker-farm.js

@@ -1,89 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const worker_pool_1 = __importDefault(require("./worker-pool"));
-const worker_thread_1 = __importDefault(require("./worker-thread"));
-const DEFAULT_PROPS = {
-    maxConcurrency: 3,
-    maxMobileConcurrency: 1,
-    reuseWorkers: true,
-    onDebug: () => { }
-};
-/**
- * Process multiple jobs with a "farm" of different workers in worker pools.
- */
-class WorkerFarm {
-    /** Checks if workers are supported on this platform */
-    static isSupported() {
-        return worker_thread_1.default.isSupported();
-    }
-    /** Get the singleton instance of the global worker farm */
-    static getWorkerFarm(props = {}) {
-        WorkerFarm._workerFarm = WorkerFarm._workerFarm || new WorkerFarm({});
-        WorkerFarm._workerFarm.setProps(props);
-        return WorkerFarm._workerFarm;
-    }
-    /** get global instance with WorkerFarm.getWorkerFarm() */
-    constructor(props) {
-        this.workerPools = new Map();
-        this.props = { ...DEFAULT_PROPS };
-        this.setProps(props);
-        /** @type Map<string, WorkerPool>} */
-        this.workerPools = new Map();
-    }
-    /**
-     * Terminate all workers in the farm
-     * @note Can free up significant memory
-     */
-    destroy() {
-        for (const workerPool of this.workerPools.values()) {
-            workerPool.destroy();
-        }
-        this.workerPools = new Map();
-    }
-    /**
-     * Set props used when initializing worker pools
-     * @param props
-     */
-    setProps(props) {
-        this.props = { ...this.props, ...props };
-        // Update worker pool props
-        for (const workerPool of this.workerPools.values()) {
-            workerPool.setProps(this._getWorkerPoolProps());
-        }
-    }
-    /**
-     * Returns a worker pool for the specified worker
-     * @param options - only used first time for a specific worker name
-     * @param options.name - the name of the worker - used to identify worker pool
-     * @param options.url -
-     * @param options.source -
-     * @example
-     *   const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
-     */
-    getWorkerPool(options) {
-        const { name, source, url } = options;
-        let workerPool = this.workerPools.get(name);
-        if (!workerPool) {
-            workerPool = new worker_pool_1.default({
-                name,
-                source,
-                url
-            });
-            workerPool.setProps(this._getWorkerPoolProps());
-            this.workerPools.set(name, workerPool);
-        }
-        return workerPool;
-    }
-    _getWorkerPoolProps() {
-        return {
-            maxConcurrency: this.props.maxConcurrency,
-            maxMobileConcurrency: this.props.maxMobileConcurrency,
-            reuseWorkers: this.props.reuseWorkers,
-            onDebug: this.props.onDebug
-        };
-    }
-}
-exports.default = WorkerFarm;

package/dist/lib/worker-farm/worker-job.js

@@ -1,47 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const assert_1 = require("../env-utils/assert");
-/**
- * Represents one Job handled by a WorkerPool or WorkerFarm
- */
-class WorkerJob {
-    constructor(jobName, workerThread) {
-        this.isRunning = true;
-        this._resolve = () => { };
-        this._reject = () => { };
-        this.name = jobName;
-        this.workerThread = workerThread;
-        this.result = new Promise((resolve, reject) => {
-            this._resolve = resolve;
-            this._reject = reject;
-        });
-    }
-    /**
-     * Send a message to the job's worker thread
-     * @param data any data structure, ideally consisting mostly of transferrable objects
-     */
-    postMessage(type, payload) {
-        this.workerThread.postMessage({
-            source: 'loaders.gl',
-            type,
-            payload
-        });
-    }
-    /**
-     * Call to resolve the `result` Promise with the supplied value
-     */
-    done(value) {
-        (0, assert_1.assert)(this.isRunning);
-        this.isRunning = false;
-        this._resolve(value);
-    }
-    /**
-     * Call to reject the `result` Promise with the supplied error
-     */
-    error(error) {
-        (0, assert_1.assert)(this.isRunning);
-        this.isRunning = false;
-        this._reject(error);
-    }
-}
-exports.default = WorkerJob;