@loaders.gl/worker-utils 4.0.0-alpha.1 → 4.0.0-alpha.11

package/dist/lib/process-utils/child-process-proxy.js

@@ -1,122 +1,136 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import * as ChildProcess from 'child_process';
-import { getAvailablePort } from './process-utils';
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-console */
+// Avoid using named imports for Node builtins to help with "empty" resolution
+// for bundlers targeting browser environments. Access imports & types
+// through the `ChildProcess` object (e.g. `ChildProcess.spawn`, `ChildProcess.ChildProcess`).
+const ChildProcess = __importStar(require("child_process"));
+const process_utils_1 = require("./process-utils");
 const DEFAULT_PROPS = {
-  command: '',
-  arguments: [],
-  port: 5000,
-  autoPort: true,
-  wait: 2000,
-  onSuccess: processProxy => {
-    console.log("Started ".concat(processProxy.props.command));
-  }
+    command: '',
+    arguments: [],
+    port: 5000,
+    autoPort: true,
+    wait: 2000,
+    onSuccess: (processProxy) => {
+        console.log(`Started ${processProxy.props.command}`);
+    }
 };
-export default class ChildProcessProxy {
-  constructor({
-    id = 'browser-driver'
-  } = {}) {
-    _defineProperty(this, "id", void 0);
-
-    _defineProperty(this, "props", { ...DEFAULT_PROPS
-    });
-
-    _defineProperty(this, "childProcess", null);
-
-    _defineProperty(this, "port", 0);
-
-    _defineProperty(this, "successTimer", void 0);
-
-    this.id = id;
-  }
-
-  async start(props) {
-    props = { ...DEFAULT_PROPS,
-      ...props
-    };
-    this.props = props;
-    const args = [...props.arguments];
-    this.port = Number(props.port);
-
-    if (props.portArg) {
-      if (props.autoPort) {
-        this.port = await getAvailablePort(props.port);
-      }
-
-      args.push(props.portArg, String(this.port));
+/**
+ * Manager for a Node.js child process
+ * Prepares arguments, starts, stops and tracks output
+ */
+class ChildProcessProxy {
+    // constructor(props?: {id?: string});
+    constructor({ id = 'browser-driver' } = {}) {
+        this.props = { ...DEFAULT_PROPS };
+        this.childProcess = null;
+        this.port = 0;
+        this.id = id;
     }
-
-    return await new Promise((resolve, reject) => {
-      try {
-        this._setTimeout(() => {
-          if (props.onSuccess) {
-            props.onSuccess(this);
-          }
-
-          resolve({});
-        });
-
-        console.log("Spawning ".concat(props.command, " ").concat(props.arguments.join(' ')));
-        const childProcess = ChildProcess.spawn(props.command, args, props.spawn);
-        this.childProcess = childProcess;
-        childProcess.stdout.on('data', data => {
-          console.log(data.toString());
+    /** Starts a child process with the provided props */
+    async start(props) {
+        props = { ...DEFAULT_PROPS, ...props };
+        this.props = props;
+        const args = [...props.arguments];
+        // If portArg is set, we can look up an available port
+        this.port = Number(props.port);
+        if (props.portArg) {
+            if (props.autoPort) {
+                this.port = await (0, process_utils_1.getAvailablePort)(props.port);
+            }
+            args.push(props.portArg, String(this.port));
+        }
+        return await new Promise((resolve, reject) => {
+            try {
+                this._setTimeout(() => {
+                    if (props.onSuccess) {
+                        props.onSuccess(this);
+                    }
+                    resolve({});
+                });
+                console.log(`Spawning ${props.command} ${props.arguments.join(' ')}`);
+                const childProcess = ChildProcess.spawn(props.command, args, props.spawn);
+                this.childProcess = childProcess;
+                childProcess.stdout.on('data', (data) => {
+                    console.log(data.toString());
+                });
+                childProcess.stderr.on('data', (data) => {
+                    console.log(`Child process wrote to stderr: "${data}".`);
+                    if (!props.ignoreStderr) {
+                        this._clearTimeout();
+                        reject(new Error(data));
+                    }
+                });
+                childProcess.on('error', (error) => {
+                    console.log(`Child process errored with ${error}`);
+                    this._clearTimeout();
+                    reject(error);
+                });
+                childProcess.on('close', (code) => {
+                    console.log(`Child process exited with ${code}`);
+                    this.childProcess = null;
+                    this._clearTimeout();
+                    resolve({});
+                });
+            }
+            catch (error) {
+                reject(error);
+            }
         });
-        childProcess.stderr.on('data', data => {
-          console.log("Child process wrote to stderr: \"".concat(data, "\"."));
-
-          this._clearTimeout();
-
-          reject(new Error(data));
-        });
-        childProcess.on('error', error => {
-          console.log("Child process errored with ".concat(error));
-
-          this._clearTimeout();
-
-          reject(error);
-        });
-        childProcess.on('close', code => {
-          console.log("Child process exited with ".concat(code));
-          this.childProcess = null;
-
-          this._clearTimeout();
-
-          resolve({});
-        });
-      } catch (error) {
-        reject(error);
-      }
-    });
-  }
-
-  async stop() {
-    if (this.childProcess) {
-      this.childProcess.kill();
-      this.childProcess = null;
     }
-  }
-
-  async exit(statusCode = 0) {
-    try {
-      await this.stop();
-      process.exit(statusCode);
-    } catch (error) {
-      console.error(error.message || error);
-      process.exit(1);
+    /** Stops a running child process */
+    async stop() {
+        if (this.childProcess) {
+            this.childProcess.kill();
+            this.childProcess = null;
+        }
+    }
+    /** Exits this process */
+    async exit(statusCode = 0) {
+        try {
+            await this.stop();
+            // eslint-disable-next-line no-process-exit
+            process.exit(statusCode);
+        }
+        catch (error) {
+            console.error(error.message || error);
+            // eslint-disable-next-line no-process-exit
+            process.exit(1);
+        }
     }
-  }
-
-  _setTimeout(callback) {
-    if (Number(this.props.wait) > 0) {
-      this.successTimer = setTimeout(callback, this.props.wait);
+    _setTimeout(callback) {
+        if (Number(this.props.wait) > 0) {
+            this.successTimer = setTimeout(callback, this.props.wait);
+        }
     }
-  }
-
-  _clearTimeout() {
-    if (this.successTimer) {
-      clearTimeout(this.successTimer);
+    _clearTimeout() {
+        if (this.successTimer) {
+            clearTimeout(this.successTimer);
+        }
     }
-  }
-
 }
-//# sourceMappingURL=child-process-proxy.js.map
+exports.default = ChildProcessProxy;

package/dist/lib/process-utils/process-utils.d.ts

@@ -0,0 +1,2 @@
+export declare function getAvailablePort(defaultPort?: number): Promise<number>;
+//# sourceMappingURL=process-utils.d.ts.map

package/dist/lib/process-utils/process-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-utils.d.ts","sourceRoot":"","sources":["../../../src/lib/process-utils/process-utils.ts"],"names":[],"mappings":"AAIA,wBAAgB,gBAAgB,CAAC,WAAW,GAAE,MAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAyB5E"}

package/dist/lib/process-utils/process-utils.js

@@ -1,29 +1,35 @@
-import ChildProcess from 'child_process';
-export function getAvailablePort(defaultPort = 3000) {
-  return new Promise(resolve => {
-    ChildProcess.exec('lsof -i -P -n | grep LISTEN', (error, stdout) => {
-      if (error) {
-        resolve(defaultPort);
-        return;
-      }
-
-      const portsInUse = [];
-      const regex = /:(\d+) \(LISTEN\)/;
-      stdout.split('\n').forEach(line => {
-        const match = regex.exec(line);
-
-        if (match) {
-          portsInUse.push(Number(match[1]));
-        }
-      });
-      let port = defaultPort;
-
-      while (portsInUse.includes(port)) {
-        port++;
-      }
-
-      resolve(port);
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAvailablePort = void 0;
+const child_process_1 = __importDefault(require("child_process"));
+// Get an available port
+// Works on Unix systems
+function getAvailablePort(defaultPort = 3000) {
+    return new Promise((resolve) => {
+        // Get a list of all ports in use
+        child_process_1.default.exec('lsof -i -P -n | grep LISTEN', (error, stdout) => {
+            if (error) {
+                // likely no permission, e.g. CI
+                resolve(defaultPort);
+                return;
+            }
+            const portsInUse = [];
+            const regex = /:(\d+) \(LISTEN\)/;
+            stdout.split('\n').forEach((line) => {
+                const match = regex.exec(line);
+                if (match) {
+                    portsInUse.push(Number(match[1]));
+                }
+            });
+            let port = defaultPort;
+            while (portsInUse.includes(port)) {
+                port++;
+            }
+            resolve(port);
+        });
     });
-  });
 }
-//# sourceMappingURL=process-utils.js.map
+exports.getAvailablePort = getAvailablePort;

package/dist/lib/worker-api/create-worker.d.ts

@@ -0,0 +1,9 @@
+import type { WorkerContext, Process, ProcessInBatches } from '../../types';
+export type ProcessOnMainThread = (data: any, options?: {
+    [key: string]: any;
+}, context?: WorkerContext) => any;
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ */
+export declare function createWorker(process: Process, processInBatches?: ProcessInBatches): void;
+//# sourceMappingURL=create-worker.d.ts.map

package/dist/lib/worker-api/create-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-worker.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-api/create-worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,aAAa,EACb,OAAO,EACP,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAUrB,MAAM,MAAM,mBAAmB,GAAG,CAChC,IAAI,EAAE,GAAG,EACT,OAAO,CAAC,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,EAC9B,OAAO,CAAC,EAAE,aAAa,KACpB,GAAG,CAAC;AAET;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAiDxF"}

package/dist/lib/worker-api/create-worker.js

@@ -1,99 +1,91 @@
-import AsyncQueue from '../async-queue/async-queue';
-import WorkerBody from '../worker-farm/worker-body';
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createWorker = void 0;
+const async_queue_1 = __importDefault(require("../async-queue/async-queue"));
+const worker_body_1 = __importDefault(require("../worker-farm/worker-body"));
+// import {validateWorkerVersion} from './validate-worker-version';
+/** Counter for jobs */
 let requestId = 0;
 let inputBatches;
 let options;
-export function createWorker(process, processInBatches) {
-  if (typeof self === 'undefined') {
-    return;
-  }
-
-  const context = {
-    process: processOnMainThread
-  };
-
-  WorkerBody.onmessage = async (type, payload) => {
-    try {
-      switch (type) {
-        case 'process':
-          if (!process) {
-            throw new Error('Worker does not support atomic processing');
-          }
-
-          const result = await process(payload.input, payload.options || {}, context);
-          WorkerBody.postMessage('done', {
-            result
-          });
-          break;
-
-        case 'process-in-batches':
-          if (!processInBatches) {
-            throw new Error('Worker does not support batched processing');
-          }
-
-          inputBatches = new AsyncQueue();
-          options = payload.options || {};
-          const resultIterator = processInBatches(inputBatches, options, context);
-
-          for await (const batch of resultIterator) {
-            WorkerBody.postMessage('output-batch', {
-              result: batch
-            });
-          }
-
-          WorkerBody.postMessage('done', {});
-          break;
-
-        case 'input-batch':
-          inputBatches.push(payload.input);
-          break;
-
-        case 'input-done':
-          inputBatches.close();
-          break;
-
-        default:
-      }
-    } catch (error) {
-      const message = error instanceof Error ? error.message : '';
-      WorkerBody.postMessage('error', {
-        error: message
-      });
-    }
-  };
-}
-
-function processOnMainThread(arrayBuffer, options = {}) {
-  return new Promise((resolve, reject) => {
-    const id = requestId++;
-
-    const onMessage = (type, payload) => {
-      if (payload.id !== id) {
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ */
+function createWorker(process, processInBatches) {
+    if (!worker_body_1.default.inWorkerThread()) {
         return;
-      }
-
-      switch (type) {
-        case 'done':
-          WorkerBody.removeEventListener(onMessage);
-          resolve(payload.result);
-          break;
-
-        case 'error':
-          WorkerBody.removeEventListener(onMessage);
-          reject(payload.error);
-          break;
-
-        default:
-      }
+    }
+    const context = {
+        process: processOnMainThread
     };
-
-    WorkerBody.addEventListener(onMessage);
-    const payload = {
-      id,
-      input: arrayBuffer,
-      options
+    // eslint-disable-next-line complexity
+    worker_body_1.default.onmessage = async (type, payload) => {
+        try {
+            switch (type) {
+                case 'process':
+                    if (!process) {
+                        throw new Error('Worker does not support atomic processing');
+                    }
+                    const result = await process(payload.input, payload.options || {}, context);
+                    worker_body_1.default.postMessage('done', { result });
+                    break;
+                case 'process-in-batches':
+                    if (!processInBatches) {
+                        throw new Error('Worker does not support batched processing');
+                    }
+                    inputBatches = new async_queue_1.default();
+                    options = payload.options || {};
+                    const resultIterator = processInBatches(inputBatches, options, context);
+                    for await (const batch of resultIterator) {
+                        worker_body_1.default.postMessage('output-batch', { result: batch });
+                    }
+                    worker_body_1.default.postMessage('done', {});
+                    break;
+                case 'input-batch':
+                    inputBatches.push(payload.input);
+                    break;
+                case 'input-done':
+                    inputBatches.close();
+                    break;
+                default:
+            }
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : '';
+            worker_body_1.default.postMessage('error', { error: message });
+        }
     };
-    WorkerBody.postMessage('process', payload);
-  });
 }
-//# sourceMappingURL=create-worker.js.map
+exports.createWorker = createWorker;
+function processOnMainThread(arrayBuffer, options = {}) {
+    return new Promise((resolve, reject) => {
+        const id = requestId++;
+        /**
+         */
+        const onMessage = (type, payload) => {
+            if (payload.id !== id) {
+                // not ours
+                return;
+            }
+            switch (type) {
+                case 'done':
+                    worker_body_1.default.removeEventListener(onMessage);
+                    resolve(payload.result);
+                    break;
+                case 'error':
+                    worker_body_1.default.removeEventListener(onMessage);
+                    reject(payload.error);
+                    break;
+                default:
+                // ignore
+            }
+        };
+        worker_body_1.default.addEventListener(onMessage);
+        // Ask the main thread to decode data
+        const payload = { id, input: arrayBuffer, options };
+        worker_body_1.default.postMessage('process', payload);
+    });
+}

package/dist/lib/worker-api/get-worker-url.d.ts

@@ -0,0 +1,14 @@
+import type { WorkerObject, WorkerOptions } from '../../types';
+/**
+ * Gets worker object's name (for debugging in Chrome thread inspector window)
+ */
+export declare function getWorkerName(worker: WorkerObject): string;
+/**
+ * Generate a worker URL based on worker object and options
+ * @returns A URL to one of the following:
+ * - a published worker on unpkg CDN
+ * - a local test worker
+ * - a URL provided by the user in options
+ */
+export declare function getWorkerURL(worker: WorkerObject, options?: WorkerOptions): string;
+//# sourceMappingURL=get-worker-url.d.ts.map

package/dist/lib/worker-api/get-worker-url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-worker-url.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-api/get-worker-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,YAAY,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAQ7D;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAG1D;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,GAAE,aAAkB,GAAG,MAAM,CAwCtF"}

package/dist/lib/worker-api/get-worker-url.js

@@ -1,32 +1,59 @@
-import { assert } from '../env-utils/assert';
-import { VERSION as __VERSION__ } from '../env-utils/version';
-const NPM_TAG = 'beta';
-const VERSION = typeof "4.0.0-alpha.1" !== 'undefined' ? "4.0.0-alpha.1" : NPM_TAG;
-export function getWorkerName(worker) {
-  const warning = worker.version !== VERSION ? " (worker-utils@".concat(VERSION, ")") : '';
-  return "".concat(worker.name, "@").concat(worker.version).concat(warning);
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getWorkerURL = exports.getWorkerName = void 0;
+const assert_1 = require("../env-utils/assert");
+const globals_1 = require("../env-utils/globals");
+const version_1 = require("../env-utils/version");
+const NPM_TAG = 'beta'; // 'beta', or 'latest' on release-branch
+const VERSION = typeof version_1.VERSION !== 'undefined' ? version_1.VERSION : NPM_TAG;
+/**
+ * Gets worker object's name (for debugging in Chrome thread inspector window)
+ */
+function getWorkerName(worker) {
+    const warning = worker.version !== VERSION ? ` (worker-utils@${VERSION})` : '';
+    return `${worker.name}@${worker.version}${warning}`;
 }
-export function getWorkerURL(worker, options = {}) {
-  const workerOptions = options[worker.id] || {};
-  const workerFile = "".concat(worker.id, "-worker.js");
-  let url = workerOptions.workerUrl;
-
-  if (options._workerType === 'test') {
-    url = "modules/".concat(worker.module, "/dist/").concat(workerFile);
-  }
-
-  if (!url) {
-    let version = worker.version;
-
-    if (version === 'latest') {
-      version = NPM_TAG;
+exports.getWorkerName = getWorkerName;
+/**
+ * Generate a worker URL based on worker object and options
+ * @returns A URL to one of the following:
+ * - a published worker on unpkg CDN
+ * - a local test worker
+ * - a URL provided by the user in options
+ */
+function getWorkerURL(worker, options = {}) {
+    const workerOptions = options[worker.id] || {};
+    const workerFile = globals_1.isBrowser ? `${worker.id}-worker.js` : `${worker.id}-worker-node.js`;
+    let url = workerOptions.workerUrl;
+    // HACK: Allow for non-nested workerUrl for the CompressionWorker.
+    // For the compression worker, workerOptions is currently not nested correctly. For most loaders,
+    // you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker
+    // puts options at the top level, not within a `compression` key (its `id`). For this reason, the
+    // above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not
+    // break backwards compatibility, we allow the CompressionWorker to have options at the top level.
+    if (!url && worker.id === 'compression') {
+        url = options.workerUrl;
     }
-
-    const versionTag = version ? "@".concat(version) : '';
-    url = "https://unpkg.com/@loaders.gl/".concat(worker.module).concat(versionTag, "/dist/").concat(workerFile);
-  }
-
-  assert(url);
-  return url;
+    // If URL is test, generate local loaders.gl url
+    // @ts-ignore _workerType
+    if (options._workerType === 'test') {
+        url = `modules/${worker.module}/dist/${workerFile}`;
+    }
+    // If url override is not provided, generate a URL to published version on npm CDN unpkg.com
+    if (!url) {
+        // GENERATE
+        let version = worker.version;
+        // On master we need to load npm alpha releases published with the `beta` tag
+        if (version === 'latest') {
+            // throw new Error('latest worker version specified');
+            version = NPM_TAG;
+        }
+        const versionTag = version ? `@${version}` : '';
+        url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+    }
+    (0, assert_1.assert)(url);
+    // Allow user to override location
+    return url;
 }
-//# sourceMappingURL=get-worker-url.js.map
+exports.getWorkerURL = getWorkerURL;

package/dist/lib/worker-api/process-on-worker.d.ts

@@ -0,0 +1,20 @@
+import type { WorkerObject, WorkerOptions, WorkerContext } from '../../types';
+type ProcessOnWorkerOptions = WorkerOptions & {
+    jobName?: string;
+    [key: string]: any;
+};
+/**
+ * Determines if we can parse with worker
+ * @param loader
+ * @param data
+ * @param options
+ */
+export declare function canProcessOnWorker(worker: WorkerObject, options?: WorkerOptions): boolean | "" | undefined;
+/**
+ * This function expects that the worker thread sends certain messages,
+ * Creating such a worker can be automated if the worker is wrapper by a call to
+ * createWorker in @loaders.gl/worker-utils.
+ */
+export declare function processOnWorker(worker: WorkerObject, data: any, options?: ProcessOnWorkerOptions, context?: WorkerContext): Promise<any>;
+export {};
+//# sourceMappingURL=process-on-worker.d.ts.map

package/dist/lib/worker-api/process-on-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-on-worker.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-api/process-on-worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,aAAa,EACb,aAAa,EAGd,MAAM,aAAa,CAAC;AAMrB,KAAK,sBAAsB,GAAG,aAAa,GAAG;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,aAAa,4BAM/E;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,GAAG,EACT,OAAO,GAAE,sBAA2B,EACpC,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,GAAG,CAAC,CAwBd"}