@loaders.gl/worker-utils 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/lib/worker-farm/worker-thread.js

@@ -1,105 +1,131 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { assert } from '../env-utils/assert';
-import { getLoadableWorkerURL } from '../worker-utils/get-loadable-worker-url';
-import { getTransferList } from '../worker-utils/get-transfer-list';
-
-const NOOP = () => {};
-
-export default class WorkerThread {
-  static isSupported() {
-    return typeof Worker !== 'undefined';
-  }
-
-  constructor(props) {
-    _defineProperty(this, "name", void 0);
-
-    _defineProperty(this, "source", void 0);
-
-    _defineProperty(this, "url", void 0);
-
-    _defineProperty(this, "terminated", false);
-
-    _defineProperty(this, "worker", void 0);
-
-    _defineProperty(this, "onMessage", void 0);
-
-    _defineProperty(this, "onError", void 0);
-
-    _defineProperty(this, "_loadableURL", '');
-
-    const {
-      name,
-      source,
-      url
-    } = props;
-    assert(source || url);
-    this.name = name;
-    this.source = source;
-    this.url = url;
-    this.onMessage = NOOP;
-
-    this.onError = error => console.log(error);
-
-    this.worker = this._createBrowserWorker();
-  }
-
-  destroy() {
-    this.onMessage = NOOP;
-    this.onError = NOOP;
-    this.worker.terminate();
-    this.terminated = true;
-  }
-
-  get isRunning() {
-    return Boolean(this.onMessage);
-  }
-
-  postMessage(data, transferList) {
-    transferList = transferList || getTransferList(data);
-    this.worker.postMessage(data, transferList);
-  }
-
-  _getErrorFromErrorEvent(event) {
-    let message = 'Failed to load ';
-    message += "worker ".concat(this.name, " from ").concat(this.url, ". ");
-
-    if (event.message) {
-      message += "".concat(event.message, " in ");
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_threads_1 = require("../node/worker_threads");
+const globals_1 = require("../env-utils/globals");
+const assert_1 = require("../env-utils/assert");
+const get_loadable_worker_url_1 = require("../worker-utils/get-loadable-worker-url");
+const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
+const NOOP = () => { };
+/**
+ * Represents one worker thread
+ */
+class WorkerThread {
+    /** Checks if workers are supported on this platform */
+    static isSupported() {
+        return ((typeof Worker !== 'undefined' && globals_1.isBrowser) ||
+            (typeof worker_threads_1.NodeWorker !== 'undefined' && !globals_1.isBrowser));
     }
-
-    if (event.lineno) {
-      message += ":".concat(event.lineno, ":").concat(event.colno);
+    constructor(props) {
+        this.terminated = false;
+        this._loadableURL = '';
+        const { name, source, url } = props;
+        (0, assert_1.assert)(source || url); // Either source or url must be defined
+        this.name = name;
+        this.source = source;
+        this.url = url;
+        this.onMessage = NOOP;
+        this.onError = (error) => console.log(error); // eslint-disable-line
+        this.worker = globals_1.isBrowser ? this._createBrowserWorker() : this._createNodeWorker();
+    }
+    /**
+     * Terminate this worker thread
+     * @note Can free up significant memory
+     */
+    destroy() {
+        this.onMessage = NOOP;
+        this.onError = NOOP;
+        this.worker.terminate(); // eslint-disable-line @typescript-eslint/no-floating-promises
+        this.terminated = true;
+    }
+    get isRunning() {
+        return Boolean(this.onMessage);
+    }
+    /**
+     * Send a message to this worker thread
+     * @param data any data structure, ideally consisting mostly of transferrable objects
+     * @param transferList If not supplied, calculated automatically by traversing data
+     */
+    postMessage(data, transferList) {
+        transferList = transferList || (0, get_transfer_list_1.getTransferList)(data);
+        // @ts-ignore
+        this.worker.postMessage(data, transferList);
+    }
+    // PRIVATE
+    /**
+     * Generate a standard Error from an ErrorEvent
+     * @param event
+     */
+    _getErrorFromErrorEvent(event) {
+        // Note Error object does not have the expected fields if loading failed completely
+        // https://developer.mozilla.org/en-US/docs/Web/API/Worker#Event_handlers
+        // https://developer.mozilla.org/en-US/docs/Web/API/ErrorEvent
+        let message = 'Failed to load ';
+        message += `worker ${this.name} from ${this.url}. `;
+        if (event.message) {
+            message += `${event.message} in `;
+        }
+        // const hasFilename = event.filename && !event.filename.startsWith('blob:');
+        // message += hasFilename ? event.filename : this.source.slice(0, 100);
+        if (event.lineno) {
+            message += `:${event.lineno}:${event.colno}`;
+        }
+        return new Error(message);
+    }
+    /**
+     * Creates a worker thread on the browser
+     */
+    _createBrowserWorker() {
+        this._loadableURL = (0, get_loadable_worker_url_1.getLoadableWorkerURL)({ source: this.source, url: this.url });
+        const worker = new Worker(this._loadableURL, { name: this.name });
+        worker.onmessage = (event) => {
+            if (!event.data) {
+                this.onError(new Error('No data received'));
+            }
+            else {
+                this.onMessage(event.data);
+            }
+        };
+        // This callback represents an uncaught exception in the worker thread
+        worker.onerror = (error) => {
+            this.onError(this._getErrorFromErrorEvent(error));
+            this.terminated = true;
+        };
+        // TODO - not clear when this would be called, for now just log in case it happens
+        worker.onmessageerror = (event) => console.error(event); // eslint-disable-line
+        return worker;
+    }
+    /**
+     * Creates a worker thread in node.js
+     * @todo https://nodejs.org/api/async_hooks.html#async-resource-worker-pool
+     */
+    _createNodeWorker() {
+        let worker;
+        if (this.url) {
+            // Make sure relative URLs start with './'
+            const absolute = this.url.includes(':/') || this.url.startsWith('/');
+            const url = absolute ? this.url : `./${this.url}`;
+            // console.log('Starting work from', url);
+            worker = new worker_threads_1.NodeWorker(url, { eval: false });
+        }
+        else if (this.source) {
+            worker = new worker_threads_1.NodeWorker(this.source, { eval: true });
+        }
+        else {
+            throw new Error('no worker');
+        }
+        worker.on('message', (data) => {
+            // console.error('message', data);
+            this.onMessage(data);
+        });
+        worker.on('error', (error) => {
+            // console.error('error', error);
+            this.onError(error);
+        });
+        worker.on('exit', (code) => {
+            // console.error('exit', code);
+        });
+        return worker;
     }
-
-    return new Error(message);
-  }
-
-  _createBrowserWorker() {
-    this._loadableURL = getLoadableWorkerURL({
-      source: this.source,
-      url: this.url
-    });
-    const worker = new Worker(this._loadableURL, {
-      name: this.name
-    });
-
-    worker.onmessage = event => {
-      if (!event.data) {
-        this.onError(new Error('No data received'));
-      } else {
-        this.onMessage(event.data);
-      }
-    };
-
-    worker.onerror = error => {
-      this.onError(this._getErrorFromErrorEvent(error));
-      this.terminated = true;
-    };
-
-    worker.onmessageerror = event => console.error(event);
-
-    return worker;
-  }
-
 }
-//# sourceMappingURL=worker-thread.js.map
+exports.default = WorkerThread;

package/dist/lib/worker-utils/get-loadable-worker-url.js

@@ -1,42 +1,72 @@
-import { assert } from '../env-utils/assert';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getLoadableWorkerURL = void 0;
+const assert_1 = require("../env-utils/assert");
 const workerURLCache = new Map();
-export function getLoadableWorkerURL(props) {
-  assert(props.source && !props.url || !props.source && props.url);
-  let workerURL = workerURLCache.get(props.source || props.url);
-
-  if (!workerURL) {
-    if (props.url) {
-      workerURL = getLoadableWorkerURLFromURL(props.url);
-      workerURLCache.set(props.url, workerURL);
+/**
+ * Creates a loadable URL from worker source or URL
+ * that can be used to create `Worker` instances.
+ * Due to CORS issues it may be necessary to wrap a URL in a small importScripts
+ * @param props
+ * @param props.source Worker source
+ * @param props.url Worker URL
+ * @returns loadable url
+ */
+function getLoadableWorkerURL(props) {
+    (0, assert_1.assert)((props.source && !props.url) || (!props.source && props.url)); // Either source or url must be defined
+    let workerURL = workerURLCache.get(props.source || props.url);
+    if (!workerURL) {
+        // Differentiate worker urls from worker source code
+        if (props.url) {
+            workerURL = getLoadableWorkerURLFromURL(props.url);
+            workerURLCache.set(props.url, workerURL);
+        }
+        if (props.source) {
+            workerURL = getLoadableWorkerURLFromSource(props.source);
+            workerURLCache.set(props.source, workerURL);
+        }
     }
-
-    if (props.source) {
-      workerURL = getLoadableWorkerURLFromSource(props.source);
-      workerURLCache.set(props.source, workerURL);
-    }
-  }
-
-  assert(workerURL);
-  return workerURL;
+    (0, assert_1.assert)(workerURL);
+    return workerURL;
 }
-
+exports.getLoadableWorkerURL = getLoadableWorkerURL;
+/**
+ * Build a loadable worker URL from worker URL
+ * @param url
+ * @returns loadable URL
+ */
 function getLoadableWorkerURLFromURL(url) {
-  if (!url.startsWith('http')) {
-    return url;
-  }
-
-  const workerSource = buildScriptSource(url);
-  return getLoadableWorkerURLFromSource(workerSource);
+    // A local script url, we can use it to initialize a Worker directly
+    if (!url.startsWith('http')) {
+        return url;
+    }
+    // A remote script, we need to use `importScripts` to load from different origin
+    const workerSource = buildScriptSource(url);
+    return getLoadableWorkerURLFromSource(workerSource);
 }
-
+/**
+ * Build a loadable worker URL from worker source
+ * @param workerSource
+ * @returns loadable url
+ */
 function getLoadableWorkerURLFromSource(workerSource) {
-  const blob = new Blob([workerSource], {
-    type: 'application/javascript'
-  });
-  return URL.createObjectURL(blob);
+    const blob = new Blob([workerSource], { type: 'application/javascript' });
+    return URL.createObjectURL(blob);
 }
-
+/**
+ * Per spec, worker cannot be initialized with a script from a different origin
+ * However a local worker script can still import scripts from other origins,
+ * so we simply build a wrapper script.
+ *
+ * @param workerUrl
+ * @returns source
+ */
 function buildScriptSource(workerUrl) {
-  return "try {\n  importScripts('".concat(workerUrl, "');\n} catch (error) {\n  console.error(error);\n  throw error;\n}");
+    return `\
+try {
+  importScripts('${workerUrl}');
+} catch (error) {
+  console.error(error);
+  throw error;
+}`;
 }
-//# sourceMappingURL=get-loadable-worker-url.js.map

package/dist/lib/worker-utils/get-transfer-list.d.ts

@@ -7,4 +7,9 @@
  * @returns a transfer list that can be passed to postMessage
  */
 export declare function getTransferList(object: any, recursive?: boolean, transfers?: Set<any>): Transferable[];
+/**
+ * Recursively drop non serializable values like functions and regexps.
+ * @param object
+ */
+export declare function getTransferListForWriter(object: object | null): object;
 //# sourceMappingURL=get-transfer-list.d.ts.map

package/dist/lib/worker-utils/get-transfer-list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"get-transfer-list.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-utils/get-transfer-list.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,GAAG,EACX,SAAS,GAAE,OAAc,EACzB,SAAS,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GACnB,YAAY,EAAE,CAwBhB"}
+{"version":3,"file":"get-transfer-list.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-utils/get-transfer-list.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,GAAG,EACX,SAAS,GAAE,OAAc,EACzB,SAAS,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GACnB,YAAY,EAAE,CAwBhB;AAuBD;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAsBtE"}

package/dist/lib/worker-utils/get-transfer-list.js

@@ -1,40 +1,87 @@
-export function getTransferList(object, recursive = true, transfers) {
-  const transfersSet = transfers || new Set();
-
-  if (!object) {} else if (isTransferable(object)) {
-    transfersSet.add(object);
-  } else if (isTransferable(object.buffer)) {
-    transfersSet.add(object.buffer);
-  } else if (ArrayBuffer.isView(object)) {} else if (recursive && typeof object === 'object') {
-    for (const key in object) {
-      getTransferList(object[key], recursive, transfersSet);
-    }
-  }
-
-  return transfers === undefined ? Array.from(transfersSet) : [];
+"use strict";
+// NOTE - there is a copy of this function is both in core and loader-utils
+// core does not need all the utils in loader-utils, just this one.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTransferListForWriter = exports.getTransferList = void 0;
+/**
+ * Returns an array of Transferrable objects that can be used with postMessage
+ * https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage
+ * @param object data to be sent via postMessage
+ * @param recursive - not for application use
+ * @param transfers - not for application use
+ * @returns a transfer list that can be passed to postMessage
+ */
+function getTransferList(object, recursive = true, transfers) {
+    // Make sure that items in the transfer list is unique
+    const transfersSet = transfers || new Set();
+    if (!object) {
+        // ignore
+    }
+    else if (isTransferable(object)) {
+        transfersSet.add(object);
+    }
+    else if (isTransferable(object.buffer)) {
+        // Typed array
+        transfersSet.add(object.buffer);
+    }
+    else if (ArrayBuffer.isView(object)) {
+        // object is a TypeArray viewing into a SharedArrayBuffer (not transferable)
+        // Do not iterate through the content in this case
+    }
+    else if (recursive && typeof object === 'object') {
+        for (const key in object) {
+            // Avoid perf hit - only go one level deep
+            getTransferList(object[key], recursive, transfersSet);
+        }
+    }
+    // If transfers is defined, is internal recursive call
+    // Otherwise it's called by the user
+    return transfers === undefined ? Array.from(transfersSet) : [];
 }
-
+exports.getTransferList = getTransferList;
+// https://developer.mozilla.org/en-US/docs/Web/API/Transferable
 function isTransferable(object) {
-  if (!object) {
+    if (!object) {
+        return false;
+    }
+    if (object instanceof ArrayBuffer) {
+        return true;
+    }
+    if (typeof MessagePort !== 'undefined' && object instanceof MessagePort) {
+        return true;
+    }
+    if (typeof ImageBitmap !== 'undefined' && object instanceof ImageBitmap) {
+        return true;
+    }
+    // @ts-ignore
+    if (typeof OffscreenCanvas !== 'undefined' && object instanceof OffscreenCanvas) {
+        return true;
+    }
     return false;
-  }
-
-  if (object instanceof ArrayBuffer) {
-    return true;
-  }
-
-  if (typeof MessagePort !== 'undefined' && object instanceof MessagePort) {
-    return true;
-  }
-
-  if (typeof ImageBitmap !== 'undefined' && object instanceof ImageBitmap) {
-    return true;
-  }
-
-  if (typeof OffscreenCanvas !== 'undefined' && object instanceof OffscreenCanvas) {
-    return true;
-  }
-
-  return false;
 }
-//# sourceMappingURL=get-transfer-list.js.map
+/**
+ * Recursively drop non serializable values like functions and regexps.
+ * @param object
+ */
+function getTransferListForWriter(object) {
+    if (object === null) {
+        return {};
+    }
+    const clone = Object.assign({}, object);
+    Object.keys(clone).forEach((key) => {
+        // Typed Arrays and Arrays are passed with no change
+        if (typeof object[key] === 'object' &&
+            !ArrayBuffer.isView(object[key]) &&
+            !(object[key] instanceof Array)) {
+            clone[key] = getTransferListForWriter(object[key]);
+        }
+        else if (typeof clone[key] === 'function' || clone[key] instanceof RegExp) {
+            clone[key] = {};
+        }
+        else {
+            clone[key] = object[key];
+        }
+    });
+    return clone;
+}
+exports.getTransferListForWriter = getTransferListForWriter;

package/dist/lib/worker-utils/remove-nontransferable-options.d.ts

@@ -1,6 +1,6 @@
 /**
- * Safely stringify JSON (drop non serializable values like functions and regexps)
- * @param value
+ * Recursively drop non serializable values like functions and regexps.
+ * @param object
  */
-export declare function removeNontransferableOptions(object: object): object;
+export declare function removeNontransferableOptions(object: object | null): object;
 //# sourceMappingURL=remove-nontransferable-options.d.ts.map

package/dist/lib/worker-utils/remove-nontransferable-options.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"remove-nontransferable-options.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-utils/remove-nontransferable-options.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAKnE"}
+{"version":3,"file":"remove-nontransferable-options.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-utils/remove-nontransferable-options.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAkB1E"}

package/dist/lib/worker-utils/remove-nontransferable-options.js

@@ -1,23 +1,27 @@
-export function removeNontransferableOptions(object) {
-  return JSON.parse(stringifyJSON(object));
-}
-
-function stringifyJSON(v) {
-  const cache = new Set();
-  return JSON.stringify(v, (key, value) => {
-    if (typeof value === 'object' && value !== null) {
-      if (cache.has(value)) {
-        try {
-          return JSON.parse(JSON.stringify(value));
-        } catch (err) {
-          return undefined;
-        }
-      }
-
-      cache.add(value);
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeNontransferableOptions = void 0;
+/**
+ * Recursively drop non serializable values like functions and regexps.
+ * @param object
+ */
+function removeNontransferableOptions(object) {
+    if (object === null) {
+        return {};
     }
-
-    return value;
-  });
+    const clone = Object.assign({}, object);
+    Object.keys(clone).forEach((key) => {
+        // Checking if it is an object and not a typed array.
+        if (typeof object[key] === 'object' && !ArrayBuffer.isView(object[key])) {
+            clone[key] = removeNontransferableOptions(object[key]);
+        }
+        else if (typeof clone[key] === 'function' || clone[key] instanceof RegExp) {
+            clone[key] = {};
+        }
+        else {
+            clone[key] = object[key];
+        }
+    });
+    return clone;
 }
-//# sourceMappingURL=remove-nontransferable-options.js.map
+exports.removeNontransferableOptions = removeNontransferableOptions;

package/dist/null-worker.js

@@ -93,16 +93,33 @@
   }
 
   // src/lib/worker-farm/worker-body.ts
+  function getParentPort() {
+    let parentPort;
+    try {
+      eval("globalThis.parentPort = require('worker_threads').parentPort");
+      parentPort = globalThis.parentPort;
+    } catch {
+    }
+    return parentPort;
+  }
   var onMessageWrapperMap = new Map();
   var WorkerBody = class {
+    static inWorkerThread() {
+      return typeof self !== "undefined" || Boolean(getParentPort());
+    }
     static set onmessage(onMessage) {
-      self.onmessage = (message) => {
-        if (!isKnownMessage(message)) {
-          return;
-        }
-        const { type, payload } = message.data;
+      function handleMessage(message) {
+        const parentPort3 = getParentPort();
+        const { type, payload } = parentPort3 ? message : message.data;
         onMessage(type, payload);
-      };
+      }
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        parentPort2.on("message", handleMessage);
+        parentPort2.on("exit", () => console.debug("Node worker closing"));
+      } else {
+        globalThis.onmessage = handleMessage;
+      }
     }
     static addEventListener(onMessage) {
       let onMessageWrapper = onMessageWrapperMap.get(onMessage);
@@ -111,22 +128,36 @@
           if (!isKnownMessage(message)) {
             return;
           }
-          const { type, payload } = message.data;
+          const parentPort3 = getParentPort();
+          const { type, payload } = parentPort3 ? message : message.data;
           onMessage(type, payload);
         };
       }
-      self.addEventListener("message", onMessageWrapper);
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        console.error("not implemented");
+      } else {
+        globalThis.addEventListener("message", onMessageWrapper);
+      }
     }
     static removeEventListener(onMessage) {
       const onMessageWrapper = onMessageWrapperMap.get(onMessage);
       onMessageWrapperMap.delete(onMessage);
-      self.removeEventListener("message", onMessageWrapper);
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        console.error("not implemented");
+      } else {
+        globalThis.removeEventListener("message", onMessageWrapper);
+      }
     }
     static postMessage(type, payload) {
-      if (self) {
-        const data = { source: "loaders.gl", type, payload };
-        const transferList = getTransferList(payload);
-        self.postMessage(data, transferList);
+      const data = { source: "loaders.gl", type, payload };
+      const transferList = getTransferList(payload);
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        parentPort2.postMessage(data, transferList);
+      } else {
+        globalThis.postMessage(data, transferList);
       }
     }
   };
@@ -140,7 +171,7 @@
   var inputBatches;
   var options;
   function createWorker(process, processInBatches) {
-    if (typeof self === "undefined") {
+    if (!WorkerBody.inWorkerThread()) {
       return;
     }
     const context = {