@loaders.gl/worker-utils 3.3.0-alpha.1 → 3.3.0-alpha.11

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,oBAAY,aAAa,GAAG;IAE1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB,CAAC;AAEF,oBAAY,aAAa,GAAG;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC,CAAC;AAEF,oBAAY,OAAO,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,EAAE,OAAO,CAAC,EAAE,aAAa,KAAK,GAAG,CAAC;AAElG,oBAAY,gBAAgB,GAAG,CAC7B,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,EAC5C,OAAO,CAAC,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,EAC9B,OAAO,CAAC,EAAE,aAAa,KACpB,aAAa,CAAC,GAAG,CAAC,CAAC;AAExB;;GAEG;AACH,oBAAY,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1B,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC,CAAC;AAmBF,oBAAY,iBAAiB,GACzB,SAAS,GACT,MAAM,GACN,OAAO,GACP,oBAAoB,GACpB,aAAa,GACb,YAAY,GACZ,cAAc,CAAC;AAEnB,oBAAY,oBAAoB,GAAG;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAC;IAC/B,OAAO,CAAC,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAC;IAC/B,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,oBAAY,iBAAiB,GAAG;IAC9B,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,iBAAiB,CAAC;IACxB,OAAO,EAAE,oBAAoB,CAAC;CAC/B,CAAC;AAEF,oBAAY,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,iBAAiB,CAAC;CACzB,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG;IAE1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,OAAO,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,EAAE,OAAO,CAAC,EAAE,aAAa,KAAK,GAAG,CAAC;AAElG,MAAM,MAAM,gBAAgB,GAAG,CAC7B,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,EAC5C,OAAO,CAAC,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,EAC9B,OAAO,CAAC,EAAE,aAAa,KACpB,aAAa,CAAC,GAAG,CAAC,CAAC;AAExB;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1B,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC,CAAC;AAmBF,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,MAAM,GACN,OAAO,GACP,oBAAoB,GACpB,aAAa,GACb,YAAY,GACZ,cAAc,CAAC;AAEnB,MAAM,MAAM,oBAAoB,GAAG;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAC;IAC/B,OAAO,CAAC,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAC;IAC/B,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,iBAAiB,CAAC;IACxB,OAAO,EAAE,oBAAoB,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,iBAAiB,CAAC;CACzB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/worker-utils",
-  "version": "3.3.0-alpha.1",
+  "version": "3.3.0-alpha.11",
   "description": "Utilities for running tasks on worker threads",
   "license": "MIT",
   "publishConfig": {
@@ -27,7 +27,7 @@
   "browser": {
     "child_process": false,
     "fs": false,
-    "./src/lib/node/worker_threads.js": "./src/lib/node/worker_threads-browser.js",
+    "./src/lib/node/worker_threads.ts": "./src/lib/node/worker_threads-browser.ts",
     "./dist/lib/node/worker_threads.js": "./dist/lib/node/worker_threads-browser.js",
     "./dist/es5/lib/node/worker_threads.js": "./dist/es5/lib/node/worker_threads-browser.js",
     "./dist/esm/lib/node/worker_threads.js": "./dist/esm/lib/node/worker_threads-browser.js",
@@ -49,5 +49,5 @@
   "dependencies": {
     "@babel/runtime": "^7.3.1"
   },
-  "gitHead": "85ed2df7d17dd0d204c8bd8feb44779f70bc693b"
+  "gitHead": "1863a8875f63d578c0fdf5bdb3b7d6827f799db9"
 }

package/src/index.ts

@@ -31,7 +31,7 @@ export {createWorker} from './lib/worker-api/create-worker';
 // WORKER UTILS - EXPORTS
 export {getWorkerURL} from './lib/worker-api/get-worker-url';
 export {validateWorkerVersion} from './lib/worker-api/validate-worker-version';
-export {getTransferList} from './lib/worker-utils/get-transfer-list';
+export {getTransferList, getTransferListForWriter} from './lib/worker-utils/get-transfer-list';
 
 // LIBRARY UTILS
 export {getLibraryUrl, loadLibrary} from './lib/library-utils/library-utils';

package/src/lib/node/{worker_threads-browser.js → worker_threads-browser.ts}

@@ -2,6 +2,11 @@
 // These fills are non-functional, and just intended to ensure that
 // `import 'worker_threads` doesn't break browser builds.
 // The replacement is done in package.json browser field
-export class Worker {}
+export class Worker {
+  terminate() {}
+}
+
+export {Worker as NodeWorker};
+export {Worker as NodeWorkerType};
 
 export const parentPort = null;

package/src/lib/node/worker_threads.ts

@@ -0,0 +1,6 @@
+// loaders.gl, MIT license
+
+import * as WorkerThreads from 'worker_threads';
+export * from 'worker_threads';
+export const NodeWorker = WorkerThreads.Worker;
+export type NodeWorkerType = WorkerThreads.Worker;

package/src/lib/worker-api/process-on-worker.ts

@@ -7,8 +7,8 @@ import type {
 } from '../../types';
 import type WorkerJob from '../worker-farm/worker-job';
 import WorkerFarm from '../worker-farm/worker-farm';
-import {removeNontransferableOptions} from '../worker-utils/remove-nontransferable-options';
 import {getWorkerURL, getWorkerName} from './get-worker-url';
+import {getTransferListForWriter} from '../worker-utils/get-transfer-list';
 
 type ProcessOnWorkerOptions = WorkerOptions & {
   jobName?: string;
@@ -58,7 +58,7 @@ export async function processOnWorker(
   );
 
   // Kick off the processing in the worker
-  const transferableOptions = removeNontransferableOptions(options);
+  const transferableOptions = getTransferListForWriter(options);
   job.postMessage('process', {input: data, options: transferableOptions});
 
   const result = await job.result;

package/src/lib/worker-farm/worker-thread.ts

@@ -1,4 +1,6 @@
-import {Worker as NodeWorker} from '../node/worker_threads';
+// loaders.gl, MIT license
+
+import {NodeWorker, NodeWorkerType} from '../node/worker_threads';
 import {isBrowser} from '../env-utils/globals';
 import {assert} from '../env-utils/assert';
 import {getLoadableWorkerURL} from '../worker-utils/get-loadable-worker-url';
@@ -20,7 +22,7 @@ export default class WorkerThread {
   readonly source: string | undefined;
   readonly url: string | undefined;
   terminated: boolean = false;
-  worker: Worker | NodeWorker;
+  worker: Worker | NodeWorkerType;
   onMessage: (message: any) => void;
   onError: (error: Error) => void;
 
@@ -28,7 +30,10 @@ export default class WorkerThread {
 
   /** Checks if workers are supported on this platform */
   static isSupported(): boolean {
-    return (typeof Worker !== 'undefined' && isBrowser) || typeof NodeWorker !== undefined;
+    return (
+      (typeof Worker !== 'undefined' && isBrowser) ||
+      (typeof NodeWorker !== 'undefined' && !isBrowser)
+    );
   }
 
   constructor(props: WorkerThreadProps) {
@@ -121,8 +126,8 @@ export default class WorkerThread {
    * Creates a worker thread in node.js
    * @todo https://nodejs.org/api/async_hooks.html#async-resource-worker-pool
    */
-  _createNodeWorker(): NodeWorker {
-    let worker: NodeWorker;
+  _createNodeWorker(): NodeWorkerType {
+    let worker: NodeWorkerType;
     if (this.url) {
       // Make sure relative URLs start with './'
       const absolute = this.url.includes(':/') || this.url.startsWith('/');

package/src/lib/worker-utils/get-transfer-list.ts

@@ -59,3 +59,31 @@ function isTransferable(object: unknown) {
   }
   return false;
 }
+
+/**
+ * Recursively drop non serializable values like functions and regexps.
+ * @param object
+ */
+export function getTransferListForWriter(object: object | null): object {
+  if (object === null) {
+    return {};
+  }
+  const clone = Object.assign({}, object);
+
+  Object.keys(clone).forEach((key) => {
+    // Typed Arrays and Arrays are passed with no change
+    if (
+      typeof object[key] === 'object' &&
+      !ArrayBuffer.isView(object[key]) &&
+      !(object[key] instanceof Array)
+    ) {
+      clone[key] = getTransferListForWriter(object[key]);
+    } else if (typeof clone[key] === 'function' || clone[key] instanceof RegExp) {
+      clone[key] = {};
+    } else {
+      clone[key] = object[key];
+    }
+  });
+
+  return clone;
+}

package/src/lib/worker-utils/remove-nontransferable-options.ts

@@ -1,31 +1,23 @@
 /**
- * Safely stringify JSON (drop non serializable values like functions and regexps)
- * @param value
+ * Recursively drop non serializable values like functions and regexps.
+ * @param object
  */
-export function removeNontransferableOptions(object: object): object {
-  // options.log object contains functions which cannot be transferred
-  // TODO - decide how to handle logging on workers
-  // TODO - warn if options stringification is long
-  return JSON.parse(stringifyJSON(object));
-}
+export function removeNontransferableOptions(object: object | null): object {
+  if (object === null) {
+    return {};
+  }
+  const clone = Object.assign({}, object);
 
-function stringifyJSON(v: unknown) {
-  const cache = new Set();
-  return JSON.stringify(v, (key, value) => {
-    if (typeof value === 'object' && value !== null) {
-      if (cache.has(value)) {
-        // Circular reference found
-        try {
-          // If this value does not reference a parent it can be deduped
-          return JSON.parse(JSON.stringify(value));
-        } catch (err) {
-          // discard key if value cannot be deduped
-          return undefined;
-        }
-      }
-      // Store value in our set
-      cache.add(value);
+  Object.keys(clone).forEach((key) => {
+    // Checking if it is an object and not a typed array.
+    if (typeof object[key] === 'object' && !ArrayBuffer.isView(object[key])) {
+      clone[key] = removeNontransferableOptions(object[key]);
+    } else if (typeof clone[key] === 'function' || clone[key] instanceof RegExp) {
+      clone[key] = {};
+    } else {
+      clone[key] = object[key];
     }
-    return value;
   });
+
+  return clone;
 }

package/src/lib/node/worker_threads.js

@@ -1 +0,0 @@
-export * from 'worker_threads';