@loaders.gl/worker-utils 4.4.0-alpha.2 → 4.4.0

package/src/index.ts

@@ -40,7 +40,12 @@ export {validateWorkerVersion} from './lib/worker-api/validate-worker-version';
 export {getTransferList, getTransferListForWriter} from './lib/worker-utils/get-transfer-list';
 
 // LIBRARY UTILS
-export {getLibraryUrl, loadLibrary} from './lib/library-utils/library-utils';
+export {
+  extractLoadLibraryOptions,
+  getLibraryUrl,
+  loadLibrary,
+  type LoadLibraryOptions
+} from './lib/library-utils/library-utils';
 
 // PARSER UTILS
 export {default as AsyncQueue} from './lib/async-queue/async-queue';

package/src/lib/env-utils/version.ts

@@ -7,16 +7,19 @@ import {NPM_TAG} from '../npm-tag';
 // Version constant cannot be imported, it needs to correspond to the build version of **this** module.
 declare let __VERSION__: string;
 
+let warningIssued = false;
+
 function getVersion() {
   if (!globalThis._loadersgl_?.version) {
     globalThis._loadersgl_ = globalThis._loadersgl_ || {};
     // __VERSION__ is injected by babel-plugin-version-inline
-    if (typeof __VERSION__ === 'undefined') {
+    if (typeof __VERSION__ === 'undefined' && !warningIssued) {
       // eslint-disable-next-line
       console.warn(
         'loaders.gl: The __VERSION__ variable is not injected using babel plugin. Latest unstable workers would be fetched from the CDN.'
       );
       globalThis._loadersgl_.version = NPM_TAG;
+      warningIssued = true;
     } else {
       globalThis._loadersgl_.version = __VERSION__;
     }

package/src/lib/library-utils/library-utils.ts

@@ -7,8 +7,40 @@ import {isBrowser, isWorker} from '../env-utils/globals';
 import {assert} from '../env-utils/assert';
 import {VERSION} from '../env-utils/version';
 
+export type LoadLibraryOptions<ModulesT extends Record<string, any> = Record<string, any>> = {
+  useLocalLibraries?: boolean;
+  CDN?: string | null;
+  modules?: ModulesT;
+  // Core must not be supplied
+  core?: never;
+};
+
+type ExtractableLoadLibraryOptions<ModulesT extends Record<string, any> = Record<string, any>> = {
+  useLocalLibraries?: boolean;
+  CDN?: string | null;
+  modules?: ModulesT;
+  core?: {
+    useLocalLibraries?: boolean;
+    CDN?: string | null;
+  } | null;
+};
+
 const loadLibraryPromises: Record<string, Promise<any>> = {}; // promises
 
+export function extractLoadLibraryOptions<
+  ModulesT extends Record<string, any> = Record<string, any>
+>(options: ExtractableLoadLibraryOptions<ModulesT> = {}): LoadLibraryOptions<ModulesT> {
+  const useLocalLibraries = options.useLocalLibraries ?? options.core?.useLocalLibraries;
+  const CDN = options.CDN ?? options.core?.CDN;
+  const modules = options.modules;
+
+  return {
+    ...(useLocalLibraries !== undefined ? {useLocalLibraries} : {}),
+    ...(CDN !== undefined ? {CDN} : {}),
+    ...(modules !== undefined ? {modules} : {})
+  };
+}
+
 /**
  * Dynamically loads a library ("module")
  *
@@ -27,7 +59,7 @@ const loadLibraryPromises: Record<string, Promise<any>> = {}; // promises
 export async function loadLibrary(
   libraryUrl: string,
   moduleName: string | null = null,
-  options: object = {},
+  options: LoadLibraryOptions = {},
   libraryName: string | null = null
 ): Promise<any> {
   if (moduleName) {
@@ -45,9 +77,13 @@ export async function loadLibrary(
 export function getLibraryUrl(
   library: string,
   moduleName?: string,
-  options: any = {},
+  options: LoadLibraryOptions = {},
   libraryName: string | null = null
 ): string {
+  if (options?.core) {
+    throw new Error('loadLibrary: options.core must be pre-normalized');
+  }
+
   // Check if already a URL
   if (!options.useLocalLibraries && library.startsWith('http')) {
     return library;
@@ -94,10 +130,21 @@ async function loadLibraryFromFile(libraryUrl: string): Promise<any> {
     // } catch (error) {
     //   console.error(error);
     // }
+    const {requireFromFile} = globalThis.loaders || {};
     try {
-      const {requireFromFile} = globalThis.loaders || {};
-      return await requireFromFile?.(libraryUrl);
+      const result = await requireFromFile?.(libraryUrl);
+      if (result || !libraryUrl.includes('/dist/libs/')) {
+        return result;
+      }
+      return await requireFromFile?.(libraryUrl.replace('/dist/libs/', '/src/libs/'));
     } catch (error) {
+      if (libraryUrl.includes('/dist/libs/')) {
+        try {
+          return await requireFromFile?.(libraryUrl.replace('/dist/libs/', '/src/libs/'));
+        } catch {
+          // ignore
+        }
+      }
       console.error(error); // eslint-disable-line no-console
       return null;
     }
@@ -114,19 +161,6 @@ async function loadLibraryFromFile(libraryUrl: string): Promise<any> {
   return loadLibraryFromString(scriptSource, libraryUrl);
 }
 
-/*
-async function loadScriptFromFile(libraryUrl) {
-  const script = document.createElement('script');
-  script.src = libraryUrl;
-  return await new Promise((resolve, reject) => {
-    script.onload = data => {
-      resolve(data);
-    };
-    script.onerror = reject;
-  });
-}
-*/
-
 // TODO - Needs security audit...
 //  - Raw eval call
 //  - Potentially bypasses CORS
@@ -158,28 +192,20 @@ function loadLibraryFromString(scriptSource: string, id: string): null | any {
   return null;
 }
 
-// TODO - technique for module injection into worker, from THREE.DracoLoader...
-/*
-function combineWorkerWithLibrary(worker, jsContent) {
-  var fn = wWorker.toString();
-  var body = [
-    '// injected',
-    jsContent,
-    '',
-    '// worker',
-    fn.substring(fn.indexOf('{') + 1, fn.lastIndexOf('}'))
-  ].join('\n');
-  this.workerSourceURL = URL.createObjectURL(new Blob([body]));
-}
-*/
-
 async function loadAsArrayBuffer(url: string): Promise<ArrayBuffer> {
   const {readFileAsArrayBuffer} = globalThis.loaders || {};
   if (isBrowser || !readFileAsArrayBuffer || url.startsWith('http')) {
     const response = await fetch(url);
     return await response.arrayBuffer();
   }
-  return await readFileAsArrayBuffer(url);
+  try {
+    return await readFileAsArrayBuffer(url);
+  } catch {
+    if (url.includes('/dist/libs/')) {
+      return await readFileAsArrayBuffer(url.replace('/dist/libs/', '/src/libs/'));
+    }
+    throw new Error(`Failed to load ArrayBuffer from ${url}`);
+  }
 }
 
 /**
@@ -193,5 +219,40 @@ async function loadAsText(url: string): Promise<string> {
     const response = await fetch(url);
     return await response.text();
   }
-  return await readFileAsText(url);
+  try {
+    return await readFileAsText(url);
+  } catch {
+    if (url.includes('/dist/libs/')) {
+      return await readFileAsText(url.replace('/dist/libs/', '/src/libs/'));
+    }
+    throw new Error(`Failed to load text from ${url}`);
+  }
+}
+
+/*
+async function loadScriptFromFile(libraryUrl) {
+  const script = document.createElement('script');
+  script.src = libraryUrl;
+  return await new Promise((resolve, reject) => {
+    script.onload = data => {
+      resolve(data);
+    };
+    script.onerror = reject;
+  });
 }
+*/
+
+// TODO - technique for module injection into worker, from THREE.DracoLoader...
+/*
+function combineWorkerWithLibrary(worker, jsContent) {
+  var fn = wWorker.toString();
+  var body = [
+    '// injected',
+    jsContent,
+    '',
+    '// worker',
+    fn.substring(fn.indexOf('{') + 1, fn.lastIndexOf('}'))
+  ].join('\n');
+  this.workerSourceURL = URL.createObjectURL(new Blob([body]));
+}
+*/

package/src/lib/process-utils/child-process-proxy.browser.ts

@@ -0,0 +1,23 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ChildProcessProxyProps} from './child-process-proxy';
+
+export default class ChildProcessProxy {
+  constructor() {
+    throw new Error('ChildProcessProxy is not available in browser environments');
+  }
+
+  async start(props?: ChildProcessProxyProps): Promise<object> {
+    return await Promise.resolve({});
+  }
+
+  async stop(): Promise<void> {
+    return await Promise.resolve();
+  }
+
+  async exit(): Promise<void> {
+    return await Promise.resolve();
+  }
+}

package/src/lib/process-utils/child-process-proxy.ts

@@ -3,12 +3,19 @@
 // Copyright (c) vis.gl contributors
 
 /* eslint-disable no-console */
-// Avoid using named imports for Node builtins to help with "empty" resolution
-// for bundlers targeting browser environments. Access imports & types
-// through the `ChildProcess` object (e.g. `ChildProcess.spawn`, `ChildProcess.ChildProcess`).
-import * as ChildProcess from 'child_process';
 import {getAvailablePort} from './process-utils';
 
+type NodeChildProcess = import('child_process').ChildProcess;
+type SpawnOptions = import('child_process').SpawnOptions;
+
+async function getChildProcessModule(): Promise<typeof import('child_process')> {
+  if (typeof process === 'undefined' || !process.versions?.node) {
+    throw new Error('ChildProcessProxy is only available in Node.js environments');
+  }
+
+  return await import('child_process');
+}
+
 export type ChildProcessProxyProps = {
   command: string;
   arguments: string[];
@@ -22,7 +29,7 @@ export type ChildProcessProxyProps = {
   /** wait: 0 - infinity */
   wait?: number;
   /** Options passed on to Node'.js `spawn` */
-  spawn?: ChildProcess.SpawnOptions;
+  spawn?: SpawnOptions;
   /** Should proceed if stderr stream recieved data */
   ignoreStderr?: boolean;
   /** Callback when the  */
@@ -48,7 +55,7 @@ const DEFAULT_PROPS: ChildProcessProxyProps = {
 export default class ChildProcessProxy {
   id: string;
   props: ChildProcessProxyProps = {...DEFAULT_PROPS};
-  private childProcess: ChildProcess.ChildProcess | null = null;
+  private childProcess: NodeChildProcess | null = null;
   private port: number = 0;
   private successTimer?: any; // NodeJS.Timeout;
 
@@ -62,6 +69,8 @@ export default class ChildProcessProxy {
     props = {...DEFAULT_PROPS, ...props};
     this.props = props;
 
+    const childProcessModule = await getChildProcessModule();
+
     const args = [...props.arguments];
 
     // If portArg is set, we can look up an available port
@@ -83,13 +92,18 @@ export default class ChildProcessProxy {
         });
 
         console.log(`Spawning ${props.command} ${props.arguments.join(' ')}`);
-        const childProcess = ChildProcess.spawn(props.command, args, props.spawn);
+        const spawnOptions: SpawnOptions = props.spawn || {};
+        const childProcess: NodeChildProcess = childProcessModule.spawn(
+          props.command,
+          args,
+          spawnOptions
+        );
         this.childProcess = childProcess;
 
-        childProcess.stdout.on('data', (data) => {
+        childProcess.stdout?.on('data', (data) => {
           console.log(data.toString());
         });
-        childProcess.stderr.on('data', (data) => {
+        childProcess.stderr?.on('data', (data) => {
           console.log(`Child process wrote to stderr: "${data}".`);
           if (!props.ignoreStderr) {
             this._clearTimeout();

package/src/lib/worker-api/get-worker-url.ts

@@ -46,7 +46,8 @@ export function getWorkerURL(worker: WorkerObject, options: WorkerOptions = {}):
 
   // If URL is test, generate local loaders.gl url
   // @ts-ignore _workerType
-  if (options._workerType === 'test') {
+  const workerType = (options as any)._workerType || (options as any)?.core?._workerType;
+  if (workerType === 'test') {
     if (isBrowser) {
       url = `modules/${worker.module}/dist/${workerFile}`;
     } else {

package/src/lib/worker-farm/worker-thread.ts

@@ -134,8 +134,10 @@ export default class WorkerThread {
       // Make sure relative URLs start with './'
       const absolute = this.url.includes(':/') || this.url.startsWith('/');
       const url = absolute ? this.url : `./${this.url}`;
+      const type = this.url.endsWith('.ts') || this.url.endsWith('.mjs') ? 'module' : 'commonjs';
       // console.log('Starting work from', url);
-      worker = new NodeWorker(url, {eval: false});
+      // @ts-expect-error type is not known
+      worker = new NodeWorker(url, {eval: false, type});
     } else if (this.source) {
       worker = new NodeWorker(this.source, {eval: true});
     } else {
@@ -146,8 +148,7 @@ export default class WorkerThread {
       this.onMessage(data);
     });
     worker.on('error', (error) => {
-      // console.error('error', error);
-      this.onError(error);
+      this.onError(error as Error);
     });
     worker.on('exit', (code) => {
       // console.error('exit', code);