@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/worker-utils/lib/library-utils/library-utils.ts

@@ -0,0 +1,303 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/* global importScripts */
+import {isBrowser, isWorker} from '../env-utils/globals';
+import {assert} from '../env-utils/assert';
+import {VERSION} from '../env-utils/version';
+
+export type LoadLibraryOptions<ModulesT extends Record<string, any> = Record<string, any>> = {
+  useLocalLibraries?: boolean;
+  CDN?: string | null;
+  modules?: ModulesT;
+  // Core must not be supplied
+  core?: never;
+};
+
+type ExtractableLoadLibraryOptions<ModulesT extends Record<string, any> = Record<string, any>> = {
+  useLocalLibraries?: boolean;
+  CDN?: string | null;
+  modules?: ModulesT;
+  core?: {
+    useLocalLibraries?: boolean;
+    CDN?: string | null;
+  } | null;
+};
+
+const loadLibraryPromises: Record<string, Promise<any>> = {}; // promises
+
+export function extractLoadLibraryOptions<
+  ModulesT extends Record<string, any> = Record<string, any>
+>(options: ExtractableLoadLibraryOptions<ModulesT> = {}): LoadLibraryOptions<ModulesT> {
+  const useLocalLibraries = options.useLocalLibraries ?? options.core?.useLocalLibraries;
+  const CDN = getCDNOption(options.CDN, options.core?.CDN);
+  const modules = options.modules;
+
+  return {
+    ...(useLocalLibraries !== undefined ? {useLocalLibraries} : {}),
+    ...(CDN !== undefined ? {CDN} : {}),
+    ...(modules !== undefined ? {modules} : {})
+  };
+}
+
+/**
+ * Extracts a valid CDN option from normalized or legacy loader options.
+ * @param cdn Deprecated top-level CDN option.
+ * @param coreCDN Nested core CDN option.
+ * @returns CDN option if it is a string or null.
+ */
+function getCDNOption(cdn: unknown, coreCDN: unknown): string | null | undefined {
+  if (typeof cdn === 'string' || cdn === null) {
+    return cdn;
+  }
+  if (typeof coreCDN === 'string' || coreCDN === null) {
+    return coreCDN;
+  }
+  return undefined;
+}
+
+/**
+ * Dynamically loads a library ("module")
+ *
+ * - wasm library: Array buffer is returned
+ * - js library: Parse JS is returned
+ *
+ * Method depends on environment
+ * - browser - script element is created and installed on document
+ * - classic worker - importScripts loads a classic wrapper
+ * - module worker - native import loads an ESM wrapper and returns its exports
+ * - node - file is required
+ *
+ * @param libraryUrl
+ * @param moduleName
+ * @param options
+ * @param libraryName Classic asset name used for URL resolution and overrides.
+ * @param moduleLibraryName ESM asset name used by module workers unless the classic URL is overridden.
+ */
+export async function loadLibrary(
+  libraryUrl: string,
+  moduleName: string | null = null,
+  options: LoadLibraryOptions = {},
+  libraryName: string | null = null,
+  moduleLibraryName: string | null = null
+): Promise<any> {
+  let moduleLibraryUrl = libraryUrl;
+  if (moduleName) {
+    const requestedLibraryName = libraryName || libraryUrl;
+    const injectedLibrary = options.modules?.[requestedLibraryName];
+    if (injectedLibrary !== undefined && typeof injectedLibrary !== 'string') {
+      return injectedLibrary;
+    }
+    libraryUrl = getLibraryUrl(libraryUrl, moduleName, options, libraryName);
+    const hasLibraryOverride = typeof injectedLibrary === 'string';
+    moduleLibraryUrl =
+      moduleLibraryName &&
+      (!hasLibraryOverride || typeof options.modules?.[moduleLibraryName] === 'string')
+        ? getLibraryUrl(moduleLibraryName, moduleName, options, moduleLibraryName)
+        : libraryUrl;
+  }
+  // Workers may select different ESM adapters for the same classic asset.
+  const cacheKey = getLibraryCacheKey(libraryUrl, moduleLibraryUrl, isWorker);
+
+  loadLibraryPromises[cacheKey] =
+    // eslint-disable-next-line @typescript-eslint/no-misused-promises
+    loadLibraryPromises[cacheKey] || loadLibraryFromFile(libraryUrl, moduleLibraryUrl);
+  return await loadLibraryPromises[cacheKey];
+}
+
+/** Identifies a library load, including the alternate adapter available in workers. */
+export function getLibraryCacheKey(
+  libraryUrl: string,
+  moduleLibraryUrl: string,
+  worker: boolean
+): string {
+  return worker ? JSON.stringify([libraryUrl, moduleLibraryUrl]) : libraryUrl;
+}
+
+// TODO - sort out how to resolve paths for main/worker and dev/prod
+export function getLibraryUrl(
+  library: string,
+  moduleName?: string,
+  options: LoadLibraryOptions = {},
+  libraryName: string | null = null
+): string {
+  if (options?.core) {
+    throw new Error('loadLibrary: options.core must be pre-normalized');
+  }
+  libraryName = libraryName || library;
+
+  // Allow application to import and supply libraries through `options.modules`
+  // TODO - See js-module-utils in loader-utils
+  const modules = options.modules || {};
+  if (typeof modules[libraryName] === 'string') {
+    return modules[libraryName];
+  }
+
+  // Check if already a URL after checking overrides so callers can replace
+  // default CDN URLs with locally bundled assets.
+  if (!options.useLocalLibraries && library.startsWith('http')) {
+    return library;
+  }
+
+  // Load from local files, not from CDN scripts in Node.js
+  // TODO - needs to locate the modules directory when installed!
+  if (!isBrowser) {
+    return `modules/${moduleName}/dist/libs/${libraryName}`;
+  }
+
+  // In browser, load from external scripts
+  if (options.CDN) {
+    if (typeof options.CDN !== 'string') {
+      throw new Error('loadLibrary: options.CDN must be a string or null');
+    }
+    assert(options.CDN.startsWith('http'));
+    return `${options.CDN}/${moduleName}@${VERSION}/dist/libs/${libraryName}`;
+  }
+
+  // TODO - loading inside workers requires paths relative to worker script location...
+  if (isWorker) {
+    return libraryName.endsWith('.mjs') ? `./libs/${libraryName}` : `../src/libs/${libraryName}`;
+  }
+
+  return `modules/${moduleName}/src/libs/${libraryName}`;
+}
+
+async function loadLibraryFromFile(libraryUrl: string, moduleLibraryUrl: string): Promise<any> {
+  if (libraryUrl.endsWith('wasm')) {
+    return await loadAsArrayBuffer(libraryUrl);
+  }
+
+  if (!isBrowser) {
+    // TODO - Node doesn't yet support dynamic import from https URLs
+    // try {
+    //   return await import(libraryUrl);
+    // } catch (error) {
+    //   console.error(error);
+    // }
+    const {requireFromFile} = globalThis.loaders || {};
+    try {
+      const result = await requireFromFile?.(libraryUrl);
+      if (result || !libraryUrl.includes('/dist/libs/')) {
+        return result;
+      }
+      return await requireFromFile?.(libraryUrl.replace('/dist/libs/', '/src/libs/'));
+    } catch (error) {
+      if (libraryUrl.includes('/dist/libs/')) {
+        try {
+          return await requireFromFile?.(libraryUrl.replace('/dist/libs/', '/src/libs/'));
+        } catch {
+          // ignore
+        }
+      }
+      console.error(error); // eslint-disable-line no-console
+      return null;
+    }
+  }
+  if (isWorker) {
+    return loadLibraryInWorker(libraryUrl, moduleLibraryUrl);
+  }
+  // TODO - fix - should be more secure than string parsing since observes CORS
+  // if (isBrowser) {
+  //   return await loadScriptFromFile(libraryUrl);
+  // }
+
+  const scriptSource = await loadAsText(libraryUrl);
+  return loadLibraryFromString(scriptSource, libraryUrl);
+}
+
+/** Loads classic scripts or native ESM exports without evaluating fetched source or masking failures. */
+export async function loadLibraryInWorker(
+  libraryUrl: string,
+  moduleLibraryUrl: string = libraryUrl
+): Promise<unknown> {
+  try {
+    // A zero-argument probe rejects in module workers without loading any script.
+    importScripts();
+  } catch {
+    // Resolve local asset paths against the same worker location used by importScripts.
+    const moduleUrl = new URL(moduleLibraryUrl, globalThis.location.href).href;
+    const library = await import(/* @vite-ignore */ moduleUrl);
+    return library.default ?? library;
+  }
+  return importScripts(libraryUrl);
+}
+
+/** Installs fetched library source on the browser main thread. Workers use script or module loading. */
+function loadLibraryFromString(scriptSource: string, id: string): null | any {
+  const script = document.createElement('script');
+  script.id = id;
+  // most browsers like a separate text node but some throw an error. The second method covers those.
+  try {
+    script.appendChild(document.createTextNode(scriptSource));
+  } catch (_e) {
+    script.text = scriptSource;
+  }
+  document.body.appendChild(script);
+  return null;
+}
+
+async function loadAsArrayBuffer(url: string): Promise<ArrayBuffer> {
+  const {readFileAsArrayBuffer} = globalThis.loaders || {};
+  if (isBrowser || !readFileAsArrayBuffer || url.startsWith('http')) {
+    const response = await fetch(url);
+    return await response.arrayBuffer();
+  }
+  try {
+    return await readFileAsArrayBuffer(url);
+  } catch {
+    if (url.includes('/dist/libs/')) {
+      return await readFileAsArrayBuffer(url.replace('/dist/libs/', '/src/libs/'));
+    }
+    throw new Error(`Failed to load ArrayBuffer from ${url}`);
+  }
+}
+
+/**
+ * Load a file from local file system
+ * @param filename
+ * @returns
+ */
+async function loadAsText(url: string): Promise<string> {
+  const {readFileAsText} = globalThis.loaders || {};
+  if (isBrowser || !readFileAsText || url.startsWith('http')) {
+    const response = await fetch(url);
+    return await response.text();
+  }
+  try {
+    return await readFileAsText(url);
+  } catch {
+    if (url.includes('/dist/libs/')) {
+      return await readFileAsText(url.replace('/dist/libs/', '/src/libs/'));
+    }
+    throw new Error(`Failed to load text from ${url}`);
+  }
+}
+
+/*
+async function loadScriptFromFile(libraryUrl) {
+  const script = document.createElement('script');
+  script.src = libraryUrl;
+  return await new Promise((resolve, reject) => {
+    script.onload = data => {
+      resolve(data);
+    };
+    script.onerror = reject;
+  });
+}
+*/
+
+// TODO - technique for module injection into worker, from THREE.DracoLoader...
+/*
+function combineWorkerWithLibrary(worker, jsContent) {
+  var fn = wWorker.toString();
+  var body = [
+    '// injected',
+    jsContent,
+    '',
+    '// worker',
+    fn.substring(fn.indexOf('{') + 1, fn.lastIndexOf('}'))
+  ].join('\n');
+  this.workerSourceURL = URL.createObjectURL(new Blob([body]));
+}
+*/

package/src/worker-utils/lib/node/worker_threads-browser.ts

@@ -0,0 +1,34 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Browser polyfill for Node.js built-in `worker_threads` module.
+ * These fills are non-functional, and just intended to ensure that
+ * `import 'worker_threads` doesn't break browser builds.
+ * The replacement is done in package.json browser field
+ */
+export class NodeWorker {
+  /** Restores the native worker constructor when this browser shim is embedded in a Node bundle. */
+  constructor(...arguments_: unknown[]) {
+    const runtimeProcess = (
+      globalThis as {
+        process?: {
+          getBuiltinModule?: (specifier: string) => {
+            Worker?: new (...workerArguments: unknown[]) => NodeWorker;
+          };
+        };
+      }
+    ).process;
+    const WorkerConstructor = runtimeProcess?.getBuiltinModule?.('node:worker_threads').Worker;
+    if (WorkerConstructor) {
+      return new WorkerConstructor(...arguments_);
+    }
+  }
+
+  /** No-op browser fallback matching the native Worker API. */
+  terminate() {}
+}
+
+export type {NodeWorker as NodeWorkerType};
+
+export const parentPort = null;

package/src/worker-utils/lib/node/worker_threads.ts

@@ -0,0 +1,58 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as WorkerThreads from 'worker_threads';
+export {
+  markAsUntransferable,
+  isMarkedAsUntransferable,
+  markAsUncloneable,
+  moveMessagePortToContext,
+  receiveMessageOnPort,
+  getEnvironmentData,
+  setEnvironmentData,
+  postMessageToThread,
+  isMainThread,
+  resourceLimits,
+  SHARE_ENV,
+  threadId,
+  workerData,
+  Worker,
+  BroadcastChannel,
+  MessageChannel,
+  MessagePort
+} from 'worker_threads';
+export type {
+  WorkerPerformance,
+  TransferListItem,
+  WorkerOptions,
+  ResourceLimits,
+  WorkerEventMap,
+  Serializable,
+  LockMode,
+  Transferable,
+  LockGrantedCallback,
+  LockInfo,
+  LockManagerSnapshot,
+  LockOptions,
+  StructuredSerializeOptions,
+  BroadcastChannelEventMap,
+  Lock,
+  LockManager,
+  MessagePortEventMap
+} from 'worker_threads';
+export const parentPort = WorkerThreads?.parentPort;
+export const NodeWorker = WorkerThreads.Worker;
+export type NodeWorkerType = WorkerThreads.Worker;
+
+/** Optional Node.js worker API, available on runtimes that provide it. */
+export const isInternalThread = WorkerThreads.isInternalThread;
+
+/** Optional Node.js worker API, available on runtimes that provide it. */
+export const threadName = WorkerThreads.threadName;
+
+/** Optional Node.js worker API, available on runtimes that provide it. */
+export const locks = WorkerThreads.locks;
+
+/** Optional Node.js worker API, available on runtimes that provide it. */
+export const structuredClone = WorkerThreads.structuredClone;

package/src/worker-utils/lib/npm-tag.ts

@@ -0,0 +1,11 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * NPM tag to use when loading modules from unpkg.com
+ * 'beta' on beta branch, 'latest' on prod branch
+ * @note Change between 'beta' and 'latest' depending on whether publishing alpha or prod releases
+ * @todo - unpkg.com doesn't seem to have a `latest` specifier for alpha releases...
+ */
+export const NPM_TAG = 'latest';

package/src/worker-utils/lib/process-utils/child-process-proxy.browser.ts

@@ -0,0 +1,23 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ChildProcessProxyProps} from './child-process-proxy';
+
+export default class ChildProcessProxy {
+  constructor() {
+    throw new Error('ChildProcessProxy is not available in browser environments');
+  }
+
+  async start(props?: ChildProcessProxyProps): Promise<object> {
+    return await Promise.resolve({});
+  }
+
+  async stop(): Promise<void> {
+    return await Promise.resolve();
+  }
+
+  async exit(): Promise<void> {
+    return await Promise.resolve();
+  }
+}

package/src/worker-utils/lib/process-utils/child-process-proxy.ts

@@ -0,0 +1,165 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/* eslint-disable no-console */
+import {getAvailablePort} from './process-utils';
+
+type NodeChildProcess = import('child_process').ChildProcess;
+type SpawnOptions = import('child_process').SpawnOptions;
+
+async function getChildProcessModule(): Promise<typeof import('child_process')> {
+  if (typeof process === 'undefined' || !process.versions?.node) {
+    throw new Error('ChildProcessProxy is only available in Node.js environments');
+  }
+
+  return await import('child_process');
+}
+
+export type ChildProcessProxyProps = {
+  command: string;
+  arguments: string[];
+  /** Whether to add a port specified arg */
+  portArg?: string;
+  /** Base port number */
+  port?: number;
+  /** Whether to search for an available port if the base port is occupied */
+  autoPort?: boolean;
+  /** Number of milliseconds to wait until concluding success */
+  /** wait: 0 - infinity */
+  wait?: number;
+  /** Options passed on to Node'.js `spawn` */
+  spawn?: SpawnOptions;
+  /** Should proceed if stderr stream recieved data */
+  ignoreStderr?: boolean;
+  /** Callback when the  */
+  onStart?: (proxy: ChildProcessProxy) => void;
+  onSuccess?: (proxy: ChildProcessProxy) => void;
+};
+
+const DEFAULT_PROPS: ChildProcessProxyProps = {
+  command: '',
+  arguments: [],
+  port: 5000,
+  autoPort: true,
+  wait: 2000,
+  onSuccess: processProxy => {
+    console.log(`Started ${processProxy.props.command}`);
+  }
+};
+
+/**
+ * Manager for a Node.js child process
+ * Prepares arguments, starts, stops and tracks output
+ */
+export default class ChildProcessProxy {
+  id: string;
+  props: ChildProcessProxyProps = {...DEFAULT_PROPS};
+  private childProcess: NodeChildProcess | null = null;
+  private port: number = 0;
+  private successTimer?: any; // NodeJS.Timeout;
+
+  // constructor(props?: {id?: string});
+  constructor({id = 'browser-driver'} = {}) {
+    this.id = id;
+  }
+
+  /** Starts a child process with the provided props */
+  async start(props: ChildProcessProxyProps): Promise<object> {
+    props = {...DEFAULT_PROPS, ...props};
+    this.props = props;
+
+    const childProcessModule = await getChildProcessModule();
+
+    const args = [...props.arguments];
+
+    // If portArg is set, we can look up an available port
+    this.port = Number(props.port);
+    if (props.portArg) {
+      if (props.autoPort) {
+        this.port = await getAvailablePort(props.port);
+      }
+      args.push(props.portArg, String(this.port));
+    }
+
+    return await new Promise((resolve, reject) => {
+      try {
+        this._setTimeout(() => {
+          if (props.onSuccess) {
+            props.onSuccess(this);
+          }
+          resolve({});
+        });
+
+        console.log(`Spawning ${props.command} ${props.arguments.join(' ')}`);
+        if (props.spawn?.shell) {
+          throw new Error('ChildProcessProxy does not support shell execution');
+        }
+        const spawnOptions: SpawnOptions = {...props.spawn, shell: false};
+        const childProcess: NodeChildProcess = childProcessModule.spawn(
+          props.command,
+          args,
+          spawnOptions
+        );
+        this.childProcess = childProcess;
+
+        childProcess.stdout?.on('data', data => {
+          console.log(data.toString());
+        });
+        childProcess.stderr?.on('data', data => {
+          console.log(`Child process wrote to stderr: "${data}".`);
+          if (!props.ignoreStderr) {
+            this._clearTimeout();
+            reject(new Error(data));
+          }
+        });
+        childProcess.on('error', error => {
+          console.log(`Child process errored with ${error}`);
+          this._clearTimeout();
+          reject(error);
+        });
+        childProcess.on('close', code => {
+          console.log(`Child process exited with ${code}`);
+          this.childProcess = null;
+          this._clearTimeout();
+          resolve({});
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+
+  /** Stops a running child process */
+  async stop(): Promise<void> {
+    if (this.childProcess) {
+      this.childProcess.kill();
+      this.childProcess = null;
+    }
+  }
+
+  /** Exits this process */
+  async exit(statusCode: number = 0): Promise<void> {
+    try {
+      await this.stop();
+      // eslint-disable-next-line no-process-exit
+      process.exit(statusCode);
+    } catch (error) {
+      console.error((error as Error).message || error);
+      // eslint-disable-next-line no-process-exit
+      process.exit(1);
+    }
+  }
+
+  _setTimeout(callback: (...args: any[]) => void) {
+    if (Number(this.props.wait) > 0) {
+      this.successTimer = setTimeout(callback, this.props.wait);
+    }
+  }
+
+  _clearTimeout() {
+    if (this.successTimer) {
+      clearTimeout(this.successTimer);
+    }
+  }
+}

package/src/worker-utils/lib/process-utils/process-utils.ts

@@ -0,0 +1,34 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import ChildProcess from 'child_process';
+
+// Get an available port
+// Works on Unix systems
+export function getAvailablePort(defaultPort: number = 3000): Promise<number> {
+  return new Promise(resolve => {
+    // Get a list of all ports in use
+    ChildProcess.exec('lsof -i -P -n | grep LISTEN', (error, stdout) => {
+      if (error) {
+        // likely no permission, e.g. CI
+        resolve(defaultPort);
+        return;
+      }
+
+      const portsInUse: number[] = [];
+      const regex = /:(\d+) \(LISTEN\)/;
+      stdout.split('\n').forEach(line => {
+        const match = regex.exec(line);
+        if (match) {
+          portsInUse.push(Number(match[1]));
+        }
+      });
+      let port = defaultPort;
+      while (portsInUse.includes(port)) {
+        port++;
+      }
+      resolve(port);
+    });
+  });
+}