@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/worker-utils/lib/worker-api/validate-worker-version.ts

@@ -0,0 +1,44 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {WorkerObject} from '../../types';
+import {assert} from '../env-utils/assert';
+import {VERSION} from '../env-utils/version';
+
+/**
+ * Check if worker is compatible with this library version
+ * @param worker
+ * @param libVersion
+ * @returns `true` if the two versions are compatible
+ */
+export function validateWorkerVersion(
+  worker: WorkerObject,
+  coreVersion: string = VERSION
+): boolean {
+  assert(worker, 'no worker provided');
+
+  const workerVersion = worker.version;
+  if (!coreVersion || !workerVersion) {
+    return false;
+  }
+
+  // TODO enable when fix the __version__ injection
+  // const coreVersions = parseVersion(coreVersion);
+  // const workerVersions = parseVersion(workerVersion);
+  // assert(
+  //   coreVersion.major === workerVersion.major && coreVersion.minor <= workerVersion.minor,
+  //   `worker: ${worker.name} is not compatible. ${coreVersion.major}.${
+  //     coreVersion.minor
+  //   }+ is required.`
+  // );
+
+  return true;
+}
+
+// @ts-ignore
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+function _parseVersion(version) {
+  const parts = version.split('.').map(Number);
+  return {major: parts[0], minor: parts[1]};
+}

package/src/worker-utils/lib/worker-farm/worker-body.ts

@@ -0,0 +1,145 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {WorkerMessageData, WorkerMessageType, WorkerMessagePayload} from '../../types';
+import {getTransferList} from '../worker-utils/get-transfer-list';
+// import type {TransferListItem} from '../node/worker_threads';
+import {parentPort} from '../node/worker_threads';
+
+type TransferListItem = any;
+type WorkerMessageEvent = MessageEvent<WorkerMessageData> | WorkerMessageData;
+type WorkerMessageListener = (type: WorkerMessageType, payload: WorkerMessagePayload) => any;
+
+/** Vile hack to defeat over-zealous bundlers from stripping out the require */
+async function getParentPort() {
+  // const isNode = globalThis.process;
+  // let parentPort;
+  // try {
+  //   // biome-ignore format: preserve intentional fixture layout
+  //   eval('globalThis.parentPort = require(\'worker_threads\').parentPort'); // eslint-disable-line no-eval
+  //   parentPort = globalThis.parentPort;
+  // } catch {
+  //   try {
+  //     // biome-ignore format: preserve intentional fixture layout
+  //     eval('globalThis.workerThreadsPromise = import(\'worker_threads\')'); // eslint-disable-line no-eval
+  //     const workerThreads = await globalThis.workerThreadsPromise;
+  //     parentPort = workerThreads.parentPort;
+  //   } catch (error) {
+  //     console.error((error as Error).message); // eslint-disable-line no-console
+  //   }
+  // }
+  return parentPort;
+}
+
+const onMessageWrapperMap = new Map<WorkerMessageListener, (message: WorkerMessageEvent) => void>();
+
+/**
+ * Type safe wrapper for worker code
+ */
+export default class WorkerBody {
+  /** Check that we are actually in a worker thread */
+  static async inWorkerThread(): Promise<boolean> {
+    return typeof self !== 'undefined' || Boolean(await getParentPort());
+  }
+
+  /*
+   * (type: WorkerMessageType, payload: WorkerMessagePayload) => any
+   */
+  static set onmessage(onMessage: (type: WorkerMessageType, payload: WorkerMessagePayload) => any) {
+    async function handleMessage(message) {
+      const parentPort = await getParentPort();
+      // Confusingly the message itself also has a 'type' field which is always set to 'message'
+      const {type, payload} = parentPort ? message : message.data;
+      // if (!isKnownMessage(message)) {
+      //   return;
+      // }
+      onMessage(type, payload);
+    }
+
+    getParentPort().then(parentPort => {
+      if (parentPort) {
+        parentPort.on('message', message => {
+          handleMessage(message);
+        });
+        // if (message == 'exit') { parentPort.unref(); }
+        // eslint-disable-next-line
+        parentPort.on('exit', () => console.debug('Node worker closing'));
+      } else {
+        // eslint-disable-next-line no-restricted-globals
+        globalThis.onmessage = handleMessage;
+      }
+    });
+  }
+
+  static async addEventListener(onMessage: WorkerMessageListener) {
+    let onMessageWrapper = onMessageWrapperMap.get(onMessage);
+
+    if (!onMessageWrapper) {
+      onMessageWrapper = (message: WorkerMessageEvent) => {
+        const messageData = getWorkerMessageData(message);
+        if (!messageData) {
+          return;
+        }
+        onMessage(messageData.type, messageData.payload);
+      };
+      onMessageWrapperMap.set(onMessage, onMessageWrapper);
+    }
+
+    const parentPort = await getParentPort();
+    if (parentPort) {
+      parentPort.on('message', onMessageWrapper);
+    } else {
+      globalThis.addEventListener('message', onMessageWrapper);
+    }
+  }
+
+  static async removeEventListener(onMessage: WorkerMessageListener) {
+    const onMessageWrapper = onMessageWrapperMap.get(onMessage);
+    onMessageWrapperMap.delete(onMessage);
+    if (!onMessageWrapper) {
+      return;
+    }
+    const parentPort = await getParentPort();
+    if (parentPort) {
+      parentPort.off('message', onMessageWrapper);
+    } else {
+      globalThis.removeEventListener('message', onMessageWrapper);
+    }
+  }
+
+  /**
+   * Send a message from a worker to creating thread (main thread)
+   * @param type
+   * @param payload
+   */
+  static async postMessage(type: WorkerMessageType, payload: WorkerMessagePayload): Promise<void> {
+    const data: WorkerMessageData = {source: 'loaders.gl', type, payload};
+    // console.log('posting message', data);
+
+    // Cast to Node compatible transfer list
+    const transferList = getTransferList(payload) as TransferListItem[];
+
+    const parentPort = await getParentPort();
+    if (parentPort) {
+      parentPort.postMessage(data, transferList);
+      // console.log('posted message', data);
+    } else {
+      // @ts-expect-error Outside of worker scopes this call has a third parameter
+      globalThis.postMessage(data, transferList);
+    }
+  }
+}
+
+// Filter out noise messages sent to workers
+function getWorkerMessageData(message: WorkerMessageEvent): WorkerMessageData | null {
+  if (!message || typeof message !== 'object') {
+    return null;
+  }
+  const messageData = 'data' in message && message.type === 'message' ? message.data : message;
+  return messageData &&
+    typeof messageData.source === 'string' &&
+    messageData.source.startsWith('loaders.gl')
+    ? messageData
+    : null;
+}

package/src/worker-utils/lib/worker-farm/worker-farm.ts

@@ -0,0 +1,144 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import WorkerPool from './worker-pool';
+import type {WorkerPoolTarget} from './worker-pool';
+import WorkerThread from './worker-thread';
+import type {LoadWorker} from '../../types';
+
+/**
+ * @param maxConcurrency - max count of workers
+ * @param maxMobileConcurrency - max count of workers on mobile
+ * @param maxConcurrency - max count of workers
+ * @param reuseWorkers - if false, destroys workers when task is completed
+ * @param onDebug - callback intended to allow application to log worker pool activity
+ */
+export type WorkerFarmProps = {
+  maxConcurrency?: number;
+  maxMobileConcurrency?: number;
+  reuseWorkers?: boolean;
+  onDebug?: () => void;
+};
+
+const DEFAULT_PROPS: Required<WorkerFarmProps> = {
+  maxConcurrency: 3,
+  maxMobileConcurrency: 1,
+  reuseWorkers: true,
+  onDebug: () => {}
+};
+
+/** Stable ids used to distinguish worker factories in the pool cache. */
+const loadWorkerIds = new WeakMap<LoadWorker, number>();
+/** Next stable worker-factory id. */
+let nextLoadWorkerId = 1;
+
+/**
+ * Process multiple jobs with a "farm" of different workers in worker pools.
+ */
+export default class WorkerFarm {
+  private props: WorkerFarmProps;
+  private workerPools = new Map<string, WorkerPool>();
+  // singleton
+  private static _workerFarm?: WorkerFarm;
+
+  /** Checks if workers are supported on this platform */
+  static isSupported(): boolean {
+    return WorkerThread.isSupported();
+  }
+
+  /** Get the singleton instance of the global worker farm */
+  static getWorkerFarm(props: WorkerFarmProps = {}): WorkerFarm {
+    WorkerFarm._workerFarm = WorkerFarm._workerFarm || new WorkerFarm({});
+    WorkerFarm._workerFarm.setProps(props);
+    return WorkerFarm._workerFarm;
+  }
+
+  /** get global instance with WorkerFarm.getWorkerFarm() */
+  private constructor(props: WorkerFarmProps) {
+    this.props = {...DEFAULT_PROPS};
+    this.setProps(props);
+    /** @type Map<string, WorkerPool>} */
+    this.workerPools = new Map();
+  }
+
+  /**
+   * Terminate all workers in the farm and abort active jobs.
+   * @param reason Optional error delivered to active jobs.
+   * @note Can free up significant memory
+   */
+  destroy(reason?: unknown): void {
+    for (const workerPool of this.workerPools.values()) {
+      workerPool.destroy(reason);
+    }
+    this.workerPools = new Map<string, WorkerPool>();
+  }
+
+  /**
+   * Set props used when initializing worker pools
+   * @param props
+   */
+  setProps(props: WorkerFarmProps): void {
+    this.props = {...this.props, ...props};
+    // Update worker pool props
+    for (const workerPool of this.workerPools.values()) {
+      workerPool.setProps(this._getWorkerPoolProps());
+    }
+  }
+
+  /**
+   * Returns a worker pool for the specified worker
+   * @param options Worker target used to identify and initialize the pool.
+   * @param options.name Worker name used as the base pool identity.
+   * @param options.url Worker script URL.
+   * @param options.source Inline worker source.
+   * @param options.loadWorker Built-in browser worker factory.
+   * @example
+   *   const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
+   */
+  getWorkerPool(options: WorkerPoolTarget): WorkerPool {
+    const {name, source, url, getUrl, loadWorker} = options;
+    const workerPoolKey = getWorkerPoolKey(options);
+    let workerPool = this.workerPools.get(workerPoolKey);
+    if (!workerPool) {
+      workerPool = new WorkerPool({
+        name,
+        source,
+        url,
+        getUrl,
+        loadWorker
+      });
+      workerPool.setProps(this._getWorkerPoolProps());
+      this.workerPools.set(workerPoolKey, workerPool);
+    }
+    return workerPool;
+  }
+
+  _getWorkerPoolProps() {
+    return {
+      maxConcurrency: this.props.maxConcurrency,
+      maxMobileConcurrency: this.props.maxMobileConcurrency,
+      reuseWorkers: this.props.reuseWorkers,
+      onDebug: this.props.onDebug
+    };
+  }
+}
+
+/** Returns a cache key that distinguishes each worker source and factory. */
+function getWorkerPoolKey(options: WorkerPoolTarget): string {
+  const {name, source, url, urlKey, loadWorker} = options;
+  const sourceKey = source ? `source:${source}` : '';
+  const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : '';
+  const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : '';
+  return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join('|');
+}
+
+/** Returns a stable identifier for a worker factory function. */
+function getLoadWorkerId(loadWorker: LoadWorker): number {
+  let loadWorkerId = loadWorkerIds.get(loadWorker);
+  if (!loadWorkerId) {
+    loadWorkerId = nextLoadWorkerId++;
+    loadWorkerIds.set(loadWorker, loadWorkerId);
+  }
+  return loadWorkerId;
+}

package/src/worker-utils/lib/worker-farm/worker-job.ts

@@ -0,0 +1,83 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {WorkerMessageType, WorkerMessagePayload} from '../../types';
+import WorkerThread from './worker-thread';
+
+/**
+ * Represents one Job handled by a WorkerPool or WorkerFarm
+ */
+export default class WorkerJob {
+  readonly name: string;
+  readonly workerThread: WorkerThread;
+  isRunning: boolean = true;
+  /** Promise that resolves when Job is done */
+  readonly result: Promise<any>;
+
+  private _resolve: (value: any) => void = () => {};
+  private _reject: (reason?: any) => void = () => {};
+
+  constructor(jobName: string, workerThread: WorkerThread) {
+    this.name = jobName;
+    this.workerThread = workerThread;
+    this.result = new Promise((resolve, reject) => {
+      this._resolve = resolve;
+      this._reject = reject;
+    });
+  }
+
+  /**
+   * Send a message to the job's worker thread
+   * @param data any data structure, ideally consisting mostly of transferrable objects
+   */
+  postMessage(type: WorkerMessageType, payload: WorkerMessagePayload): void {
+    this.workerThread.postMessage({
+      source: 'loaders.gl', // Lets worker ignore unrelated messages
+      type,
+      payload
+    });
+  }
+
+  /**
+   * Call to resolve the `result` Promise with the supplied value
+   */
+  done(value: any): void {
+    if (!this.isRunning) {
+      return;
+    }
+    this.isRunning = false;
+    this._resolve(value);
+  }
+
+  /**
+   * Call to reject the `result` Promise with the supplied error
+   */
+  error(error: unknown): void {
+    if (!this.isRunning) {
+      return;
+    }
+    this.isRunning = false;
+    this._reject(error);
+  }
+
+  /**
+   * Terminates the worker executing this job and rejects its result.
+   * @param reason Abort reason supplied by the caller, or a cross-runtime `AbortError`.
+   */
+  abort(reason?: unknown): void {
+    if (!this.isRunning) {
+      return;
+    }
+    const error = reason ?? createAbortError(`Worker job "${this.name}" was aborted`);
+    this.workerThread.destroy();
+    this.error(error);
+  }
+}
+
+/** Creates an abort error without requiring the DOMException global. */
+function createAbortError(message: string): Error {
+  const error = new Error(message);
+  error.name = 'AbortError';
+  return error;
+}

package/src/worker-utils/lib/worker-farm/worker-pool.ts

@@ -0,0 +1,263 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoadWorker, WorkerMessageType, WorkerMessagePayload} from '../../types';
+import {isMobile} from '../env-utils/globals';
+import WorkerThread from './worker-thread';
+import WorkerJob from './worker-job';
+
+/** WorkerPool onDebug Callback Parameters */
+type OnDebugParameters = {
+  message: string;
+  worker: string;
+  name: string;
+  job: string;
+  backlog: number;
+  workerThread: WorkerThread;
+};
+
+/** WorkerPool Properties */
+export type WorkerPoolProps = {
+  name?: string;
+  source?: string; // | Function;
+  url?: string;
+  /** Lazily resolves a classic worker fallback URL. */
+  getUrl?: () => string;
+  /** Creates a browser Worker directly. */
+  loadWorker?: LoadWorker;
+  maxConcurrency?: number;
+  maxMobileConcurrency?: number;
+  onDebug?: (options: OnDebugParameters) => any;
+  reuseWorkers?: boolean;
+};
+
+/** Worker-pool target, including values used only to distinguish cached pools. */
+export type WorkerPoolTarget = WorkerPoolProps & {
+  /** Worker name used as the base pool identity. */
+  name: string;
+  /** Stable identity for a lazily resolved URL. */
+  urlKey?: string;
+};
+
+/** Private helper types */
+type OnMessage = (job: WorkerJob, type: WorkerMessageType, payload: WorkerMessagePayload) => void;
+type OnError = (job: WorkerJob, error: Error) => void;
+
+type QueuedJob = {
+  name: string;
+  onMessage: OnMessage;
+  onError: OnError;
+  onStart: (value: any) => void; // Resolve job start promise
+  onReject: (reason: unknown) => void; // Reject job start promise
+};
+
+/**
+ * Process multiple data messages with small pool of identical workers
+ */
+export default class WorkerPool {
+  name: string = 'unnamed';
+  source?: string; // | Function;
+  url?: string;
+  /** Lazily resolves a classic worker fallback URL. */
+  getUrl?: () => string;
+  /** Creates a browser Worker directly. */
+  loadWorker?: LoadWorker;
+  maxConcurrency: number = 1;
+  maxMobileConcurrency: number = 1;
+  onDebug: (options: OnDebugParameters) => any = () => {};
+  reuseWorkers: boolean = true;
+
+  private props: WorkerPoolProps = {};
+  private jobQueue: QueuedJob[] = [];
+  private idleQueue: WorkerThread[] = [];
+  private activeJobs = new Set<WorkerJob>();
+  private count = 0;
+  private isDestroyed = false;
+
+  /** Checks if workers are supported on this platform */
+  static isSupported(): boolean {
+    return WorkerThread.isSupported();
+  }
+
+  /**
+   * @param processor - worker function
+   * @param maxConcurrency - max count of workers
+   */
+  constructor(props: WorkerPoolProps) {
+    this.source = props.source;
+    this.url = props.url;
+    this.getUrl = props.getUrl;
+    this.loadWorker = props.loadWorker;
+    this.setProps(props);
+  }
+
+  /**
+   * Terminates all workers in the pool and aborts active jobs.
+   * @param reason Optional error delivered to active jobs.
+   * @note Can free up significant memory
+   */
+  destroy(reason: unknown = new Error('Worker pool was destroyed')): void {
+    // Abort active jobs so callers receive an error when a development worker is invalidated.
+    this.activeJobs.forEach(job => job.abort(reason));
+    const queuedJobs = this.jobQueue.splice(0);
+    queuedJobs.forEach(queuedJob => queuedJob.onReject(reason));
+    this.idleQueue.forEach(worker => worker.destroy());
+    this.isDestroyed = true;
+  }
+
+  setProps(props: WorkerPoolProps) {
+    this.props = {...this.props, ...props};
+
+    if (props.name !== undefined) {
+      this.name = props.name;
+    }
+    if (props.maxConcurrency !== undefined) {
+      this.maxConcurrency = props.maxConcurrency;
+    }
+    if (props.maxMobileConcurrency !== undefined) {
+      this.maxMobileConcurrency = props.maxMobileConcurrency;
+    }
+    if (props.reuseWorkers !== undefined) {
+      this.reuseWorkers = props.reuseWorkers;
+    }
+    if (props.onDebug !== undefined) {
+      this.onDebug = props.onDebug;
+    }
+  }
+
+  async startJob(
+    name: string,
+    onMessage: OnMessage = (job, type, data) => job.done(data),
+    onError: OnError = (job, error) => job.error(error)
+  ): Promise<WorkerJob> {
+    if (this.isDestroyed) {
+      return Promise.reject(new Error('Worker pool was destroyed'));
+    }
+    // Promise resolves when thread starts working on this job
+    const startPromise = new Promise<WorkerJob>((onStart, onReject) => {
+      // Promise resolves when thread completes or fails working on this job
+      this.jobQueue.push({name, onMessage, onError, onStart, onReject});
+      return this;
+    });
+    this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
+    return await startPromise;
+  }
+
+  // PRIVATE
+
+  /**
+   * Starts first queued job if worker is available or can be created
+   * Called when job is started and whenever a worker returns to the idleQueue
+   */
+  async _startQueuedJob(): Promise<void> {
+    if (!this.jobQueue.length) {
+      return;
+    }
+
+    const workerThread = this._getAvailableWorker();
+    if (!workerThread) {
+      return;
+    }
+
+    // We have a worker, dequeue and start the job
+    const queuedJob = this.jobQueue.shift();
+    if (queuedJob) {
+      // Emit a debug event
+      // @ts-ignore
+      this.onDebug({
+        message: 'Starting job',
+        name: queuedJob.name,
+        workerThread,
+        backlog: this.jobQueue.length
+      });
+
+      // Create a worker job to let the app access thread and manage job completion
+      const job = new WorkerJob(queuedJob.name, workerThread);
+      this.activeJobs.add(job);
+      workerThread.ref();
+
+      // Set the worker thread's message handlers
+      workerThread.onMessage = data => queuedJob.onMessage(job, data.type, data.payload);
+      workerThread.onError = error => queuedJob.onError(job, error);
+
+      // Resolve the start promise so that the app can start sending messages to worker
+      queuedJob.onStart(job);
+
+      // Wait for the app to signal that the job is complete, then return worker to queue
+      try {
+        await job.result;
+      } catch {
+        // The job result promise carries worker errors back to the caller; do not duplicate-log
+        // handled rejections here.
+      } finally {
+        this.activeJobs.delete(job);
+        this.returnWorkerToQueue(workerThread);
+      }
+    }
+  }
+
+  /**
+   * Returns a worker to the idle queue
+   * Destroys the worker if
+   *  - pool is destroyed
+   *  - if this pool doesn't reuse workers
+   *  - if maxConcurrency has been lowered
+   * @param worker
+   */
+  returnWorkerToQueue(worker: WorkerThread) {
+    const shouldDestroyWorker =
+      // Aborted jobs terminate their worker immediately.
+      worker.terminated ||
+      // If the pool is destroyed, there is no reason to keep the worker around
+      this.isDestroyed ||
+      // If the app has disabled worker reuse, any completed workers should be destroyed
+      !this.reuseWorkers ||
+      // If concurrency has been lowered, this worker might be surplus to requirements
+      this.count > this._getMaxConcurrency();
+
+    if (shouldDestroyWorker) {
+      if (!worker.terminated) {
+        worker.destroy();
+      }
+      this.count--;
+    } else {
+      worker.unref();
+      this.idleQueue.push(worker);
+    }
+
+    if (!this.isDestroyed) {
+      this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
+    }
+  }
+
+  /**
+   * Returns idle worker or creates new worker if maxConcurrency has not been reached
+   */
+  _getAvailableWorker(): WorkerThread | null {
+    // If a worker has completed and returned to the queue, it can be used
+    if (this.idleQueue.length > 0) {
+      return this.idleQueue.shift() || null;
+    }
+
+    // Create fresh worker if we haven't yet created the max amount of worker threads for this worker source
+    if (this.count < this._getMaxConcurrency()) {
+      this.count++;
+      const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
+      return new WorkerThread({
+        name,
+        source: this.source,
+        url: this.url,
+        getUrl: this.getUrl,
+        loadWorker: this.loadWorker
+      });
+    }
+
+    // No worker available, have to wait
+    return null;
+  }
+
+  _getMaxConcurrency() {
+    return isMobile ? this.maxMobileConcurrency : this.maxConcurrency;
+  }
+}