@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/worker-utils/lib/worker-api/create-worker.ts

@@ -0,0 +1,166 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  WorkerMessageType,
+  WorkerMessagePayload,
+  WorkerContext,
+  WorkerJobContext,
+  Process,
+  ProcessInBatches
+} from '../../types';
+import AsyncQueue from '../async-queue/async-queue';
+import WorkerBody from '../worker-farm/worker-body';
+// import {validateWorkerVersion} from './validate-worker-version';
+
+/** Counter for jobs */
+let requestId = 0;
+let activeInputBatches: AsyncQueue<any> | null = null;
+let activeOutputAcknowledgements: AsyncQueue<void> | null = null;
+
+export type ProcessOnMainThread = (
+  data: any,
+  options?: {[key: string]: any},
+  jobContext?: WorkerJobContext
+) => any;
+
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ */
+export async function createWorker(
+  process: Process,
+  processInBatches?: ProcessInBatches
+): Promise<void> {
+  if (!(await WorkerBody.inWorkerThread())) {
+    return;
+  }
+
+  const context: WorkerContext = {
+    process: processOnMainThread
+  };
+
+  // eslint-disable-next-line complexity
+  WorkerBody.onmessage = async (type: WorkerMessageType, payload: WorkerMessagePayload) => {
+    try {
+      switch (type) {
+        case 'preload':
+          WorkerBody.postMessage('done', {});
+          break;
+
+        case 'process':
+          if (!process) {
+            throw new Error('Worker does not support atomic processing');
+          }
+          const result = await process(
+            payload.input,
+            payload.options || {},
+            context,
+            payload.context || {}
+          );
+          WorkerBody.postMessage('done', {result});
+          break;
+
+        case 'process-in-batches':
+          if (!processInBatches) {
+            throw new Error('Worker does not support batched processing');
+          }
+          const inputBatches = new AsyncQueue<any>();
+          const outputAcknowledgements = new AsyncQueue<void>();
+          activeInputBatches = inputBatches;
+          activeOutputAcknowledgements = outputAcknowledgements;
+          try {
+            const resultIterator = processInBatches(
+              createDemandDrivenIterator(inputBatches),
+              payload.options || {},
+              context,
+              payload.context || {}
+            );
+            for await (const batch of resultIterator) {
+              await WorkerBody.postMessage('output-batch', {result: batch});
+              await outputAcknowledgements.next();
+            }
+            await WorkerBody.postMessage('done', {});
+          } finally {
+            activeInputBatches = null;
+            activeOutputAcknowledgements = null;
+          }
+          break;
+
+        case 'input-batch':
+          if (!activeInputBatches) {
+            throw new Error('Worker has no active batched processing session');
+          }
+          activeInputBatches.push(payload.input);
+          break;
+
+        case 'input-done':
+          if (!activeInputBatches) {
+            throw new Error('Worker has no active batched processing session');
+          }
+          activeInputBatches.close();
+          break;
+
+        case 'output-ack':
+          if (!activeOutputAcknowledgements) {
+            throw new Error('Worker has no active batched processing session');
+          }
+          activeOutputAcknowledgements.push(undefined);
+          break;
+
+        default:
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : '';
+      WorkerBody.postMessage('error', {error: message});
+    }
+  };
+}
+
+/** Requests exactly one input batch whenever the worker-side processor advances its iterator. */
+async function* createDemandDrivenIterator(inputBatches: AsyncQueue<any>): AsyncIterable<any> {
+  while (true) {
+    await WorkerBody.postMessage('input-request', {});
+    const nextBatch = await inputBatches.next();
+    if (nextBatch.done) {
+      return;
+    }
+    yield nextBatch.value;
+  }
+}
+
+function processOnMainThread(arrayBuffer: ArrayBuffer, options = {}, jobContext = {}) {
+  return new Promise((resolve, reject) => {
+    const id = requestId++;
+
+    /**
+     */
+    const onMessage = (type: string, payload: WorkerMessagePayload) => {
+      if (payload.id !== id) {
+        // not ours
+        return;
+      }
+
+      switch (type) {
+        case 'done':
+          WorkerBody.removeEventListener(onMessage);
+          resolve(payload.result);
+          break;
+
+        case 'error':
+          WorkerBody.removeEventListener(onMessage);
+          reject(payload.error);
+          break;
+
+        default:
+        // ignore
+      }
+    };
+
+    WorkerBody.addEventListener(onMessage);
+
+    // Ask the main thread to decode data
+    const payload = {id, input: arrayBuffer, options, context: jobContext};
+    WorkerBody.postMessage('process', payload);
+  });
+}

package/src/worker-utils/lib/worker-api/get-worker-url.ts

@@ -0,0 +1,129 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {WorkerObject, WorkerOptions} from '../../types';
+import {assert} from '../env-utils/assert';
+import {isBrowser} from '../env-utils/globals';
+import {VERSION} from '../env-utils/version';
+import {NPM_TAG} from '../npm-tag';
+
+const warnedWorkerVersionFallbacks = new Set<string>();
+
+/**
+ * Gets worker object's name (for debugging in Chrome thread inspector window)
+ */
+export function getWorkerName(worker: WorkerObject): string {
+  const warning = worker.version !== VERSION ? ` (worker-utils@${VERSION})` : '';
+  return `${worker.name}@${worker.version}${warning}`;
+}
+
+/**
+ * Returns an explicitly configured or test worker URL.
+ * @param worker Worker descriptor to resolve.
+ * @param options Worker options that can override the descriptor.
+ * @returns A worker URL, or `null` when URL-based fallback resolution is still required.
+ */
+export function getCustomWorkerURL(
+  worker: WorkerObject,
+  options: WorkerOptions = {}
+): string | null {
+  const workerOptions = options[worker.id] || {};
+
+  const workerFile = isBrowser
+    ? worker.workerFile || `${worker.id}-worker.js`
+    : worker.workerNode || `${worker.id}-worker-node.js`;
+
+  let url = workerOptions.workerUrl;
+
+  // HACK: Allow for non-nested workerUrl for the CompressionWorker.
+  // For the compression worker, workerOptions is currently not nested correctly. For most loaders,
+  // you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker
+  // puts options at the top level, not within a `compression` key (its `id`). For this reason, the
+  // above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not
+  // break backwards compatibility, we allow the CompressionWorker to have options at the top level.
+  if (!url && worker.id === 'compression') {
+    url = options.workerUrl;
+  }
+
+  // If URL is test, generate local loaders.gl url
+  // @ts-ignore _workerType
+  const workerType = (options as any)._workerType || (options as any)?.core?._workerType;
+  if (!url && workerType === 'test') {
+    if (isBrowser) {
+      url = `modules/${worker.module}/dist/${workerFile}`;
+    } else {
+      // In the test environment the ts-node loader requires TypeScript code
+      const workerSourceDirectory =
+        worker.module === 'core' && worker.workerNode?.startsWith('worker-utils/')
+          ? 'worker-utils/workers'
+          : 'workers';
+      const workerSourceNode =
+        worker.workerSourceNode || `${workerSourceDirectory}/${worker.id}-worker-node.ts`;
+      url = `modules/${worker.module}/src/${workerSourceNode}`;
+    }
+  }
+
+  return url || null;
+}
+
+/**
+ * Generates a worker URL using overrides first and the published CDN artifact as a fallback.
+ * @param worker Worker descriptor to resolve.
+ * @param options Worker options that can override the descriptor.
+ * @returns A loadable worker URL.
+ */
+export function getWorkerURL(worker: WorkerObject, options: WorkerOptions = {}): string {
+  const customWorkerUrl = getCustomWorkerURL(worker, options);
+  const descriptorWorkerUrl = typeof worker.worker === 'string' ? worker.worker : null;
+  const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
+
+  assert(url);
+
+  // Allow user to override location
+  return url;
+}
+
+/**
+ * Returns the generated URL for a published pre-built worker.
+ * @param worker Worker descriptor to resolve.
+ * @param warn Whether to warn when an uninjected development version uses the npm tag.
+ * @returns The CDN worker URL.
+ */
+export function getDefaultWorkerURL(worker: WorkerObject, warn: boolean = false): string {
+  const workerFile = isBrowser
+    ? worker.workerFile || `${worker.id}-worker.js`
+    : worker.workerNode || `${worker.id}-worker-node.js`;
+  let version = worker.version;
+  if (version === 'latest') {
+    version = NPM_TAG;
+  }
+  const versionTag = version ? `@${version}` : '';
+  const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+  if (warn) {
+    warnIfUsingNpmTagFallback(worker, url);
+  }
+  return url;
+}
+
+/** Warn once when a worker falls back to the npm tag because __VERSION__ was not injected. */
+function warnIfUsingNpmTagFallback(worker: WorkerObject, url: string): void {
+  if (worker.version !== 'latest') {
+    return;
+  }
+
+  const workerId = `${worker.module}:${worker.id}`;
+  if (warnedWorkerVersionFallbacks.has(workerId)) {
+    return;
+  }
+
+  warnedWorkerVersionFallbacks.add(workerId);
+  // eslint-disable-next-line no-console
+  console.warn(
+    `loaders.gl: ${worker.name} loader worker version is "latest" because __VERSION__ was not injected. Fetching ${url} from CDN.`
+  );
+}

package/src/worker-utils/lib/worker-api/process-on-worker.ts

@@ -0,0 +1,413 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  WorkerObject,
+  WorkerOptions,
+  WorkerContext,
+  WorkerJobContext,
+  WorkerMessageType,
+  WorkerMessagePayload
+} from '../../types';
+import type WorkerJob from '../worker-farm/worker-job';
+import AsyncQueue from '../async-queue/async-queue';
+import WorkerFarm from '../worker-farm/worker-farm';
+import type {WorkerPoolTarget} from '../worker-farm/worker-pool';
+import {
+  getCustomWorkerURL,
+  getDefaultWorkerURL,
+  getWorkerURL,
+  getWorkerName
+} from './get-worker-url';
+import {getTransferListForWriter} from '../worker-utils/get-transfer-list';
+import {isBrowser} from '../env-utils/globals';
+
+/** Options for worker processing */
+export type ProcessOnWorkerOptions = WorkerOptions & {
+  /** Cancels the job by terminating its worker. */
+  signal?: AbortSignal;
+  /** Diagnostic name shown for the worker job. */
+  jobName?: string;
+  [key: string]: any;
+};
+
+/** Options for preloading workers. */
+export type PreloadWorkerOptions = {
+  /** Number of workers to warm in the worker pool. */
+  count?: number;
+};
+
+/**
+ * Determines if we can parse with worker
+ * @param loader
+ * @param data
+ * @param options
+ */
+export function canProcessOnWorker(worker: WorkerObject, options?: WorkerOptions) {
+  if (!WorkerFarm.isSupported()) {
+    return false;
+  }
+
+  const workerOptions = options?.[worker.id];
+  return Boolean(
+    (worker.worker || worker.loadWorker || workerOptions?.workerUrl) && options?.worker
+  );
+}
+
+/**
+ * This function expects that the worker thread sends certain messages,
+ * Creating such a worker can be automated if the worker is wrapper by a call to
+ * createWorker in @loaders.gl/core/worker-utils.
+ */
+export async function processOnWorker(
+  worker: WorkerObject,
+  data: any,
+  options: ProcessOnWorkerOptions = {},
+  context: WorkerContext = {},
+  jobContext: WorkerJobContext = {}
+): Promise<any> {
+  throwIfAborted(options.signal);
+  const name = getWorkerName(worker);
+
+  const workerFarm = WorkerFarm.getWorkerFarm(options);
+  const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
+
+  const jobName = options.jobName || worker.name;
+  const job = await workerPool.startJob(
+    jobName,
+    // eslint-disable-next-line
+    onMessage.bind(null, context)
+  );
+
+  const abortJob = (): void => job.abort(getAbortReason(options.signal));
+  options.signal?.addEventListener('abort', abortJob, {once: true});
+  if (options.signal?.aborted) {
+    abortJob();
+  }
+
+  try {
+    // AbortSignal cannot be structured-cloned. It controls the job on this thread only.
+    const {signal: _signal, ...workerOptions} = options;
+    const transferableOptions = getTransferListForWriter(workerOptions);
+    const transferableContext = getTransferListForWriter(jobContext);
+    if (job.isRunning) {
+      job.postMessage('process', {
+        input: data,
+        options: transferableOptions,
+        context: transferableContext
+      });
+    }
+
+    const result = await job.result;
+    return result.result;
+  } finally {
+    options.signal?.removeEventListener('abort', abortJob);
+  }
+}
+
+/**
+ * Processes an input iterator on one leased worker and streams output batches back.
+ *
+ * The worker remains assigned to this job for the complete iterator lifetime, allowing
+ * `processInBatches` implementations to retain parser or encoder state between input batches.
+ * Input is demand-driven: the main thread advances the source only when that worker requests the
+ * next batch.
+ */
+export function processOnWorkerInBatches<InputBatch = any, OutputBatch = any>(
+  worker: WorkerObject,
+  input: AsyncIterable<InputBatch> | Iterable<InputBatch>,
+  options: ProcessOnWorkerOptions = {},
+  context: WorkerContext = {},
+  jobContext: WorkerJobContext = {}
+): AsyncIterable<OutputBatch> {
+  return processOnWorkerInBatchesIterator(worker, input, options, context, jobContext);
+}
+
+/** Implements stateful worker batch processing as an async generator. */
+async function* processOnWorkerInBatchesIterator<InputBatch, OutputBatch>(
+  worker: WorkerObject,
+  input: AsyncIterable<InputBatch> | Iterable<InputBatch>,
+  options: ProcessOnWorkerOptions,
+  context: WorkerContext,
+  jobContext: WorkerJobContext
+): AsyncIterable<OutputBatch> {
+  throwIfAborted(options.signal);
+  const name = getWorkerName(worker);
+  const workerFarm = WorkerFarm.getWorkerFarm(options);
+  const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
+  const outputBatches = new AsyncQueue<OutputBatch>();
+  const inputIterator = getAsyncIterator(input);
+  let outputFinished = false;
+  let inputFinished = false;
+  let inputRequest = Promise.resolve();
+  const batchJobFailed = new Error('Worker batch job failed');
+
+  const finishOutput = (): void => {
+    if (!outputFinished) {
+      outputFinished = true;
+      outputBatches.close();
+    }
+  };
+  const failOutput = (_error: unknown): void => {
+    if (!outputFinished) {
+      outputFinished = true;
+      outputBatches.enqueue(batchJobFailed);
+      outputBatches.close();
+    }
+  };
+
+  const job = await workerPool.startJob(
+    options.jobName || worker.name,
+    (activeJob, type, payload) => {
+      switch (type) {
+        case 'input-request':
+          inputRequest = inputRequest
+            .then(async () => {
+              const nextBatch = await inputIterator.next();
+              if (!activeJob.isRunning) {
+                return;
+              }
+              if (nextBatch.done) {
+                inputFinished = true;
+                activeJob.postMessage('input-done', {});
+              } else {
+                activeJob.postMessage('input-batch', {input: nextBatch.value});
+              }
+            })
+            .catch(error => activeJob.abort(error));
+          break;
+
+        case 'output-batch':
+          outputBatches.push(payload.result as OutputBatch);
+          break;
+
+        case 'done':
+          activeJob.done(payload);
+          finishOutput();
+          break;
+
+        case 'error': {
+          const error = new Error(payload.error || 'Worker batch processing failed');
+          activeJob.error(error);
+          failOutput(error);
+          break;
+        }
+
+        case 'process':
+          void onMessage(context, activeJob, type, payload);
+          break;
+
+        default: {
+          const error = new Error(`Unexpected worker batch message: ${type}`);
+          activeJob.error(error);
+          failOutput(error);
+          break;
+        }
+      }
+    }
+  );
+
+  const abortJob = (): void => job.abort(getAbortReason(options.signal));
+  options.signal?.addEventListener('abort', abortJob, {once: true});
+  if (options.signal?.aborted) {
+    abortJob();
+  }
+  void job.result.catch(failOutput);
+
+  try {
+    const {signal: _signal, ...workerOptions} = options;
+    if (job.isRunning) {
+      job.postMessage('process-in-batches', {
+        options: getTransferListForWriter(workerOptions),
+        context: getTransferListForWriter(jobContext)
+      });
+    }
+
+    try {
+      for await (const outputBatch of outputBatches) {
+        yield outputBatch;
+        if (job.isRunning) {
+          job.postMessage('output-ack', {});
+        }
+      }
+    } catch (error) {
+      if (error === batchJobFailed) {
+        await job.result;
+      }
+      throw error;
+    }
+    await job.result;
+  } finally {
+    options.signal?.removeEventListener('abort', abortJob);
+    if (job.isRunning) {
+      job.abort(createAbortError('Worker batch iterator was closed'));
+    }
+    if (!inputFinished) {
+      closeInputIterator(inputIterator);
+    }
+  }
+}
+
+/** Throws a cross-runtime abort error when a signal is already aborted. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (!signal?.aborted) {
+    return;
+  }
+  throw getAbortReason(signal);
+}
+
+/** Returns a caller-provided abort reason or a cross-runtime `AbortError`. */
+function getAbortReason(signal?: AbortSignal): unknown {
+  return signal?.reason ?? createAbortError('Worker job was aborted');
+}
+
+/** Creates an abort error without requiring the DOMException global. */
+function createAbortError(message: string): Error {
+  const error = new Error(message);
+  error.name = 'AbortError';
+  return error;
+}
+
+/** Returns an asynchronous iterator for either accepted source kind. */
+function getAsyncIterator<T>(input: AsyncIterable<T> | Iterable<T>): AsyncIterator<T> {
+  if (Symbol.asyncIterator in input) {
+    return input[Symbol.asyncIterator]();
+  }
+  return (async function* iterateSynchronously() {
+    yield* input;
+  })();
+}
+
+/** Closes an input iterator without allowing a blocked return to delay cancellation. */
+function closeInputIterator<InputBatch>(inputIterator: AsyncIterator<InputBatch>): void {
+  try {
+    void Promise.resolve(inputIterator.return?.()).catch(() => undefined);
+  } catch {
+    // The worker job has already been aborted; iterator cleanup is best effort.
+  }
+}
+
+/**
+ * Warm-start one or more workers in the same pool used by processOnWorker.
+ * @param worker Worker object to preload.
+ * @param options Worker options used to resolve the worker pool.
+ * @param preloadOptions Preload options.
+ */
+export async function preloadWorker(
+  worker: WorkerObject,
+  options: ProcessOnWorkerOptions = {},
+  preloadOptions: PreloadWorkerOptions = {}
+): Promise<void> {
+  const name = getWorkerName(worker);
+  const workerFarm = WorkerFarm.getWorkerFarm(options);
+  const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
+  const count = preloadOptions.count ?? options.maxConcurrency ?? options.core?.maxConcurrency ?? 1;
+
+  const preloadJobs = Array.from({length: count}, async () => {
+    const job = await workerPool.startJob(`${worker.name} preload`, onPreloadMessage);
+    job.postMessage('preload', {});
+    return await job.result;
+  });
+
+  await Promise.all(preloadJobs);
+}
+
+/**
+ * Resolves the worker source while preserving URL overrides and classic-worker fallbacks.
+ * @param worker Worker descriptor to resolve.
+ * @param options Options for the active worker operation.
+ * @param name Versioned worker name.
+ * @returns A target suitable for worker-pool lookup.
+ */
+function getWorkerPoolTarget(
+  worker: WorkerObject,
+  options: ProcessOnWorkerOptions,
+  name: string
+): WorkerPoolTarget {
+  if (options.source) {
+    return {name, source: options.source};
+  }
+
+  const customWorkerUrl = getCustomWorkerURL(worker, options);
+  if (customWorkerUrl) {
+    return {name, url: customWorkerUrl};
+  }
+
+  if (isBrowser && worker.loadWorker) {
+    return {
+      name,
+      loadWorker: worker.loadWorker,
+      getUrl: () => getWorkerURL(worker, options),
+      urlKey: getDefaultWorkerURL(worker)
+    };
+  }
+
+  return {name, url: getWorkerURL(worker, options)};
+}
+
+/**
+ * Job completes when we receive the result
+ * @param job
+ * @param message
+ */
+async function onMessage(
+  context: WorkerContext,
+  job: WorkerJob,
+  type: WorkerMessageType,
+  payload: WorkerMessagePayload
+) {
+  switch (type) {
+    case 'done':
+      // Worker is done
+      job.done(payload);
+      break;
+
+    case 'error':
+      // Worker encountered an error
+      job.error(new Error(payload.error));
+      break;
+
+    case 'process':
+      // Worker is asking for us (main thread) to process something
+      const {id, input, options} = payload;
+      try {
+        if (!context.process) {
+          job.postMessage('error', {id, error: 'Worker not set up to process on main thread'});
+          return;
+        }
+        const result = await context.process(input, options, undefined, payload.context || {});
+        job.postMessage('done', {id, result});
+      } catch (error) {
+        const message = error instanceof Error ? error.message : 'unknown error';
+        job.postMessage('error', {id, error: message});
+      }
+      break;
+
+    default:
+      // eslint-disable-next-line
+      console.warn(`process-on-worker: unknown message ${type}`);
+  }
+}
+
+/**
+ * Completes a preload job when the worker acknowledges the preload message.
+ * @param job Worker job.
+ * @param type Worker message type.
+ * @param payload Worker message payload.
+ */
+function onPreloadMessage(job: WorkerJob, type: WorkerMessageType, payload: WorkerMessagePayload) {
+  switch (type) {
+    case 'done':
+      job.done(payload);
+      break;
+
+    case 'error':
+      job.error(new Error(payload.error));
+      break;
+
+    default:
+      // eslint-disable-next-line
+      console.warn(`process-on-worker: unknown preload message ${type}`);
+  }
+}