@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/loader-utils/lib/worker-loader-utils/encode-with-worker.ts

@@ -0,0 +1,28 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {WorkerFarm} from '@loaders.gl/core/worker-utils';
+import {Writer, WriterOptions} from '../../writer-types';
+import {isBrowser} from '../env-utils/globals';
+
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+export function canEncodeWithWorker(writer: Writer, options?: WriterOptions) {
+  if (!WorkerFarm.isSupported()) {
+    return false;
+  }
+
+  const nodeWorkers = options?._nodeWorkers ?? options?.core?._nodeWorkers;
+  const useWorkers = options?.worker ?? options?.core?.worker;
+
+  // Node workers are still experimental
+  if (!isBrowser && !nodeWorkers) {
+    return false;
+  }
+
+  return Boolean(writer.worker && useWorkers);
+}

package/src/loader-utils/lib/worker-loader-utils/parse-with-worker.ts

@@ -0,0 +1,347 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {
+  canProcessOnWorker,
+  isBrowser,
+  processOnWorker,
+  processOnWorkerInBatches
+} from '@loaders.gl/core/worker-utils';
+import type {DataType} from '../../types';
+import type {
+  Loader,
+  LoaderWithParser,
+  StrictLoaderOptions,
+  LoaderContext
+} from '../../loader-types';
+
+type ParseOnMainThread = (
+  arrayBuffer: ArrayBuffer,
+  loaders?: Loader | Loader[] | StrictLoaderOptions,
+  options?: StrictLoaderOptions,
+  context?: LoaderContext
+) => Promise<unknown>;
+
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+export function canParseWithWorker(loader: Loader, options?: StrictLoaderOptions) {
+  // Caller-only loaders do not need worker transport or JSON-serializable limits.
+  const workerUrl = (options?.[loader.id] as {workerUrl?: string} | undefined)?.workerUrl;
+  if (options?.core?.worker === false || !(loader.worker || loader.loadWorker || workerUrl))
+    return false;
+  // Request signers and token providers cannot be cloned into worker runtimes.
+  // Keep parsing on the caller so nested fetches retain these application callbacks.
+  if (
+    options?.core?.credentials?.some(
+      credential =>
+        credential.type === 'request' ||
+        ('token' in credential && typeof credential.token === 'function')
+    )
+  )
+    return false;
+  const workerOptions = getWorkerOptions(options);
+  const nodeWorkers = workerOptions._nodeWorkers;
+  if (!isBrowser && !nodeWorkers) {
+    return false;
+  }
+
+  // LAS has no Arrow worker transport yet, so keep its Arrow path on the main thread.
+  if (
+    loader.id === 'las' &&
+    ((options as {las?: {shape?: string}} | undefined)?.las?.shape ?? 'arrow-table') ===
+      'arrow-table'
+  ) {
+    return false;
+  }
+
+  if (loader.id === 'csv' && !shouldParseCSVWithWorker(options)) {
+    return false;
+  }
+
+  return Boolean(canProcessOnWorker(loader, workerOptions));
+}
+
+/**
+ * Determines whether an atomic parse should use a worker, including optional loader work estimates.
+ * @param loader Loader metadata or parser-bearing loader.
+ * @param data Original parse input, before materialization.
+ * @param options Loader and worker options.
+ * @param context Loader context.
+ * @param estimateLoader Optional metadata loader that owns the estimate hook.
+ */
+export function shouldParseWithWorker(
+  loader: Loader,
+  data: DataType,
+  options?: StrictLoaderOptions,
+  context?: LoaderContext,
+  estimateLoader: Loader = loader
+): boolean {
+  if (!canParseWithWorker(loader, options)) {
+    return false;
+  }
+
+  if (options?.core?.worker !== 'auto' || !estimateLoader.getWorkerEstimate) {
+    return true;
+  }
+
+  try {
+    const estimate = estimateLoader.getWorkerEstimate(data, options, context);
+    if (estimate === undefined || !Number.isFinite(estimate) || estimate < 0 || estimate > 1) {
+      return true;
+    }
+    return estimate >= (options.core?.workerThreshold ?? 0.1);
+  } catch {
+    // Estimation must never make a previously worker-capable parse fail.
+    return true;
+  }
+}
+
+/**
+ * this function expects that the worker function sends certain messages,
+ * this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/core/loader-utils.
+ * @param loader Loader metadata used to select the worker pool.
+ * @param data Input transferred to the worker.
+ * @param options Loader and worker options.
+ * @param context Serializable loader context.
+ * @param parseOnMainThread Callback for worker requests that must run on the calling thread.
+ * @param signal Optional cancellation signal for this worker job.
+ */
+export async function parseWithWorker(
+  loader: Loader,
+  data: any,
+  options?: StrictLoaderOptions,
+  context?: LoaderContext,
+  parseOnMainThread?: ParseOnMainThread,
+  signal?: AbortSignal
+) {
+  const workerSignal = signal || getWorkerAbortSignal(options);
+  const result = await processOnWorker(
+    loader,
+    data,
+    {...getWorkerOptions(options, loader), signal: workerSignal},
+    {
+      process: async (input, processOptions, _workerContext, parseContext) => {
+        if (!parseOnMainThread) {
+          throw new Error('Worker not set up to parse on main thread');
+        }
+        const {_loaderIds, _loaderIsArray, ...workerParseContext} = parseContext || {};
+        const mainThreadContext =
+          context || parseContext
+            ? ({...context, ...workerParseContext} as LoaderContext)
+            : undefined;
+        return await callParseOnMainThread(
+          parseOnMainThread,
+          input,
+          processOptions,
+          mainThreadContext,
+          resolveWorkerLoaders(loader, _loaderIds, _loaderIsArray, context, options)
+        );
+      }
+    },
+    getSerializableLoaderContext(context)
+  );
+  return isLoaderWithWorkerResultDeserializer(loader)
+    ? loader.deserializeWorkerResult(result, options, context)
+    : result;
+}
+
+/**
+ * Parses an input iterator through one stateful worker session.
+ * @param loader Parser-bearing loader with a batched worker implementation.
+ * @param inputIterator Input fragments to stream to the worker.
+ * @param options Loader and worker options.
+ * @param context Serializable loader context.
+ * @param parseOnMainThread Callback for worker requests that must run on the calling thread.
+ * @param signal Optional cancellation signal for this worker job.
+ */
+export async function* parseWithWorkerInBatches(
+  loader: LoaderWithParser,
+  inputIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
+  options?: StrictLoaderOptions,
+  context?: LoaderContext,
+  parseOnMainThread?: ParseOnMainThread,
+  signal?: AbortSignal
+): AsyncIterable<unknown> {
+  const workerSignal = signal || getWorkerAbortSignal(options);
+  const outputIterator = processOnWorkerInBatches(
+    loader,
+    inputIterator,
+    {...getWorkerOptions(options), signal: workerSignal},
+    {
+      process: async (input, processOptions, _workerContext, parseContext) => {
+        if (!parseOnMainThread) {
+          throw new Error('Worker not set up to parse on main thread');
+        }
+        const {_loaderIds, _loaderIsArray, ...workerParseContext} = parseContext || {};
+        const mainThreadContext =
+          context || parseContext
+            ? ({...context, ...workerParseContext} as LoaderContext)
+            : undefined;
+        return await callParseOnMainThread(
+          parseOnMainThread,
+          input,
+          processOptions,
+          mainThreadContext,
+          resolveWorkerLoaders(loader, _loaderIds, _loaderIsArray, context, options)
+        );
+      }
+    },
+    getSerializableLoaderContext(context)
+  );
+
+  for await (const batch of outputIterator) {
+    yield loader.deserializeWorkerBatch
+      ? loader.deserializeWorkerBatch(batch, options, context)
+      : batch;
+  }
+}
+
+/** Returns the loader-specific abort signal used to cancel a worker job. */
+function getWorkerAbortSignal(options?: StrictLoaderOptions): AbortSignal | undefined {
+  const parquetSignal = (options as {parquet?: {signal?: unknown}} | undefined)?.parquet?.signal;
+  if (typeof AbortSignal !== 'undefined' && parquetSignal instanceof AbortSignal) {
+    return parquetSignal;
+  }
+  return undefined;
+}
+
+/**
+ * Calls either the legacy two-argument parse callback or the loader-utils callback.
+ * @param parseOnMainThread Main-thread parse callback.
+ * @param input Data to parse on the main thread.
+ * @param options Loader options from the worker.
+ * @param context Loader context merged from the worker and caller.
+ * @param loaders Caller-local loaders selected by the worker.
+ */
+function callParseOnMainThread(
+  parseOnMainThread: ParseOnMainThread,
+  input: ArrayBuffer,
+  options?: StrictLoaderOptions,
+  context?: LoaderContext,
+  loaders?: Loader | Loader[]
+): Promise<unknown> {
+  if (!loaders && parseOnMainThread.length <= 2) {
+    return parseOnMainThread(input, options);
+  }
+  return parseOnMainThread(input, loaders, options, context);
+}
+
+/** Resolves worker-supplied loader identifiers to caller-local loaders without transferring functions. */
+function resolveWorkerLoaders(
+  loader: Loader,
+  loaderIds: string[] | undefined,
+  loaderIsArray: boolean | undefined,
+  context?: LoaderContext,
+  options?: StrictLoaderOptions
+): Loader | Loader[] | undefined {
+  if (!loaderIds) return undefined;
+  const availableLoaders = new Map<string, Loader>();
+  const visitedLoaders = new Set<Loader>();
+  for (const candidate of [loader, ...(context?.loaders || [])]) {
+    visitLoader(candidate);
+  }
+  const selectedLoaders = loaderIds.map(loaderId => {
+    const selectedLoader = availableLoaders.get(loaderId);
+    if (!selectedLoader) {
+      throw new Error(
+        `Worker requested unknown loader "${loaderId}". Declare it in loader.subloaders or pass it in context.loaders.`
+      );
+    }
+    return selectedLoader;
+  });
+  return loaderIsArray ? selectedLoaders : selectedLoaders[0];
+
+  /** Collects dependency metadata while retaining caller-provided parser implementations. */
+  function visitLoader(candidate: Loader): void {
+    if (visitedLoaders.has(candidate)) return;
+    visitedLoaders.add(candidate);
+    availableLoaders.set(candidate.id, candidate);
+    for (const [name, dependency] of Object.entries(candidate.subloaders || {})) {
+      const replacement = options?.core?.subloaders?.[name] || dependency;
+      visitLoader(replacement);
+      availableLoaders.set(dependency.id, replacement);
+    }
+  }
+}
+
+/**
+ * Create worker options with deprecated top-level worker fields available to worker-utils.
+ * @param options
+ * @param loader Loader whose parser will handle the worker request.
+ */
+function getWorkerOptions(options: StrictLoaderOptions = {}, loader?: Loader) {
+  const optionsToSerialize = {
+    ...options,
+    ...(options.core ? {core: {...options.core, authentications: undefined}} : {})
+  };
+  const serializedOptions = JSON.parse(
+    JSON.stringify(optionsToSerialize, (_key, value) => (typeof value === 'bigint' ? null : value))
+  );
+  restoreBigIntOptions(optionsToSerialize, serializedOptions);
+  const workerOptions: Record<string, any> = {
+    ...serializedOptions.core,
+    ...serializedOptions
+  };
+  // The decision has already been made before dispatch, so worker runtimes should
+  // receive the established boolean form even when the caller selected `auto`.
+  if (workerOptions.worker === 'auto') {
+    workerOptions.worker = true;
+  }
+  if (loader) {
+    workerOptions._workerLoaderId = loader.id;
+  }
+  return workerOptions;
+}
+
+/** Restores exact bigint leaves after JSON removes callbacks; ordinary JSON values keep their shape. */
+function restoreBigIntOptions(original: any, serialized: any): void {
+  if (!serialized || typeof serialized !== 'object') return;
+  for (const key of Object.keys(serialized)) {
+    const value = original?.[key];
+    if (typeof value === 'bigint') serialized[key] = value;
+    else restoreBigIntOptions(value, serialized[key]);
+  }
+}
+
+/**
+ * Create a serializable loader context for worker jobs.
+ * @param context
+ */
+function getSerializableLoaderContext(context?: LoaderContext) {
+  if (!context) {
+    return {};
+  }
+  const {fetch, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext} =
+    context;
+  return JSON.parse(JSON.stringify(serializableContext));
+}
+
+/**
+ * Checks whether CSV options request Arrow output that can be transported from a worker.
+ * @param options Loader options.
+ * @returns True when CSV should parse on a worker.
+ */
+function shouldParseCSVWithWorker(options?: StrictLoaderOptions): boolean {
+  const csvOptions = options as {csv?: {shape?: string}; core?: {shape?: string}} | undefined;
+  return (csvOptions?.csv?.shape ?? csvOptions?.core?.shape) === 'arrow-table';
+}
+
+/**
+ * Tests whether a loader can deserialize worker results.
+ * @param loader Loader object.
+ * @returns True when the loader exposes a worker result deserializer.
+ */
+function isLoaderWithWorkerResultDeserializer(
+  loader: Loader
+): loader is Loader & Required<Pick<Loader, 'deserializeWorkerResult'>> {
+  return (
+    typeof (loader as Loader & {deserializeWorkerResult?: unknown}).deserializeWorkerResult ===
+    'function'
+  );
+}