@loaders.gl/core 4.4.0-alpha.2 → 4.4.0

package/src/lib/loader-utils/get-data.ts

@@ -9,21 +9,30 @@ import type {
   Loader,
   LoaderOptions
 } from '@loaders.gl/loader-utils';
-import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
 import {
+  concatenateArrayBuffersAsync,
+  isPromise,
   isResponse,
   isReadableStream,
   isAsyncIterable,
   isIterable,
   isIterator,
   isBlob,
-  isBuffer
-} from '../../javascript-utils/is-type';
+  isBuffer,
+  isArrayBufferLike,
+  toArrayBuffer,
+  toArrayBufferView
+} from '@loaders.gl/loader-utils';
 import {makeIterator} from '../../iterators/make-iterator/make-iterator';
 import {checkResponse, makeResponse} from '../utils/response-utils';
 
 const ERR_DATA = 'Cannot convert supplied data type';
 
+/**
+ * Returns an {@link ArrayBuffer} or string from the provided data synchronously.
+ * Supports `ArrayBuffer`, `ArrayBufferView`, and `ArrayBufferLike` (e.g. `SharedArrayBuffer`)
+ * while preserving typed array view offsets.
+ */
 // eslint-disable-next-line complexity
 export function getArrayBufferOrStringFromDataSync(
   data: SyncDataType,
@@ -35,52 +44,32 @@ export function getArrayBufferOrStringFromDataSync(
   }
 
   if (isBuffer(data)) {
-    // @ts-ignore
     data = data.buffer;
   }
 
-  if (data instanceof ArrayBuffer) {
-    const arrayBuffer = data;
-    if (loader.text && !loader.binary) {
-      const textDecoder = new TextDecoder('utf8');
-      return textDecoder.decode(arrayBuffer);
-    }
-    return arrayBuffer;
-  }
-
-  // We may need to handle offsets
-  if (ArrayBuffer.isView(data)) {
-    // TextDecoder is invoked on typed arrays and will handle offsets
+  if (isArrayBufferLike(data)) {
+    const bufferSource = toArrayBufferView(data);
     if (loader.text && !loader.binary) {
       const textDecoder = new TextDecoder('utf8');
-      return textDecoder.decode(data);
-    }
-
-    let arrayBuffer = data.buffer;
-
-    // Since we are returning the underlying arrayBuffer, we must create a new copy
-    // if this typed array / Buffer is a partial view into the ArryayBuffer
-    // TODO - this is a potentially unnecessary copy
-    const byteLength = data.byteLength || data.length;
-    if (data.byteOffset !== 0 || byteLength !== arrayBuffer.byteLength) {
-      // console.warn(`loaders.gl copying arraybuffer of length ${byteLength}`);
-      arrayBuffer = arrayBuffer.slice(data.byteOffset, data.byteOffset + byteLength);
+      return textDecoder.decode(bufferSource);
     }
-    return arrayBuffer;
+    return toArrayBuffer(bufferSource);
   }
 
   throw new Error(ERR_DATA);
 }
 
-// Convert async iterator to a promise
+/**
+ * Resolves the provided data into an {@link ArrayBuffer} or string asynchronously.
+ * Accepts the full {@link DataType} surface including responses and async iterables.
+ */
 export async function getArrayBufferOrStringFromData(
   data: DataType,
   loader: Loader,
   options: LoaderOptions
 ): Promise<ArrayBuffer | string> {
-  const isArrayBuffer = data instanceof ArrayBuffer || ArrayBuffer.isView(data);
-  if (typeof data === 'string' || isArrayBuffer) {
-    return getArrayBufferOrStringFromDataSync(data as string | ArrayBuffer, loader, options);
+  if (typeof data === 'string' || isArrayBufferLike(data)) {
+    return getArrayBufferOrStringFromDataSync(data as SyncDataType, loader, options);
   }
 
   // Blobs and files are FileReader compatible
@@ -89,9 +78,8 @@ export async function getArrayBufferOrStringFromData(
   }
 
   if (isResponse(data)) {
-    const response = data as Response;
-    await checkResponse(response);
-    return loader.binary ? await response.arrayBuffer() : await response.text();
+    await checkResponse(data);
+    return loader.binary ? await data.arrayBuffer() : await data.text();
   }
 
   if (isReadableStream(data)) {
@@ -101,29 +89,41 @@ export async function getArrayBufferOrStringFromData(
 
   if (isIterable(data) || isAsyncIterable(data)) {
     // Assume arrayBuffer iterator - attempt to concatenate
-    return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBuffer>);
+    return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);
   }
 
   throw new Error(ERR_DATA);
 }
 
+/**
+ * Normalizes batchable inputs into async iterables for batch parsing flows.
+ * Supports synchronous iterables, async iterables, fetch responses, readable streams, and
+ * single binary chunks (including typed array views and `ArrayBufferLike` values).
+ */
 export async function getAsyncIterableFromData(
   data: BatchableDataType,
   options: LoaderOptions
-): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {
+): Promise<
+  AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>
+> {
+  if (isPromise(data)) {
+    data = await data;
+  }
+
   if (isIterator(data)) {
     return data as AsyncIterable<ArrayBuffer>;
   }
 
   if (isResponse(data)) {
-    const response = data as Response;
     // Note Since this function is not async, we currently can't load error message, just status
-    await checkResponse(response);
+    await checkResponse(data);
     // TODO - bug in polyfill, body can be a Promise under Node.js
     // eslint-disable-next-line @typescript-eslint/await-thenable
-    const body = await response.body;
-    // TODO - body can be null?
-    return makeIterator(body as ReadableStream<Uint8Array>, options as any);
+    const body = await data.body;
+    if (!body) {
+      throw new Error(ERR_DATA);
+    }
+    return makeIterator(body, options as any);
   }
 
   if (isBlob(data) || isReadableStream(data)) {
@@ -131,38 +131,52 @@ export async function getAsyncIterableFromData(
   }
 
   if (isAsyncIterable(data)) {
-    return data as AsyncIterable<ArrayBuffer>;
+    return data as AsyncIterable<ArrayBufferLike | ArrayBufferView>;
   }
 
+  if (isIterable(data)) {
+    return data as Iterable<ArrayBufferLike | ArrayBufferView>;
+  }
+
+  // @ts-expect-error TODO - fix type mess
   return getIterableFromData(data);
 }
 
+/**
+ * Returns a readable stream for streaming loader inputs when available.
+ */
 export async function getReadableStream(data: BatchableDataType): Promise<ReadableStream> {
   if (isReadableStream(data)) {
     return data as ReadableStream;
   }
   if (isResponse(data)) {
     // @ts-ignore
+    if (!data.body) {
+      throw new Error(ERR_DATA);
+    }
     return data.body;
   }
   const response = await makeResponse(data);
   // @ts-ignore
+  if (!response.body) {
+    throw new Error(ERR_DATA);
+  }
   return response.body;
 }
 
 // HELPERS
 
-function getIterableFromData(data) {
+function getIterableFromData(data: string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView) {
   // generate an iterator that emits a single chunk
   if (ArrayBuffer.isView(data)) {
     return (function* oneChunk() {
-      yield data.buffer;
+      yield toArrayBuffer(data);
     })();
   }
 
-  if (data instanceof ArrayBuffer) {
+  if (isArrayBufferLike(data)) {
     return (function* oneChunk() {
-      yield data;
+      yield toArrayBuffer(data);
     })();
   }
 

package/src/lib/loader-utils/get-fetch-function.ts

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 
 import type {LoaderContext, LoaderOptions, FetchLike} from '@loaders.gl/loader-utils';
-import {isObject} from '../../javascript-utils/is-type';
+import {isObject} from '@loaders.gl/loader-utils';
 import {fetchFile} from '../fetch/fetch-file';
 import {getGlobalLoaderOptions} from './option-utils';
 
@@ -19,15 +19,16 @@ export function getFetchFunction(
   const globalOptions = getGlobalLoaderOptions();
 
   const loaderOptions = options || globalOptions;
+  const fetchOption = loaderOptions.fetch ?? loaderOptions.core?.fetch;
 
   // options.fetch can be a function
-  if (typeof loaderOptions.fetch === 'function') {
-    return loaderOptions.fetch;
+  if (typeof fetchOption === 'function') {
+    return fetchOption;
   }
 
   // options.fetch can be an options object
-  if (isObject(loaderOptions.fetch)) {
-    return (url) => fetchFile(url, loaderOptions.fetch as RequestInit);
+  if (isObject(fetchOption)) {
+    return (url) => fetchFile(url, fetchOption as RequestInit);
   }
 
   // else refer to context (from parent loader) if available

package/src/lib/loader-utils/loader-context.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Loader, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderContext, StrictLoaderOptions} from '@loaders.gl/loader-utils';
 import {getFetchFunction} from './get-fetch-function';
 import {extractQueryString, stripQueryString} from '../utils/url-utils';
 import {path} from '@loaders.gl/loader-utils';
@@ -21,7 +21,7 @@ type LoaderContextProps = Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderCont
  */
 export function getLoaderContext(
   context: LoaderContextProps,
-  options: LoaderOptions,
+  options: StrictLoaderOptions,
   parentContext: LoaderContext | null
 ): LoaderContext {
   // For recursive calls, we already have a context

package/src/lib/loader-utils/option-defaults.ts

@@ -7,44 +7,77 @@ import {isBrowser} from '@loaders.gl/loader-utils';
 import {ConsoleLog} from './loggers';
 
 export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
-  // baseUri
-  fetch: null,
-  mimeType: undefined,
-  nothrow: false,
-  log: new ConsoleLog(), // A probe.gl compatible (`log.log()()` syntax) that just logs to console
-  useLocalLibraries: false,
-
-  CDN: 'https://unpkg.com/@loaders.gl',
-  worker: true, // By default, use worker if provided by loader.
-  maxConcurrency: 3, // How many worker instances should be created for each loader.
-  maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
-  reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
-  _nodeWorkers: false, // By default do not support node workers
-  _workerType: '', // 'test' to use locally generated workers
-
-  limit: 0,
-  _limitMB: 0,
-  batchSize: 'auto',
-  batchDebounceMs: 0,
-  metadata: false, // TODO - currently only implemented for parseInBatches, adds initial metadata batch,
-  transforms: []
+  core: {
+    baseUrl: undefined,
+    // baseUrl
+    fetch: null,
+    mimeType: undefined,
+    fallbackMimeType: undefined,
+    ignoreRegisteredLoaders: undefined,
+    nothrow: false,
+    log: new ConsoleLog(), // A probe.gl compatible (`log.log()()` syntax) that just logs to console
+    useLocalLibraries: false,
+
+    CDN: 'https://unpkg.com/@loaders.gl',
+    worker: true, // By default, use worker if provided by loader.
+    maxConcurrency: 3, // How many worker instances should be created for each loader.
+    maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
+    reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
+    _nodeWorkers: false, // By default do not support node workers
+    _workerType: '', // 'test' to use locally generated workers
+
+    limit: 0,
+    _limitMB: 0,
+    batchSize: 'auto',
+    batchDebounceMs: 0,
+    metadata: false, // TODO - currently only implemented for parseInBatches, adds initial metadata batch,
+    transforms: []
+  }
 };
 
 export const REMOVED_LOADER_OPTIONS = {
+  // deprecated top-level alias
+  baseUri: 'core.baseUrl',
+  fetch: 'core.fetch',
+  mimeType: 'core.mimeType',
+  fallbackMimeType: 'core.fallbackMimeType',
+  ignoreRegisteredLoaders: 'core.ignoreRegisteredLoaders',
+  nothrow: 'core.nothrow',
+  log: 'core.log',
+  useLocalLibraries: 'core.useLocalLibraries',
+
+  CDN: 'core.CDN',
+  worker: 'core.worker',
+  maxConcurrency: 'core.maxConcurrency',
+  maxMobileConcurrency: 'core.maxMobileConcurrency',
+  reuseWorkers: 'core.reuseWorkers',
+  _nodeWorkers: 'core.nodeWorkers',
+  _workerType: 'core._workerType',
+  _worker: 'core._workerType',
+
+  limit: 'core.limit',
+  _limitMB: 'core._limitMB',
+  batchSize: 'core.batchSize',
+  batchDebounceMs: 'core.batchDebounceMs',
+  metadata: 'core.metadata',
+  transforms: 'core.transforms',
+
+  // Older deprecations
   throws: 'nothrow',
   dataType: '(no longer used)',
-  uri: 'baseUri',
-  // Warn if fetch options are used on top-level
-  method: 'fetch.method',
-  headers: 'fetch.headers',
-  body: 'fetch.body',
-  mode: 'fetch.mode',
-  credentials: 'fetch.credentials',
-  cache: 'fetch.cache',
-  redirect: 'fetch.redirect',
-  referrer: 'fetch.referrer',
-  referrerPolicy: 'fetch.referrerPolicy',
-  integrity: 'fetch.integrity',
-  keepalive: 'fetch.keepalive',
-  signal: 'fetch.signal'
+  uri: 'core.baseUrl',
+
+  // Warn if fetch options are used on toplevel
+  method: 'core.fetch.method',
+  headers: 'core.fetch.headers',
+  body: 'core.fetch.body',
+  mode: 'core.fetch.mode',
+  credentials: 'core.fetch.credentials',
+  cache: 'core.fetch.cache',
+  redirect: 'core.fetch.redirect',
+  referrer: 'core.fetch.referrer',
+  referrerPolicy: 'core.fetch.referrerPolicy',
+  integrity: 'core.fetch.integrity',
+  keepalive: 'core.fetch.keepalive',
+  signal: 'core.fetch.signal'
 };

package/src/lib/loader-utils/option-utils.ts

@@ -2,10 +2,42 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {Loader, LoaderOptions, registerJSModules} from '@loaders.gl/loader-utils';
-import {isPureObject, isObject} from '../../javascript-utils/is-type';
+import {
+  Loader,
+  LoaderOptions,
+  registerJSModules,
+  isPureObject,
+  isObject,
+  StrictLoaderOptions,
+  path
+} from '@loaders.gl/loader-utils';
 import {probeLog, NullLog} from './loggers';
 import {DEFAULT_LOADER_OPTIONS, REMOVED_LOADER_OPTIONS} from './option-defaults';
+import {stripQueryString} from '../utils/url-utils';
+
+const CORE_LOADER_OPTION_KEYS = [
+  'baseUrl',
+  'fetch',
+  'mimeType',
+  'fallbackMimeType',
+  'ignoreRegisteredLoaders',
+  'nothrow',
+  'log',
+  'useLocalLibraries',
+  'CDN',
+  'worker',
+  'maxConcurrency',
+  'maxMobileConcurrency',
+  'reuseWorkers',
+  '_nodeWorkers',
+  '_workerType',
+  'limit',
+  '_limitMB',
+  'batchSize',
+  'batchDebounceMs',
+  'metadata',
+  'transforms'
+] as const;
 
 /**
  * Global state for loaders.gl. Stored on `globalThis.loaders._state`
@@ -37,11 +69,14 @@ export function getGlobalLoaderState(): GlobalLoaderState {
  * NOTE: This use case is not reliable but can help when testing new versions of loaders.gl with existing frameworks
  * @returns global loader options merged with default loader options
  */
-export function getGlobalLoaderOptions(): LoaderOptions {
+export function getGlobalLoaderOptions(): StrictLoaderOptions {
   const state = getGlobalLoaderState();
   // Ensure all default loader options from this library are mentioned
-  state.globalOptions = state.globalOptions || {...DEFAULT_LOADER_OPTIONS};
-  return state.globalOptions;
+  state.globalOptions = state.globalOptions || {
+    ...DEFAULT_LOADER_OPTIONS,
+    core: {...DEFAULT_LOADER_OPTIONS.core}
+  };
+  return normalizeLoaderOptions(state.globalOptions);
 }
 
 /**
@@ -58,7 +93,7 @@ export function setGlobalOptions(options: LoaderOptions): void {
 }
 
 /**
- * Merges options with global opts and loader defaults, also injects baseUri
+ * Merges options with global opts and loader defaults, also injects baseUrl
  * @param options
  * @param loader
  * @param loaders
@@ -69,12 +104,31 @@ export function normalizeOptions(
   loader: Loader,
   loaders?: Loader[],
   url?: string
-): LoaderOptions {
+): StrictLoaderOptions {
   loaders = loaders || [];
   loaders = Array.isArray(loaders) ? loaders : [loaders];
 
   validateOptions(options, loaders);
-  return normalizeOptionsInternal(loader, options, url);
+  return normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
+}
+
+/**
+ * Returns a copy of the provided options with deprecated top-level core fields moved into `core`
+ * and removed from the top level. This keeps global options from leaking deprecated aliases into
+ * loader-specific option maps during normalization.
+ */
+export function normalizeLoaderOptions(options: LoaderOptions): StrictLoaderOptions {
+  const normalized = cloneLoaderOptions(options);
+  moveDeprecatedTopLevelOptionsToCore(normalized);
+  for (const key of CORE_LOADER_OPTION_KEYS) {
+    if (normalized.core && normalized.core[key] !== undefined) {
+      delete (normalized as Record<string, unknown>)[key];
+    }
+  }
+  if (normalized.core && normalized.core._workerType !== undefined) {
+    delete (normalized as any)._worker;
+  }
+  return normalized as StrictLoaderOptions;
 }
 
 // VALIDATE OPTIONS
@@ -89,10 +143,8 @@ function validateOptions(options: LoaderOptions, loaders: Loader[]): void {
   validateOptionsObject(options, null, DEFAULT_LOADER_OPTIONS, REMOVED_LOADER_OPTIONS, loaders);
   for (const loader of loaders) {
     // Get the scoped, loader specific options from the user supplied options
-    const idOptions: Record<string, unknown> = ((options && options[loader.id]) || {}) as Record<
-      string,
-      unknown
-    >;
+    const idOptions: Record<string, unknown> =
+      ((options && options[loader.id]) as Record<string, unknown>) || {};
 
     // Get scoped, loader specific default and deprecated options from the selected loader
     const loaderOptions = (loader.options && loader.options[loader.id]) || {};
@@ -125,14 +177,18 @@ function validateOptionsObject(
     if (!(key in defaultOptions) && !isBaseUriOption && !isWorkerUrlOption) {
       // Issue deprecation warnings
       if (key in deprecatedOptions) {
-        probeLog.warn(
-          `${loaderName} loader option \'${prefix}${key}\' no longer supported, use \'${deprecatedOptions[key]}\'`
-        )();
+        if (probeLog.level > 0) {
+          probeLog.warn(
+            `${loaderName} loader option \'${prefix}${key}\' no longer supported, use \'${deprecatedOptions[key]}\'`
+          )();
+        }
       } else if (!isSubOptions) {
-        const suggestion = findSimilarOption(key, loaders);
-        probeLog.warn(
-          `${loaderName} loader option \'${prefix}${key}\' not recognized. ${suggestion}`
-        )();
+        if (probeLog.level > 0) {
+          const suggestion = findSimilarOption(key, loaders);
+          probeLog.warn(
+            `${loaderName} loader option \'${prefix}${key}\' not recognized. ${suggestion}`
+          )();
+        }
       }
     }
   }
@@ -165,16 +221,23 @@ function normalizeOptionsInternal(
   const loaderDefaultOptions = loader.options || {};
 
   const mergedOptions = {...loaderDefaultOptions};
-
-  addUrlOptions(mergedOptions, url);
+  if (loaderDefaultOptions.core) {
+    mergedOptions.core = {...loaderDefaultOptions.core};
+  }
+  moveDeprecatedTopLevelOptionsToCore(mergedOptions);
 
   // LOGGING: options.log can be set to `null` to defeat logging
-  if (mergedOptions.log === null) {
-    mergedOptions.log = new NullLog();
+  if (mergedOptions.core?.log === null) {
+    mergedOptions.core = {...mergedOptions.core, log: new NullLog()};
   }
 
-  mergeNestedFields(mergedOptions, getGlobalLoaderOptions());
-  mergeNestedFields(mergedOptions, options);
+  mergeNestedFields(mergedOptions, normalizeLoaderOptions(getGlobalLoaderOptions()));
+
+  const userOptions = normalizeLoaderOptions(options);
+  mergeNestedFields(mergedOptions, userOptions);
+
+  addUrlOptions(mergedOptions, url);
+  addDeprecatedTopLevelOptions(mergedOptions);
 
   return mergedOptions;
 }
@@ -201,14 +264,68 @@ function mergeNestedFields(mergedOptions: LoaderOptions, options: LoaderOptions)
 
 /**
  * Harvest information from the url
- * @deprecated This is mainly there to support a hack in the GLTFLoader
- * TODO - baseUri should be a directory, i.e. remove file component from baseUri
+ * @deprecated This is mainly there to support loaders that still resolve from options
  * TODO - extract extension?
  * TODO - extract query parameters?
  * TODO - should these be injected on context instead of options?
  */
 function addUrlOptions(options: LoaderOptions, url?: string): void {
-  if (url && !('baseUri' in options)) {
-    options.baseUri = url;
+  if (!url) {
+    return;
+  }
+  const hasCoreBaseUrl = options.core?.baseUrl !== undefined;
+  if (!hasCoreBaseUrl) {
+    options.core ||= {};
+    options.core.baseUrl = path.dirname(stripQueryString(url));
+  }
+}
+
+function cloneLoaderOptions(options: LoaderOptions): LoaderOptions {
+  const clonedOptions = {...options};
+  if (options.core) {
+    clonedOptions.core = {...options.core};
+  }
+  return clonedOptions;
+}
+
+function moveDeprecatedTopLevelOptionsToCore(options: LoaderOptions): void {
+  if (options.baseUri !== undefined) {
+    options.core ||= {};
+    if (options.core.baseUrl === undefined) {
+      options.core.baseUrl = options.baseUri;
+    }
+  }
+
+  for (const key of CORE_LOADER_OPTION_KEYS) {
+    if ((options as Record<string, unknown>)[key] !== undefined) {
+      const coreOptions = (options.core = options.core || {});
+      const coreRecord = coreOptions as Record<string, unknown>;
+      // Treat deprecated top-level core options as aliases to `options.core`, but never override an explicitly
+      // provided `options.core` value.
+      if (coreRecord[key] === undefined) {
+        coreRecord[key] = (options as Record<string, unknown>)[key];
+      }
+    }
+  }
+
+  // Support the older internal `_worker` alias (used by some tests and integrations) for `_workerType`.
+  const workerTypeAlias = (options as any)._worker;
+  if (workerTypeAlias !== undefined) {
+    options.core ||= {};
+    if (options.core._workerType === undefined) {
+      options.core._workerType = workerTypeAlias;
+    }
+  }
+}
+
+function addDeprecatedTopLevelOptions(options: LoaderOptions): void {
+  const coreOptions = options.core as Record<string, unknown> | undefined;
+  if (!coreOptions) {
+    return;
+  }
+  for (const key of CORE_LOADER_OPTION_KEYS) {
+    if (coreOptions[key] !== undefined) {
+      (options as Record<string, unknown>)[key] = coreOptions[key];
+    }
   }
 }