@loaders.gl/core 4.5.3 → 5.0.0-alpha.10

package/src/lib/api/select-source.ts

@@ -2,26 +2,23 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Source} from '@loaders.gl/loader-utils';
+import type {SourceLoader} from '@loaders.gl/loader-utils';
+import {selectLoaderSync} from './select-loader';
 
 /** Guess service type from URL */
 export function selectSource(
   url: string | Blob,
-  sources: Source[],
+  sources: SourceLoader[],
   options?: {
     /** Provide id of a source to select that source. Omit or provide 'auto' to test the source*/
     type?: string;
     nothrow?: boolean;
   }
-): Source | null {
+): SourceLoader | null {
   const type = options?.type || 'auto';
-  let selectedSource: Source | null = null;
+  let selectedSource: SourceLoader | null = null;
   if (type === 'auto') {
-    for (const source of sources) {
-      if (typeof url === 'string' && source.testURL && source.testURL(url)) {
-        return source;
-      }
-    }
+    selectedSource = selectLoaderSync(url, sources, {core: {nothrow: true}}) as SourceLoader | null;
   } else {
     selectedSource = getSourceOfType(type, sources);
   }
@@ -34,7 +31,7 @@ export function selectSource(
 }
 
 /** Guess service type from URL */
-function getSourceOfType(type: string, sources: Source[]): Source | null {
+function getSourceOfType(type: string, sources: SourceLoader[]): SourceLoader | null {
   for (const service of sources) {
     if (service.type === type) {
       return service;

package/src/lib/fetch/fetch-error-message.ts

@@ -17,7 +17,7 @@ export async function getErrorMessageFromResponse(response: Response): Promise<s
     } else {
       message += response.statusText;
     }
-  } catch (error) {
+  } catch (_error) {
     // eslint forbids return in finally statement
     return message;
   }

package/src/lib/filesystems/browser-filesystem.ts

@@ -50,23 +50,34 @@ export class BrowserFileSystem implements FileSystem {
     }
 
     // Local fetches are served from the list of files
-    const file = this.files[path];
+    const file = this._getFile(path, true);
     if (!file) {
       return new Response(path, {status: 400, statusText: 'NOT FOUND'});
     }
 
     const headers = new Headers(options?.headers);
     const range = headers.get('Range');
-    const bytes = range && /bytes=($1)-($2)/.exec(range);
+    const bytes = range && /bytes=(\d+)-(\d+)/.exec(range);
 
     if (bytes) {
       const start = parseInt(bytes[1]);
-      const end = parseInt(bytes[2]);
+      const end = Math.min(parseInt(bytes[2]), file.size - 1);
+      if (start > end) {
+        const response = new Response(null, {
+          status: 416,
+          headers: {'Content-Range': `bytes */${file.size}`}
+        });
+        Object.defineProperty(response, 'url', {value: path});
+        return response;
+      }
       // The trick when reading File objects is to read successive "slices" of the File
       // Per spec https://w3c.github.io/FileAPI/, slicing a File should only update the start and end fields
       // Actually reading from file should happen in `readAsArrayBuffer` (and as far we can tell it does)
-      const data = await file.slice(start, end).arrayBuffer();
-      const response = new Response(data);
+      const data = await file.slice(start, end + 1).arrayBuffer();
+      const response = new Response(data, {
+        status: 206,
+        headers: {'Content-Range': `bytes ${start}-${end}/${file.size}`}
+      });
       Object.defineProperty(response, 'url', {value: path});
       return response;
     }
@@ -95,7 +106,7 @@ export class BrowserFileSystem implements FileSystem {
    * Return information (size) about files in this file system
    */
   async stat(path: string, options?: object): Promise<{size: number}> {
-    const file = this.files[path];
+    const file = this._getFile(path, false);
     if (!file) {
       throw new Error(path);
     }
@@ -106,16 +117,19 @@ export class BrowserFileSystem implements FileSystem {
    * Just removes the file from the list
    */
   async unlink(path: string): Promise<void> {
-    delete this.files[path];
-    delete this.lowerCaseFiles[path];
-    this.usedFiles[path] = true;
+    const file = this._getFile(path, false);
+    if (file) {
+      delete this.files[file.name];
+      delete this.lowerCaseFiles[file.name.toLowerCase()];
+      this.usedFiles[file.name] = true;
+    }
   }
 
   // implements IRandomAccessFileSystem
 
   // RANDOM ACCESS
   async openReadableFile(pathname: string, flags: unknown): Promise<ReadableFile> {
-    return new BlobFile(this.files[pathname]);
+    return new BlobFile(this._getFile(pathname, true));
   }
 
   // PRIVATE
@@ -123,7 +137,7 @@ export class BrowserFileSystem implements FileSystem {
   // Supports case independent paths, and file usage tracking
   _getFile(path: string, used: boolean): File {
     // Prefer case match, but fall back to case independent.
-    const file = this.files[path] || this.lowerCaseFiles[path];
+    const file = this.files[path] || this.lowerCaseFiles[path.toLowerCase()];
     if (file && used) {
       this.usedFiles[path] = true;
     }

package/src/lib/loader-utils/check-errors.ts

@@ -14,7 +14,7 @@ export async function checkFetchResponseStatus(response: Response): Promise<void
       if (text) {
         errorMessage += `: ${getErrorText(text)}`;
       }
-    } catch (error) {
+    } catch (_error) {
       // ignore error
     }
     throw new Error(errorMessage);

package/src/lib/loader-utils/get-data.ts

@@ -95,6 +95,42 @@ export async function getArrayBufferOrStringFromData(
   throw new Error(ERR_DATA);
 }
 
+/**
+ * Resolves the provided data into an {@link ArrayBuffer}, preserving bytes for worker transfer.
+ */
+export async function getArrayBufferFromData(
+  data: DataType,
+  options: LoaderOptions
+): Promise<ArrayBuffer> {
+  if (typeof data === 'string') {
+    return new TextEncoder().encode(data).buffer;
+  }
+
+  if (isArrayBufferLike(data)) {
+    return toArrayBuffer(toArrayBufferView(data));
+  }
+
+  if (isBlob(data)) {
+    data = await makeResponse(data);
+  }
+
+  if (isResponse(data)) {
+    await checkResponse(data);
+    return await data.arrayBuffer();
+  }
+
+  if (isReadableStream(data)) {
+    // @ts-expect-error TS2559 options type
+    data = makeIterator(data as ReadableStream, options);
+  }
+
+  if (isIterable(data) || isAsyncIterable(data)) {
+    return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);
+  }
+
+  throw new Error(ERR_DATA);
+}
+
 /**
  * Normalizes batchable inputs into async iterables for batch parsing flows.
  * Supports synchronous iterables, async iterables, fetch responses, readable streams, and

package/src/lib/loader-utils/get-fetch-function.ts

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 
 import type {LoaderContext, LoaderOptions, FetchLike} from '@loaders.gl/loader-utils';
-import {isObject} from '@loaders.gl/loader-utils';
+import {createAuthenticatedFetch, isObject} from '@loaders.gl/loader-utils';
 import {fetchFile} from '../fetch/fetch-file';
 import {getGlobalLoaderOptions} from './option-utils';
 
@@ -14,28 +14,30 @@ import {getGlobalLoaderOptions} from './option-utils';
  */
 export function getFetchFunction(
   options?: LoaderOptions,
-  context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>
+  context?: Omit<LoaderContext, 'fetch' | 'coreApi'> &
+    Partial<Pick<LoaderContext, 'fetch' | 'coreApi'>>
 ): FetchLike {
   const globalOptions = getGlobalLoaderOptions();
 
   const loaderOptions = options || globalOptions;
   const fetchOption = loaderOptions.fetch ?? loaderOptions.core?.fetch;
+  let fetchFunction: FetchLike;
 
   // options.fetch can be a function
   if (typeof fetchOption === 'function') {
-    return fetchOption;
+    fetchFunction = fetchOption;
+  } else if (isObject(fetchOption)) {
+    fetchFunction = context?.fetch || fetchFile;
+  } else if (context?.fetch) {
+    fetchFunction = context.fetch;
+  } else {
+    fetchFunction = fetchFile;
   }
 
-  // options.fetch can be an options object
-  if (isObject(fetchOption)) {
-    return (url) => fetchFile(url, fetchOption as RequestInit);
-  }
-
-  // else refer to context (from parent loader) if available
-  if (context?.fetch) {
-    return context?.fetch;
-  }
-
-  // else return the default fetch function
-  return fetchFile;
+  return createAuthenticatedFetch({
+    fetch: fetchFunction,
+    credentials: loaderOptions.core?.credentials || [],
+    authentications: loaderOptions.core?.authentications,
+    fetchOptions: isObject(fetchOption) ? (fetchOption as RequestInit) : undefined
+  });
 }

package/src/lib/loader-utils/loader-context.ts

@@ -6,9 +6,11 @@ import type {Loader, LoaderContext, StrictLoaderOptions} from '@loaders.gl/loade
 import {getFetchFunction} from './get-fetch-function';
 import {extractQueryString, stripQueryString} from '../utils/url-utils';
 import {path} from '@loaders.gl/loader-utils';
+import {coreApi} from '../api/core-api';
 
 /** Properties for creating an updated context */
-type LoaderContextProps = Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>;
+type LoaderContextProps = Omit<LoaderContext, 'fetch' | 'coreApi'> &
+  Partial<Pick<LoaderContext, 'fetch' | 'coreApi'>>;
 
 /**
  * "sub" loaders invoked by other loaders get a "context" injected on `this`
@@ -17,22 +19,28 @@ type LoaderContextProps = Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderCont
  *
  * @param context
  * @param options
- * @param previousContext
+ * @param parentContext
  */
 export function getLoaderContext(
   context: LoaderContextProps,
   options: StrictLoaderOptions,
   parentContext: LoaderContext | null
 ): LoaderContext {
-  // For recursive calls, we already have a context
+  // For recursive calls, preserve the context while applying this loader's options
   // TODO - add any additional loaders to context?
   if (parentContext) {
-    return parentContext;
+    return {
+      ...parentContext,
+      coreApi: parentContext.coreApi || context.coreApi || coreApi,
+      _parse: parentContext._parse || context._parse,
+      fetch: getFetchFunction(options, parentContext)
+    };
   }
 
   const newContext: LoaderContext = {
+    ...context,
     fetch: getFetchFunction(options, context),
-    ...context
+    coreApi: context.coreApi || coreApi
   };
 
   // Parse URLs so that subloaders can easily generate correct strings

package/src/lib/loader-utils/loggers.ts

@@ -6,42 +6,68 @@
 
 import {Log} from '@probe.gl/log';
 
+/** Default probe.gl logger used by loaders.gl. */
 export const probeLog = new Log({id: 'loaders.gl'});
 
 type LogFunction = () => void;
 
-// Logs nothing
+/** Logger implementation that intentionally drops all messages. */
 export class NullLog {
+  /** Drops a log message. */
   log(): LogFunction {
     return () => {};
   }
+  /** Drops an info message. */
   info(): LogFunction {
     return () => {};
   }
+  /** Logs one message once; no-op for the null logger. */
+  once(): LogFunction {
+    return () => {};
+  }
+  /** Drops a warning message. */
   warn(): LogFunction {
     return () => {};
   }
+  /** Drops an error message. */
   error(): LogFunction {
     return () => {};
   }
 }
 
-// Logs to console
+/** Logger implementation that forwards messages to the global console. */
 export class ConsoleLog {
+  /** Console-like object used for message output. */
   console;
+  /** Messages already emitted through `once`. */
+  private onceMessages = new Set<string>();
 
+  /** Creates a console-backed logger. */
   constructor() {
     this.console = console; // eslint-disable-line
   }
+  /** Logs a message to the console. */
   log(...args: unknown[]): LogFunction {
     return this.console.log.bind(this.console, ...args);
   }
+  /** Logs an informational message to the console. */
   info(...args: unknown[]): LogFunction {
     return this.console.info.bind(this.console, ...args);
   }
+  /** Logs one message once to the console. */
+  once(...args: unknown[]): LogFunction {
+    const message = String(args[0]);
+    if (this.onceMessages.has(message)) {
+      return () => {};
+    }
+    this.onceMessages.add(message);
+    return this.console.info.bind(this.console, ...args);
+  }
+  /** Logs a warning message to the console. */
   warn(...args: unknown[]): LogFunction {
     return this.console.warn.bind(this.console, ...args);
   }
+  /** Logs an error message to the console. */
   error(...args: unknown[]): LogFunction {
     return this.console.error.bind(this.console, ...args);
   }

package/src/lib/loader-utils/option-defaults.ts

@@ -7,6 +7,7 @@ import {isBrowser} from '@loaders.gl/loader-utils';
 import {ConsoleLog} from './loggers';
 
 export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
+  searchParams: undefined,
   core: {
     baseUrl: undefined,
     // baseUrl
@@ -15,11 +16,14 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
     fallbackMimeType: undefined,
     ignoreRegisteredLoaders: undefined,
     nothrow: false,
+    shape: undefined,
     log: new ConsoleLog(), // A probe.gl compatible (`log.log()()` syntax) that just logs to console
     useLocalLibraries: false,
+    subloaders: undefined,
 
     CDN: 'https://unpkg.com/@loaders.gl',
     worker: true, // By default, use worker if provided by loader.
+    workerThreshold: 0.1,
     maxConcurrency: 3, // How many worker instances should be created for each loader.
     maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
     reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
@@ -57,6 +61,7 @@ export const REMOVED_LOADER_OPTIONS = {
 
   limit: 'core.limit',
   _limitMB: 'core._limitMB',
+  shape: 'core.shape',
   batchSize: 'core.batchSize',
   batchDebounceMs: 'core.batchDebounceMs',
   metadata: 'core.metadata',

package/src/lib/loader-utils/option-utils.ts

@@ -26,19 +26,56 @@ const CORE_LOADER_OPTION_KEYS = [
   'useLocalLibraries',
   'CDN',
   'worker',
+  'workerThreshold',
   'maxConcurrency',
   'maxMobileConcurrency',
   'reuseWorkers',
+  'workerTransferBufferCopy',
   '_nodeWorkers',
   '_workerType',
   'limit',
   '_limitMB',
+  'shape',
   'batchSize',
   'batchDebounceMs',
   'metadata',
   'transforms'
 ] as const;
 
+/**
+ * Appends loader search parameters to a URL while preserving existing parameters and fragments.
+ * @param url URL to update
+ * @param options Loader options containing optional search parameters
+ * @returns URL with the configured parameters appended
+ */
+export function applySearchParamsToUrl(url: string, options?: LoaderOptions): string {
+  const searchParams = options?.searchParams;
+  if (!searchParams || Object.keys(searchParams).length === 0 || /^(?:data|blob):/i.test(url)) {
+    return url;
+  }
+
+  const hashIndex = url.indexOf('#');
+  const hash = hashIndex >= 0 ? url.slice(hashIndex) : '';
+  const urlWithoutHash = hashIndex >= 0 ? url.slice(0, hashIndex) : url;
+  const queryIndex = urlWithoutHash.indexOf('?');
+  const path = queryIndex >= 0 ? urlWithoutHash.slice(0, queryIndex) : urlWithoutHash;
+  const query = queryIndex >= 0 ? urlWithoutHash.slice(queryIndex + 1) : '';
+  const existingSearchParams = new URLSearchParams(query);
+  const appendedSearchParams = new URLSearchParams();
+  for (const [key, value] of Object.entries(searchParams)) {
+    if (!existingSearchParams.has(key)) {
+      appendedSearchParams.set(key, String(value));
+    }
+  }
+
+  const serializedAppendedSearchParams = appendedSearchParams.toString();
+  return `${path}${queryIndex >= 0 ? `?${query}` : ''}${
+    serializedAppendedSearchParams
+      ? `${queryIndex >= 0 ? '&' : '?'}${serializedAppendedSearchParams}`
+      : ''
+  }${hash}`;
+}
+
 /**
  * Global state for loaders.gl. Stored on `globalThis.loaders._state`
  */
@@ -109,7 +146,10 @@ export function normalizeOptions(
   loaders = Array.isArray(loaders) ? loaders : [loaders];
 
   validateOptions(options, loaders);
-  return normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
+  validateWorkerThreshold(options);
+  const normalizedOptions = normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
+  validateWorkerThreshold(normalizedOptions);
+  return normalizedOptions;
 }
 
 /**
@@ -157,6 +197,14 @@ function validateOptions(options: LoaderOptions, loaders: Loader[]): void {
   }
 }
 
+/** Validates the automatic worker-selection threshold. */
+function validateWorkerThreshold(options: LoaderOptions): void {
+  const threshold = options.core?.workerThreshold;
+  if (threshold !== undefined && (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)) {
+    throw new Error('core.workerThreshold must be a finite number between 0 and 1');
+  }
+}
+
 // eslint-disable-next-line max-params, complexity
 function validateOptionsObject(
   options: LoaderOptions,
@@ -235,6 +283,7 @@ function normalizeOptionsInternal(
 
   const userOptions = normalizeLoaderOptions(options);
   mergeNestedFields(mergedOptions, userOptions);
+  applyCoreShapeDefault(mergedOptions, userOptions, loader);
 
   addUrlOptions(mergedOptions, url);
   addDeprecatedTopLevelOptions(mergedOptions);
@@ -242,7 +291,50 @@ function normalizeOptionsInternal(
   return mergedOptions;
 }
 
-// Merge nested options objects
+/**
+ * Applies `options.core.shape` to a loader's scoped `shape` option when that loader declares
+ * shape support and neither global nor user-supplied loader-scoped options override it.
+ * @param mergedOptions fully merged options object being normalized
+ * @param userOptions normalized user-supplied options
+ * @param loader selected loader
+ */
+function applyCoreShapeDefault(
+  mergedOptions: LoaderOptions,
+  userOptions: LoaderOptions,
+  loader: Loader
+): void {
+  const coreShape = mergedOptions.core?.shape;
+  if (coreShape === undefined) {
+    return;
+  }
+
+  const loaderScopedDefaults = loader.options?.[loader.id];
+  if (!isPureObject(loaderScopedDefaults) || !('shape' in loaderScopedDefaults)) {
+    return;
+  }
+
+  const globalLoaderScopedOptions = getGlobalLoaderOptions()[loader.id];
+  if (isPureObject(globalLoaderScopedOptions) && 'shape' in globalLoaderScopedOptions) {
+    return;
+  }
+
+  const userLoaderScopedOptions = userOptions[loader.id];
+  if (isPureObject(userLoaderScopedOptions) && 'shape' in userLoaderScopedOptions) {
+    return;
+  }
+
+  const loaderScopedOptions = mergedOptions[loader.id];
+  mergedOptions[loader.id] = {
+    ...(isPureObject(loaderScopedOptions) ? loaderScopedOptions : {}),
+    shape: coreShape
+  };
+}
+
+/**
+ * Merges one level of nested option objects into the target options record.
+ * @param mergedOptions target options object to mutate
+ * @param options source options object
+ */
 function mergeNestedFields(mergedOptions: LoaderOptions, options: LoaderOptions): void {
   for (const key in options) {
     // Check for nested options
@@ -288,6 +380,11 @@ function cloneLoaderOptions(options: LoaderOptions): LoaderOptions {
   return clonedOptions;
 }
 
+/**
+ * Moves deprecated top-level core option aliases into `options.core` without overwriting
+ * explicitly provided `options.core` values.
+ * @param options loader options to normalize in place
+ */
 function moveDeprecatedTopLevelOptionsToCore(options: LoaderOptions): void {
   if (options.baseUri !== undefined) {
     options.core ||= {};
@@ -298,7 +395,8 @@ function moveDeprecatedTopLevelOptionsToCore(options: LoaderOptions): void {
 
   for (const key of CORE_LOADER_OPTION_KEYS) {
     if ((options as Record<string, unknown>)[key] !== undefined) {
-      const coreOptions = (options.core = options.core || {});
+      options.core ||= {};
+      const coreOptions = options.core;
       const coreRecord = coreOptions as Record<string, unknown>;
       // Treat deprecated top-level core options as aliases to `options.core`, but never override an explicitly
       // provided `options.core` value.
@@ -318,6 +416,11 @@ function moveDeprecatedTopLevelOptionsToCore(options: LoaderOptions): void {
   }
 }
 
+/**
+ * Rehydrates deprecated top-level aliases from `options.core` for compatibility with older
+ * call sites that still read normalized options from the top level.
+ * @param options normalized loader options to mutate in place
+ */
 function addDeprecatedTopLevelOptions(options: LoaderOptions): void {
   const coreOptions = options.core as Record<string, unknown> | undefined;
   if (!coreOptions) {

package/src/lib/utils/response-utils.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {isResponse} from '@loaders.gl/loader-utils';
+import {encodeArrayBufferToBase64, isResponse} from '@loaders.gl/loader-utils';
 import {FetchError} from '../fetch/fetch-error';
 import {getResourceContentLength, getResourceUrl, getResourceMIMEType} from './resource-utils';
 import {shortenUrlForDisplay} from './url-utils';
@@ -97,7 +97,7 @@ async function getResponseError(response: Response): Promise<Error> {
       !response.bodyUsed && contentType?.includes('application/json')
         ? await response.json()
         : await response.text();
-  } catch (error) {
+  } catch (_error) {
     // eslint forbids return in a finally statement, so we just catch here
   }
   return new FetchError(message, info);
@@ -112,26 +112,16 @@ async function getInitialDataUrl(
   }
   if (resource instanceof Blob) {
     const blobSlice = resource.slice(0, 5);
-    return await new Promise((resolve) => {
+    return await new Promise(resolve => {
       const reader = new FileReader();
-      reader.onload = (event) => resolve(event?.target?.result as string);
+      reader.onload = event => resolve(event?.target?.result as string);
       reader.readAsDataURL(blobSlice);
     });
   }
   if (resource instanceof ArrayBuffer) {
     const slice = resource.slice(0, INITIAL_DATA_LENGTH);
-    const base64 = arrayBufferToBase64(slice);
+    const base64 = encodeArrayBufferToBase64(slice);
     return `data:base64,${base64}`;
   }
   return null;
 }
-
-// https://stackoverflow.com/questions/9267899/arraybuffer-to-base64-encoded-string
-function arrayBufferToBase64(buffer: ArrayBuffer): string {
-  let binary = '';
-  const bytes = new Uint8Array(buffer);
-  for (let i = 0; i < bytes.byteLength; i++) {
-    binary += String.fromCharCode(bytes[i]);
-  }
-  return btoa(binary);
-}

package/src/null-loader.ts

@@ -2,13 +2,13 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import {LoaderContext} from '@loaders.gl/loader-utils';
+
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import {LoaderContext} from '@loaders.gl/loader-utils';
-
 export type NullLoaderOptions = LoaderOptions & {
   null?: {};
 };